This commit is contained in:
Aliaksandr Valialkin 2024-05-15 02:52:23 +02:00
parent 33a01c659b
commit edd493431e
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
3 changed files with 14 additions and 14 deletions

View file

@ -1199,19 +1199,19 @@ _time:5m | sort by (foo, bar) desc
``` ```
Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest
or smallest values. This can be done by adding ``first N` to the end of `sort ...` pipe. or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe.
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries. Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
For example, the following query returns top 10 log entries with the biggest values For example, the following query returns top 10 log entries with the biggest values
for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour: for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour:
```logsql ```logsql
_time:1h | sort by (request_duration desc) first 10 _time:1h | sort by (request_duration desc) limit 10
``` ```
Note that sorting of big number of logs can be slow and can consume a lot of additional memory. Note that sorting of big number of logs can be slow and can consume a lot of additional memory.
It is recommended limiting the number of logs before sorting with the following approaches: It is recommended limiting the number of logs before sorting with the following approaches:
- Adding `first N` to the end of `sort ...` pipe. - Adding `limit N` to the end of `sort ...` pipe.
- Reducing the selected time range with [time filter](#time-filter). - Reducing the selected time range with [time filter](#time-filter).
- Using more specific [filters](#filters), so they select less logs. - Using more specific [filters](#filters), so they select less logs.
- Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe). - Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe).

View file

@ -974,10 +974,10 @@ func TestParseQuerySuccess(t *testing.T) {
f(`* | sort bY (foo)`, `* | sort by (foo)`) f(`* | sort bY (foo)`, `* | sort by (foo)`)
f(`* | sORt bY (_time, _stream DEsc, host)`, `* | sort by (_time, _stream desc, host)`) f(`* | sORt bY (_time, _stream DEsc, host)`, `* | sort by (_time, _stream desc, host)`)
f(`* | sort bY (foo desc, bar,) desc`, `* | sort by (foo desc, bar) desc`) f(`* | sort bY (foo desc, bar,) desc`, `* | sort by (foo desc, bar) desc`)
f(`* | sort first 10`, `* | sort first 10`) f(`* | sort limit 10`, `* | sort limit 10`)
f(`* | sort desc first 10`, `* | sort desc first 10`) f(`* | sort desc limit 10`, `* | sort desc limit 10`)
f(`* | sort by (foo desc, bar) first 10`, `* | sort by (foo desc, bar) first 10`) f(`* | sort by (foo desc, bar) limit 10`, `* | sort by (foo desc, bar) limit 10`)
f(`* | sort by (foo desc, bar) desc first 10`, `* | sort by (foo desc, bar) desc first 10`) f(`* | sort by (foo desc, bar) desc limit 10`, `* | sort by (foo desc, bar) desc limit 10`)
// uniq pipe // uniq pipe
f(`* | uniq`, `* | uniq`) f(`* | uniq`, `* | uniq`)
@ -1334,10 +1334,10 @@ func TestParseQueryFailure(t *testing.T) {
f(`foo | sort by(baz`) f(`foo | sort by(baz`)
f(`foo | sort by(baz,`) f(`foo | sort by(baz,`)
f(`foo | sort by(bar) foo`) f(`foo | sort by(bar) foo`)
f(`foo | sort by(bar) first`) f(`foo | sort by(bar) limit`)
f(`foo | sort by(bar) first foo`) f(`foo | sort by(bar) limit foo`)
f(`foo | sort by(bar) first -1234`) f(`foo | sort by(bar) limit -1234`)
f(`foo | sort by(bar) first 12.34`) f(`foo | sort by(bar) limit 12.34`)
// invalid uniq pipe // invalid uniq pipe
f(`foo | uniq bar`) f(`foo | uniq bar`)

View file

@ -45,7 +45,7 @@ func (ps *pipeSort) String() string {
s += " desc" s += " desc"
} }
if ps.limit > 0 { if ps.limit > 0 {
s += fmt.Sprintf(" first %d", ps.limit) s += fmt.Sprintf(" limit %d", ps.limit)
} }
return s return s
} }
@ -689,12 +689,12 @@ func parsePipeSort(lex *lexer) (*pipeSort, error) {
} }
switch { switch {
case lex.isKeyword("first"): case lex.isKeyword("limit"):
lex.nextToken() lex.nextToken()
n, ok := tryParseUint64(lex.token) n, ok := tryParseUint64(lex.token)
lex.nextToken() lex.nextToken()
if !ok { if !ok {
return nil, fmt.Errorf("cannot parse 'first %s'", lex.token) return nil, fmt.Errorf("cannot parse 'limit %s'", lex.token)
} }
ps.limit = n ps.limit = n
} }