This commit is contained in:
Aliaksandr Valialkin 2024-05-15 02:52:23 +02:00
parent 33a01c659b
commit edd493431e
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
3 changed files with 14 additions and 14 deletions

View file

@ -1199,19 +1199,19 @@ _time:5m | sort by (foo, bar) desc
```
Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest
or smallest values. This can be done by adding ``first N` to the end of `sort ...` pipe.
or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe.
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
For example, the following query returns top 10 log entries with the biggest values
for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour:
```logsql
_time:1h | sort by (request_duration desc) first 10
_time:1h | sort by (request_duration desc) limit 10
```
Note that sorting of big number of logs can be slow and can consume a lot of additional memory.
It is recommended limiting the number of logs before sorting with the following approaches:
- Adding `first N` to the end of `sort ...` pipe.
- Adding `limit N` to the end of `sort ...` pipe.
- Reducing the selected time range with [time filter](#time-filter).
- Using more specific [filters](#filters), so they select less logs.
- Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe).

View file

@ -974,10 +974,10 @@ func TestParseQuerySuccess(t *testing.T) {
f(`* | sort bY (foo)`, `* | sort by (foo)`)
f(`* | sORt bY (_time, _stream DEsc, host)`, `* | sort by (_time, _stream desc, host)`)
f(`* | sort bY (foo desc, bar,) desc`, `* | sort by (foo desc, bar) desc`)
f(`* | sort first 10`, `* | sort first 10`)
f(`* | sort desc first 10`, `* | sort desc first 10`)
f(`* | sort by (foo desc, bar) first 10`, `* | sort by (foo desc, bar) first 10`)
f(`* | sort by (foo desc, bar) desc first 10`, `* | sort by (foo desc, bar) desc first 10`)
f(`* | sort limit 10`, `* | sort limit 10`)
f(`* | sort desc limit 10`, `* | sort desc limit 10`)
f(`* | sort by (foo desc, bar) limit 10`, `* | sort by (foo desc, bar) limit 10`)
f(`* | sort by (foo desc, bar) desc limit 10`, `* | sort by (foo desc, bar) desc limit 10`)
// uniq pipe
f(`* | uniq`, `* | uniq`)
@ -1334,10 +1334,10 @@ func TestParseQueryFailure(t *testing.T) {
f(`foo | sort by(baz`)
f(`foo | sort by(baz,`)
f(`foo | sort by(bar) foo`)
f(`foo | sort by(bar) first`)
f(`foo | sort by(bar) first foo`)
f(`foo | sort by(bar) first -1234`)
f(`foo | sort by(bar) first 12.34`)
f(`foo | sort by(bar) limit`)
f(`foo | sort by(bar) limit foo`)
f(`foo | sort by(bar) limit -1234`)
f(`foo | sort by(bar) limit 12.34`)
// invalid uniq pipe
f(`foo | uniq bar`)

View file

@ -45,7 +45,7 @@ func (ps *pipeSort) String() string {
s += " desc"
}
if ps.limit > 0 {
s += fmt.Sprintf(" first %d", ps.limit)
s += fmt.Sprintf(" limit %d", ps.limit)
}
return s
}
@ -689,12 +689,12 @@ func parsePipeSort(lex *lexer) (*pipeSort, error) {
}
switch {
case lex.isKeyword("first"):
case lex.isKeyword("limit"):
lex.nextToken()
n, ok := tryParseUint64(lex.token)
lex.nextToken()
if !ok {
return nil, fmt.Errorf("cannot parse 'first %s'", lex.token)
return nil, fmt.Errorf("cannot parse 'limit %s'", lex.token)
}
ps.limit = n
}