mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-31 15:06:26 +00:00
wip
This commit is contained in:
parent
33a01c659b
commit
edd493431e
3 changed files with 14 additions and 14 deletions
|
@ -1199,19 +1199,19 @@ _time:5m | sort by (foo, bar) desc
|
||||||
```
|
```
|
||||||
|
|
||||||
Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest
|
Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest
|
||||||
or smallest values. This can be done by adding ``first N` to the end of `sort ...` pipe.
|
or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe.
|
||||||
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
|
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
|
||||||
For example, the following query returns top 10 log entries with the biggest values
|
For example, the following query returns top 10 log entries with the biggest values
|
||||||
for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour:
|
for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
_time:1h | sort by (request_duration desc) first 10
|
_time:1h | sort by (request_duration desc) limit 10
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that sorting of big number of logs can be slow and can consume a lot of additional memory.
|
Note that sorting of big number of logs can be slow and can consume a lot of additional memory.
|
||||||
It is recommended limiting the number of logs before sorting with the following approaches:
|
It is recommended limiting the number of logs before sorting with the following approaches:
|
||||||
|
|
||||||
- Adding `first N` to the end of `sort ...` pipe.
|
- Adding `limit N` to the end of `sort ...` pipe.
|
||||||
- Reducing the selected time range with [time filter](#time-filter).
|
- Reducing the selected time range with [time filter](#time-filter).
|
||||||
- Using more specific [filters](#filters), so they select less logs.
|
- Using more specific [filters](#filters), so they select less logs.
|
||||||
- Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe).
|
- Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe).
|
||||||
|
|
|
@ -974,10 +974,10 @@ func TestParseQuerySuccess(t *testing.T) {
|
||||||
f(`* | sort bY (foo)`, `* | sort by (foo)`)
|
f(`* | sort bY (foo)`, `* | sort by (foo)`)
|
||||||
f(`* | sORt bY (_time, _stream DEsc, host)`, `* | sort by (_time, _stream desc, host)`)
|
f(`* | sORt bY (_time, _stream DEsc, host)`, `* | sort by (_time, _stream desc, host)`)
|
||||||
f(`* | sort bY (foo desc, bar,) desc`, `* | sort by (foo desc, bar) desc`)
|
f(`* | sort bY (foo desc, bar,) desc`, `* | sort by (foo desc, bar) desc`)
|
||||||
f(`* | sort first 10`, `* | sort first 10`)
|
f(`* | sort limit 10`, `* | sort limit 10`)
|
||||||
f(`* | sort desc first 10`, `* | sort desc first 10`)
|
f(`* | sort desc limit 10`, `* | sort desc limit 10`)
|
||||||
f(`* | sort by (foo desc, bar) first 10`, `* | sort by (foo desc, bar) first 10`)
|
f(`* | sort by (foo desc, bar) limit 10`, `* | sort by (foo desc, bar) limit 10`)
|
||||||
f(`* | sort by (foo desc, bar) desc first 10`, `* | sort by (foo desc, bar) desc first 10`)
|
f(`* | sort by (foo desc, bar) desc limit 10`, `* | sort by (foo desc, bar) desc limit 10`)
|
||||||
|
|
||||||
// uniq pipe
|
// uniq pipe
|
||||||
f(`* | uniq`, `* | uniq`)
|
f(`* | uniq`, `* | uniq`)
|
||||||
|
@ -1334,10 +1334,10 @@ func TestParseQueryFailure(t *testing.T) {
|
||||||
f(`foo | sort by(baz`)
|
f(`foo | sort by(baz`)
|
||||||
f(`foo | sort by(baz,`)
|
f(`foo | sort by(baz,`)
|
||||||
f(`foo | sort by(bar) foo`)
|
f(`foo | sort by(bar) foo`)
|
||||||
f(`foo | sort by(bar) first`)
|
f(`foo | sort by(bar) limit`)
|
||||||
f(`foo | sort by(bar) first foo`)
|
f(`foo | sort by(bar) limit foo`)
|
||||||
f(`foo | sort by(bar) first -1234`)
|
f(`foo | sort by(bar) limit -1234`)
|
||||||
f(`foo | sort by(bar) first 12.34`)
|
f(`foo | sort by(bar) limit 12.34`)
|
||||||
|
|
||||||
// invalid uniq pipe
|
// invalid uniq pipe
|
||||||
f(`foo | uniq bar`)
|
f(`foo | uniq bar`)
|
||||||
|
|
|
@ -45,7 +45,7 @@ func (ps *pipeSort) String() string {
|
||||||
s += " desc"
|
s += " desc"
|
||||||
}
|
}
|
||||||
if ps.limit > 0 {
|
if ps.limit > 0 {
|
||||||
s += fmt.Sprintf(" first %d", ps.limit)
|
s += fmt.Sprintf(" limit %d", ps.limit)
|
||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
@ -689,12 +689,12 @@ func parsePipeSort(lex *lexer) (*pipeSort, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case lex.isKeyword("first"):
|
case lex.isKeyword("limit"):
|
||||||
lex.nextToken()
|
lex.nextToken()
|
||||||
n, ok := tryParseUint64(lex.token)
|
n, ok := tryParseUint64(lex.token)
|
||||||
lex.nextToken()
|
lex.nextToken()
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("cannot parse 'first %s'", lex.token)
|
return nil, fmt.Errorf("cannot parse 'limit %s'", lex.token)
|
||||||
}
|
}
|
||||||
ps.limit = n
|
ps.limit = n
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue