From d50273d6c0c103da2a457232a7410a4e674893ba Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 15 May 2024 22:31:21 +0200 Subject: [PATCH] wip --- docs/VictoriaLogs/LogsQL.md | 18 ++++++++++++++++++ lib/logstorage/parser_test.go | 3 +++ lib/logstorage/pipe_sort.go | 6 ++++-- lib/logstorage/pipe_stats.go | 6 ++++-- lib/logstorage/pipe_uniq.go | 6 ++++-- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 4418248ed..700cb130b 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1200,6 +1200,12 @@ The reverse order can be applied globally via `desc` keyword after `by(...)` cla _time:5m | sort by (foo, bar) desc ``` +The `by` keyword can be skipped in `sort ...` pipe. For example, the following query is equivalent to the previous one: + +```logsql +_time:5m | sort (foo, bar) desc +``` + Sorting of big number of logs can consume a lot of CPU time and memory. Sometimes it is enough to return the first `N` entries with the biggest or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe. Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries. @@ -1257,6 +1263,12 @@ This allows limiting memory usage. For example, the following query returns up t _time:5m | uniq by (host, path) limit 100 ``` +The `by` keyword can be skipped in `uniq ...` pipe. For example, the following query is equivalent to the previous one: + +```logsql +_time:5m | uniq (host, path) limit 100 +``` + See also: - [`uniq_values` stats function](#uniq_values-stats) @@ -1316,6 +1328,12 @@ grouped by `(host, path)` fields: _time:5m | stats by (host, path) count() logs_total, count_uniq(ip) ips_total ``` +The `by` keyword can be skipped in `stats ...` pipe. For example, the following query is equvalent to the previous one: + +```logsql +_time:5m | stats (host, path) count() logs_total, count_uniq(ip) ips_total +``` + #### Stats by time buckets The following syntax can be used for calculating stats grouped by time buckets: diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 1f0289e58..2a021908a 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -966,6 +966,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`* | stats by (_time:week) count() foo`, `* | stats by (_time:week) count(*) as foo`) f(`* | stats by (_time:month) count() foo`, `* | stats by (_time:month) count(*) as foo`) f(`* | stats by (_time:year offset 6.5h) count() foo`, `* | stats by (_time:year offset 6.5h) count(*) as foo`) + f(`* | stats (_time:year offset 6.5h) count() foo`, `* | stats by (_time:year offset 6.5h) count(*) as foo`) // sort pipe f(`* | sort`, `* | sort`) @@ -983,6 +984,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`* | sort by (foo desc, bar) desc limit 10`, `* | sort by (foo desc, bar) desc limit 10`) f(`* | sort by (foo desc, bar) desc OFFSET 30 limit 10`, `* | sort by (foo desc, bar) desc offset 30 limit 10`) f(`* | sort by (foo desc, bar) desc limit 10 OFFSET 30`, `* | sort by (foo desc, bar) desc offset 30 limit 10`) + f(`* | sort (foo desc, bar) desc limit 10 OFFSET 30`, `* | sort by (foo desc, bar) desc offset 30 limit 10`) // uniq pipe f(`* | uniq`, `* | uniq`) @@ -991,6 +993,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`* | uniq by(foo,*,bar)`, `* | uniq`) f(`* | uniq by(f1,f2)`, `* | uniq by (f1, f2)`) f(`* | uniq by(f1,f2) limit 10`, `* | uniq by (f1, f2) limit 10`) + f(`* | uniq (f1,f2) limit 10`, `* | uniq by (f1, f2) limit 10`) f(`* | uniq limit 10`, `* | uniq limit 10`) // multiple different pipes diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go index 55b74b5dd..f1bcc522a 100644 --- a/lib/logstorage/pipe_sort.go +++ b/lib/logstorage/pipe_sort.go @@ -689,8 +689,10 @@ func parsePipeSort(lex *lexer) (*pipeSort, error) { lex.nextToken() var ps pipeSort - if lex.isKeyword("by") { - lex.nextToken() + if lex.isKeyword("by", "(") { + if lex.isKeyword("by") { + lex.nextToken() + } bfs, err := parseBySortFields(lex) if err != nil { return nil, fmt.Errorf("cannot parse 'by' clause: %w", err) diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index efb5a9cfd..62b3def5b 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -443,8 +443,10 @@ func parsePipeStats(lex *lexer) (*pipeStats, error) { lex.nextToken() var ps pipeStats - if lex.isKeyword("by") { - lex.nextToken() + if lex.isKeyword("by", "(") { + if lex.isKeyword("by") { + lex.nextToken() + } bfs, err := parseByStatsFields(lex) if err != nil { return nil, fmt.Errorf("cannot parse 'by' clause: %w", err) diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go index 3b72aaac2..3b2fcc66d 100644 --- a/lib/logstorage/pipe_uniq.go +++ b/lib/logstorage/pipe_uniq.go @@ -360,8 +360,10 @@ func parsePipeUniq(lex *lexer) (*pipeUniq, error) { lex.nextToken() var pu pipeUniq - if lex.isKeyword("by") { - lex.nextToken() + if lex.isKeyword("by", "(") { + if lex.isKeyword("by") { + lex.nextToken() + } bfs, err := parseFieldNamesInParens(lex) if err != nil { return nil, fmt.Errorf("cannot parse 'by' clause: %w", err)