diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 9ad8b073b..26c09f65f 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* FEATURE: allow omitting `stats` prefix in [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe). For example, `_time:5m | count() rows` is a valid query now. It is equivalent to `_time:5m | stats count() as rows`. +* FEATURE: allow omitting `filter` prefix in [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe) if the filter doesn't clash with [pipe names](#https://docs.victoriametrics.com/victorialogs/logsql/#pipes). For example, `_time:5m | stats by (host) count() rows | rows:>1000` is a valid query now. It is equivalent to `_time:5m | stats by (host) count() rows | filter rows:>1000`. * FEATURE: allow [`head` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#limit-pipe) without number. For example, `error | head`. In this case 10 last values are returned as `head` Unix command does by default. * FEATURE: allow using [comparison filters](https://docs.victoriametrics.com/victorialogs/logsql/#range-comparison-filters) with strings. For example, `some_text_field:>="foo"` matches [log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) with `some_text_field` field values bigger or equal to `foo`. diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 3a8c41da6..6a09f1521 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1391,8 +1391,7 @@ See also: ### filter pipe -Sometimes it is needed to apply additional filters on the calculated results. This can be done with `| filter ...` [pipe](#pipes). -The `filter` pipe can contain arbitrary [filters](#filters). +The `| filter ...` [pipe](#pipes) allows filtering the selected logs entries with arbitrary [filters](#filters). For example, the following query returns `host` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) values if the number of log messages with the `error` [word](#word) for them over the last hour exceeds `1_000`: @@ -1401,6 +1400,13 @@ if the number of log messages with the `error` [word](#word) for them over the l _time:1h error | stats by (host) count() logs_count | filter logs_count:> 1_000 ``` +It is allowed to omit `filter` prefix if the used filters do not clash with [pipe names](#pipes). +So the following query is equivalent to the previous one: + +```logsql +_time:1h error | stats by (host) count() logs_count | logs_count:> 1_000 +``` + See also: - [`stats` pipe](#stats-pipe) @@ -1761,6 +1767,12 @@ For example, the following query calculates the following stats for logs over th _time:5m | stats count() logs_total, count_uniq(_stream) streams_total ``` +It is allowed to omit `stats` prefix for convenience. So the following query is equivalent to the previous one: + +```logsql +_time:5m | count() logs_total, count_uniq(_stream) streams_total +``` + See also: - [stats by fields](#stats-by-fields) diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 5ea4e164e..87e4ecc00 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -252,7 +252,7 @@ func (q *Query) AddCountByTimePipe(step, off int64, fields []string) { s := fmt.Sprintf("stats by (%s) count() hits", byFieldsStr) lex := newLexer(s) - ps, err := parsePipeStats(lex) + ps, err := parsePipeStats(lex, true) if err != nil { logger.Panicf("BUG: unexpected error when parsing [%s]: %s", s, err) } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index e4ec7cc0d..e7e773a7d 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -1094,7 +1094,7 @@ func TestParseQueryFailure(t *testing.T) { f("") f("|") f("foo|") - f("foo|bar") + f("foo|bar(") f("foo and") f("foo OR ") f("not") @@ -1163,7 +1163,7 @@ func TestParseQueryFailure(t *testing.T) { f(`very long query with error aaa ffdfd fdfdfd fdfd:( ffdfdfdfdfd`) // query with unexpected tail - f(`foo | bar`) + f(`foo | bar(`) // unexpected comma f(`foo,bar`) @@ -1284,9 +1284,9 @@ func TestParseQueryFailure(t *testing.T) { // missing pipe keyword f(`foo |`) - // unknown pipe keyword - f(`foo | bar`) - f(`foo | fields bar | baz`) + // invlaid pipe + f(`foo | bar(`) + f(`foo | fields bar | baz(`) // missing field in fields pipe f(`foo | fields`) diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go index d01fdc4be..464c4e5c2 100644 --- a/lib/logstorage/pipe.go +++ b/lib/logstorage/pipe.go @@ -86,6 +86,8 @@ func parsePipes(lex *lexer) ([]pipe, error) { lex.nextToken() case lex.isKeyword(")", ""): return pipes, nil + default: + return nil, fmt.Errorf("unexpected token after [%s]: %q; expecting '|' or ')'", pipes[len(pipes)-1], lex.token) } } } @@ -123,7 +125,7 @@ func parsePipe(lex *lexer) (pipe, error) { } return pf, nil case lex.isKeyword("filter"): - pf, err := parsePipeFilter(lex) + pf, err := parsePipeFilter(lex, true) if err != nil { return nil, fmt.Errorf("cannot parse 'filter' pipe: %w", err) } @@ -177,7 +179,7 @@ func parsePipe(lex *lexer) (pipe, error) { } return ps, nil case lex.isKeyword("stats"): - ps, err := parsePipeStats(lex) + ps, err := parsePipeStats(lex, true) if err != nil { return nil, fmt.Errorf("cannot parse 'stats' pipe: %w", err) } @@ -207,6 +209,22 @@ func parsePipe(lex *lexer) (pipe, error) { } return pu, nil default: + lexState := lex.backupState() + + // Try parsing stats pipe without 'stats' keyword + ps, err := parsePipeStats(lex, false) + if err == nil { + return ps, nil + } + lex.restoreState(lexState) + + // Try parsing filter pipe without 'filter' keyword + pf, err := parsePipeFilter(lex, false) + if err == nil { + return pf, nil + } + lex.restoreState(lexState) + return nil, fmt.Errorf("unexpected pipe %q", lex.token) } } diff --git a/lib/logstorage/pipe_filter.go b/lib/logstorage/pipe_filter.go index daba8f5bc..c1f418f60 100644 --- a/lib/logstorage/pipe_filter.go +++ b/lib/logstorage/pipe_filter.go @@ -108,11 +108,13 @@ func (pfp *pipeFilterProcessor) flush() error { return nil } -func parsePipeFilter(lex *lexer) (*pipeFilter, error) { - if !lex.isKeyword("filter") { - return nil, fmt.Errorf("expecting 'filter'; got %q", lex.token) +func parsePipeFilter(lex *lexer, needFilterKeyword bool) (*pipeFilter, error) { + if needFilterKeyword { + if !lex.isKeyword("filter") { + return nil, fmt.Errorf("expecting 'filter'; got %q", lex.token) + } + lex.nextToken() } - lex.nextToken() f, err := parseFilter(lex) if err != nil { diff --git a/lib/logstorage/pipe_filter_test.go b/lib/logstorage/pipe_filter_test.go index 0c3183019..dc244ffb0 100644 --- a/lib/logstorage/pipe_filter_test.go +++ b/lib/logstorage/pipe_filter_test.go @@ -32,6 +32,14 @@ func TestPipeFilter(t *testing.T) { expectPipeResults(t, pipeStr, rows, rowsExpected) } + // filter mismatch, missing 'filter' prefix + f("abc", [][]Field{ + { + {"_msg", `{"foo":"bar"}`}, + {"a", `test`}, + }, + }, [][]Field{}) + // filter mismatch f("filter abc", [][]Field{ { @@ -40,6 +48,19 @@ func TestPipeFilter(t *testing.T) { }, }, [][]Field{}) + // filter match, missing 'filter' prefix + f("foo", [][]Field{ + { + {"_msg", `{"foo":"bar"}`}, + {"a", `test`}, + }, + }, [][]Field{ + { + {"_msg", `{"foo":"bar"}`}, + {"a", `test`}, + }, + }) + // filter match f("filter foo", [][]Field{ { diff --git a/lib/logstorage/pipe_rename.go b/lib/logstorage/pipe_rename.go index 44ded34ac..6911413fc 100644 --- a/lib/logstorage/pipe_rename.go +++ b/lib/logstorage/pipe_rename.go @@ -62,7 +62,7 @@ func (pr *pipeRename) hasFilterInWithQuery() bool { return false } -func (pr *pipeRename) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pr *pipeRename) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pr, nil } diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index 90fdb0c73..ab3852e95 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -537,13 +537,14 @@ func (psp *pipeStatsProcessor) flush() error { return nil } -func parsePipeStats(lex *lexer) (*pipeStats, error) { - if !lex.isKeyword("stats") { - return nil, fmt.Errorf("expecting 'stats'; got %q", lex.token) +func parsePipeStats(lex *lexer, needStatsKeyword bool) (*pipeStats, error) { + if needStatsKeyword { + if !lex.isKeyword("stats") { + return nil, fmt.Errorf("expecting 'stats'; got %q", lex.token) + } + lex.nextToken() } - lex.nextToken() - var ps pipeStats if lex.isKeyword("by", "(") { if lex.isKeyword("by") { diff --git a/lib/logstorage/pipe_stats_test.go b/lib/logstorage/pipe_stats_test.go index 0d2cdd4c9..5363518db 100644 --- a/lib/logstorage/pipe_stats_test.go +++ b/lib/logstorage/pipe_stats_test.go @@ -39,6 +39,70 @@ func TestPipeStats(t *testing.T) { expectPipeResults(t, pipeStr, rows, rowsExpected) } + // missing 'stats' keyword + f("count(*) as rows", [][]Field{ + { + {"_msg", `abc`}, + {"a", `2`}, + {"b", `3`}, + }, + { + {"_msg", `def`}, + {"a", `1`}, + }, + { + {"a", `2`}, + {"b", `54`}, + }, + }, [][]Field{ + { + {"rows", "3"}, + }, + }) + + // missing 'stats' keyword + f("count() as rows, count() if (a:2) rows2", [][]Field{ + { + {"_msg", `abc`}, + {"a", `2`}, + {"b", `3`}, + }, + { + {"_msg", `def`}, + {"a", `1`}, + }, + { + {"a", `2`}, + {"b", `54`}, + }, + }, [][]Field{ + { + {"rows", "3"}, + {"rows2", "2"}, + }, + }) + + f("stats count() as rows, count() if (a:2) rows2", [][]Field{ + { + {"_msg", `abc`}, + {"a", `2`}, + {"b", `3`}, + }, + { + {"_msg", `def`}, + {"a", `1`}, + }, + { + {"a", `2`}, + {"b", `54`}, + }, + }, [][]Field{ + { + {"rows", "3"}, + {"rows2", "2"}, + }, + }) + f("stats count(*) as rows", [][]Field{ { {"_msg", `abc`}, @@ -141,6 +205,32 @@ func TestPipeStats(t *testing.T) { }, }) + // missing 'stats' keyword + f("by (a) count(*) as rows", [][]Field{ + { + {"_msg", `abc`}, + {"a", `2`}, + {"b", `3`}, + }, + { + {"_msg", `def`}, + {"a", `1`}, + }, + { + {"a", `2`}, + {"b", `54`}, + }, + }, [][]Field{ + { + {"a", "1"}, + {"rows", "1"}, + }, + { + {"a", "2"}, + {"rows", "2"}, + }, + }) + f("stats by (a) count(*) as rows", [][]Field{ { {"_msg", `abc`},