From a4337149a24321e0ad2becbc70ee4e54ee0c2253 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 23 May 2024 12:01:15 +0200 Subject: [PATCH] wip --- docs/VictoriaLogs/CHANGELOG.md | 2 ++ docs/VictoriaLogs/LogsQL.md | 24 +++++++++++++++++++++++- lib/logstorage/filter_regexp.go | 2 +- lib/logstorage/filter_stream.go | 6 +----- lib/logstorage/parser.go | 26 ++++++++++++++++++++++++++ lib/logstorage/parser_test.go | 24 +++++++++++++++++------- 6 files changed, 70 insertions(+), 14 deletions(-) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 0e4ee919d..2c0d1903a 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta ## tip +* BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter). + ## [v0.9.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.9.1-victorialogs) Released at 2024-05-22 diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index dee882f90..af3dff3cc 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -403,6 +403,13 @@ This query doesn't match the following log messages: - `SSH: login fail`, since the `SSH` word is in capital letters. Use `i("ssh: login fail")` for case-insensitive search. See [these docs](#case-insensitive-filter) for details. +If the phrase contains double quotes, then either put `\` in front of double quotes or put the phrase inside single quotes. For example, the following filter searches +logs with `"foo":"bar"` phrase: + +```logsql +'"foo":"bar"' +``` + By default the given phrase is searched in the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field). Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the phrase and put a colon after it if it must be searched in the given field. For example, the following query returns log entries containing the `cannot open file` phrase in the `event.original` field: @@ -470,6 +477,13 @@ This query doesn't match the following log messages: - `failed to open file: unexpected EOF`, since `failed` [word](#word) occurs before the `unexpected` word. Use `unexpected AND fail*` for this case. See [these docs](#logical-filter) for details. +If the prefix contains double quotes, then either put `\` in front of double quotes or put the prefix inside single quotes. For example, the following filter searches +logs with `"foo":"bar` prefix: + +```logsql +'"foo":"bar'* +``` + By default the prefix filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field). Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the prefix filter in order to apply it to the given field. For example, the following query matches `log.level` field containing any word with the `err` prefix: @@ -783,6 +797,13 @@ The query doesn't match the following log messages: See [these docs](https://github.com/google/re2/wiki/Syntax) for details. See also [case-insenstive filter docs](#case-insensitive-filter). - `it is warmer than usual`, since it doesn't contain neither `err` nor `warn` substrings. +If the regexp contains double quotes, then either put `\` in front of double quotes or put the regexp inside single quotes. For example, the following regexp searches +logs matching `"foo":"(bar|baz)"` regexp: + +```logsql +'"foo":"(bar|baz)"' +``` + By default the regexp filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field). Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the filter in order to apply it to the given field. For example, the following query matches `event.original` field containing either `err` or `warn` substrings: @@ -1134,7 +1155,8 @@ For example, the following query is equivalent to the previous one: _time:1d error | extract "ip= " | stats by (ip) count() logs | sort by (logs) desc limit 10 ``` -If the `pattern` contains double quotes, then it can be quoted into single quotes. For example, the following query extracts `ip` from the corresponding JSON field: +If the `pattern` contains double quotes, then either put `\` in front of double quotes or put the `pattern` inside single quotes. +For example, the following query extracts `ip` from the corresponding JSON field: ```logsql _time:5m | extract '"ip":""' diff --git a/lib/logstorage/filter_regexp.go b/lib/logstorage/filter_regexp.go index 4d8ad0245..adfb49337 100644 --- a/lib/logstorage/filter_regexp.go +++ b/lib/logstorage/filter_regexp.go @@ -16,7 +16,7 @@ type filterRegexp struct { } func (fr *filterRegexp) String() string { - return fmt.Sprintf("%s~%q", quoteFieldNameIfNeeded(fr.fieldName), fr.re.String()) + return fmt.Sprintf("%s~%s", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.re.String())) } func (fr *filterRegexp) updateNeededFields(neededFields fieldsSet) { diff --git a/lib/logstorage/filter_stream.go b/lib/logstorage/filter_stream.go index 73aa49367..2359d7f36 100644 --- a/lib/logstorage/filter_stream.go +++ b/lib/logstorage/filter_stream.go @@ -22,11 +22,7 @@ type filterStream struct { } func (fs *filterStream) String() string { - s := fs.f.String() - if s == "{}" { - return "" - } - return "_stream:" + s + return "_stream:" + fs.f.String() } func (fs *filterStream) updateNeededFields(neededFields fieldsSet) { diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 49b677b68..4a62f1889 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -597,8 +597,12 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) { return parseFilterLT(lex, fieldName) case lex.isKeyword("="): return parseFilterEQ(lex, fieldName) + case lex.isKeyword("!="): + return parseFilterNEQ(lex, fieldName) case lex.isKeyword("~"): return parseFilterTilda(lex, fieldName) + case lex.isKeyword("!~"): + return parseFilterNotTilda(lex, fieldName) case lex.isKeyword("not", "!"): return parseFilterNot(lex, fieldName) case lex.isKeyword("exact"): @@ -1033,6 +1037,17 @@ func parseFilterTilda(lex *lexer, fieldName string) (filter, error) { return fr, nil } +func parseFilterNotTilda(lex *lexer, fieldName string) (filter, error) { + f, err := parseFilterTilda(lex, fieldName) + if err != nil { + return nil, err + } + fn := &filterNot{ + f: f, + } + return fn, nil +} + func parseFilterEQ(lex *lexer, fieldName string) (filter, error) { lex.nextToken() phrase := getCompoundFuncArg(lex) @@ -1051,6 +1066,17 @@ func parseFilterEQ(lex *lexer, fieldName string) (filter, error) { return f, nil } +func parseFilterNEQ(lex *lexer, fieldName string) (filter, error) { + f, err := parseFilterEQ(lex, fieldName) + if err != nil { + return nil, err + } + fn := &filterNot{ + f: f, + } + return fn, nil +} + func parseFilterGT(lex *lexer, fieldName string) (filter, error) { lex.nextToken() diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 824133796..0efde5c88 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -544,6 +544,16 @@ func TestParseQuerySuccess(t *testing.T) { if result != resultExpected { t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected) } + + // verify that the marshaled query is parsed to the same query + qParsed, err := ParseQuery(result) + if err != nil { + t.Fatalf("cannot parse marshaled query: %s", err) + } + qStr := qParsed.String() + if qStr != result { + t.Fatalf("unexpected marshaled query\ngot\n%s\nwant\n%s", qStr, result) + } } f("foo", "foo") @@ -586,7 +596,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`foo:(bar baz or not :xxx)`, `foo:bar foo:baz or !foo:xxx`) f(`(foo:bar and (foo:baz or aa:bb) and xx) and y`, `foo:bar (foo:baz or aa:bb) xx y`) f("level:error and _msg:(a or b)", "level:error (a or b)") - f("level: ( ((error or warn*) and re(foo))) (not (bar))", `(level:error or level:warn*) level:~"foo" !bar`) + f("level: ( ((error or warn*) and re(foo))) (not (bar))", `(level:error or level:warn*) level:~foo !bar`) f("!(foo bar or baz and not aa*)", `!(foo bar or baz !aa*)`) // prefix search @@ -600,7 +610,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`"" or foo:"" and not bar:""`, `"" or foo:"" !bar:""`) // _stream filters - f(`_stream:{}`, ``) + f(`_stream:{}`, `_stream:{}`) f(`_stream:{foo="bar", baz=~"x" OR or!="b", "x=},"="d}{"}`, `_stream:{foo="bar",baz=~"x" or "or"!="b","x=},"="d}{"}`) f(`_stream:{or=a or ","="b"}`, `_stream:{"or"="a" or ","="b"}`) f("_stream : { foo = bar , } ", `_stream:{foo="bar"}`) @@ -713,7 +723,7 @@ func TestParseQuerySuccess(t *testing.T) { f(`exact("foo/bar")`, `="foo/bar"`) f(`exact('foo/bar')`, `="foo/bar"`) f(`="foo/bar"`, `="foo/bar"`) - f("=foo=bar =b<=a>z ='abc'*", `="foo=bar" ="b<=a>z" =abc*`) + f("=foo=bar !=b<=a>z foo:!='abc'*", `="foo=bar" !="b<=a>z" !foo:=abc*`) f("==foo =>=bar x : ( = =a=20)", `="=foo" =">=bar" x:="=a=20`) // i filter @@ -772,14 +782,14 @@ func TestParseQuerySuccess(t *testing.T) { f(`foo: >= 10.5M`, `foo:>=10.5M`) f(`foo: < 10.5M`, `foo:<10.5M`) f(`foo: <= 10.5M`, `foo:<=10.5M`) - f(`foo:(>10 <=20)`, `foo:>10 foo:<=20`) - f(`>=10 <20`, `>=10 <20`) + f(`foo:(>10 !<=20)`, `foo:>10 !foo:<=20`) + f(`>=10 !<20`, `>=10 !<20`) // re filter f("re('foo|ba(r.+)')", `~"foo|ba(r.+)"`) - f("re(foo)", `~"foo"`) + f("re(foo)", `~foo`) f(`foo:re(foo-bar/baz.)`, `foo:~"foo-bar/baz."`) - f(`~foo.bar.baz`, `~"foo.bar.baz"`) + f(`~foo.bar.baz !~bar`, `~foo.bar.baz !~bar`) f(`foo:~~foo~ba/ba>z`, `foo:~"~foo~ba/ba>z"`) f(`foo:~'.*'`, `foo:~".*"`)