wip

docs/VictoriaLogs/CHANGELOG.md

@@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
 
 ## tip
 
+* BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter).
+
 ## [v0.9.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.9.1-victorialogs)
 
 Released at 2024-05-22

docs/VictoriaLogs/LogsQL.md

@@ -403,6 +403,13 @@ This query doesn't match the following log messages:
 - `SSH: login fail`, since the `SSH` word is in capital letters. Use `i("ssh: login fail")` for case-insensitive search.
   See [these docs](#case-insensitive-filter) for details.
 
+If the phrase contains double quotes, then either put `\` in front of double quotes or put the phrase inside single quotes. For example, the following filter searches
+logs with `"foo":"bar"` phrase:
+
+```logsql
+'"foo":"bar"'
+```
+
 By default the given phrase is searched in the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
 Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the phrase and put a colon after it
 if it must be searched in the given field. For example, the following query returns log entries containing the `cannot open file` phrase in the `event.original` field:
@@ -470,6 +477,13 @@ This query doesn't match the following log messages:
 - `failed to open file: unexpected EOF`, since `failed` [word](#word) occurs before the `unexpected` word. Use `unexpected AND fail*` for this case.
   See [these docs](#logical-filter) for details.
 
+If the prefix contains double quotes, then either put `\` in front of double quotes or put the prefix inside single quotes. For example, the following filter searches
+logs with `"foo":"bar` prefix:
+
+```logsql
+'"foo":"bar'*
+```
+
 By default the prefix filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
 Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the prefix filter
 in order to apply it to the given field. For example, the following query matches `log.level` field containing any word with the `err` prefix:
@@ -783,6 +797,13 @@ The query doesn't match the following log messages:
   See [these docs](https://github.com/google/re2/wiki/Syntax) for details. See also [case-insenstive filter docs](#case-insensitive-filter).
 - `it is warmer than usual`, since it doesn't contain neither `err` nor `warn` substrings.
 
+If the regexp contains double quotes, then either put `\` in front of double quotes or put the regexp inside single quotes. For example, the following regexp searches
+logs matching `"foo":"(bar|baz)"` regexp:
+
+```logsql
+'"foo":"(bar|baz)"'
+```
+
 By default the regexp filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
 Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the filter
 in order to apply it to the given field. For example, the following query matches `event.original` field containing either `err` or `warn` substrings:
@@ -1134,7 +1155,8 @@ For example, the following query is equivalent to the previous one:
 _time:1d error | extract "ip=<ip> " | stats by (ip) count() logs | sort by (logs) desc limit 10
 ```
 
-If the `pattern` contains double quotes, then it can be quoted into single quotes. For example, the following query extracts `ip` from the corresponding JSON field:
+If the `pattern` contains double quotes, then either put `\` in front of double quotes or put the `pattern` inside single quotes.
+For example, the following query extracts `ip` from the corresponding JSON field:
 
 ```logsql
 _time:5m | extract '"ip":"<ip>"'

lib/logstorage/filter_regexp.go

@@ -16,7 +16,7 @@ type filterRegexp struct {
 }
 
 func (fr *filterRegexp) String() string {
-	return fmt.Sprintf("%s~%q", quoteFieldNameIfNeeded(fr.fieldName), fr.re.String())
+	return fmt.Sprintf("%s~%s", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.re.String()))
 }
 
 func (fr *filterRegexp) updateNeededFields(neededFields fieldsSet) {

lib/logstorage/filter_stream.go

@@ -22,11 +22,7 @@ type filterStream struct {
 }
 
 func (fs *filterStream) String() string {
-	s := fs.f.String()
-	if s == "{}" {
-		return ""
-	}
-	return "_stream:" + s
+	return "_stream:" + fs.f.String()
 }
 
 func (fs *filterStream) updateNeededFields(neededFields fieldsSet) {

lib/logstorage/parser.go

@@ -597,8 +597,12 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
 		return parseFilterLT(lex, fieldName)
 	case lex.isKeyword("="):
 		return parseFilterEQ(lex, fieldName)
+	case lex.isKeyword("!="):
+		return parseFilterNEQ(lex, fieldName)
 	case lex.isKeyword("~"):
 		return parseFilterTilda(lex, fieldName)
+	case lex.isKeyword("!~"):
+		return parseFilterNotTilda(lex, fieldName)
 	case lex.isKeyword("not", "!"):
 		return parseFilterNot(lex, fieldName)
 	case lex.isKeyword("exact"):
@@ -1033,6 +1037,17 @@ func parseFilterTilda(lex *lexer, fieldName string) (filter, error) {
 	return fr, nil
 }
 
+func parseFilterNotTilda(lex *lexer, fieldName string) (filter, error) {
+	f, err := parseFilterTilda(lex, fieldName)
+	if err != nil {
+		return nil, err
+	}
+	fn := &filterNot{
+		f: f,
+	}
+	return fn, nil
+}
+
 func parseFilterEQ(lex *lexer, fieldName string) (filter, error) {
 	lex.nextToken()
 	phrase := getCompoundFuncArg(lex)
@@ -1051,6 +1066,17 @@ func parseFilterEQ(lex *lexer, fieldName string) (filter, error) {
 	return f, nil
 }
 
+func parseFilterNEQ(lex *lexer, fieldName string) (filter, error) {
+	f, err := parseFilterEQ(lex, fieldName)
+	if err != nil {
+		return nil, err
+	}
+	fn := &filterNot{
+		f: f,
+	}
+	return fn, nil
+}
+
 func parseFilterGT(lex *lexer, fieldName string) (filter, error) {
 	lex.nextToken()
 

lib/logstorage/parser_test.go

@@ -544,6 +544,16 @@ func TestParseQuerySuccess(t *testing.T) {
 		if result != resultExpected {
 			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
+
+		// verify that the marshaled query is parsed to the same query
+		qParsed, err := ParseQuery(result)
+		if err != nil {
+			t.Fatalf("cannot parse marshaled query: %s", err)
+		}
+		qStr := qParsed.String()
+		if qStr != result {
+			t.Fatalf("unexpected marshaled query\ngot\n%s\nwant\n%s", qStr, result)
+		}
 	}
 
 	f("foo", "foo")
@@ -586,7 +596,7 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`foo:(bar baz or not :xxx)`, `foo:bar foo:baz or !foo:xxx`)
 	f(`(foo:bar and (foo:baz or aa:bb) and xx) and y`, `foo:bar (foo:baz or aa:bb) xx y`)
 	f("level:error and _msg:(a or b)", "level:error (a or b)")
-	f("level: ( ((error or warn*) and re(foo))) (not (bar))", `(level:error or level:warn*) level:~"foo" !bar`)
+	f("level: ( ((error or warn*) and re(foo))) (not (bar))", `(level:error or level:warn*) level:~foo !bar`)
 	f("!(foo bar or baz and not aa*)", `!(foo bar or baz !aa*)`)
 
 	// prefix search
@@ -600,7 +610,7 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`"" or foo:"" and not bar:""`, `"" or foo:"" !bar:""`)
 
 	// _stream filters
-	f(`_stream:{}`, ``)
+	f(`_stream:{}`, `_stream:{}`)
 	f(`_stream:{foo="bar", baz=~"x" OR or!="b", "x=},"="d}{"}`, `_stream:{foo="bar",baz=~"x" or "or"!="b","x=},"="d}{"}`)
 	f(`_stream:{or=a or ","="b"}`, `_stream:{"or"="a" or ","="b"}`)
 	f("_stream : { foo =  bar , }  ", `_stream:{foo="bar"}`)
@@ -713,7 +723,7 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`exact("foo/bar")`, `="foo/bar"`)
 	f(`exact('foo/bar')`, `="foo/bar"`)
 	f(`="foo/bar"`, `="foo/bar"`)
-	f("=foo=bar =b<=a>z ='abc'*", `="foo=bar" ="b<=a>z" =abc*`)
+	f("=foo=bar !=b<=a>z foo:!='abc'*", `="foo=bar" !="b<=a>z" !foo:=abc*`)
 	f("==foo =>=bar x : ( = =a<b*='c*' >=20)", `="=foo" =">=bar" x:="=a<b"* x:="c*" x:>=20`)
 
 	// i filter
@@ -772,14 +782,14 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`foo: >= 10.5M`, `foo:>=10.5M`)
 	f(`foo: < 10.5M`, `foo:<10.5M`)
 	f(`foo: <= 10.5M`, `foo:<=10.5M`)
-	f(`foo:(>10 <=20)`, `foo:>10 foo:<=20`)
-	f(`>=10 <20`, `>=10 <20`)
+	f(`foo:(>10 !<=20)`, `foo:>10 !foo:<=20`)
+	f(`>=10 !<20`, `>=10 !<20`)
 
 	// re filter
 	f("re('foo|ba(r.+)')", `~"foo|ba(r.+)"`)
-	f("re(foo)", `~"foo"`)
+	f("re(foo)", `~foo`)
 	f(`foo:re(foo-bar/baz.)`, `foo:~"foo-bar/baz."`)
-	f(`~foo.bar.baz`, `~"foo.bar.baz"`)
+	f(`~foo.bar.baz !~bar`, `~foo.bar.baz !~bar`)
 	f(`foo:~~foo~ba/ba>z`, `foo:~"~foo~ba/ba>z"`)
 	f(`foo:~'.*'`, `foo:~".*"`)