diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index ecee1de7a..93a2e6005 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -24,32 +24,6 @@ type filter interface { apply(bs *blockSearch, bm *bitmap) } -// notFilter negates the filter. -// -// It is expressed as `NOT f` or `!f` in LogsQL. -type notFilter struct { - f filter -} - -func (fn *notFilter) String() string { - s := fn.f.String() - switch fn.f.(type) { - case *filterAnd, *filterOr: - s = "(" + s + ")" - } - return "!" + s -} - -func (fn *notFilter) apply(bs *blockSearch, bm *bitmap) { - // Minimize the number of rows to check by the filter by applying it - // only to the rows, which match the bm, e.g. they may change the bm result. - bmTmp := getBitmap(bm.bitsLen) - bmTmp.copyFrom(bm) - fn.f.apply(bs, bmTmp) - bm.andNot(bmTmp) - putBitmap(bmTmp) -} - // streamFilter is the filter for `_stream:{...}` type streamFilter struct { // f is the filter to apply diff --git a/lib/logstorage/filter_not.go b/lib/logstorage/filter_not.go new file mode 100644 index 000000000..37b36f2e7 --- /dev/null +++ b/lib/logstorage/filter_not.go @@ -0,0 +1,27 @@ +package logstorage + +// filterNot negates the filter. +// +// It is expressed as `NOT f` or `!f` in LogsQL. +type filterNot struct { + f filter +} + +func (fn *filterNot) String() string { + s := fn.f.String() + switch fn.f.(type) { + case *filterAnd, *filterOr: + s = "(" + s + ")" + } + return "!" + s +} + +func (fn *filterNot) apply(bs *blockSearch, bm *bitmap) { + // Minimize the number of rows to check by the filter by applying it + // only to the rows, which match the bm, e.g. they may change the bm result. + bmTmp := getBitmap(bm.bitsLen) + bmTmp.copyFrom(bm) + fn.f.apply(bs, bmTmp) + bm.andNot(bmTmp) + putBitmap(bmTmp) +} diff --git a/lib/logstorage/filter_not_test.go b/lib/logstorage/filter_not_test.go new file mode 100644 index 000000000..722dc7f43 --- /dev/null +++ b/lib/logstorage/filter_not_test.go @@ -0,0 +1,75 @@ +package logstorage + +import ( + "testing" +) + +func TestFilterNot(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "a foo", + "a foobar", + "aa abc a", + "ca afdf a,foobar baz", + "a fddf foobarbaz", + "", + "a foobar", + "a kjlkjf dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fn := &filterNot{ + f: &phraseFilter{ + fieldName: "foo", + phrase: "", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9}) + + fn = &filterNot{ + f: &phraseFilter{ + fieldName: "foo", + phrase: "a", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", []int{5}) + + fn = &filterNot{ + f: &phraseFilter{ + fieldName: "non-existing-field", + phrase: "foobar", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fn = &filterNot{ + f: &prefixFilter{ + fieldName: "non-existing-field", + prefix: "", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fn = &filterNot{ + f: &prefixFilter{ + fieldName: "foo", + prefix: "", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", []int{5}) + + // mismatch + fn = &filterNot{ + f: &phraseFilter{ + fieldName: "non-existing-field", + phrase: "", + }, + } + testFilterMatchForColumns(t, columns, fn, "foo", nil) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index 88df031d2..2779f5095 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -362,7 +362,7 @@ func TestComplexFilters(t *testing.T) { fieldName: "foo", phrase: "foobar", }, - ¬Filter{ + &filterNot{ f: &phraseFilter{ fieldName: "foo", phrase: "baz", @@ -391,7 +391,7 @@ func TestComplexFilters(t *testing.T) { fieldName: "foo", phrase: "foobaz", }, - ¬Filter{ + &filterNot{ f: &phraseFilter{ fieldName: "foo", phrase: "baz", @@ -420,7 +420,7 @@ func TestComplexFilters(t *testing.T) { fieldName: "foo", phrase: "foobar", }, - ¬Filter{ + &filterNot{ f: &phraseFilter{ fieldName: "foo", phrase: "baz", @@ -453,7 +453,7 @@ func TestComplexFilters(t *testing.T) { fieldName: "foo", phrase: "foobar", }, - ¬Filter{ + &filterNot{ f: &phraseFilter{ fieldName: "foo", phrase: "qwert", @@ -480,76 +480,6 @@ func TestComplexFilters(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", []int{1, 3, 6}) } -func TestNotFilter(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "a foo", - "a foobar", - "aa abc a", - "ca afdf a,foobar baz", - "a fddf foobarbaz", - "", - "a foobar", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - nf := ¬Filter{ - f: &phraseFilter{ - fieldName: "foo", - phrase: "", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9}) - - nf = ¬Filter{ - f: &phraseFilter{ - fieldName: "foo", - phrase: "a", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", []int{5}) - - nf = ¬Filter{ - f: &phraseFilter{ - fieldName: "non-existing-field", - phrase: "foobar", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - nf = ¬Filter{ - f: &prefixFilter{ - fieldName: "non-existing-field", - prefix: "", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - nf = ¬Filter{ - f: &prefixFilter{ - fieldName: "foo", - prefix: "", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", []int{5}) - - // mismatch - nf = ¬Filter{ - f: &phraseFilter{ - fieldName: "non-existing-field", - phrase: "", - }, - } - testFilterMatchForColumns(t, columns, nf, "foo", nil) -} - func TestTimeFilter(t *testing.T) { timestamps := []int64{ 1, diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 5acb4ab4b..2072f33dd 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -242,17 +242,17 @@ func parseFilter(lex *lexer) (filter, error) { if !lex.mustNextToken() || lex.isKeyword("|") { return nil, fmt.Errorf("missing query") } - fo, err := parseOrFilter(lex, "") + fo, err := parseFilterOr(lex, "") if err != nil { return nil, err } return fo, nil } -func parseOrFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterOr(lex *lexer, fieldName string) (filter, error) { var filters []filter for { - f, err := parseAndFilter(lex, fieldName) + f, err := parseFilterAnd(lex, fieldName) if err != nil { return nil, err } @@ -274,7 +274,7 @@ func parseOrFilter(lex *lexer, fieldName string) (filter, error) { } } -func parseAndFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterAnd(lex *lexer, fieldName string) (filter, error) { var filters []filter for { f, err := parseGenericFilter(lex, fieldName) @@ -320,7 +320,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) { } return parseParensFilter(lex, fieldName) case lex.isKeyword("not", "!"): - return parseNotFilter(lex, fieldName) + return parseFilterNot(lex, fieldName) case lex.isKeyword("exact"): return parseExactFilter(lex, fieldName) case lex.isKeyword("i"): @@ -444,7 +444,7 @@ func parseParensFilter(lex *lexer, fieldName string) (filter, error) { if !lex.mustNextToken() { return nil, fmt.Errorf("missing filter after '('") } - f, err := parseOrFilter(lex, fieldName) + f, err := parseFilterOr(lex, fieldName) if err != nil { return nil, err } @@ -455,7 +455,7 @@ func parseParensFilter(lex *lexer, fieldName string) (filter, error) { return f, nil } -func parseNotFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterNot(lex *lexer, fieldName string) (filter, error) { notKeyword := lex.token if !lex.mustNextToken() { return nil, fmt.Errorf("missing filters after '%s'", notKeyword) @@ -464,14 +464,14 @@ func parseNotFilter(lex *lexer, fieldName string) (filter, error) { if err != nil { return nil, err } - nf, ok := f.(*notFilter) + fn, ok := f.(*filterNot) if ok { - return nf.f, nil + return fn.f, nil } - nf = ¬Filter{ + fn = &filterNot{ f: f, } - return nf, nil + return fn, nil } func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) { diff --git a/lib/logstorage/storage_search.go b/lib/logstorage/storage_search.go index 29f614d6a..b09bfba85 100644 --- a/lib/logstorage/storage_search.go +++ b/lib/logstorage/storage_search.go @@ -343,7 +343,7 @@ func hasStreamFilters(f filter) bool { return hasStreamFiltersInList(t.filters) case *filterOr: return hasStreamFiltersInList(t.filters) - case *notFilter: + case *filterNot: return hasStreamFilters(t.f) case *streamFilter: return true @@ -371,8 +371,8 @@ func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter { return &filterOr{ filters: initStreamFiltersList(tenantIDs, idb, t.filters), } - case *notFilter: - return ¬Filter{ + case *filterNot: + return &filterNot{ f: initStreamFilters(tenantIDs, idb, t.f), } case *streamFilter: