From 70baaace98ff3cf2c10833431f19b816cc36f7c2 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 29 Apr 2024 04:07:51 +0200 Subject: [PATCH] wip --- lib/logstorage/filter.go | 176 +++++++------- lib/logstorage/filter_test.go | 332 +++++++++++++------------- lib/logstorage/parser.go | 8 +- lib/logstorage/storage_search.go | 14 +- lib/logstorage/storage_search_test.go | 4 +- 5 files changed, 267 insertions(+), 267 deletions(-) diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 534b22829..ae15dad64 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -76,18 +76,18 @@ func (fo *filterOr) apply(bs *blockSearch, bm *bitmap) { putBitmap(bmResult) } -// andFilter contains filters joined by AND opertor. +// filterAnd contains filters joined by AND opertor. // // It is expressed as `f1 AND f2 ... AND fN` in LogsQL. -type andFilter struct { +type filterAnd struct { filters []filter msgTokensOnce sync.Once msgTokens []string } -func (af *andFilter) String() string { - filters := af.filters +func (fa *filterAnd) String() string { + filters := fa.filters a := make([]string, len(filters)) for i, f := range filters { s := f.String() @@ -100,9 +100,9 @@ func (af *andFilter) String() string { return strings.Join(a, " ") } -func (af *andFilter) apply(bs *blockSearch, bm *bitmap) { - if tokens := af.getMsgTokens(); len(tokens) > 0 { - // Verify whether af tokens for the _msg field match bloom filter. +func (fa *filterAnd) apply(bs *blockSearch, bm *bitmap) { + if tokens := fa.getMsgTokens(); len(tokens) > 0 { + // Verify whether fa tokens for the _msg field match bloom filter. ch := bs.csh.getColumnHeader("_msg") if ch == nil { // Fast path - there is no _msg field in the block. @@ -110,14 +110,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) { return } if !matchBloomFilterAllTokens(bs, ch, tokens) { - // Fast path - af tokens for the _msg field do not match bloom filter. + // Fast path - fa tokens for the _msg field do not match bloom filter. bm.resetBits() return } } // Slow path - verify every filter separately. - for _, f := range af.filters { + for _, f := range fa.filters { f.apply(bs, bm) if bm.isZero() { // Shortcut - there is no need in applying the remaining filters, @@ -127,14 +127,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) { } } -func (af *andFilter) getMsgTokens() []string { - af.msgTokensOnce.Do(af.initMsgTokens) - return af.msgTokens +func (fa *filterAnd) getMsgTokens() []string { + fa.msgTokensOnce.Do(fa.initMsgTokens) + return fa.msgTokens } -func (af *andFilter) initMsgTokens() { +func (fa *filterAnd) initMsgTokens() { var a []string - for _, f := range af.filters { + for _, f := range fa.filters { switch t := f.(type) { case *phraseFilter: if isMsgFieldName(t.fieldName) { @@ -158,7 +158,7 @@ func (af *andFilter) initMsgTokens() { } } } - af.msgTokens = a + fa.msgTokens = a } // notFilter negates the filter. @@ -171,7 +171,7 @@ type notFilter struct { func (fn *notFilter) String() string { s := fn.f.String() switch fn.f.(type) { - case *andFilter, *filterOr: + case *filterAnd, *filterOr: s = "(" + s + ")" } return "!" + s @@ -557,25 +557,25 @@ type inFilter struct { timestampISO8601Values map[string]struct{} } -func (af *inFilter) String() string { - values := af.values +func (fi *inFilter) String() string { + values := fi.values a := make([]string, len(values)) for i, value := range values { a[i] = quoteTokenIfNeeded(value) } - return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(af.fieldName), strings.Join(a, ",")) + return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ",")) } -func (af *inFilter) getTokenSets() [][]string { - af.tokenSetsOnce.Do(af.initTokenSets) - return af.tokenSets +func (fi *inFilter) getTokenSets() [][]string { + fi.tokenSetsOnce.Do(fi.initTokenSets) + return fi.tokenSets } // It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter. const maxTokenSetsToInit = 1000 -func (af *inFilter) initTokenSets() { - values := af.values +func (fi *inFilter) initTokenSets() { + values := fi.values tokenSetsLen := len(values) if tokenSetsLen > maxTokenSetsToInit { tokenSetsLen = maxTokenSetsToInit @@ -588,30 +588,30 @@ func (af *inFilter) initTokenSets() { break } } - af.tokenSets = tokenSets + fi.tokenSets = tokenSets } -func (af *inFilter) getStringValues() map[string]struct{} { - af.stringValuesOnce.Do(af.initStringValues) - return af.stringValues +func (fi *inFilter) getStringValues() map[string]struct{} { + fi.stringValuesOnce.Do(fi.initStringValues) + return fi.stringValues } -func (af *inFilter) initStringValues() { - values := af.values +func (fi *inFilter) initStringValues() { + values := fi.values m := make(map[string]struct{}, len(values)) for _, v := range values { m[v] = struct{}{} } - af.stringValues = m + fi.stringValues = m } -func (af *inFilter) getUint8Values() map[string]struct{} { - af.uint8ValuesOnce.Do(af.initUint8Values) - return af.uint8Values +func (fi *inFilter) getUint8Values() map[string]struct{} { + fi.uint8ValuesOnce.Do(fi.initUint8Values) + return fi.uint8Values } -func (af *inFilter) initUint8Values() { - values := af.values +func (fi *inFilter) initUint8Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*1) for _, v := range values { @@ -624,16 +624,16 @@ func (af *inFilter) initUint8Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.uint8Values = m + fi.uint8Values = m } -func (af *inFilter) getUint16Values() map[string]struct{} { - af.uint16ValuesOnce.Do(af.initUint16Values) - return af.uint16Values +func (fi *inFilter) getUint16Values() map[string]struct{} { + fi.uint16ValuesOnce.Do(fi.initUint16Values) + return fi.uint16Values } -func (af *inFilter) initUint16Values() { - values := af.values +func (fi *inFilter) initUint16Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*2) for _, v := range values { @@ -646,16 +646,16 @@ func (af *inFilter) initUint16Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.uint16Values = m + fi.uint16Values = m } -func (af *inFilter) getUint32Values() map[string]struct{} { - af.uint32ValuesOnce.Do(af.initUint32Values) - return af.uint32Values +func (fi *inFilter) getUint32Values() map[string]struct{} { + fi.uint32ValuesOnce.Do(fi.initUint32Values) + return fi.uint32Values } -func (af *inFilter) initUint32Values() { - values := af.values +func (fi *inFilter) initUint32Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*4) for _, v := range values { @@ -668,16 +668,16 @@ func (af *inFilter) initUint32Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.uint32Values = m + fi.uint32Values = m } -func (af *inFilter) getUint64Values() map[string]struct{} { - af.uint64ValuesOnce.Do(af.initUint64Values) - return af.uint64Values +func (fi *inFilter) getUint64Values() map[string]struct{} { + fi.uint64ValuesOnce.Do(fi.initUint64Values) + return fi.uint64Values } -func (af *inFilter) initUint64Values() { - values := af.values +func (fi *inFilter) initUint64Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*8) for _, v := range values { @@ -690,16 +690,16 @@ func (af *inFilter) initUint64Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.uint64Values = m + fi.uint64Values = m } -func (af *inFilter) getFloat64Values() map[string]struct{} { - af.float64ValuesOnce.Do(af.initFloat64Values) - return af.float64Values +func (fi *inFilter) getFloat64Values() map[string]struct{} { + fi.float64ValuesOnce.Do(fi.initFloat64Values) + return fi.float64Values } -func (af *inFilter) initFloat64Values() { - values := af.values +func (fi *inFilter) initFloat64Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*8) for _, v := range values { @@ -713,16 +713,16 @@ func (af *inFilter) initFloat64Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.float64Values = m + fi.float64Values = m } -func (af *inFilter) getIPv4Values() map[string]struct{} { - af.ipv4ValuesOnce.Do(af.initIPv4Values) - return af.ipv4Values +func (fi *inFilter) getIPv4Values() map[string]struct{} { + fi.ipv4ValuesOnce.Do(fi.initIPv4Values) + return fi.ipv4Values } -func (af *inFilter) initIPv4Values() { - values := af.values +func (fi *inFilter) initIPv4Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*4) for _, v := range values { @@ -735,16 +735,16 @@ func (af *inFilter) initIPv4Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.ipv4Values = m + fi.ipv4Values = m } -func (af *inFilter) getTimestampISO8601Values() map[string]struct{} { - af.timestampISO8601ValuesOnce.Do(af.initTimestampISO8601Values) - return af.timestampISO8601Values +func (fi *inFilter) getTimestampISO8601Values() map[string]struct{} { + fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values) + return fi.timestampISO8601Values } -func (af *inFilter) initTimestampISO8601Values() { - values := af.values +func (fi *inFilter) initTimestampISO8601Values() { + values := fi.values m := make(map[string]struct{}, len(values)) buf := make([]byte, 0, len(values)*8) for _, v := range values { @@ -757,20 +757,20 @@ func (af *inFilter) initTimestampISO8601Values() { s := bytesutil.ToUnsafeString(buf[bufLen:]) m[s] = struct{}{} } - af.timestampISO8601Values = m + fi.timestampISO8601Values = m } -func (af *inFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := af.fieldName +func (fi *inFilter) apply(bs *blockSearch, bm *bitmap) { + fieldName := fi.fieldName - if len(af.values) == 0 { + if len(fi.values) == 0 { bm.resetBits() return } v := bs.csh.getConstColumnValue(fieldName) if v != "" { - stringValues := af.getStringValues() + stringValues := fi.getStringValues() if _, ok := stringValues[v]; !ok { bm.resetBits() } @@ -782,42 +782,42 @@ func (af *inFilter) apply(bs *blockSearch, bm *bitmap) { if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty phrase. - stringValues := af.getStringValues() + stringValues := fi.getStringValues() if _, ok := stringValues[""]; !ok { bm.resetBits() } return } - tokenSets := af.getTokenSets() + tokenSets := fi.getTokenSets() switch ch.valueType { case valueTypeString: - stringValues := af.getStringValues() + stringValues := fi.getStringValues() matchAnyValue(bs, ch, bm, stringValues, tokenSets) case valueTypeDict: - stringValues := af.getStringValues() + stringValues := fi.getStringValues() matchValuesDictByAnyValue(bs, ch, bm, stringValues) case valueTypeUint8: - binValues := af.getUint8Values() + binValues := fi.getUint8Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeUint16: - binValues := af.getUint16Values() + binValues := fi.getUint16Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeUint32: - binValues := af.getUint32Values() + binValues := fi.getUint32Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeUint64: - binValues := af.getUint64Values() + binValues := fi.getUint64Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeFloat64: - binValues := af.getFloat64Values() + binValues := fi.getFloat64Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeIPv4: - binValues := af.getIPv4Values() + binValues := fi.getIPv4Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) case valueTypeTimestampISO8601: - binValues := af.getTimestampISO8601Values() + binValues := fi.getTimestampISO8601Values() matchAnyValue(bs, ch, bm, binValues, tokenSets) default: logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index 4b398598a..137b8b924 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -356,7 +356,7 @@ func TestComplexFilters(t *testing.T) { } // (foobar AND NOT baz AND (abcdef OR xyz)) - f := &andFilter{ + f := &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -385,7 +385,7 @@ func TestComplexFilters(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", []int{6}) // (foobaz AND NOT baz AND (abcdef OR xyz)) - f = &andFilter{ + f = &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -414,7 +414,7 @@ func TestComplexFilters(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", nil) // (foobaz AND NOT baz AND (abcdef OR xyz OR a)) - f = &andFilter{ + f = &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -447,7 +447,7 @@ func TestComplexFilters(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", []int{1, 6}) // (foobaz AND NOT qwert AND (abcdef OR xyz OR a)) - f = &andFilter{ + f = &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -625,7 +625,7 @@ func TestAndFilter(t *testing.T) { } // non-empty intersection - af := &andFilter{ + fa := &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -637,10 +637,10 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6}) + testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6}) // reverse non-empty intersection - af = &andFilter{ + fa = &filterAnd{ filters: []filter{ &prefixFilter{ fieldName: "foo", @@ -652,10 +652,10 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6}) + testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6}) // the first filter mismatch - af = &andFilter{ + fa = &filterAnd{ filters: []filter{ &prefixFilter{ fieldName: "foo", @@ -667,10 +667,10 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fa, "foo", nil) // the last filter mismatch - af = &andFilter{ + fa = &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -682,10 +682,10 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fa, "foo", nil) // empty intersection - af = &andFilter{ + fa = &filterAnd{ filters: []filter{ &phraseFilter{ fieldName: "foo", @@ -697,10 +697,10 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fa, "foo", nil) // reverse empty intersection - af = &andFilter{ + fa = &filterAnd{ filters: []filter{ &prefixFilter{ fieldName: "foo", @@ -712,7 +712,7 @@ func TestAndFilter(t *testing.T) { }, }, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fa, "foo", nil) } func TestNotFilter(t *testing.T) { @@ -2845,54 +2845,54 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"abc def", "abc", "foobar"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - af = &inFilter{ + fi = &inFilter{ fieldName: "other column", values: []string{"asdfdsf", ""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{"", "foo"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"abc", "def"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"", "abc"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "other column", values: []string{"sd"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing column", values: []string{"abc", "def"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("const-column", func(t *testing.T) { @@ -2908,42 +2908,42 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"aaaa", "abc def", "foobar"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{"", "abc"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"abc def ", "foobar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing column", values: []string{"x"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("dict", func(t *testing.T) { @@ -2963,42 +2963,42 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"foobar", "aaaa", "abc", "baz"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bbbb", "", "aaaa"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar", "aaaa"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing column", values: []string{"foobar", "aaaa"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("strings", func(t *testing.T) { @@ -3021,36 +3021,36 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"a foobar", "aa abc a"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"aa a"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("uint8", func(t *testing.T) { @@ -3074,48 +3074,48 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"12", "32"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"0"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"33"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"1234"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("uint16", func(t *testing.T) { @@ -3139,48 +3139,48 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"12", "32"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"0"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"33"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"123456"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("uint32", func(t *testing.T) { @@ -3204,48 +3204,48 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"12", "32"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"0"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"33"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"12345678901"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("uint64", func(t *testing.T) { @@ -3269,42 +3269,42 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"12", "32"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"0"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"33"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("float64", func(t *testing.T) { @@ -3326,66 +3326,66 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"1234", "1", "foobar", "123211"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 5}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"1234.5678901"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{4}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"-65536"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{3}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{3}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"65536"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"123"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"12345678901234567890"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("ipv4", func(t *testing.T) { @@ -3410,48 +3410,48 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "foo", values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{2, 4, 5, 6, 7}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) // mismatch - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"5"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "foo", values: []string{"255.255.255.255"}, } - testFilterMatchForColumns(t, columns, af, "foo", nil) + testFilterMatchForColumns(t, columns, fi, "foo", nil) }) t.Run("timestamp-iso8601", func(t *testing.T) { @@ -3473,42 +3473,42 @@ func TestInFilter(t *testing.T) { } // match - af := &inFilter{ + fi := &inFilter{ fieldName: "_msg", values: []string{"2006-01-02T15:04:05.005Z", "foobar"}, } - testFilterMatchForColumns(t, columns, af, "_msg", []int{4}) + testFilterMatchForColumns(t, columns, fi, "_msg", []int{4}) - af = &inFilter{ + fi = &inFilter{ fieldName: "non-existing-column", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) // mimatch - af = &inFilter{ + fi = &inFilter{ fieldName: "_msg", values: []string{"bar"}, } - testFilterMatchForColumns(t, columns, af, "_msg", nil) + testFilterMatchForColumns(t, columns, fi, "_msg", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "_msg", values: []string{}, } - testFilterMatchForColumns(t, columns, af, "_msg", nil) + testFilterMatchForColumns(t, columns, fi, "_msg", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "_msg", values: []string{""}, } - testFilterMatchForColumns(t, columns, af, "_msg", nil) + testFilterMatchForColumns(t, columns, fi, "_msg", nil) - af = &inFilter{ + fi = &inFilter{ fieldName: "_msg", values: []string{"2006-04-02T15:04:05.005Z"}, } - testFilterMatchForColumns(t, columns, af, "_msg", nil) + testFilterMatchForColumns(t, columns, fi, "_msg", nil) }) } diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index d65ebc226..5acb4ab4b 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -242,11 +242,11 @@ func parseFilter(lex *lexer) (filter, error) { if !lex.mustNextToken() || lex.isKeyword("|") { return nil, fmt.Errorf("missing query") } - af, err := parseOrFilter(lex, "") + fo, err := parseOrFilter(lex, "") if err != nil { return nil, err } - return af, nil + return fo, nil } func parseOrFilter(lex *lexer, fieldName string) (filter, error) { @@ -287,10 +287,10 @@ func parseAndFilter(lex *lexer, fieldName string) (filter, error) { if len(filters) == 1 { return filters[0], nil } - af := &andFilter{ + fa := &filterAnd{ filters: filters, } - return af, nil + return fa, nil case lex.isKeyword("and"): if !lex.mustNextToken() { return nil, fmt.Errorf("missing filter after 'and'") diff --git a/lib/logstorage/storage_search.go b/lib/logstorage/storage_search.go index f62a6babe..29f614d6a 100644 --- a/lib/logstorage/storage_search.go +++ b/lib/logstorage/storage_search.go @@ -339,7 +339,7 @@ func (pt *partition) search(tf *timeFilter, sf *StreamFilter, f filter, so *gene func hasStreamFilters(f filter) bool { switch t := f.(type) { - case *andFilter: + case *filterAnd: return hasStreamFiltersInList(t.filters) case *filterOr: return hasStreamFiltersInList(t.filters) @@ -363,8 +363,8 @@ func hasStreamFiltersInList(filters []filter) bool { func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter { switch t := f.(type) { - case *andFilter: - return &andFilter{ + case *filterAnd: + return &filterAnd{ filters: initStreamFiltersList(tenantIDs, idb, t.filters), } case *filterOr: @@ -695,16 +695,16 @@ func appendPartsInTimeRange(dst, src []*partWrapper, minTimestamp, maxTimestamp func getCommonStreamFilter(f filter) (*StreamFilter, filter) { switch t := f.(type) { - case *andFilter: + case *filterAnd: filters := t.filters for i, filter := range filters { sf, ok := filter.(*streamFilter) if ok && !sf.f.isEmpty() { // Remove sf from filters, since it doesn't filter out anything then. - af := &andFilter{ + fa := &filterAnd{ filters: append(filters[:i:i], filters[i+1:]...), } - return sf.f, af + return sf.f, fa } } case *streamFilter: @@ -715,7 +715,7 @@ func getCommonStreamFilter(f filter) (*StreamFilter, filter) { func getCommonTimeFilter(f filter) (*timeFilter, filter) { switch t := f.(type) { - case *andFilter: + case *filterAnd: for _, filter := range t.filters { tf, ok := filter.(*timeFilter) if ok { diff --git a/lib/logstorage/storage_search_test.go b/lib/logstorage/storage_search_test.go index 49499dfdb..067b143ee 100644 --- a/lib/logstorage/storage_search_test.go +++ b/lib/logstorage/storage_search_test.go @@ -388,7 +388,7 @@ func TestStorageSearch(t *testing.T) { f: sf, }) } - return &andFilter{ + return &filterAnd{ filters: filters, } } @@ -578,7 +578,7 @@ func TestStorageSearch(t *testing.T) { minTimestamp := baseTimestamp maxTimestamp := baseTimestamp + rowsPerBlock*1e9 + blocksPerStream f := getBaseFilter(minTimestamp, maxTimestamp, sf) - f = &andFilter{ + f = &filterAnd{ filters: []filter{ f, ®expFilter{