diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 035426f6a..18efac831 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -71,95 +71,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { } } -// anyCasePrefixFilter matches the given prefix in lower, upper and mixed case. -// -// Example LogsQL: `fieldName:i(prefix*)` or `fieldName:i("some prefix"*)` -// -// A special case `fieldName:i(*)` equals to `fieldName:*` and matches non-emtpy value for the given `fieldName` field. -type anyCasePrefixFilter struct { - fieldName string - prefix string - - prefixLowercaseOnce sync.Once - prefixLowercase string - - tokensOnce sync.Once - tokens []string -} - -func (pf *anyCasePrefixFilter) String() string { - if pf.prefix == "" { - return quoteFieldNameIfNeeded(pf.fieldName) + "i(*)" - } - return fmt.Sprintf("%si(%s*)", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.prefix)) -} - -func (pf *anyCasePrefixFilter) getTokens() []string { - pf.tokensOnce.Do(pf.initTokens) - return pf.tokens -} - -func (pf *anyCasePrefixFilter) initTokens() { - pf.tokens = getTokensSkipLast(pf.prefix) -} - -func (pf *anyCasePrefixFilter) getPrefixLowercase() string { - pf.prefixLowercaseOnce.Do(pf.initPrefixLowercase) - return pf.prefixLowercase -} - -func (pf *anyCasePrefixFilter) initPrefixLowercase() { - pf.prefixLowercase = strings.ToLower(pf.prefix) -} - -func (pf *anyCasePrefixFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := pf.fieldName - prefixLowercase := pf.getPrefixLowercase() - - // Verify whether pf matches const column - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - if !matchAnyCasePrefix(v, prefixLowercase) { - bm.resetBits() - } - return - } - - // Verify whether pf matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - // Fast path - there are no matching columns. - bm.resetBits() - return - } - - tokens := pf.getTokens() - - switch ch.valueType { - case valueTypeString: - matchStringByAnyCasePrefix(bs, ch, bm, prefixLowercase) - case valueTypeDict: - matchValuesDictByAnyCasePrefix(bs, ch, bm, prefixLowercase) - case valueTypeUint8: - matchUint8ByPrefix(bs, ch, bm, prefixLowercase) - case valueTypeUint16: - matchUint16ByPrefix(bs, ch, bm, prefixLowercase) - case valueTypeUint32: - matchUint32ByPrefix(bs, ch, bm, prefixLowercase) - case valueTypeUint64: - matchUint64ByPrefix(bs, ch, bm, prefixLowercase) - case valueTypeFloat64: - matchFloat64ByPrefix(bs, ch, bm, prefixLowercase, tokens) - case valueTypeIPv4: - matchIPv4ByPrefix(bs, ch, bm, prefixLowercase, tokens) - case valueTypeTimestampISO8601: - prefixUppercase := strings.ToUpper(pf.prefix) - matchTimestampISO8601ByPrefix(bs, ch, bm, prefixUppercase, tokens) - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // prefixFilter matches the given prefix. // // Example LogsQL: `fieldName:prefix*` or `fieldName:"some prefix"*` @@ -173,27 +84,27 @@ type prefixFilter struct { tokens []string } -func (pf *prefixFilter) String() string { - if pf.prefix == "" { - return quoteFieldNameIfNeeded(pf.fieldName) + "*" +func (fp *prefixFilter) String() string { + if fp.prefix == "" { + return quoteFieldNameIfNeeded(fp.fieldName) + "*" } - return fmt.Sprintf("%s%s*", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.prefix)) + return fmt.Sprintf("%s%s*", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.prefix)) } -func (pf *prefixFilter) getTokens() []string { - pf.tokensOnce.Do(pf.initTokens) - return pf.tokens +func (fp *prefixFilter) getTokens() []string { + fp.tokensOnce.Do(fp.initTokens) + return fp.tokens } -func (pf *prefixFilter) initTokens() { - pf.tokens = getTokensSkipLast(pf.prefix) +func (fp *prefixFilter) initTokens() { + fp.tokens = getTokensSkipLast(fp.prefix) } -func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := pf.fieldName - prefix := pf.prefix +func (fp *prefixFilter) apply(bs *blockSearch, bm *bitmap) { + fieldName := fp.fieldName + prefix := fp.prefix - // Verify whether pf matches const column + // Verify whether fp matches const column v := bs.csh.getConstColumnValue(fieldName) if v != "" { if !matchPrefix(v, prefix) { @@ -202,7 +113,7 @@ func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) { return } - // Verify whether pf matches other columns + // Verify whether fp matches other columns ch := bs.csh.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. @@ -210,7 +121,7 @@ func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) { return } - tokens := pf.getTokens() + tokens := fp.getTokens() switch ch.valueType { case valueTypeString: @@ -250,33 +161,33 @@ type anyCasePhraseFilter struct { tokens []string } -func (pf *anyCasePhraseFilter) String() string { - return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.phrase)) +func (fp *anyCasePhraseFilter) String() string { + return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase)) } -func (pf *anyCasePhraseFilter) getTokens() []string { - pf.tokensOnce.Do(pf.initTokens) - return pf.tokens +func (fp *anyCasePhraseFilter) getTokens() []string { + fp.tokensOnce.Do(fp.initTokens) + return fp.tokens } -func (pf *anyCasePhraseFilter) initTokens() { - pf.tokens = tokenizeStrings(nil, []string{pf.phrase}) +func (fp *anyCasePhraseFilter) initTokens() { + fp.tokens = tokenizeStrings(nil, []string{fp.phrase}) } -func (pf *anyCasePhraseFilter) getPhraseLowercase() string { - pf.phraseLowercaseOnce.Do(pf.initPhraseLowercase) - return pf.phraseLowercase +func (fp *anyCasePhraseFilter) getPhraseLowercase() string { + fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase) + return fp.phraseLowercase } -func (pf *anyCasePhraseFilter) initPhraseLowercase() { - pf.phraseLowercase = strings.ToLower(pf.phrase) +func (fp *anyCasePhraseFilter) initPhraseLowercase() { + fp.phraseLowercase = strings.ToLower(fp.phrase) } -func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := pf.fieldName - phraseLowercase := pf.getPhraseLowercase() +func (fp *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { + fieldName := fp.fieldName + phraseLowercase := fp.getPhraseLowercase() - // Verify whether pf matches const column + // Verify whether fp matches const column v := bs.csh.getConstColumnValue(fieldName) if v != "" { if !matchAnyCasePhrase(v, phraseLowercase) { @@ -285,7 +196,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { return } - // Verify whether pf matches other columns + // Verify whether fp matches other columns ch := bs.csh.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. @@ -296,7 +207,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { return } - tokens := pf.getTokens() + tokens := fp.getTokens() switch ch.valueType { case valueTypeString: @@ -316,7 +227,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { case valueTypeIPv4: matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens) case valueTypeTimestampISO8601: - phraseUppercase := strings.ToUpper(pf.phrase) + phraseUppercase := strings.ToUpper(fp.phrase) matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens) default: logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) @@ -341,24 +252,24 @@ type phraseFilter struct { tokens []string } -func (pf *phraseFilter) String() string { - return quoteFieldNameIfNeeded(pf.fieldName) + quoteTokenIfNeeded(pf.phrase) +func (fp *phraseFilter) String() string { + return quoteFieldNameIfNeeded(fp.fieldName) + quoteTokenIfNeeded(fp.phrase) } -func (pf *phraseFilter) getTokens() []string { - pf.tokensOnce.Do(pf.initTokens) - return pf.tokens +func (fp *phraseFilter) getTokens() []string { + fp.tokensOnce.Do(fp.initTokens) + return fp.tokens } -func (pf *phraseFilter) initTokens() { - pf.tokens = tokenizeStrings(nil, []string{pf.phrase}) +func (fp *phraseFilter) initTokens() { + fp.tokens = tokenizeStrings(nil, []string{fp.phrase}) } -func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := pf.fieldName - phrase := pf.phrase +func (fp *phraseFilter) apply(bs *blockSearch, bm *bitmap) { + fieldName := fp.fieldName + phrase := fp.phrase - // Verify whether pf matches const column + // Verify whether fp matches const column v := bs.csh.getConstColumnValue(fieldName) if v != "" { if !matchPhrase(v, phrase) { @@ -367,7 +278,7 @@ func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) { return } - // Verify whether pf matches other columns + // Verify whether fp matches other columns ch := bs.csh.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. @@ -378,7 +289,7 @@ func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) { return } - tokens := pf.getTokens() + tokens := fp.getTokens() switch ch.valueType { case valueTypeString: @@ -549,17 +460,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase bbPool.Put(bb) } -func matchValuesDictByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) { - bb := bbPool.Get() - for i, v := range ch.valuesDict.values { - if matchAnyCasePrefix(v, prefixLowercase) { - bb.B = append(bb.B, byte(i)) - } - } - matchEncodedValuesDict(bs, ch, bm, bb.B) - bbPool.Put(bb) -} - func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { bb := bbPool.Get() for i, v := range ch.valuesDict.values { @@ -620,12 +520,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod }) } -func matchStringByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) { - visitValues(bs, ch, bm, func(v string) bool { - return matchAnyCasePrefix(v, prefixLowercase) - }) -} - func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { visitValues(bs, ch, bm, func(v string) bool { return matchAnyCasePhrase(v, phraseLowercase) @@ -777,30 +671,6 @@ func visitValues(bs *blockSearch, ch *columnHeader, bm *bitmap, f func(value str }) } -func matchAnyCasePrefix(s, prefixLowercase string) bool { - if len(prefixLowercase) == 0 { - // Special case - empty prefix matches any non-empty string. - return len(s) > 0 - } - if len(prefixLowercase) > len(s) { - return false - } - - if isASCIILowercase(s) { - // Fast path - s is in lowercase - return matchPrefix(s, prefixLowercase) - } - - // Slow path - convert s to lowercase before matching - bb := bbPool.Get() - bb.B = stringsutil.AppendLowercase(bb.B, s) - sLowercase := bytesutil.ToUnsafeString(bb.B) - ok := matchPrefix(sLowercase, prefixLowercase) - bbPool.Put(bb) - - return ok -} - func isASCIILowercase(s string) bool { for i := 0; i < len(s); i++ { c := s[i] diff --git a/lib/logstorage/filter_any_case_prefix.go b/lib/logstorage/filter_any_case_prefix.go new file mode 100644 index 000000000..fd89d43b6 --- /dev/null +++ b/lib/logstorage/filter_any_case_prefix.go @@ -0,0 +1,141 @@ +package logstorage + +import ( + "fmt" + "strings" + "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil" +) + +// filterAnyCasePrefix matches the given prefix in lower, upper and mixed case. +// +// Example LogsQL: `fieldName:i(prefix*)` or `fieldName:i("some prefix"*)` +// +// A special case `fieldName:i(*)` equals to `fieldName:*` and matches non-emtpy value for the given `fieldName` field. +type filterAnyCasePrefix struct { + fieldName string + prefix string + + prefixLowercaseOnce sync.Once + prefixLowercase string + + tokensOnce sync.Once + tokens []string +} + +func (fp *filterAnyCasePrefix) String() string { + if fp.prefix == "" { + return quoteFieldNameIfNeeded(fp.fieldName) + "i(*)" + } + return fmt.Sprintf("%si(%s*)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.prefix)) +} + +func (fp *filterAnyCasePrefix) getTokens() []string { + fp.tokensOnce.Do(fp.initTokens) + return fp.tokens +} + +func (fp *filterAnyCasePrefix) initTokens() { + fp.tokens = getTokensSkipLast(fp.prefix) +} + +func (fp *filterAnyCasePrefix) getPrefixLowercase() string { + fp.prefixLowercaseOnce.Do(fp.initPrefixLowercase) + return fp.prefixLowercase +} + +func (fp *filterAnyCasePrefix) initPrefixLowercase() { + fp.prefixLowercase = strings.ToLower(fp.prefix) +} + +func (fp *filterAnyCasePrefix) apply(bs *blockSearch, bm *bitmap) { + fieldName := fp.fieldName + prefixLowercase := fp.getPrefixLowercase() + + // Verify whether fp matches const column + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + if !matchAnyCasePrefix(v, prefixLowercase) { + bm.resetBits() + } + return + } + + // Verify whether fp matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + // Fast path - there are no matching columns. + bm.resetBits() + return + } + + tokens := fp.getTokens() + + switch ch.valueType { + case valueTypeString: + matchStringByAnyCasePrefix(bs, ch, bm, prefixLowercase) + case valueTypeDict: + matchValuesDictByAnyCasePrefix(bs, ch, bm, prefixLowercase) + case valueTypeUint8: + matchUint8ByPrefix(bs, ch, bm, prefixLowercase) + case valueTypeUint16: + matchUint16ByPrefix(bs, ch, bm, prefixLowercase) + case valueTypeUint32: + matchUint32ByPrefix(bs, ch, bm, prefixLowercase) + case valueTypeUint64: + matchUint64ByPrefix(bs, ch, bm, prefixLowercase) + case valueTypeFloat64: + matchFloat64ByPrefix(bs, ch, bm, prefixLowercase, tokens) + case valueTypeIPv4: + matchIPv4ByPrefix(bs, ch, bm, prefixLowercase, tokens) + case valueTypeTimestampISO8601: + prefixUppercase := strings.ToUpper(fp.prefix) + matchTimestampISO8601ByPrefix(bs, ch, bm, prefixUppercase, tokens) + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchValuesDictByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) { + bb := bbPool.Get() + for i, v := range ch.valuesDict.values { + if matchAnyCasePrefix(v, prefixLowercase) { + bb.B = append(bb.B, byte(i)) + } + } + matchEncodedValuesDict(bs, ch, bm, bb.B) + bbPool.Put(bb) +} + +func matchStringByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) { + visitValues(bs, ch, bm, func(v string) bool { + return matchAnyCasePrefix(v, prefixLowercase) + }) +} + +func matchAnyCasePrefix(s, prefixLowercase string) bool { + if len(prefixLowercase) == 0 { + // Special case - empty prefix matches any non-empty string. + return len(s) > 0 + } + if len(prefixLowercase) > len(s) { + return false + } + + if isASCIILowercase(s) { + // Fast path - s is in lowercase + return matchPrefix(s, prefixLowercase) + } + + // Slow path - convert s to lowercase before matching + bb := bbPool.Get() + bb.B = stringsutil.AppendLowercase(bb.B, s) + sLowercase := bytesutil.ToUnsafeString(bb.B) + ok := matchPrefix(sLowercase, prefixLowercase) + bbPool.Put(bb) + + return ok +} diff --git a/lib/logstorage/filter_any_case_prefix_test.go b/lib/logstorage/filter_any_case_prefix_test.go new file mode 100644 index 000000000..2528d2e8e --- /dev/null +++ b/lib/logstorage/filter_any_case_prefix_test.go @@ -0,0 +1,930 @@ +package logstorage + +import ( + "testing" +) + +func TestMatchAnyCasePrefix(t *testing.T) { + f := func(s, prefixLowercase string, resultExpected bool) { + t.Helper() + result := matchAnyCasePrefix(s, prefixLowercase) + if result != resultExpected { + t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) + } + } + + // empty prefix matches non-empty strings + f("", "", false) + f("foo", "", true) + f("тест", "", true) + + // empty string doesn't match non-empty prefix + f("", "foo", false) + f("", "тест", false) + + // full match + f("foo", "foo", true) + f("FOo", "foo", true) + f("Test ТЕСт 123", "test тест 123", true) + + // prefix match + f("foo", "f", true) + f("foo тест bar", "те", true) + f("foo ТЕСТ bar", "те", true) + + // mismatch + f("foo", "o", false) + f("тест", "foo", false) + f("Тест", "ест", false) +} + +func TestFilterAnyCasePrefix(t *testing.T) { + t.Run("single-row", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "aBc DEf", + }, + }, + { + name: "other column", + values: []string{ + "aSDfdsf", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "abc", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "ABC", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "ab", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "abc def", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "def", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "other column", + prefix: "asdfdSF", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bc", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "other column", + prefix: "sd", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "abc", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "other-column", + values: []string{ + "x", + "X", + "X", + }, + }, + { + name: "foo", + values: []string{ + "abc def", + "ABC DEF", + "AbC Def", + }, + }, + { + name: "_msg", + values: []string{ + "1 2 3", + "1 2 3", + "1 2 3", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "Abc", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "AB", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "abc de", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: " de", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "abc def", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "other-column", + prefix: "x", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: " 2 ", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "abc def ", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "x", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "other-column", + prefix: "foo", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "x", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "foo", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "fOObar", + "Abc", + "aFDf FooBar baz", + "fddf FOObarBAZ", + "AFoobarbaz", + "foobar", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "FooBar", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 3, 4, 6}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 3, 4, 5, 6}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "ba", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{3}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "foobar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "A FOO", + "a fOoBar", + "aA aBC A", + "ca afdf a,foobar baz", + "a fddf foobarbaz", + "a afoobarbaz", + "a fooBaR", + "a kjlkjf dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "a", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "нГк", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{8}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "aa a", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{2}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "!,", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{9}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "aa ax", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "qwe rty abc", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "@", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 5}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "0", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{3, 4}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "33", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "1234", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65535", + "1234", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "123", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "0", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{1}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "33", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "123456", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65536", + "1234", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "123", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "65536", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{3}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "33", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12345678901", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65536", + "12345678901", + "1", + "2", + "3", + "4", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "1234", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12345678901", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{4}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "33", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12345678901234567890", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "-65536", + "1234.5678901", + "1", + "0.0002", + "-320001", + "4", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "123", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "1234.5678901", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "56789", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{4}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "-6553", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{3}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "65536", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{3}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "7344.8943", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "-1234", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "+1234", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "23", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "678", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "33", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12345678901234567890", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "12.0.127.6", + "55.55.12.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "127.0.0.1", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "12", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 6, 7, 8, 9}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "127.0.0", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "2.3.", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{0}) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "0", + } + testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 4, 5, 6, 7, 8}) + + // mismatch + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "8", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "127.1", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "27.0", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "foo", + prefix: "255.255.255.255", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2006-01-02T15:04:05.001Z", + "2006-01-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2006-01-02T15:04:05.005Z", + "2006-01-02T15:04:05.006Z", + "2006-01-02T15:04:05.007Z", + "2006-01-02T15:04:05.008Z", + "2006-01-02T15:04:05.009Z", + }, + }, + } + + // match + fp := &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "2006-01-02t15:04:05.005z", + } + testFilterMatchForColumns(t, columns, fp, "_msg", []int{4}) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "2006-01-0", + } + testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "002", + } + testFilterMatchForColumns(t, columns, fp, "_msg", []int{1}) + + // mimatch + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "bar", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "2006-03-02T15:04:05.005Z", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "06", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + + // This filter shouldn't match row=4, since it has different string representation of the timestamp + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "2006-01-02T16:04:05.005+01:00", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + + // This filter shouldn't match row=4, since it contains too many digits for millisecond part + fp = &filterAnyCasePrefix{ + fieldName: "_msg", + prefix: "2006-01-02T15:04:05.00500Z", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + + fp = &filterAnyCasePrefix{ + fieldName: "non-existing-column", + prefix: "", + } + testFilterMatchForColumns(t, columns, fp, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index 933040765..de301aa3b 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -8,40 +8,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" ) -func TestMatchAnyCasePrefix(t *testing.T) { - f := func(s, prefixLowercase string, resultExpected bool) { - t.Helper() - result := matchAnyCasePrefix(s, prefixLowercase) - if result != resultExpected { - t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) - } - } - - // empty prefix matches non-empty strings - f("", "", false) - f("foo", "", true) - f("тест", "", true) - - // empty string doesn't match non-empty prefix - f("", "foo", false) - f("", "тест", false) - - // full match - f("foo", "foo", true) - f("FOo", "foo", true) - f("Test ТЕСт 123", "test тест 123", true) - - // prefix match - f("foo", "f", true) - f("foo тест bar", "те", true) - f("foo ТЕСТ bar", "те", true) - - // mismatch - f("foo", "o", false) - f("тест", "foo", false) - f("Тест", "ест", false) -} - func TestMatchAnyCasePhrase(t *testing.T) { f := func(s, phraseLowercase string, resultExpected bool) { t.Helper() @@ -332,897 +298,6 @@ func TestStreamFilter(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", nil) } -func TestAnyCasePrefixFilter(t *testing.T) { - t.Run("single-row", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "aBc DEf", - }, - }, - { - name: "other column", - values: []string{ - "aSDfdsf", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "ABC", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "ab", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "abc def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "other column", - prefix: "asdfdSF", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bc", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "other column", - prefix: "sd", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "other-column", - values: []string{ - "x", - "X", - "X", - }, - }, - { - name: "foo", - values: []string{ - "abc def", - "ABC DEF", - "AbC Def", - }, - }, - { - name: "_msg", - values: []string{ - "1 2 3", - "1 2 3", - "1 2 3", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "Abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "AB", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "abc de", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: " de", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "abc def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "other-column", - prefix: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: " 2 ", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "abc def ", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "other-column", - prefix: "foo", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "foo", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "fOObar", - "Abc", - "aFDf FooBar baz", - "fddf FOObarBAZ", - "AFoobarbaz", - "foobar", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "FooBar", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 4, 6}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 3, 4, 5, 6}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "ba", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "foobar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "A FOO", - "a fOoBar", - "aA aBC A", - "ca afdf a,foobar baz", - "a fddf foobarbaz", - "a afoobarbaz", - "a fooBaR", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "a", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "нГк", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{8}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "aa a", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "!,", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{9}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "aa ax", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "qwe rty abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "@", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 5}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65535", - "1234", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "123", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "123456", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65536", - "1234", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "123", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "65536", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12345678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65536", - "12345678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12345678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12345678901234567890", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "-65536", - "1234.5678901", - "1", - "0.0002", - "-320001", - "4", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "123", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "1234.5678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "56789", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "-6553", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "65536", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "7344.8943", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "-1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "+1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "23", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "678", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12345678901234567890", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.12.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "127.0.0.1", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "127.0.0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "2.3.", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8}) - - // mismatch - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "8", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "127.1", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "27.0", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "foo", - prefix: "255.255.255.255", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2006-01-02T15:04:05.005Z", - "2006-01-02T15:04:05.006Z", - "2006-01-02T15:04:05.007Z", - "2006-01-02T15:04:05.008Z", - "2006-01-02T15:04:05.009Z", - }, - }, - } - - // match - pf := &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "2006-01-02t15:04:05.005z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{4}) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "2006-01-0", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "002", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{1}) - - // mimatch - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "2006-03-02T15:04:05.005Z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "06", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - // This filter shouldn't match row=4, since it has different string representation of the timestamp - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "2006-01-02T16:04:05.005+01:00", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - // This filter shouldn't match row=4, since it contains too many digits for millisecond part - pf = &anyCasePrefixFilter{ - fieldName: "_msg", - prefix: "2006-01-02T15:04:05.00500Z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - }) -} - func TestPrefixFilter(t *testing.T) { t.Run("single-row", func(t *testing.T) { columns := []column{ diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 6171e2415..5b1724471 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -477,7 +477,7 @@ func parseFilterNot(lex *lexer, fieldName string) (filter, error) { func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) { return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) { if isPrefixFilter { - f := &anyCasePrefixFilter{ + f := &filterAnyCasePrefix{ fieldName: fieldName, prefix: phrase, } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 15c0cb88f..824a17892 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -411,15 +411,15 @@ func TestParseAnyCasePhraseFilter(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - pf, ok := q.f.(*anyCasePhraseFilter) + fp, ok := q.f.(*anyCasePhraseFilter) if !ok { t.Fatalf("unexpected filter type; got %T; want *anyCasePhraseFilter; filter: %s", q.f, q.f) } - if pf.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) + if fp.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) } - if pf.phrase != phraseExpected { - t.Fatalf("unexpected phrase; got %q; want %q", pf.phrase, phraseExpected) + if fp.phrase != phraseExpected { + t.Fatalf("unexpected phrase; got %q; want %q", fp.phrase, phraseExpected) } } @@ -436,15 +436,15 @@ func TestParseAnyCasePrefixFilter(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - pf, ok := q.f.(*anyCasePrefixFilter) + fp, ok := q.f.(*filterAnyCasePrefix) if !ok { - t.Fatalf("unexpected filter type; got %T; want *anyCasePrefixFilter; filter: %s", q.f, q.f) + t.Fatalf("unexpected filter type; got %T; want *filterAnyCasePrefix; filter: %s", q.f, q.f) } - if pf.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) + if fp.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) } - if pf.prefix != prefixExpected { - t.Fatalf("unexpected prefix; got %q; want %q", pf.prefix, prefixExpected) + if fp.prefix != prefixExpected { + t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected) } } @@ -463,15 +463,15 @@ func TestParsePhraseFilter(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - pf, ok := q.f.(*phraseFilter) + fp, ok := q.f.(*phraseFilter) if !ok { t.Fatalf("unexpected filter type; got %T; want *phraseFilter; filter: %s", q.f, q.f) } - if pf.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) + if fp.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) } - if pf.phrase != phraseExpected { - t.Fatalf("unexpected prefix; got %q; want %q", pf.phrase, phraseExpected) + if fp.phrase != phraseExpected { + t.Fatalf("unexpected prefix; got %q; want %q", fp.phrase, phraseExpected) } } @@ -490,15 +490,15 @@ func TestParsePrefixFilter(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - pf, ok := q.f.(*prefixFilter) + fp, ok := q.f.(*prefixFilter) if !ok { t.Fatalf("unexpected filter type; got %T; want *prefixFilter; filter: %s", q.f, q.f) } - if pf.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) + if fp.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) } - if pf.prefix != prefixExpected { - t.Fatalf("unexpected prefix; got %q; want %q", pf.prefix, prefixExpected) + if fp.prefix != prefixExpected { + t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected) } }