diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 254015eaf..52e1ebd96 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -2,7 +2,6 @@ package logstorage import ( "bytes" - "fmt" "math" "strconv" "strings" @@ -12,7 +11,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil" ) type filter interface { @@ -71,93 +69,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { } } -// anyCasePhraseFilter filters field entries by case-insensitive phrase match. -// -// An example LogsQL query: `fieldName:i(word)` or `fieldName:i("word1 ... wordN")` -type anyCasePhraseFilter struct { - fieldName string - phrase string - - phraseLowercaseOnce sync.Once - phraseLowercase string - - tokensOnce sync.Once - tokens []string -} - -func (fp *anyCasePhraseFilter) String() string { - return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase)) -} - -func (fp *anyCasePhraseFilter) getTokens() []string { - fp.tokensOnce.Do(fp.initTokens) - return fp.tokens -} - -func (fp *anyCasePhraseFilter) initTokens() { - fp.tokens = tokenizeStrings(nil, []string{fp.phrase}) -} - -func (fp *anyCasePhraseFilter) getPhraseLowercase() string { - fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase) - return fp.phraseLowercase -} - -func (fp *anyCasePhraseFilter) initPhraseLowercase() { - fp.phraseLowercase = strings.ToLower(fp.phrase) -} - -func (fp *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := fp.fieldName - phraseLowercase := fp.getPhraseLowercase() - - // Verify whether fp matches const column - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - if !matchAnyCasePhrase(v, phraseLowercase) { - bm.resetBits() - } - return - } - - // Verify whether fp matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - // Fast path - there are no matching columns. - // It matches anything only for empty phrase. - if len(phraseLowercase) > 0 { - bm.resetBits() - } - return - } - - tokens := fp.getTokens() - - switch ch.valueType { - case valueTypeString: - matchStringByAnyCasePhrase(bs, ch, bm, phraseLowercase) - case valueTypeDict: - matchValuesDictByAnyCasePhrase(bs, ch, bm, phraseLowercase) - case valueTypeUint8: - matchUint8ByExactValue(bs, ch, bm, phraseLowercase, tokens) - case valueTypeUint16: - matchUint16ByExactValue(bs, ch, bm, phraseLowercase, tokens) - case valueTypeUint32: - matchUint32ByExactValue(bs, ch, bm, phraseLowercase, tokens) - case valueTypeUint64: - matchUint64ByExactValue(bs, ch, bm, phraseLowercase, tokens) - case valueTypeFloat64: - matchFloat64ByPhrase(bs, ch, bm, phraseLowercase, tokens) - case valueTypeIPv4: - matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens) - case valueTypeTimestampISO8601: - phraseUppercase := strings.ToUpper(fp.phrase) - matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens) - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // phraseFilter filters field entries by phrase match (aka full text search). // // A phrase consists of any number of words with delimiters between them. @@ -314,17 +225,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase bbPool.Put(bb) } -func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { - bb := bbPool.Get() - for i, v := range ch.valuesDict.values { - if matchAnyCasePhrase(v, phraseLowercase) { - bb.B = append(bb.B, byte(i)) - } - } - matchEncodedValuesDict(bs, ch, bm, bb.B) - bbPool.Put(bb) -} - func matchValuesDictByAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}) { bb := bbPool.Get() for i, v := range ch.valuesDict.values { @@ -363,12 +263,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod }) } -func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { - visitValues(bs, ch, bm, func(v string) bool { - return matchAnyCasePhrase(v, phraseLowercase) - }) -} - func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) { if !matchBloomFilterAllTokens(bs, ch, tokens) { bm.resetBits() @@ -422,30 +316,6 @@ func isASCIILowercase(s string) bool { return true } -func matchAnyCasePhrase(s, phraseLowercase string) bool { - if len(phraseLowercase) == 0 { - // Special case - empty phrase matches only empty string. - return len(s) == 0 - } - if len(phraseLowercase) > len(s) { - return false - } - - if isASCIILowercase(s) { - // Fast path - s is in lowercase - return matchPhrase(s, phraseLowercase) - } - - // Slow path - convert s to lowercase before matching - bb := bbPool.Get() - bb.B = stringsutil.AppendLowercase(bb.B, s) - sLowercase := bytesutil.ToUnsafeString(bb.B) - ok := matchPhrase(sLowercase, phraseLowercase) - bbPool.Put(bb) - - return ok -} - func matchPhrase(s, phrase string) bool { if len(phrase) == 0 { // Special case - empty phrase matches only empty string. diff --git a/lib/logstorage/filter_any_case_phrase.go b/lib/logstorage/filter_any_case_phrase.go new file mode 100644 index 000000000..0febdf341 --- /dev/null +++ b/lib/logstorage/filter_any_case_phrase.go @@ -0,0 +1,139 @@ +package logstorage + +import ( + "fmt" + "strings" + "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil" +) + +// filterAnyCasePhrase filters field entries by case-insensitive phrase match. +// +// An example LogsQL query: `fieldName:i(word)` or `fieldName:i("word1 ... wordN")` +type filterAnyCasePhrase struct { + fieldName string + phrase string + + phraseLowercaseOnce sync.Once + phraseLowercase string + + tokensOnce sync.Once + tokens []string +} + +func (fp *filterAnyCasePhrase) String() string { + return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase)) +} + +func (fp *filterAnyCasePhrase) getTokens() []string { + fp.tokensOnce.Do(fp.initTokens) + return fp.tokens +} + +func (fp *filterAnyCasePhrase) initTokens() { + fp.tokens = tokenizeStrings(nil, []string{fp.phrase}) +} + +func (fp *filterAnyCasePhrase) getPhraseLowercase() string { + fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase) + return fp.phraseLowercase +} + +func (fp *filterAnyCasePhrase) initPhraseLowercase() { + fp.phraseLowercase = strings.ToLower(fp.phrase) +} + +func (fp *filterAnyCasePhrase) apply(bs *blockSearch, bm *bitmap) { + fieldName := fp.fieldName + phraseLowercase := fp.getPhraseLowercase() + + // Verify whether fp matches const column + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + if !matchAnyCasePhrase(v, phraseLowercase) { + bm.resetBits() + } + return + } + + // Verify whether fp matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + // Fast path - there are no matching columns. + // It matches anything only for empty phrase. + if len(phraseLowercase) > 0 { + bm.resetBits() + } + return + } + + tokens := fp.getTokens() + + switch ch.valueType { + case valueTypeString: + matchStringByAnyCasePhrase(bs, ch, bm, phraseLowercase) + case valueTypeDict: + matchValuesDictByAnyCasePhrase(bs, ch, bm, phraseLowercase) + case valueTypeUint8: + matchUint8ByExactValue(bs, ch, bm, phraseLowercase, tokens) + case valueTypeUint16: + matchUint16ByExactValue(bs, ch, bm, phraseLowercase, tokens) + case valueTypeUint32: + matchUint32ByExactValue(bs, ch, bm, phraseLowercase, tokens) + case valueTypeUint64: + matchUint64ByExactValue(bs, ch, bm, phraseLowercase, tokens) + case valueTypeFloat64: + matchFloat64ByPhrase(bs, ch, bm, phraseLowercase, tokens) + case valueTypeIPv4: + matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens) + case valueTypeTimestampISO8601: + phraseUppercase := strings.ToUpper(fp.phrase) + matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens) + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { + bb := bbPool.Get() + for i, v := range ch.valuesDict.values { + if matchAnyCasePhrase(v, phraseLowercase) { + bb.B = append(bb.B, byte(i)) + } + } + matchEncodedValuesDict(bs, ch, bm, bb.B) + bbPool.Put(bb) +} + +func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { + visitValues(bs, ch, bm, func(v string) bool { + return matchAnyCasePhrase(v, phraseLowercase) + }) +} + +func matchAnyCasePhrase(s, phraseLowercase string) bool { + if len(phraseLowercase) == 0 { + // Special case - empty phrase matches only empty string. + return len(s) == 0 + } + if len(phraseLowercase) > len(s) { + return false + } + + if isASCIILowercase(s) { + // Fast path - s is in lowercase + return matchPhrase(s, phraseLowercase) + } + + // Slow path - convert s to lowercase before matching + bb := bbPool.Get() + bb.B = stringsutil.AppendLowercase(bb.B, s) + sLowercase := bytesutil.ToUnsafeString(bb.B) + ok := matchPhrase(sLowercase, phraseLowercase) + bbPool.Put(bb) + + return ok +} diff --git a/lib/logstorage/filter_any_case_phrase_test.go b/lib/logstorage/filter_any_case_phrase_test.go new file mode 100644 index 000000000..99a6516df --- /dev/null +++ b/lib/logstorage/filter_any_case_phrase_test.go @@ -0,0 +1,888 @@ +package logstorage + +import ( + "testing" +) + +func TestMatchAnyCasePhrase(t *testing.T) { + f := func(s, phraseLowercase string, resultExpected bool) { + t.Helper() + result := matchAnyCasePhrase(s, phraseLowercase) + if result != resultExpected { + t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) + } + } + + // empty phrase matches only empty string + f("", "", true) + f("foo", "", false) + f("тест", "", false) + + // empty string doesn't match non-empty phrase + f("", "foo", false) + f("", "тест", false) + + // full match + f("foo", "foo", true) + f("FOo", "foo", true) + f("Test ТЕСт 123", "test тест 123", true) + + // phrase match + f("a foo", "foo", true) + f("foo тест bar", "тест", true) + f("foo ТЕСТ bar", "тест bar", true) + + // mismatch + f("foo", "fo", false) + f("тест", "foo", false) + f("Тест", "ест", false) +} + +func TestFilterAnyCasePhrase(t *testing.T) { + t.Run("single-row", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "aBc DEf", + }, + }, + { + name: "other column", + values: []string{ + "aSDfdsF", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "Abc", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "abc def", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "def", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "other column", + phrase: "ASdfdsf", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "ab", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "other column", + phrase: "sd", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing column", + phrase: "abc", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "other-column", + values: []string{ + "X", + "x", + "x", + }, + }, + { + name: "foo", + values: []string{ + "aBC def", + "abc DEf", + "Abc deF", + }, + }, + { + name: "_msg", + values: []string{ + "1 2 3", + "1 2 3", + "1 2 3", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "abc", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "def", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: " def", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "abc def", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "other-column", + phrase: "x", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: " 2 ", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "abc def ", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "x", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "other-column", + phrase: "foo", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing column", + phrase: "x", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "foo", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "fooBar", + "ABc", + "afdf foobar BAz", + "fddf fOObARbaz", + "AfooBarbaz", + "foobar", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "FoobAr", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "baZ", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing column", + phrase: "foobar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "a foo", + "A Foobar", + "aA aBC a", + "ca afdf a,foobar baz", + "a fddf foobarbaz", + "a aFOObarbaz", + "a foobar", + "a kjlkjf dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "A", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "НгкШ", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{8}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "!,", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{9}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "aa a", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "@", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "12", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "0", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "33", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65535", + "1234", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "0", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{1}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "33", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "123456", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65536", + "1234", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "65536", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "33", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "12345678901", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "65536", + "12345678901", + "1", + "2", + "3", + "4", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "12345678901", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "33", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "12345678901234567890", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "-65536", + "1234.5678901", + "1", + "2", + "3", + "4", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "1234.5678901", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "5678901", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "-65536", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "65536", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "-1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "+1234", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "123", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "5678", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "33", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "12345678901234567890", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "12.0.127.6", + "55.55.55.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "127.0.0.1", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "127", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "127.0.0", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "2.3", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "0", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + // mismatch + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "5", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "127.1", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "27.0", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "foo", + phrase: "255.255.255.255", + } + testFilterMatchForColumns(t, columns, pf, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2006-01-02T15:04:05.001Z", + "2006-01-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2006-01-02T15:04:05.005Z", + "2006-01-02T15:04:05.006Z", + "2006-01-02T15:04:05.007Z", + "2006-01-02T15:04:05.008Z", + "2006-01-02T15:04:05.009Z", + }, + }, + } + + // match + pf := &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "2006-01-02t15:04:05.005z", + } + testFilterMatchForColumns(t, columns, pf, "_msg", []int{4}) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "2006-01", + } + testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "002Z", + } + testFilterMatchForColumns(t, columns, pf, "_msg", []int{1}) + + pf = &filterAnyCasePhrase{ + fieldName: "non-existing-column", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mimatch + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "bar", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "2006-03-02T15:04:05.005Z", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "06", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + + // This filter shouldn't match row=4, since it has different string representation of the timestamp + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "2006-01-02T16:04:05.005+01:00", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + + // This filter shouldn't match row=4, since it contains too many digits for millisecond part + pf = &filterAnyCasePhrase{ + fieldName: "_msg", + phrase: "2006-01-02T15:04:05.00500Z", + } + testFilterMatchForColumns(t, columns, pf, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index ab518ebf4..aab49a682 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -8,40 +8,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" ) -func TestMatchAnyCasePhrase(t *testing.T) { - f := func(s, phraseLowercase string, resultExpected bool) { - t.Helper() - result := matchAnyCasePhrase(s, phraseLowercase) - if result != resultExpected { - t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) - } - } - - // empty phrase matches only empty string - f("", "", true) - f("foo", "", false) - f("тест", "", false) - - // empty string doesn't match non-empty phrase - f("", "foo", false) - f("", "тест", false) - - // full match - f("foo", "foo", true) - f("FOo", "foo", true) - f("Test ТЕСт 123", "test тест 123", true) - - // phrase match - f("a foo", "foo", true) - f("foo тест bar", "тест", true) - f("foo ТЕСТ bar", "тест bar", true) - - // mismatch - f("foo", "fo", false) - f("тест", "foo", false) - f("Тест", "ест", false) -} - func TestMatchPhrase(t *testing.T) { f := func(s, phrase string, resultExpected bool) { t.Helper() @@ -259,855 +225,6 @@ func TestStreamFilter(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", nil) } -func TestAnyCasePhraseFilter(t *testing.T) { - t.Run("single-row", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "aBc DEf", - }, - }, - { - name: "other column", - values: []string{ - "aSDfdsF", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "Abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "abc def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "other column", - phrase: "ASdfdsf", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "ab", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "other column", - phrase: "sd", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing column", - phrase: "abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "other-column", - values: []string{ - "X", - "x", - "x", - }, - }, - { - name: "foo", - values: []string{ - "aBC def", - "abc DEf", - "Abc deF", - }, - }, - { - name: "_msg", - values: []string{ - "1 2 3", - "1 2 3", - "1 2 3", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "abc", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: " def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "abc def", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "other-column", - phrase: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: " 2 ", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "abc def ", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "other-column", - phrase: "foo", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing column", - phrase: "x", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "foo", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "fooBar", - "ABc", - "afdf foobar BAz", - "fddf fOObARbaz", - "AfooBarbaz", - "foobar", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "FoobAr", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "baZ", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing column", - phrase: "foobar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "a foo", - "A Foobar", - "aA aBC a", - "ca afdf a,foobar baz", - "a fddf foobarbaz", - "a aFOObarbaz", - "a foobar", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "A", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "НгкШ", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{8}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "!,", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{9}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "aa a", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "@", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "12", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65535", - "1234", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "123456", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65536", - "1234", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "65536", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "12345678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "65536", - "12345678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "12345678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "12345678901234567890", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "-65536", - "1234.5678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "1234.5678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "5678901", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "-65536", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "65536", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "-1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "+1234", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "123", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "5678", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "33", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "12345678901234567890", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.55.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "127.0.0.1", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "127", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "127.0.0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "2.3", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "0", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - // mismatch - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "5", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "127.1", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "27.0", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "foo", - phrase: "255.255.255.255", - } - testFilterMatchForColumns(t, columns, pf, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2006-01-02T15:04:05.005Z", - "2006-01-02T15:04:05.006Z", - "2006-01-02T15:04:05.007Z", - "2006-01-02T15:04:05.008Z", - "2006-01-02T15:04:05.009Z", - }, - }, - } - - // match - pf := &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "2006-01-02t15:04:05.005z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{4}) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "2006-01", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "002Z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{1}) - - pf = &anyCasePhraseFilter{ - fieldName: "non-existing-column", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mimatch - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "bar", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "2006-03-02T15:04:05.005Z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "06", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - // This filter shouldn't match row=4, since it has different string representation of the timestamp - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "2006-01-02T16:04:05.005+01:00", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - - // This filter shouldn't match row=4, since it contains too many digits for millisecond part - pf = &anyCasePhraseFilter{ - fieldName: "_msg", - phrase: "2006-01-02T15:04:05.00500Z", - } - testFilterMatchForColumns(t, columns, pf, "_msg", nil) - }) -} - func TestPhraseFilter(t *testing.T) { t.Run("single-row", func(t *testing.T) { columns := []column{ diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 9f0ce18ea..fc8e4f286 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -483,7 +483,7 @@ func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) { } return f, nil } - f := &anyCasePhraseFilter{ + f := &filterAnyCasePhrase{ fieldName: fieldName, phrase: phrase, } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index a43c0fb4f..616e57cf2 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -411,9 +411,9 @@ func TestParseAnyCasePhraseFilter(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - fp, ok := q.f.(*anyCasePhraseFilter) + fp, ok := q.f.(*filterAnyCasePhrase) if !ok { - t.Fatalf("unexpected filter type; got %T; want *anyCasePhraseFilter; filter: %s", q.f, q.f) + t.Fatalf("unexpected filter type; got %T; want *filterAnyCasePhrase; filter: %s", q.f, q.f) } if fp.fieldName != fieldNameExpected { t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)