From 68ec1cb1ddbc81530ae9530af3e556c51a55dff0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 29 Apr 2024 05:04:20 +0200 Subject: [PATCH] wip --- lib/logstorage/filter.go | 247 +---- lib/logstorage/filter_and.go | 2 +- lib/logstorage/filter_sequence.go | 234 +++++ lib/logstorage/filter_sequence_test.go | 821 +++++++++++++++ lib/logstorage/filter_test.go | 1273 ------------------------ lib/logstorage/filter_time.go | 40 + lib/logstorage/parser.go | 8 +- lib/logstorage/parser_test.go | 14 +- 8 files changed, 1118 insertions(+), 1521 deletions(-) create mode 100644 lib/logstorage/filter_sequence.go create mode 100644 lib/logstorage/filter_sequence_test.go create mode 100644 lib/logstorage/filter_time.go diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index ba4e445e2..7bf63a9f8 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -39,142 +39,39 @@ type streamFilter struct { streamIDs map[streamID]struct{} } -func (sf *streamFilter) String() string { - s := sf.f.String() +func (fs *streamFilter) String() string { + s := fs.f.String() if s == "{}" { return "" } return "_stream:" + s } -func (sf *streamFilter) getStreamIDs() map[streamID]struct{} { - sf.streamIDsOnce.Do(sf.initStreamIDs) - return sf.streamIDs +func (fs *streamFilter) getStreamIDs() map[streamID]struct{} { + fs.streamIDsOnce.Do(fs.initStreamIDs) + return fs.streamIDs } -func (sf *streamFilter) initStreamIDs() { - streamIDs := sf.idb.searchStreamIDs(sf.tenantIDs, sf.f) +func (fs *streamFilter) initStreamIDs() { + streamIDs := fs.idb.searchStreamIDs(fs.tenantIDs, fs.f) m := make(map[streamID]struct{}, len(streamIDs)) for i := range streamIDs { m[streamIDs[i]] = struct{}{} } - sf.streamIDs = m + fs.streamIDs = m } -func (sf *streamFilter) apply(bs *blockSearch, bm *bitmap) { - if sf.f.isEmpty() { +func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { + if fs.f.isEmpty() { return } - streamIDs := sf.getStreamIDs() + streamIDs := fs.getStreamIDs() if _, ok := streamIDs[bs.bsw.bh.streamID]; !ok { bm.resetBits() return } } -// sequenceFilter matches an ordered sequence of phrases -// -// Example LogsQL: `fieldName:seq(foo, "bar baz")` -type sequenceFilter struct { - fieldName string - phrases []string - - tokensOnce sync.Once - tokens []string - - nonEmptyPhrasesOnce sync.Once - nonEmptyPhrases []string -} - -func (sf *sequenceFilter) String() string { - phrases := sf.phrases - a := make([]string, len(phrases)) - for i, phrase := range phrases { - a[i] = quoteTokenIfNeeded(phrase) - } - return fmt.Sprintf("%sseq(%s)", quoteFieldNameIfNeeded(sf.fieldName), strings.Join(a, ",")) -} - -func (sf *sequenceFilter) getTokens() []string { - sf.tokensOnce.Do(sf.initTokens) - return sf.tokens -} - -func (sf *sequenceFilter) initTokens() { - phrases := sf.getNonEmptyPhrases() - tokens := tokenizeStrings(nil, phrases) - sf.tokens = tokens -} - -func (sf *sequenceFilter) getNonEmptyPhrases() []string { - sf.nonEmptyPhrasesOnce.Do(sf.initNonEmptyPhrases) - return sf.nonEmptyPhrases -} - -func (sf *sequenceFilter) initNonEmptyPhrases() { - phrases := sf.phrases - result := make([]string, 0, len(phrases)) - for _, phrase := range phrases { - if phrase != "" { - result = append(result, phrase) - } - } - sf.nonEmptyPhrases = result -} - -func (sf *sequenceFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := sf.fieldName - phrases := sf.getNonEmptyPhrases() - - if len(phrases) == 0 { - return - } - - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - if !matchSequence(v, phrases) { - bm.resetBits() - } - return - } - - // Verify whether filter matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - // Fast path - there are no matching columns. - // It matches anything only for empty phrase. - if !matchSequence("", phrases) { - bm.resetBits() - } - return - } - - tokens := sf.getTokens() - - switch ch.valueType { - case valueTypeString: - matchStringBySequence(bs, ch, bm, phrases, tokens) - case valueTypeDict: - matchValuesDictBySequence(bs, ch, bm, phrases) - case valueTypeUint8: - matchUint8BySequence(bs, ch, bm, phrases, tokens) - case valueTypeUint16: - matchUint16BySequence(bs, ch, bm, phrases, tokens) - case valueTypeUint32: - matchUint32BySequence(bs, ch, bm, phrases, tokens) - case valueTypeUint64: - matchUint64BySequence(bs, ch, bm, phrases, tokens) - case valueTypeFloat64: - matchFloat64BySequence(bs, ch, bm, phrases, tokens) - case valueTypeIPv4: - matchIPv4BySequence(bs, ch, bm, phrases, tokens) - case valueTypeTimestampISO8601: - matchTimestampISO8601BySequence(bs, ch, bm, phrases, tokens) - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // exactPrefixFilter matches the exact prefix. // // Example LogsQL: `fieldName:exact("foo bar"*) @@ -1329,25 +1226,6 @@ func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap bbPool.Put(bb) } -func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) == 1 { - matchTimestampISO8601ByPhrase(bs, ch, bm, phrases[0], tokens) - return - } - if !matchBloomFilterAllTokens(bs, ch, tokens) { - bm.resetBits() - return - } - - // Slow path - phrases contain incomplete timestamp. Search over string representation of the timestamp. - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toTimestampISO8601StringExt(bs, bb, v) - return matchSequence(s, phrases) - }) - bbPool.Put(bb) -} - func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) { if prefix == "" { return @@ -1473,27 +1351,6 @@ func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix str bbPool.Put(bb) } -func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) == 1 { - matchIPv4ByPhrase(bs, ch, bm, phrases[0], tokens) - return - } - if !matchBloomFilterAllTokens(bs, ch, tokens) { - bm.resetBits() - return - } - - // Slow path - phrases contain parts of IP address. For example, `1.23` should match `1.23.4.5` and `4.1.23.54`. - // We cannot compare binary represetnation of ip address and need converting - // the ip to string before searching for prefix there. - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toIPv4StringExt(bs, bb, v) - return matchSequence(s, phrases) - }) - bbPool.Put(bb) -} - func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) { if prefix == "" { return @@ -1629,24 +1486,6 @@ func matchFloat64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix bbPool.Put(bb) } -func matchFloat64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if !matchBloomFilterAllTokens(bs, ch, tokens) { - bm.resetBits() - return - } - // The phrase may contain a part of the floating-point number. - // For example, `foo:"123"` must match `123`, `123.456` and `-0.123`. - // This means we cannot search in binary representation of floating-point numbers. - // Instead, we need searching for the whole phrase in string representation - // of floating-point numbers :( - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toFloat64StringExt(bs, bb, v) - return matchSequence(s, phrases) - }) - bbPool.Put(bb) -} - func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) { if prefix == "" { // An empty prefix matches all the values @@ -1795,17 +1634,6 @@ func matchValuesDictByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pref bbPool.Put(bb) } -func matchValuesDictBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string) { - bb := bbPool.Get() - for i, v := range ch.valuesDict.values { - if matchSequence(v, phrases) { - bb.B = append(bb.B, byte(i)) - } - } - matchEncodedValuesDict(bs, ch, bm, bb.B) - bbPool.Put(bb) -} - func matchValuesDictByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string) { bb := bbPool.Get() for i, v := range ch.valuesDict.values { @@ -1918,16 +1746,6 @@ func matchStringByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s }) } -func matchStringBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []string) { - if !matchBloomFilterAllTokens(bs, ch, tokens) { - bm.resetBits() - return - } - visitValues(bs, ch, bm, func(v string) bool { - return matchSequence(v, phrases) - }) -} - func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) { if !matchBloomFilterAllTokens(bs, ch, tokens) { bm.resetBits() @@ -2279,38 +2097,6 @@ func matchUint64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s bbPool.Put(bb) } -func matchUint8BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) > 1 { - bm.resetBits() - return - } - matchUint8ByExactValue(bs, ch, bm, phrases[0], tokens) -} - -func matchUint16BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) > 1 { - bm.resetBits() - return - } - matchUint16ByExactValue(bs, ch, bm, phrases[0], tokens) -} - -func matchUint32BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) > 1 { - bm.resetBits() - return - } - matchUint32ByExactValue(bs, ch, bm, phrases[0], tokens) -} - -func matchUint64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { - if len(phrases) > 1 { - bm.resetBits() - return - } - matchUint64ByExactValue(bs, ch, bm, phrases[0], tokens) -} - func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) { if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) { return @@ -2583,17 +2369,6 @@ func matchRange(s string, minValue, maxValue float64) bool { return f >= minValue && f <= maxValue } -func matchSequence(s string, phrases []string) bool { - for _, phrase := range phrases { - n := getPhrasePos(s, phrase) - if n < 0 { - return false - } - s = s[n+len(phrase):] - } - return true -} - func matchAnyCasePhrase(s, phraseLowercase string) bool { if len(phraseLowercase) == 0 { // Special case - empty phrase matches only empty string. diff --git a/lib/logstorage/filter_and.go b/lib/logstorage/filter_and.go index 3b639303e..6cc2f82cd 100644 --- a/lib/logstorage/filter_and.go +++ b/lib/logstorage/filter_and.go @@ -69,7 +69,7 @@ func (fa *filterAnd) initMsgTokens() { if isMsgFieldName(t.fieldName) { a = append(a, t.getTokens()...) } - case *sequenceFilter: + case *filterSequence: if isMsgFieldName(t.fieldName) { a = append(a, t.getTokens()...) } diff --git a/lib/logstorage/filter_sequence.go b/lib/logstorage/filter_sequence.go new file mode 100644 index 000000000..075999593 --- /dev/null +++ b/lib/logstorage/filter_sequence.go @@ -0,0 +1,234 @@ +package logstorage + +import ( + "fmt" + "strings" + "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" +) + +// filterSequence matches an ordered sequence of phrases +// +// Example LogsQL: `fieldName:seq(foo, "bar baz")` +type filterSequence struct { + fieldName string + phrases []string + + tokensOnce sync.Once + tokens []string + + nonEmptyPhrasesOnce sync.Once + nonEmptyPhrases []string +} + +func (fs *filterSequence) String() string { + phrases := fs.phrases + a := make([]string, len(phrases)) + for i, phrase := range phrases { + a[i] = quoteTokenIfNeeded(phrase) + } + return fmt.Sprintf("%sseq(%s)", quoteFieldNameIfNeeded(fs.fieldName), strings.Join(a, ",")) +} + +func (fs *filterSequence) getTokens() []string { + fs.tokensOnce.Do(fs.initTokens) + return fs.tokens +} + +func (fs *filterSequence) initTokens() { + phrases := fs.getNonEmptyPhrases() + tokens := tokenizeStrings(nil, phrases) + fs.tokens = tokens +} + +func (fs *filterSequence) getNonEmptyPhrases() []string { + fs.nonEmptyPhrasesOnce.Do(fs.initNonEmptyPhrases) + return fs.nonEmptyPhrases +} + +func (fs *filterSequence) initNonEmptyPhrases() { + phrases := fs.phrases + result := make([]string, 0, len(phrases)) + for _, phrase := range phrases { + if phrase != "" { + result = append(result, phrase) + } + } + fs.nonEmptyPhrases = result +} + +func (fs *filterSequence) apply(bs *blockSearch, bm *bitmap) { + fieldName := fs.fieldName + phrases := fs.getNonEmptyPhrases() + + if len(phrases) == 0 { + return + } + + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + if !matchSequence(v, phrases) { + bm.resetBits() + } + return + } + + // Verify whether filter matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + // Fast path - there are no matching columns. + // It matches anything only for empty phrase. + if !matchSequence("", phrases) { + bm.resetBits() + } + return + } + + tokens := fs.getTokens() + + switch ch.valueType { + case valueTypeString: + matchStringBySequence(bs, ch, bm, phrases, tokens) + case valueTypeDict: + matchValuesDictBySequence(bs, ch, bm, phrases) + case valueTypeUint8: + matchUint8BySequence(bs, ch, bm, phrases, tokens) + case valueTypeUint16: + matchUint16BySequence(bs, ch, bm, phrases, tokens) + case valueTypeUint32: + matchUint32BySequence(bs, ch, bm, phrases, tokens) + case valueTypeUint64: + matchUint64BySequence(bs, ch, bm, phrases, tokens) + case valueTypeFloat64: + matchFloat64BySequence(bs, ch, bm, phrases, tokens) + case valueTypeIPv4: + matchIPv4BySequence(bs, ch, bm, phrases, tokens) + case valueTypeTimestampISO8601: + matchTimestampISO8601BySequence(bs, ch, bm, phrases, tokens) + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) == 1 { + matchTimestampISO8601ByPhrase(bs, ch, bm, phrases[0], tokens) + return + } + if !matchBloomFilterAllTokens(bs, ch, tokens) { + bm.resetBits() + return + } + + // Slow path - phrases contain incomplete timestamp. Search over string representation of the timestamp. + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toTimestampISO8601StringExt(bs, bb, v) + return matchSequence(s, phrases) + }) + bbPool.Put(bb) +} + +func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) == 1 { + matchIPv4ByPhrase(bs, ch, bm, phrases[0], tokens) + return + } + if !matchBloomFilterAllTokens(bs, ch, tokens) { + bm.resetBits() + return + } + + // Slow path - phrases contain parts of IP address. For example, `1.23` should match `1.23.4.5` and `4.1.23.54`. + // We cannot compare binary represetnation of ip address and need converting + // the ip to string before searching for prefix there. + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toIPv4StringExt(bs, bb, v) + return matchSequence(s, phrases) + }) + bbPool.Put(bb) +} + +func matchFloat64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if !matchBloomFilterAllTokens(bs, ch, tokens) { + bm.resetBits() + return + } + // The phrase may contain a part of the floating-point number. + // For example, `foo:"123"` must match `123`, `123.456` and `-0.123`. + // This means we cannot search in binary representation of floating-point numbers. + // Instead, we need searching for the whole phrase in string representation + // of floating-point numbers :( + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toFloat64StringExt(bs, bb, v) + return matchSequence(s, phrases) + }) + bbPool.Put(bb) +} + +func matchValuesDictBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string) { + bb := bbPool.Get() + for i, v := range ch.valuesDict.values { + if matchSequence(v, phrases) { + bb.B = append(bb.B, byte(i)) + } + } + matchEncodedValuesDict(bs, ch, bm, bb.B) + bbPool.Put(bb) +} + +func matchStringBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []string) { + if !matchBloomFilterAllTokens(bs, ch, tokens) { + bm.resetBits() + return + } + visitValues(bs, ch, bm, func(v string) bool { + return matchSequence(v, phrases) + }) +} + +func matchUint8BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) > 1 { + bm.resetBits() + return + } + matchUint8ByExactValue(bs, ch, bm, phrases[0], tokens) +} + +func matchUint16BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) > 1 { + bm.resetBits() + return + } + matchUint16ByExactValue(bs, ch, bm, phrases[0], tokens) +} + +func matchUint32BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) > 1 { + bm.resetBits() + return + } + matchUint32ByExactValue(bs, ch, bm, phrases[0], tokens) +} + +func matchUint64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) { + if len(phrases) > 1 { + bm.resetBits() + return + } + matchUint64ByExactValue(bs, ch, bm, phrases[0], tokens) +} + +func matchSequence(s string, phrases []string) bool { + for _, phrase := range phrases { + n := getPhrasePos(s, phrase) + if n < 0 { + return false + } + s = s[n+len(phrase):] + } + return true +} diff --git a/lib/logstorage/filter_sequence_test.go b/lib/logstorage/filter_sequence_test.go new file mode 100644 index 000000000..6f642c3a8 --- /dev/null +++ b/lib/logstorage/filter_sequence_test.go @@ -0,0 +1,821 @@ +package logstorage + +import ( + "testing" +) + +func TestMatchSequence(t *testing.T) { + f := func(s string, phrases []string, resultExpected bool) { + t.Helper() + result := matchSequence(s, phrases) + if result != resultExpected { + t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) + } + } + + f("", []string{""}, true) + f("foo", []string{""}, true) + f("", []string{"foo"}, false) + f("foo", []string{"foo"}, true) + f("foo bar", []string{"foo"}, true) + f("foo bar", []string{"bar"}, true) + f("foo bar", []string{"foo bar"}, true) + f("foo bar", []string{"foo", "bar"}, true) + f("foo bar", []string{"foo", " bar"}, true) + f("foo bar", []string{"foo ", "bar"}, true) + f("foo bar", []string{"foo ", " bar"}, false) + f("foo bar", []string{"bar", "foo"}, false) +} + +func TestFilterSequence(t *testing.T) { + t.Run("single-row", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "abc def", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"abc"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"def"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc def"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc ", "", "def", ""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"ab"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc", "abc"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc", "def", "foo"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "abc def", + "abc def", + "abc def", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"abc", " def"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc ", ""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{"", ""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"abc def ", "foobar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "non-existing column", + phrases: []string{"x", "yz"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "baz foobar", + "abc", + "afdf foobar baz", + "fddf foobarbaz", + "afoobarbaz", + "foobar", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"foobar", "baz"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{3}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"baz", "aaaa"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "non-existing column", + phrases: []string{"foobar", "aaaa"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "a bb foo", + "bb a foobar", + "aa abc a", + "ca afdf a,foobar baz", + "a fddf foobarbaz", + "a afoobarbaz", + "a foobar bb", + "a kjlkjf dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"a", "bb"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 6}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"НГКШ", " "}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{8}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"!,", "(!1)"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{9}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"aa a", "bcdasqq"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"@", "!!!!"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"12"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{1, 5}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", "bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234", "567"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "256", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"12"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{1, 5}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", "bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234", "567"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "65536", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"12"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{1, 5}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", "bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234", "567"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "12345678901", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"12"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{1, 5}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", "bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234", "567"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "-65536", + "1234.5678901", + "1", + "2", + "3", + "4", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"-", "65536"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{3}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"1234.", "5678901"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{4}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", "5678901"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{4}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"", ""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"65536", "-"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"5678901", "1234"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"12345678901234567890"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "1.0.127.6", + "55.55.55.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "foo", + phrases: []string{"127.0.0.1"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{2, 4, 5, 7}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"127", "1"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{2, 4, 5, 7}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"127.0.0"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{2, 4, 5, 7}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"2.3", ".4"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + // mismatch + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"5"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"127.", "1", "1", "345"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"27.0"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + + fs = &filterSequence{ + fieldName: "foo", + phrases: []string{"255.255.255.255"}, + } + testFilterMatchForColumns(t, columns, fs, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2006-01-02T15:04:05.001Z", + "2006-01-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2006-01-02T15:04:05.005Z", + "2006-01-02T15:04:05.006Z", + "2006-01-02T15:04:05.007Z", + "2006-01-02T15:04:05.008Z", + "2006-01-02T15:04:05.009Z", + }, + }, + } + + // match + fs := &filterSequence{ + fieldName: "_msg", + phrases: []string{"2006-01-02T15:04:05.005Z"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{4}) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"2006-01", "04:05."}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"2006", "002Z"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{1}) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + fs = &filterSequence{ + fieldName: "non-existing-column", + phrases: []string{""}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mimatch + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", nil) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"002Z", "2006"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", nil) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"2006-04-02T15:04:05.005Z", "2023"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", nil) + + fs = &filterSequence{ + fieldName: "_msg", + phrases: []string{"06"}, + } + testFilterMatchForColumns(t, columns, fs, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index ece7cffbb..86accca1a 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -182,29 +182,6 @@ func TestMatchPrefix(t *testing.T) { f("255.255.255.255", "255.255", true) } -func TestMatchSequence(t *testing.T) { - f := func(s string, phrases []string, resultExpected bool) { - t.Helper() - result := matchSequence(s, phrases) - if result != resultExpected { - t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) - } - } - - f("", []string{""}, true) - f("foo", []string{""}, true) - f("", []string{"foo"}, false) - f("foo", []string{"foo"}, true) - f("foo bar", []string{"foo"}, true) - f("foo bar", []string{"bar"}, true) - f("foo bar", []string{"foo bar"}, true) - f("foo bar", []string{"foo", "bar"}, true) - f("foo bar", []string{"foo", " bar"}, true) - f("foo bar", []string{"foo ", "bar"}, true) - f("foo bar", []string{"foo ", " bar"}, false) - f("foo bar", []string{"bar", "foo"}, false) -} - func TestMatchStringRange(t *testing.T) { f := func(s, minValue, maxValue string, resultExpected bool) { t.Helper() @@ -514,1256 +491,6 @@ func TestStreamFilter(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", nil) } -func TestSequenceFilter(t *testing.T) { - t.Run("single-row", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "abc def", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"def"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc def"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc ", "", "def", ""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"ab"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc", "abc"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc", "def", "foo"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "abc def", - "abc def", - "abc def", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc", " def"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc ", ""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{"", ""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"abc def ", "foobar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "non-existing column", - phrases: []string{"x", "yz"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "baz foobar", - "abc", - "afdf foobar baz", - "fddf foobarbaz", - "afoobarbaz", - "foobar", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"foobar", "baz"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{3}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"baz", "aaaa"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "non-existing column", - phrases: []string{"foobar", "aaaa"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "a bb foo", - "bb a foobar", - "aa abc a", - "ca afdf a,foobar baz", - "a fddf foobarbaz", - "a afoobarbaz", - "a foobar bb", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"a", "bb"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 6}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"НГКШ", " "}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{8}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"!,", "(!1)"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{9}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"aa a", "bcdasqq"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"@", "!!!!"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"12"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{1, 5}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", "bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234", "567"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "256", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"12"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{1, 5}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", "bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234", "567"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "65536", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"12"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{1, 5}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", "bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234", "567"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "12345678901", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"12"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{1, 5}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", "bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234", "567"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "-65536", - "1234.5678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"-", "65536"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{3}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"1234.", "5678901"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{4}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", "5678901"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{4}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"", ""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"65536", "-"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"5678901", "1234"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"12345678901234567890"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "1.0.127.6", - "55.55.55.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "foo", - phrases: []string{"127.0.0.1"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{2, 4, 5, 7}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"127", "1"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{2, 4, 5, 7}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"127.0.0"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{2, 4, 5, 7}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"2.3", ".4"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - // mismatch - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"5"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"127.", "1", "1", "345"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"27.0"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - - sf = &sequenceFilter{ - fieldName: "foo", - phrases: []string{"255.255.255.255"}, - } - testFilterMatchForColumns(t, columns, sf, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2006-01-02T15:04:05.005Z", - "2006-01-02T15:04:05.006Z", - "2006-01-02T15:04:05.007Z", - "2006-01-02T15:04:05.008Z", - "2006-01-02T15:04:05.009Z", - }, - }, - } - - // match - sf := &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"2006-01-02T15:04:05.005Z"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{4}) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"2006-01", "04:05."}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"2006", "002Z"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{1}) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - sf = &sequenceFilter{ - fieldName: "non-existing-column", - phrases: []string{""}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mimatch - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", nil) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"002Z", "2006"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", nil) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"2006-04-02T15:04:05.005Z", "2023"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", nil) - - sf = &sequenceFilter{ - fieldName: "_msg", - phrases: []string{"06"}, - } - testFilterMatchForColumns(t, columns, sf, "_msg", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "foobar", - "abc", - "afdf foobar baz", - "fddf foobarbaz", - "foobarbaz", - "foobar", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "foobar", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 5, 6}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "baz", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "non-existing column", - prefix: "foobar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "a foo", - "a foobar", - "aa abc a", - "ca afdf a,foobar baz", - "aa fddf foobarbaz", - "a afoobarbaz", - "a foobar baz", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "aa ", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{2, 4}) - - ef = &exactPrefixFilter{ - fieldName: "non-existing-column", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "aa b", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "fobar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "non-existing-column", - prefix: "aa", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 5}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "999", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "7", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "467", - "5", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 5}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "999", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "7", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "65536", - "5", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 5}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "99999", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "7", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "123456789012", - "5", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "12", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 5}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "1234567890123", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "7", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "-65536", - "1234.5678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "123", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 4}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "1234.567", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{4}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "-65536", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{3}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "6511", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.2", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.55.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "foo", - prefix: "127.0.", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{2, 4, 5, 7}) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - // mismatch - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - - ef = &exactPrefixFilter{ - fieldName: "foo", - prefix: "255", - } - testFilterMatchForColumns(t, columns, ef, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:06.004Z", - "2006-01-02T15:04:06.005Z", - "2006-01-02T15:04:07.006Z", - "2006-01-02T15:04:10.007Z", - "2006-01-02T15:04:12.008Z", - "2006-01-02T15:04:15.009Z", - }, - }, - } - - // match - ef := &exactPrefixFilter{ - fieldName: "_msg", - prefix: "2006-01-02T15:04:05", - } - testFilterMatchForColumns(t, columns, ef, "_msg", []int{0, 1, 2}) - - ef = &exactPrefixFilter{ - fieldName: "_msg", - prefix: "", - } - testFilterMatchForColumns(t, columns, ef, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mimatch - ef = &exactPrefixFilter{ - fieldName: "_msg", - prefix: "bar", - } - testFilterMatchForColumns(t, columns, ef, "_msg", nil) - - ef = &exactPrefixFilter{ - fieldName: "_msg", - prefix: "0", - } - testFilterMatchForColumns(t, columns, ef, "_msg", nil) - }) -} - func TestExactPrefixFilter(t *testing.T) { t.Run("single-row", func(t *testing.T) { columns := []column{ diff --git a/lib/logstorage/filter_time.go b/lib/logstorage/filter_time.go new file mode 100644 index 000000000..381b3de61 --- /dev/null +++ b/lib/logstorage/filter_time.go @@ -0,0 +1,40 @@ +package logstorage + +// filterTime filters by time. +// +// It is expressed as `_time:(start, end]` in LogsQL. +type filterTime struct { + minTimestamp int64 + maxTimestamp int64 + + stringRepr string +} + +func (ft *filterTime) String() string { + return "_time:" + ft.stringRepr +} + +func (ft *filterTime) apply(bs *blockSearch, bm *bitmap) { + minTimestamp := ft.minTimestamp + maxTimestamp := ft.maxTimestamp + + if minTimestamp > maxTimestamp { + bm.resetBits() + return + } + + th := bs.bsw.bh.timestampsHeader + if minTimestamp > th.maxTimestamp || maxTimestamp < th.minTimestamp { + bm.resetBits() + return + } + if minTimestamp <= th.minTimestamp && maxTimestamp >= th.maxTimestamp { + return + } + + timestamps := bs.getTimestamps() + bm.forEachSetBit(func(idx int) bool { + ts := timestamps[idx] + return ts >= minTimestamp && ts <= maxTimestamp + }) +} diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index e8cd88711..4f20136c8 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -336,7 +336,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) { case lex.isKeyword("re"): return parseRegexpFilter(lex, fieldName) case lex.isKeyword("seq"): - return parseSequenceFilter(lex, fieldName) + return parseFilterSequence(lex, fieldName) case lex.isKeyword("string_range"): return parseStringRangeFilter(lex, fieldName) case lex.isKeyword(`"`, "'", "`"): @@ -622,13 +622,13 @@ func parseInFilter(lex *lexer, fieldName string) (filter, error) { }) } -func parseSequenceFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterSequence(lex *lexer, fieldName string) (filter, error) { return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) { - sf := &sequenceFilter{ + fs := &filterSequence{ fieldName: fieldName, phrases: args, } - return sf, nil + return fs, nil }) } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 25e1846a0..d02e9e50f 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -274,22 +274,22 @@ func TestParseTimeRange(t *testing.T) { f(`[2023-03-01+02:20,2023-04-06T23] offset 30m5s`, minTimestamp, maxTimestamp) } -func TestParseSequenceFilter(t *testing.T) { +func TestParseFilterSequence(t *testing.T) { f := func(s, fieldNameExpected string, phrasesExpected []string) { t.Helper() q, err := ParseQuery(s) if err != nil { t.Fatalf("unexpected error: %s", err) } - sf, ok := q.f.(*sequenceFilter) + fs, ok := q.f.(*filterSequence) if !ok { - t.Fatalf("unexpected filter type; got %T; want *sequenceFilter; filter: %s", q.f, q.f) + t.Fatalf("unexpected filter type; got %T; want *filterSequence; filter: %s", q.f, q.f) } - if sf.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", sf.fieldName, fieldNameExpected) + if fs.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", fs.fieldName, fieldNameExpected) } - if !reflect.DeepEqual(sf.phrases, phrasesExpected) { - t.Fatalf("unexpected phrases\ngot\n%q\nwant\n%q", sf.phrases, phrasesExpected) + if !reflect.DeepEqual(fs.phrases, phrasesExpected) { + t.Fatalf("unexpected phrases\ngot\n%q\nwant\n%q", fs.phrases, phrasesExpected) } }