From 2a1cede9f7e826fd52201233a22d9cb0f4ec71ee Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 29 Apr 2024 06:01:06 +0200 Subject: [PATCH] wip --- lib/logstorage/filter.go | 331 --------------- lib/logstorage/filter_in.go | 343 +++++++++++++++ lib/logstorage/filter_in_test.go | 690 +++++++++++++++++++++++++++++++ lib/logstorage/filter_test.go | 685 ------------------------------ lib/logstorage/parser.go | 6 +- lib/logstorage/parser_test.go | 6 +- 6 files changed, 1039 insertions(+), 1022 deletions(-) create mode 100644 lib/logstorage/filter_in.go create mode 100644 lib/logstorage/filter_in_test.go diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 5d57324d3..8c48dec4e 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -72,308 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { } } -// inFilter matches any exact value from the values map. -// -// Example LogsQL: `fieldName:in("foo", "bar baz")` -type inFilter struct { - fieldName string - values []string - - tokenSetsOnce sync.Once - tokenSets [][]string - - stringValuesOnce sync.Once - stringValues map[string]struct{} - - uint8ValuesOnce sync.Once - uint8Values map[string]struct{} - - uint16ValuesOnce sync.Once - uint16Values map[string]struct{} - - uint32ValuesOnce sync.Once - uint32Values map[string]struct{} - - uint64ValuesOnce sync.Once - uint64Values map[string]struct{} - - float64ValuesOnce sync.Once - float64Values map[string]struct{} - - ipv4ValuesOnce sync.Once - ipv4Values map[string]struct{} - - timestampISO8601ValuesOnce sync.Once - timestampISO8601Values map[string]struct{} -} - -func (fi *inFilter) String() string { - values := fi.values - a := make([]string, len(values)) - for i, value := range values { - a[i] = quoteTokenIfNeeded(value) - } - return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ",")) -} - -func (fi *inFilter) getTokenSets() [][]string { - fi.tokenSetsOnce.Do(fi.initTokenSets) - return fi.tokenSets -} - -// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter. -const maxTokenSetsToInit = 1000 - -func (fi *inFilter) initTokenSets() { - values := fi.values - tokenSetsLen := len(values) - if tokenSetsLen > maxTokenSetsToInit { - tokenSetsLen = maxTokenSetsToInit - } - tokenSets := make([][]string, 0, tokenSetsLen+1) - for _, v := range values { - tokens := tokenizeStrings(nil, []string{v}) - tokenSets = append(tokenSets, tokens) - if len(tokens) > maxTokenSetsToInit { - break - } - } - fi.tokenSets = tokenSets -} - -func (fi *inFilter) getStringValues() map[string]struct{} { - fi.stringValuesOnce.Do(fi.initStringValues) - return fi.stringValues -} - -func (fi *inFilter) initStringValues() { - values := fi.values - m := make(map[string]struct{}, len(values)) - for _, v := range values { - m[v] = struct{}{} - } - fi.stringValues = m -} - -func (fi *inFilter) getUint8Values() map[string]struct{} { - fi.uint8ValuesOnce.Do(fi.initUint8Values) - return fi.uint8Values -} - -func (fi *inFilter) initUint8Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*1) - for _, v := range values { - n, ok := tryParseUint64(v) - if !ok || n >= (1<<8) { - continue - } - bufLen := len(buf) - buf = append(buf, byte(n)) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.uint8Values = m -} - -func (fi *inFilter) getUint16Values() map[string]struct{} { - fi.uint16ValuesOnce.Do(fi.initUint16Values) - return fi.uint16Values -} - -func (fi *inFilter) initUint16Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*2) - for _, v := range values { - n, ok := tryParseUint64(v) - if !ok || n >= (1<<16) { - continue - } - bufLen := len(buf) - buf = encoding.MarshalUint16(buf, uint16(n)) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.uint16Values = m -} - -func (fi *inFilter) getUint32Values() map[string]struct{} { - fi.uint32ValuesOnce.Do(fi.initUint32Values) - return fi.uint32Values -} - -func (fi *inFilter) initUint32Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*4) - for _, v := range values { - n, ok := tryParseUint64(v) - if !ok || n >= (1<<32) { - continue - } - bufLen := len(buf) - buf = encoding.MarshalUint32(buf, uint32(n)) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.uint32Values = m -} - -func (fi *inFilter) getUint64Values() map[string]struct{} { - fi.uint64ValuesOnce.Do(fi.initUint64Values) - return fi.uint64Values -} - -func (fi *inFilter) initUint64Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*8) - for _, v := range values { - n, ok := tryParseUint64(v) - if !ok { - continue - } - bufLen := len(buf) - buf = encoding.MarshalUint64(buf, n) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.uint64Values = m -} - -func (fi *inFilter) getFloat64Values() map[string]struct{} { - fi.float64ValuesOnce.Do(fi.initFloat64Values) - return fi.float64Values -} - -func (fi *inFilter) initFloat64Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*8) - for _, v := range values { - f, ok := tryParseFloat64(v) - if !ok { - continue - } - n := math.Float64bits(f) - bufLen := len(buf) - buf = encoding.MarshalUint64(buf, n) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.float64Values = m -} - -func (fi *inFilter) getIPv4Values() map[string]struct{} { - fi.ipv4ValuesOnce.Do(fi.initIPv4Values) - return fi.ipv4Values -} - -func (fi *inFilter) initIPv4Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*4) - for _, v := range values { - n, ok := tryParseIPv4(v) - if !ok { - continue - } - bufLen := len(buf) - buf = encoding.MarshalUint32(buf, uint32(n)) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.ipv4Values = m -} - -func (fi *inFilter) getTimestampISO8601Values() map[string]struct{} { - fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values) - return fi.timestampISO8601Values -} - -func (fi *inFilter) initTimestampISO8601Values() { - values := fi.values - m := make(map[string]struct{}, len(values)) - buf := make([]byte, 0, len(values)*8) - for _, v := range values { - n, ok := tryParseTimestampISO8601(v) - if !ok { - continue - } - bufLen := len(buf) - buf = encoding.MarshalUint64(buf, n) - s := bytesutil.ToUnsafeString(buf[bufLen:]) - m[s] = struct{}{} - } - fi.timestampISO8601Values = m -} - -func (fi *inFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := fi.fieldName - - if len(fi.values) == 0 { - bm.resetBits() - return - } - - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - stringValues := fi.getStringValues() - if _, ok := stringValues[v]; !ok { - bm.resetBits() - } - return - } - - // Verify whether filter matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - // Fast path - there are no matching columns. - // It matches anything only for empty phrase. - stringValues := fi.getStringValues() - if _, ok := stringValues[""]; !ok { - bm.resetBits() - } - return - } - - tokenSets := fi.getTokenSets() - - switch ch.valueType { - case valueTypeString: - stringValues := fi.getStringValues() - matchAnyValue(bs, ch, bm, stringValues, tokenSets) - case valueTypeDict: - stringValues := fi.getStringValues() - matchValuesDictByAnyValue(bs, ch, bm, stringValues) - case valueTypeUint8: - binValues := fi.getUint8Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeUint16: - binValues := fi.getUint16Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeUint32: - binValues := fi.getUint32Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeUint64: - binValues := fi.getUint64Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeFloat64: - binValues := fi.getFloat64Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeIPv4: - binValues := fi.getIPv4Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - case valueTypeTimestampISO8601: - binValues := fi.getTimestampISO8601Values() - matchAnyValue(bs, ch, bm, binValues, tokenSets) - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // ipv4RangeFilter matches the given ipv4 range [minValue..maxValue]. // // Example LogsQL: `fieldName:ipv4_range(127.0.0.1, 127.0.0.255)` @@ -1821,35 +1519,6 @@ func matchUint64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s bbPool.Put(bb) } -func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, tokenSets [][]string) { - if !matchBloomFilterAnyTokenSet(bs, ch, tokenSets) { - bm.resetBits() - return - } - visitValues(bs, ch, bm, func(v string) bool { - _, ok := values[v] - return ok - }) -} - -func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, tokenSets [][]string) bool { - if len(tokenSets) == 0 { - return false - } - if len(tokenSets) > maxTokenSetsToInit || uint64(len(tokenSets)) > 10*bs.bsw.bh.rowsCount { - // It is faster to match every row in the block against all the values - // instead of using bloom filter for too big number of tokenSets. - return true - } - bf := bs.getBloomFilterForColumn(ch) - for _, tokens := range tokenSets { - if bf.containsAll(tokens) { - return true - } - } - return false -} - func matchBloomFilterAllTokens(bs *blockSearch, ch *columnHeader, tokens []string) bool { if len(tokens) == 0 { return true diff --git a/lib/logstorage/filter_in.go b/lib/logstorage/filter_in.go new file mode 100644 index 000000000..cd5742a6c --- /dev/null +++ b/lib/logstorage/filter_in.go @@ -0,0 +1,343 @@ +package logstorage + +import ( + "fmt" + "math" + "strings" + "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" +) + +// filterIn matches any exact value from the values map. +// +// Example LogsQL: `fieldName:in("foo", "bar baz")` +type filterIn struct { + fieldName string + values []string + + tokenSetsOnce sync.Once + tokenSets [][]string + + stringValuesOnce sync.Once + stringValues map[string]struct{} + + uint8ValuesOnce sync.Once + uint8Values map[string]struct{} + + uint16ValuesOnce sync.Once + uint16Values map[string]struct{} + + uint32ValuesOnce sync.Once + uint32Values map[string]struct{} + + uint64ValuesOnce sync.Once + uint64Values map[string]struct{} + + float64ValuesOnce sync.Once + float64Values map[string]struct{} + + ipv4ValuesOnce sync.Once + ipv4Values map[string]struct{} + + timestampISO8601ValuesOnce sync.Once + timestampISO8601Values map[string]struct{} +} + +func (fi *filterIn) String() string { + values := fi.values + a := make([]string, len(values)) + for i, value := range values { + a[i] = quoteTokenIfNeeded(value) + } + return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ",")) +} + +func (fi *filterIn) getTokenSets() [][]string { + fi.tokenSetsOnce.Do(fi.initTokenSets) + return fi.tokenSets +} + +// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter. +const maxTokenSetsToInit = 1000 + +func (fi *filterIn) initTokenSets() { + values := fi.values + tokenSetsLen := len(values) + if tokenSetsLen > maxTokenSetsToInit { + tokenSetsLen = maxTokenSetsToInit + } + tokenSets := make([][]string, 0, tokenSetsLen+1) + for _, v := range values { + tokens := tokenizeStrings(nil, []string{v}) + tokenSets = append(tokenSets, tokens) + if len(tokens) > maxTokenSetsToInit { + break + } + } + fi.tokenSets = tokenSets +} + +func (fi *filterIn) getStringValues() map[string]struct{} { + fi.stringValuesOnce.Do(fi.initStringValues) + return fi.stringValues +} + +func (fi *filterIn) initStringValues() { + values := fi.values + m := make(map[string]struct{}, len(values)) + for _, v := range values { + m[v] = struct{}{} + } + fi.stringValues = m +} + +func (fi *filterIn) getUint8Values() map[string]struct{} { + fi.uint8ValuesOnce.Do(fi.initUint8Values) + return fi.uint8Values +} + +func (fi *filterIn) initUint8Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*1) + for _, v := range values { + n, ok := tryParseUint64(v) + if !ok || n >= (1<<8) { + continue + } + bufLen := len(buf) + buf = append(buf, byte(n)) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.uint8Values = m +} + +func (fi *filterIn) getUint16Values() map[string]struct{} { + fi.uint16ValuesOnce.Do(fi.initUint16Values) + return fi.uint16Values +} + +func (fi *filterIn) initUint16Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*2) + for _, v := range values { + n, ok := tryParseUint64(v) + if !ok || n >= (1<<16) { + continue + } + bufLen := len(buf) + buf = encoding.MarshalUint16(buf, uint16(n)) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.uint16Values = m +} + +func (fi *filterIn) getUint32Values() map[string]struct{} { + fi.uint32ValuesOnce.Do(fi.initUint32Values) + return fi.uint32Values +} + +func (fi *filterIn) initUint32Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*4) + for _, v := range values { + n, ok := tryParseUint64(v) + if !ok || n >= (1<<32) { + continue + } + bufLen := len(buf) + buf = encoding.MarshalUint32(buf, uint32(n)) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.uint32Values = m +} + +func (fi *filterIn) getUint64Values() map[string]struct{} { + fi.uint64ValuesOnce.Do(fi.initUint64Values) + return fi.uint64Values +} + +func (fi *filterIn) initUint64Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*8) + for _, v := range values { + n, ok := tryParseUint64(v) + if !ok { + continue + } + bufLen := len(buf) + buf = encoding.MarshalUint64(buf, n) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.uint64Values = m +} + +func (fi *filterIn) getFloat64Values() map[string]struct{} { + fi.float64ValuesOnce.Do(fi.initFloat64Values) + return fi.float64Values +} + +func (fi *filterIn) initFloat64Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*8) + for _, v := range values { + f, ok := tryParseFloat64(v) + if !ok { + continue + } + n := math.Float64bits(f) + bufLen := len(buf) + buf = encoding.MarshalUint64(buf, n) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.float64Values = m +} + +func (fi *filterIn) getIPv4Values() map[string]struct{} { + fi.ipv4ValuesOnce.Do(fi.initIPv4Values) + return fi.ipv4Values +} + +func (fi *filterIn) initIPv4Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*4) + for _, v := range values { + n, ok := tryParseIPv4(v) + if !ok { + continue + } + bufLen := len(buf) + buf = encoding.MarshalUint32(buf, uint32(n)) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.ipv4Values = m +} + +func (fi *filterIn) getTimestampISO8601Values() map[string]struct{} { + fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values) + return fi.timestampISO8601Values +} + +func (fi *filterIn) initTimestampISO8601Values() { + values := fi.values + m := make(map[string]struct{}, len(values)) + buf := make([]byte, 0, len(values)*8) + for _, v := range values { + n, ok := tryParseTimestampISO8601(v) + if !ok { + continue + } + bufLen := len(buf) + buf = encoding.MarshalUint64(buf, n) + s := bytesutil.ToUnsafeString(buf[bufLen:]) + m[s] = struct{}{} + } + fi.timestampISO8601Values = m +} + +func (fi *filterIn) apply(bs *blockSearch, bm *bitmap) { + fieldName := fi.fieldName + + if len(fi.values) == 0 { + bm.resetBits() + return + } + + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + stringValues := fi.getStringValues() + if _, ok := stringValues[v]; !ok { + bm.resetBits() + } + return + } + + // Verify whether filter matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + // Fast path - there are no matching columns. + // It matches anything only for empty phrase. + stringValues := fi.getStringValues() + if _, ok := stringValues[""]; !ok { + bm.resetBits() + } + return + } + + tokenSets := fi.getTokenSets() + + switch ch.valueType { + case valueTypeString: + stringValues := fi.getStringValues() + matchAnyValue(bs, ch, bm, stringValues, tokenSets) + case valueTypeDict: + stringValues := fi.getStringValues() + matchValuesDictByAnyValue(bs, ch, bm, stringValues) + case valueTypeUint8: + binValues := fi.getUint8Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeUint16: + binValues := fi.getUint16Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeUint32: + binValues := fi.getUint32Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeUint64: + binValues := fi.getUint64Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeFloat64: + binValues := fi.getFloat64Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeIPv4: + binValues := fi.getIPv4Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + case valueTypeTimestampISO8601: + binValues := fi.getTimestampISO8601Values() + matchAnyValue(bs, ch, bm, binValues, tokenSets) + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, tokenSets [][]string) { + if !matchBloomFilterAnyTokenSet(bs, ch, tokenSets) { + bm.resetBits() + return + } + visitValues(bs, ch, bm, func(v string) bool { + _, ok := values[v] + return ok + }) +} + +func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, tokenSets [][]string) bool { + if len(tokenSets) == 0 { + return false + } + if len(tokenSets) > maxTokenSetsToInit || uint64(len(tokenSets)) > 10*bs.bsw.bh.rowsCount { + // It is faster to match every row in the block against all the values + // instead of using bloom filter for too big number of tokenSets. + return true + } + bf := bs.getBloomFilterForColumn(ch) + for _, tokens := range tokenSets { + if bf.containsAll(tokens) { + return true + } + } + return false +} diff --git a/lib/logstorage/filter_in_test.go b/lib/logstorage/filter_in_test.go new file mode 100644 index 000000000..ffe8e944b --- /dev/null +++ b/lib/logstorage/filter_in_test.go @@ -0,0 +1,690 @@ +package logstorage + +import ( + "testing" +) + +func TestFilterIn(t *testing.T) { + t.Run("single-row", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "abc def", + }, + }, + { + name: "other column", + values: []string{ + "asdfdsf", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"abc def", "abc", "foobar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) + + fi = &filterIn{ + fieldName: "other column", + values: []string{"asdfdsf", ""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{"", "foo"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"abc", "def"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"", "abc"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "other column", + values: []string{"sd"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "non-existing column", + values: []string{"abc", "def"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "abc def", + "abc def", + "abc def", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"aaaa", "abc def", "foobar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{"", "abc"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"abc def ", "foobar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "non-existing column", + values: []string{"x"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "foobar", + "abc", + "afdf foobar baz", + "fddf foobarbaz", + "afoobarbaz", + "foobar", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"foobar", "aaaa", "abc", "baz"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"bbbb", "", "aaaa"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar", "aaaa"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "non-existing column", + values: []string{"foobar", "aaaa"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "a foo", + "a foobar", + "aa abc a", + "ca afdf a,foobar baz", + "a fddf foobarbaz", + "a afoobarbaz", + "a foobar", + "a kjlkjf dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"a foobar", "aa abc a"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"aa a"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"12", "32"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"0"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"33"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"1234"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "256", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"12", "32"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"0"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"33"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"123456"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "65536", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"12", "32"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"0"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"33"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"12345678901"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "12345678901", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"12", "32"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"0"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"33"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1234", + "0", + "3454", + "-65536", + "1234.5678901", + "1", + "2", + "3", + "4", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"1234", "1", "foobar", "123211"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"1234.5678901"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{4}) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"-65536"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{3}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"65536"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"123"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"12345678901234567890"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "12.0.127.6", + "55.55.55.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "foo", + values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + + // mismatch + fi = &filterIn{ + fieldName: "foo", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"5"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + + fi = &filterIn{ + fieldName: "foo", + values: []string{"255.255.255.255"}, + } + testFilterMatchForColumns(t, columns, fi, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2006-01-02T15:04:05.001Z", + "2006-01-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2006-01-02T15:04:05.005Z", + "2006-01-02T15:04:05.006Z", + "2006-01-02T15:04:05.007Z", + "2006-01-02T15:04:05.008Z", + "2006-01-02T15:04:05.009Z", + }, + }, + } + + // match + fi := &filterIn{ + fieldName: "_msg", + values: []string{"2006-01-02T15:04:05.005Z", "foobar"}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", []int{4}) + + fi = &filterIn{ + fieldName: "non-existing-column", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) + + // mimatch + fi = &filterIn{ + fieldName: "_msg", + values: []string{"bar"}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", nil) + + fi = &filterIn{ + fieldName: "_msg", + values: []string{}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", nil) + + fi = &filterIn{ + fieldName: "_msg", + values: []string{""}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", nil) + + fi = &filterIn{ + fieldName: "_msg", + values: []string{"2006-04-02T15:04:05.005Z"}, + } + testFilterMatchForColumns(t, columns, fi, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index b2883f538..0baee038d 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -491,691 +491,6 @@ func TestStreamFilter(t *testing.T) { testFilterMatchForColumns(t, columns, f, "foo", nil) } -func TestInFilter(t *testing.T) { - t.Run("single-row", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "abc def", - }, - }, - { - name: "other column", - values: []string{ - "asdfdsf", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"abc def", "abc", "foobar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - - fi = &inFilter{ - fieldName: "other column", - values: []string{"asdfdsf", ""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{"", "foo"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"abc", "def"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"", "abc"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "other column", - values: []string{"sd"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "non-existing column", - values: []string{"abc", "def"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "abc def", - "abc def", - "abc def", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"aaaa", "abc def", "foobar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{"", "abc"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"abc def ", "foobar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "non-existing column", - values: []string{"x"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "foobar", - "abc", - "afdf foobar baz", - "fddf foobarbaz", - "afoobarbaz", - "foobar", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"foobar", "aaaa", "abc", "baz"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"bbbb", "", "aaaa"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar", "aaaa"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "non-existing column", - values: []string{"foobar", "aaaa"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "a foo", - "a foobar", - "aa abc a", - "ca afdf a,foobar baz", - "a fddf foobarbaz", - "a afoobarbaz", - "a foobar", - "a kjlkjf dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"a foobar", "aa abc a"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"aa a"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"12", "32"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"0"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"33"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"1234"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "256", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"12", "32"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"0"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"33"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"123456"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "65536", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"12", "32"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"0"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"33"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"12345678901"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "12345678901", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"12", "32"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"0"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"33"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1234", - "0", - "3454", - "-65536", - "1234.5678901", - "1", - "2", - "3", - "4", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"1234", "1", "foobar", "123211"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"1234.5678901"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{4}) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"-65536"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{3}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"65536"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"123"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"12345678901234567890"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.55.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "foo", - values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) - - // mismatch - fi = &inFilter{ - fieldName: "foo", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"5"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - - fi = &inFilter{ - fieldName: "foo", - values: []string{"255.255.255.255"}, - } - testFilterMatchForColumns(t, columns, fi, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2006-01-02T15:04:05.005Z", - "2006-01-02T15:04:05.006Z", - "2006-01-02T15:04:05.007Z", - "2006-01-02T15:04:05.008Z", - "2006-01-02T15:04:05.009Z", - }, - }, - } - - // match - fi := &inFilter{ - fieldName: "_msg", - values: []string{"2006-01-02T15:04:05.005Z", "foobar"}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", []int{4}) - - fi = &inFilter{ - fieldName: "non-existing-column", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) - - // mimatch - fi = &inFilter{ - fieldName: "_msg", - values: []string{"bar"}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", nil) - - fi = &inFilter{ - fieldName: "_msg", - values: []string{}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", nil) - - fi = &inFilter{ - fieldName: "_msg", - values: []string{""}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", nil) - - fi = &inFilter{ - fieldName: "_msg", - values: []string{"2006-04-02T15:04:05.005Z"}, - } - testFilterMatchForColumns(t, columns, fi, "_msg", nil) - }) -} - func TestRegexpFilter(t *testing.T) { t.Run("const-column", func(t *testing.T) { columns := []column{ diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 51d07bf46..0a69636dc 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -326,7 +326,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) { case lex.isKeyword("i"): return parseAnyCaseFilter(lex, fieldName) case lex.isKeyword("in"): - return parseInFilter(lex, fieldName) + return parseFilterIn(lex, fieldName) case lex.isKeyword("ipv4_range"): return parseIPv4RangeFilter(lex, fieldName) case lex.isKeyword("len_range"): @@ -612,9 +612,9 @@ func tryParseIPv4CIDR(s string) (uint32, uint32, bool) { return minValue, maxValue, true } -func parseInFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterIn(lex *lexer, fieldName string) (filter, error) { return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) { - f := &inFilter{ + f := &filterIn{ fieldName: fieldName, values: args, } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index d02e9e50f..12fa1e6f0 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -299,16 +299,16 @@ func TestParseFilterSequence(t *testing.T) { f(`seq(foo,bar-baz.aa"bb","c,)d")`, ``, []string{"foo", `bar-baz.aa"bb"`, "c,)d"}) } -func TestParseInFilter(t *testing.T) { +func TestParseFilterIn(t *testing.T) { f := func(s, fieldNameExpected string, valuesExpected []string) { t.Helper() q, err := ParseQuery(s) if err != nil { t.Fatalf("unexpected error: %s", err) } - f, ok := q.f.(*inFilter) + f, ok := q.f.(*filterIn) if !ok { - t.Fatalf("unexpected filter type; got %T; want *inFilter; filter: %s", q.f, q.f) + t.Fatalf("unexpected filter type; got %T; want *filterIn; filter: %s", q.f, q.f) } if f.fieldName != fieldNameExpected { t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected)