From 75838f0a87cfc49a2a66fb631918d6ec0c1c3a31 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 29 Apr 2024 06:27:35 +0200 Subject: [PATCH] wip --- lib/logstorage/bitmap_test.go | 96 ++ lib/logstorage/filter.go | 182 ---- lib/logstorage/filter_ipv4_range_test.go | 402 ++++++++ lib/logstorage/filter_string_range.go | 189 ++++ lib/logstorage/filter_string_range_test.go | 548 +++++++++++ lib/logstorage/filter_test.go | 1030 -------------------- lib/logstorage/parser.go | 6 +- lib/logstorage/parser_test.go | 6 +- 8 files changed, 1241 insertions(+), 1218 deletions(-) create mode 100644 lib/logstorage/bitmap_test.go create mode 100644 lib/logstorage/filter_ipv4_range_test.go create mode 100644 lib/logstorage/filter_string_range.go create mode 100644 lib/logstorage/filter_string_range_test.go diff --git a/lib/logstorage/bitmap_test.go b/lib/logstorage/bitmap_test.go new file mode 100644 index 000000000..90ed0fffc --- /dev/null +++ b/lib/logstorage/bitmap_test.go @@ -0,0 +1,96 @@ +package logstorage + +import ( + "testing" +) + +func TestBitmap(t *testing.T) { + for i := 0; i < 100; i++ { + bm := getBitmap(i) + if bm.bitsLen != i { + t.Fatalf("unexpected bits length: %d; want %d", bm.bitsLen, i) + } + + if !bm.isZero() { + t.Fatalf("all the bits must be zero for bitmap with %d bits", i) + } + if i == 0 && !bm.areAllBitsSet() { + t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits") + } + if i > 0 && bm.areAllBitsSet() { + t.Fatalf("areAllBitsSet() must return false on new bitmap with %d bits; %#v", i, bm) + } + + bm.setBits() + + // Make sure that all the bits are set. + nextIdx := 0 + bm.forEachSetBit(func(idx int) bool { + if idx >= i { + t.Fatalf("index must be smaller than %d", i) + } + if idx != nextIdx { + t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx) + } + nextIdx++ + return true + }) + + if !bm.areAllBitsSet() { + t.Fatalf("all bits must be set for bitmap with %d bits", i) + } + + // Clear a part of bits + bm.forEachSetBit(func(idx int) bool { + return idx%2 != 0 + }) + + if i <= 1 && !bm.isZero() { + t.Fatalf("bm.isZero() must return true for bitmap with %d bits", i) + } + if i > 1 && bm.isZero() { + t.Fatalf("bm.isZero() must return false, since some bits are set for bitmap with %d bits", i) + } + if i == 0 && !bm.areAllBitsSet() { + t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits") + } + if i > 0 && bm.areAllBitsSet() { + t.Fatalf("some bits mustn't be set for bitmap with %d bits", i) + } + + nextIdx = 1 + bm.forEachSetBit(func(idx int) bool { + if idx != nextIdx { + t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx) + } + nextIdx += 2 + return true + }) + + // Clear all the bits + bm.forEachSetBit(func(_ int) bool { + return false + }) + + if !bm.isZero() { + t.Fatalf("all the bits must be reset for bitmap with %d bits", i) + } + if i == 0 && !bm.areAllBitsSet() { + t.Fatalf("allAllBitsSet() must return true for bitmap with 0 bits") + } + if i > 0 && bm.areAllBitsSet() { + t.Fatalf("areAllBitsSet() must return false for bitmap with %d bits", i) + } + + bitsCount := 0 + bm.forEachSetBit(func(_ int) bool { + bitsCount++ + return true + }) + if bitsCount != 0 { + t.Fatalf("unexpected non-zero number of set bits remained: %d", bitsCount) + } + + putBitmap(bm) + } +} diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 92d04ea3e..c0692f216 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -72,73 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { } } -// stringRangeFilter matches tie given string range [minValue..maxValue) -// -// Note that the minValue is included in the range, while the maxValue isn't included in the range. -// This simplifies querying distincts log sets with string_range(A, B), string_range(B, C), etc. -// -// Example LogsQL: `fieldName:string_range(minValue, maxValue)` -type stringRangeFilter struct { - fieldName string - minValue string - maxValue string -} - -func (fr *stringRangeFilter) String() string { - return fmt.Sprintf("%sstring_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.minValue), quoteTokenIfNeeded(fr.maxValue)) -} - -func (fr *stringRangeFilter) apply(bs *blockSearch, bm *bitmap) { - fieldName := fr.fieldName - minValue := fr.minValue - maxValue := fr.maxValue - - if minValue > maxValue { - bm.resetBits() - return - } - - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - if !matchStringRange(v, minValue, maxValue) { - bm.resetBits() - } - return - } - - // Verify whether filter matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - if !matchStringRange("", minValue, maxValue) { - bm.resetBits() - } - return - } - - switch ch.valueType { - case valueTypeString: - matchStringByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeDict: - matchValuesDictByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeUint8: - matchUint8ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeUint16: - matchUint16ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeUint32: - matchUint32ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeUint64: - matchUint64ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeFloat64: - matchFloat64ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeIPv4: - matchIPv4ByStringRange(bs, ch, bm, minValue, maxValue) - case valueTypeTimestampISO8601: - matchTimestampISO8601ByStringRange(bs, ch, bm, minValue, maxValue) - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // lenRangeFilter matches field values with the length in the given range [minLen, maxLen]. // // Example LogsQL: `fieldName:len_range(10, 20)` @@ -670,20 +603,6 @@ func matchTimestampISO8601ByLenRange(bm *bitmap, minLen, maxLen uint64) { } } -func matchTimestampISO8601ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toTimestampISO8601StringExt(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) { bb := bbPool.Get() visitValues(bs, ch, bm, func(v string) bool { @@ -736,20 +655,6 @@ func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap bbPool.Put(bb) } -func matchIPv4ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toIPv4StringExt(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - func matchIPv4ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) { if minLen > uint64(len("255.255.255.255")) || maxLen < uint64(len("0.0.0.0")) { bm.resetBits() @@ -834,20 +739,6 @@ func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase str bbPool.Put(bb) } -func matchFloat64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "+" { - bm.resetBits() - return - } - - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toFloat64StringExt(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - func matchFloat64ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) { if minLen > 24 || maxLen == 0 { bm.resetBits() @@ -945,17 +836,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase bbPool.Put(bb) } -func matchValuesDictByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - bb := bbPool.Get() - for i, v := range ch.valuesDict.values { - if matchStringRange(v, minValue, maxValue) { - bb.B = append(bb.B, byte(i)) - } - } - matchEncodedValuesDict(bs, ch, bm, bb.B) - bbPool.Put(bb) -} - func matchValuesDictByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) { bb := bbPool.Get() for i, v := range ch.valuesDict.values { @@ -1060,12 +940,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod }) } -func matchStringByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - visitValues(bs, ch, bm, func(v string) bool { - return matchStringRange(v, minValue, maxValue) - }) -} - func matchStringByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) { visitValues(bs, ch, bm, func(v string) bool { return matchLenRange(v, minLen, maxLen) @@ -1116,58 +990,6 @@ func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase s }) } -func matchUint8ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toUint8String(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - -func matchUint16ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toUint16String(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - -func matchUint32ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toUint32String(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - -func matchUint64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { - if minValue > "9" || maxValue < "0" { - bm.resetBits() - return - } - bb := bbPool.Get() - visitValues(bs, ch, bm, func(v string) bool { - s := toUint64String(bs, bb, v) - return matchStringRange(s, minValue, maxValue) - }) - bbPool.Put(bb) -} - func matchMinMaxValueLen(ch *columnHeader, minLen, maxLen uint64) bool { bb := bbPool.Get() defer bbPool.Put(bb) @@ -1526,10 +1348,6 @@ func matchPrefix(s, prefix string) bool { } } -func matchStringRange(s, minValue, maxValue string) bool { - return s >= minValue && s < maxValue -} - func matchLenRange(s string, minLen, maxLen uint64) bool { sLen := uint64(utf8.RuneCountInString(s)) return sLen >= minLen && sLen <= maxLen diff --git a/lib/logstorage/filter_ipv4_range_test.go b/lib/logstorage/filter_ipv4_range_test.go new file mode 100644 index 000000000..98541cdf8 --- /dev/null +++ b/lib/logstorage/filter_ipv4_range_test.go @@ -0,0 +1,402 @@ +package logstorage + +import ( + "testing" +) + +func TestMatchIPv4Range(t *testing.T) { + f := func(s string, minValue, maxValue uint32, resultExpected bool) { + t.Helper() + result := matchIPv4Range(s, minValue, maxValue) + if result != resultExpected { + t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) + } + } + + // Invalid IP + f("", 0, 1000, false) + f("123", 0, 1000, false) + + // range mismatch + f("0.0.0.1", 2, 100, false) + f("127.0.0.1", 0x6f000000, 0x7f000000, false) + + // range match + f("0.0.0.1", 1, 1, true) + f("0.0.0.1", 0, 100, true) + f("127.0.0.1", 0x7f000000, 0x7f000001, true) +} + +func TestFilterIPv4Range(t *testing.T) { + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "127.0.0.1", + "127.0.0.1", + "127.0.0.1", + }, + }, + } + + // match + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0x80000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x7f000001, + maxValue: 0x7f000001, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) + + // mismatch + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0x7f000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "non-existing-column", + minValue: 0, + maxValue: 20000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x80000000, + maxValue: 0, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "127.0.0.1", + "Abc", + "127.255.255.255", + "10.4", + "foo 127.0.0.1", + "127.0.0.1 bar", + "127.0.0.1", + }, + }, + } + + // match + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0x7f000000, + maxValue: 0x80000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 7}) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0x7f000001, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7}) + + // mismatch + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 1000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x7f000002, + maxValue: 0x7f7f0000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x80000000, + maxValue: 0x7f000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "A FOO", + "a 10", + "127.0.0.1", + "20", + "15.5", + "-5", + "a fooBaR", + "a 127.0.0.1 dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0x7f000000, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{2}) + + // mismatch + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 10000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0xffffffff, + maxValue: 0x7f000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "65535", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "65536", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12345678901", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "123456.78901", + "-0.2", + "2", + "-334", + "4", + "5", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "12.0.127.6", + "55.55.12.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + fr := &filterIPv4Range{ + fieldName: "foo", + minValue: 0, + maxValue: 0x08000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 11}) + + // mismatch + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x80000000, + maxValue: 0x90000000, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0xff000000, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterIPv4Range{ + fieldName: "foo", + minValue: 0x08000000, + maxValue: 0, + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2006-01-02T15:04:05.001Z", + "2006-01-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2006-01-02T15:04:05.005Z", + "2006-01-02T15:04:05.006Z", + "2006-01-02T15:04:05.007Z", + "2006-01-02T15:04:05.008Z", + "2006-01-02T15:04:05.009Z", + }, + }, + } + + // mismatch + fr := &filterIPv4Range{ + fieldName: "_msg", + minValue: 0, + maxValue: 0xffffffff, + } + testFilterMatchForColumns(t, columns, fr, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_string_range.go b/lib/logstorage/filter_string_range.go new file mode 100644 index 000000000..0bb77e16f --- /dev/null +++ b/lib/logstorage/filter_string_range.go @@ -0,0 +1,189 @@ +package logstorage + +import ( + "fmt" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" +) + +// filterStringRange matches tie given string range [minValue..maxValue) +// +// Note that the minValue is included in the range, while the maxValue isn't included in the range. +// This simplifies querying distincts log sets with string_range(A, B), string_range(B, C), etc. +// +// Example LogsQL: `fieldName:string_range(minValue, maxValue)` +type filterStringRange struct { + fieldName string + minValue string + maxValue string +} + +func (fr *filterStringRange) String() string { + return fmt.Sprintf("%sstring_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.minValue), quoteTokenIfNeeded(fr.maxValue)) +} + +func (fr *filterStringRange) apply(bs *blockSearch, bm *bitmap) { + fieldName := fr.fieldName + minValue := fr.minValue + maxValue := fr.maxValue + + if minValue > maxValue { + bm.resetBits() + return + } + + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + if !matchStringRange(v, minValue, maxValue) { + bm.resetBits() + } + return + } + + // Verify whether filter matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + if !matchStringRange("", minValue, maxValue) { + bm.resetBits() + } + return + } + + switch ch.valueType { + case valueTypeString: + matchStringByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeDict: + matchValuesDictByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeUint8: + matchUint8ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeUint16: + matchUint16ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeUint32: + matchUint32ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeUint64: + matchUint64ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeFloat64: + matchFloat64ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeIPv4: + matchIPv4ByStringRange(bs, ch, bm, minValue, maxValue) + case valueTypeTimestampISO8601: + matchTimestampISO8601ByStringRange(bs, ch, bm, minValue, maxValue) + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchTimestampISO8601ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toTimestampISO8601StringExt(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchIPv4ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toIPv4StringExt(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchFloat64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "+" { + bm.resetBits() + return + } + + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toFloat64StringExt(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchValuesDictByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + bb := bbPool.Get() + for i, v := range ch.valuesDict.values { + if matchStringRange(v, minValue, maxValue) { + bb.B = append(bb.B, byte(i)) + } + } + matchEncodedValuesDict(bs, ch, bm, bb.B) + bbPool.Put(bb) +} + +func matchStringByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + visitValues(bs, ch, bm, func(v string) bool { + return matchStringRange(v, minValue, maxValue) + }) +} + +func matchUint8ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toUint8String(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchUint16ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toUint16String(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchUint32ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toUint32String(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchUint64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { + if minValue > "9" || maxValue < "0" { + bm.resetBits() + return + } + bb := bbPool.Get() + visitValues(bs, ch, bm, func(v string) bool { + s := toUint64String(bs, bb, v) + return matchStringRange(s, minValue, maxValue) + }) + bbPool.Put(bb) +} + +func matchStringRange(s, minValue, maxValue string) bool { + return s >= minValue && s < maxValue +} diff --git a/lib/logstorage/filter_string_range_test.go b/lib/logstorage/filter_string_range_test.go new file mode 100644 index 000000000..02bf269cf --- /dev/null +++ b/lib/logstorage/filter_string_range_test.go @@ -0,0 +1,548 @@ +package logstorage + +import ( + "testing" +) + +func TestMatchStringRange(t *testing.T) { + f := func(s, minValue, maxValue string, resultExpected bool) { + t.Helper() + result := matchStringRange(s, minValue, maxValue) + if result != resultExpected { + t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) + } + } + + f("foo", "a", "b", false) + f("foo", "a", "foa", false) + f("foo", "a", "foz", true) + f("foo", "foo", "foo", false) + f("foo", "foo", "fooa", true) + f("foo", "fooa", "foo", false) +} + +func TestFilterStringRange(t *testing.T) { + t.Run("const-column", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "127.0.0.1", + "127.0.0.1", + "127.0.0.1", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.1", + maxValue: "255.", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.1", + maxValue: "127.0.0.1", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "", + maxValue: "127.0.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "non-existing-column", + minValue: "1", + maxValue: "2", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.2", + maxValue: "", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("dict", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "", + "127.0.0.1", + "Abc", + "127.255.255.255", + "10.4", + "foo 127.0.0.1", + "127.0.0.1 bar", + "127.0.0.1", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.0", + maxValue: "128.0.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 6, 7}) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "127", + maxValue: "127.0.0.1", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "0", + maxValue: "10", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.2", + maxValue: "127.127.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "128.0.0.0", + maxValue: "127.0.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("strings", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "A FOO", + "a 10", + "127.0.0.1", + "20", + "15.5", + "-5", + "a fooBaR", + "a 127.0.0.1 dfff", + "a ТЕСТЙЦУК НГКШ ", + "a !!,23.(!1)", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0.1", + maxValue: "255.255.255.255", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "0", + maxValue: "10", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "255.255.255.255", + maxValue: "127.0.0.1", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint8", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "33", + maxValue: "5", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "100", + maxValue: "101", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "5", + maxValue: "33", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint16", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "65535", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "33", + maxValue: "5", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "100", + maxValue: "101", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "5", + maxValue: "33", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint32", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "65536", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "33", + maxValue: "5", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "100", + maxValue: "101", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "5", + maxValue: "33", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("uint64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "12345678901", + "1", + "2", + "3", + "4", + "5", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "33", + maxValue: "5", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "100", + maxValue: "101", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "5", + maxValue: "33", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("float64", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "123", + "12", + "32", + "0", + "0", + "123456.78901", + "-0.2", + "2", + "-334", + "4", + "5", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "33", + maxValue: "5", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) + fr = &filterStringRange{ + fieldName: "foo", + minValue: "-0", + maxValue: "-1", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{6}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "100", + maxValue: "101", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "5", + maxValue: "33", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("ipv4", func(t *testing.T) { + columns := []column{ + { + name: "foo", + values: []string{ + "1.2.3.4", + "0.0.0.0", + "127.0.0.1", + "254.255.255.255", + "127.0.0.1", + "127.0.0.1", + "127.0.4.2", + "127.0.0.1", + "12.0.127.6", + "55.55.12.55", + "66.66.66.66", + "7.7.7.7", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "foo", + minValue: "127.0.0", + maxValue: "128.0.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 4, 5, 6, 7}) + + // mismatch + fr = &filterStringRange{ + fieldName: "foo", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "128.0.0.0", + maxValue: "129.0.0.0", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "255.0.0.0", + maxValue: "255.255.255.255", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + + fr = &filterStringRange{ + fieldName: "foo", + minValue: "128.0.0.0", + maxValue: "", + } + testFilterMatchForColumns(t, columns, fr, "foo", nil) + }) + + t.Run("timestamp-iso8601", func(t *testing.T) { + columns := []column{ + { + name: "_msg", + values: []string{ + "2005-01-02T15:04:05.001Z", + "2006-02-02T15:04:05.002Z", + "2006-01-02T15:04:05.003Z", + "2006-01-02T15:04:05.004Z", + "2026-01-02T15:04:05.005Z", + "2026-01-02T15:04:05.006Z", + "2026-01-02T15:04:05.007Z", + "2026-01-02T15:04:05.008Z", + "2026-01-02T15:04:05.009Z", + }, + }, + } + + // match + fr := &filterStringRange{ + fieldName: "_msg", + minValue: "2006-01-02", + maxValue: "2006-01-03", + } + testFilterMatchForColumns(t, columns, fr, "_msg", []int{2, 3}) + + fr = &filterStringRange{ + fieldName: "_msg", + minValue: "", + maxValue: "2006", + } + testFilterMatchForColumns(t, columns, fr, "_msg", []int{0}) + + // mismatch + fr = &filterStringRange{ + fieldName: "_msg", + minValue: "3", + maxValue: "4", + } + testFilterMatchForColumns(t, columns, fr, "_msg", nil) + + fr = &filterStringRange{ + fieldName: "_msg", + minValue: "a", + maxValue: "b", + } + testFilterMatchForColumns(t, columns, fr, "_msg", nil) + + fr = &filterStringRange{ + fieldName: "_msg", + minValue: "2006-01-03", + maxValue: "2006-01-02", + } + testFilterMatchForColumns(t, columns, fr, "_msg", nil) + }) +} diff --git a/lib/logstorage/filter_test.go b/lib/logstorage/filter_test.go index c5d9fcbf4..695b974c8 100644 --- a/lib/logstorage/filter_test.go +++ b/lib/logstorage/filter_test.go @@ -182,137 +182,6 @@ func TestMatchPrefix(t *testing.T) { f("255.255.255.255", "255.255", true) } -func TestMatchStringRange(t *testing.T) { - f := func(s, minValue, maxValue string, resultExpected bool) { - t.Helper() - result := matchStringRange(s, minValue, maxValue) - if result != resultExpected { - t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) - } - } - - f("foo", "a", "b", false) - f("foo", "a", "foa", false) - f("foo", "a", "foz", true) - f("foo", "foo", "foo", false) - f("foo", "foo", "fooa", true) - f("foo", "fooa", "foo", false) -} - -func TestMatchIPv4Range(t *testing.T) { - f := func(s string, minValue, maxValue uint32, resultExpected bool) { - t.Helper() - result := matchIPv4Range(s, minValue, maxValue) - if result != resultExpected { - t.Fatalf("unexpected result; got %v; want %v", result, resultExpected) - } - } - - // Invalid IP - f("", 0, 1000, false) - f("123", 0, 1000, false) - - // range mismatch - f("0.0.0.1", 2, 100, false) - f("127.0.0.1", 0x6f000000, 0x7f000000, false) - - // range match - f("0.0.0.1", 1, 1, true) - f("0.0.0.1", 0, 100, true) - f("127.0.0.1", 0x7f000000, 0x7f000001, true) -} - -func TestBitmap(t *testing.T) { - for i := 0; i < 100; i++ { - bm := getBitmap(i) - if bm.bitsLen != i { - t.Fatalf("unexpected bits length: %d; want %d", bm.bitsLen, i) - } - - if !bm.isZero() { - t.Fatalf("all the bits must be zero for bitmap with %d bits", i) - } - if i == 0 && !bm.areAllBitsSet() { - t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits") - } - if i > 0 && bm.areAllBitsSet() { - t.Fatalf("areAllBitsSet() must return false on new bitmap with %d bits; %#v", i, bm) - } - - bm.setBits() - - // Make sure that all the bits are set. - nextIdx := 0 - bm.forEachSetBit(func(idx int) bool { - if idx >= i { - t.Fatalf("index must be smaller than %d", i) - } - if idx != nextIdx { - t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx) - } - nextIdx++ - return true - }) - - if !bm.areAllBitsSet() { - t.Fatalf("all bits must be set for bitmap with %d bits", i) - } - - // Clear a part of bits - bm.forEachSetBit(func(idx int) bool { - return idx%2 != 0 - }) - - if i <= 1 && !bm.isZero() { - t.Fatalf("bm.isZero() must return true for bitmap with %d bits", i) - } - if i > 1 && bm.isZero() { - t.Fatalf("bm.isZero() must return false, since some bits are set for bitmap with %d bits", i) - } - if i == 0 && !bm.areAllBitsSet() { - t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits") - } - if i > 0 && bm.areAllBitsSet() { - t.Fatalf("some bits mustn't be set for bitmap with %d bits", i) - } - - nextIdx = 1 - bm.forEachSetBit(func(idx int) bool { - if idx != nextIdx { - t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx) - } - nextIdx += 2 - return true - }) - - // Clear all the bits - bm.forEachSetBit(func(_ int) bool { - return false - }) - - if !bm.isZero() { - t.Fatalf("all the bits must be reset for bitmap with %d bits", i) - } - if i == 0 && !bm.areAllBitsSet() { - t.Fatalf("allAllBitsSet() must return true for bitmap with 0 bits") - } - if i > 0 && bm.areAllBitsSet() { - t.Fatalf("areAllBitsSet() must return false for bitmap with %d bits", i) - } - - bitsCount := 0 - bm.forEachSetBit(func(_ int) bool { - bitsCount++ - return true - }) - if bitsCount != 0 { - t.Fatalf("unexpected non-zero number of set bits remained: %d", bitsCount) - } - - putBitmap(bm) - } -} - func TestComplexFilters(t *testing.T) { columns := []column{ { @@ -854,905 +723,6 @@ func TestRegexpFilter(t *testing.T) { }) } -func TestStringRangeFilter(t *testing.T) { - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "127.0.0.1", - "127.0.0.1", - "127.0.0.1", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.1", - maxValue: "255.", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.1", - maxValue: "127.0.0.1", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "", - maxValue: "127.0.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "non-existing-column", - minValue: "1", - maxValue: "2", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.2", - maxValue: "", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "127.0.0.1", - "Abc", - "127.255.255.255", - "10.4", - "foo 127.0.0.1", - "127.0.0.1 bar", - "127.0.0.1", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.0", - maxValue: "128.0.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 6, 7}) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "127", - maxValue: "127.0.0.1", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "0", - maxValue: "10", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.2", - maxValue: "127.127.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "128.0.0.0", - maxValue: "127.0.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "A FOO", - "a 10", - "127.0.0.1", - "20", - "15.5", - "-5", - "a fooBaR", - "a 127.0.0.1 dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0.1", - maxValue: "255.255.255.255", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "0", - maxValue: "10", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "255.255.255.255", - maxValue: "127.0.0.1", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "33", - maxValue: "5", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "100", - maxValue: "101", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "5", - maxValue: "33", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "65535", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "33", - maxValue: "5", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "100", - maxValue: "101", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "5", - maxValue: "33", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "65536", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "33", - maxValue: "5", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "100", - maxValue: "101", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "5", - maxValue: "33", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12345678901", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "33", - maxValue: "5", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "100", - maxValue: "101", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "5", - maxValue: "33", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "123456.78901", - "-0.2", - "2", - "-334", - "4", - "5", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "33", - maxValue: "5", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10}) - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "-0", - maxValue: "-1", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{6}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "100", - maxValue: "101", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "5", - maxValue: "33", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.12.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "foo", - minValue: "127.0.0", - maxValue: "128.0.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 4, 5, 6, 7}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "128.0.0.0", - maxValue: "129.0.0.0", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "255.0.0.0", - maxValue: "255.255.255.255", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &stringRangeFilter{ - fieldName: "foo", - minValue: "128.0.0.0", - maxValue: "", - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2005-01-02T15:04:05.001Z", - "2006-02-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2026-01-02T15:04:05.005Z", - "2026-01-02T15:04:05.006Z", - "2026-01-02T15:04:05.007Z", - "2026-01-02T15:04:05.008Z", - "2026-01-02T15:04:05.009Z", - }, - }, - } - - // match - fr := &stringRangeFilter{ - fieldName: "_msg", - minValue: "2006-01-02", - maxValue: "2006-01-03", - } - testFilterMatchForColumns(t, columns, fr, "_msg", []int{2, 3}) - - fr = &stringRangeFilter{ - fieldName: "_msg", - minValue: "", - maxValue: "2006", - } - testFilterMatchForColumns(t, columns, fr, "_msg", []int{0}) - - // mismatch - fr = &stringRangeFilter{ - fieldName: "_msg", - minValue: "3", - maxValue: "4", - } - testFilterMatchForColumns(t, columns, fr, "_msg", nil) - - fr = &stringRangeFilter{ - fieldName: "_msg", - minValue: "a", - maxValue: "b", - } - testFilterMatchForColumns(t, columns, fr, "_msg", nil) - - fr = &stringRangeFilter{ - fieldName: "_msg", - minValue: "2006-01-03", - maxValue: "2006-01-02", - } - testFilterMatchForColumns(t, columns, fr, "_msg", nil) - }) -} - -func TestFilterIPv4Range(t *testing.T) { - t.Run("const-column", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "127.0.0.1", - "127.0.0.1", - "127.0.0.1", - }, - }, - } - - // match - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0x80000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x7f000001, - maxValue: 0x7f000001, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2}) - - // mismatch - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0x7f000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "non-existing-column", - minValue: 0, - maxValue: 20000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x80000000, - maxValue: 0, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("dict", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "", - "127.0.0.1", - "Abc", - "127.255.255.255", - "10.4", - "foo 127.0.0.1", - "127.0.0.1 bar", - "127.0.0.1", - }, - }, - } - - // match - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0x7f000000, - maxValue: 0x80000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 7}) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0x7f000001, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7}) - - // mismatch - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 1000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x7f000002, - maxValue: 0x7f7f0000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x80000000, - maxValue: 0x7f000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("strings", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "A FOO", - "a 10", - "127.0.0.1", - "20", - "15.5", - "-5", - "a fooBaR", - "a 127.0.0.1 dfff", - "a ТЕСТЙЦУК НГКШ ", - "a !!,23.(!1)", - }, - }, - } - - // match - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0x7f000000, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{2}) - - // mismatch - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 10000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0xffffffff, - maxValue: 0x7f000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint8", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint16", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "65535", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint32", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "65536", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("uint64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "12345678901", - "1", - "2", - "3", - "4", - "5", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("float64", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "123", - "12", - "32", - "0", - "0", - "123456.78901", - "-0.2", - "2", - "-334", - "4", - "5", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("ipv4", func(t *testing.T) { - columns := []column{ - { - name: "foo", - values: []string{ - "1.2.3.4", - "0.0.0.0", - "127.0.0.1", - "254.255.255.255", - "127.0.0.1", - "127.0.0.1", - "127.0.4.2", - "127.0.0.1", - "12.0.127.6", - "55.55.12.55", - "66.66.66.66", - "7.7.7.7", - }, - }, - } - - // match - fr := &filterIPv4Range{ - fieldName: "foo", - minValue: 0, - maxValue: 0x08000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 11}) - - // mismatch - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x80000000, - maxValue: 0x90000000, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0xff000000, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - - fr = &filterIPv4Range{ - fieldName: "foo", - minValue: 0x08000000, - maxValue: 0, - } - testFilterMatchForColumns(t, columns, fr, "foo", nil) - }) - - t.Run("timestamp-iso8601", func(t *testing.T) { - columns := []column{ - { - name: "_msg", - values: []string{ - "2006-01-02T15:04:05.001Z", - "2006-01-02T15:04:05.002Z", - "2006-01-02T15:04:05.003Z", - "2006-01-02T15:04:05.004Z", - "2006-01-02T15:04:05.005Z", - "2006-01-02T15:04:05.006Z", - "2006-01-02T15:04:05.007Z", - "2006-01-02T15:04:05.008Z", - "2006-01-02T15:04:05.009Z", - }, - }, - } - - // mismatch - fr := &filterIPv4Range{ - fieldName: "_msg", - minValue: 0, - maxValue: 0xffffffff, - } - testFilterMatchForColumns(t, columns, fr, "_msg", nil) - }) -} - func TestLenRangeFilter(t *testing.T) { t.Run("const-column", func(t *testing.T) { columns := []column{ diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 9f2991cef..dffed80a3 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -338,7 +338,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) { case lex.isKeyword("seq"): return parseFilterSequence(lex, fieldName) case lex.isKeyword("string_range"): - return parseStringRangeFilter(lex, fieldName) + return parseFilterStringRange(lex, fieldName) case lex.isKeyword(`"`, "'", "`"): return nil, fmt.Errorf("improperly quoted string") case lex.isKeyword(",", ")", "[", "]"): @@ -542,13 +542,13 @@ func parseLenRangeFilter(lex *lexer, fieldName string) (filter, error) { }) } -func parseStringRangeFilter(lex *lexer, fieldName string) (filter, error) { +func parseFilterStringRange(lex *lexer, fieldName string) (filter, error) { funcName := lex.token return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) { if len(args) != 2 { return nil, fmt.Errorf("unexpected number of args for %s(); got %d; want 2", funcName, len(args)) } - fr := &stringRangeFilter{ + fr := &filterStringRange{ fieldName: fieldName, minValue: args[0], maxValue: args[1], diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index e1b238f77..dbcec4b72 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -356,16 +356,16 @@ func TestParseFilterIPv4Range(t *testing.T) { f(`ipv4_range(1.2.3.34/0)`, ``, 0, 0xffffffff) } -func TestParseStringRangeFilter(t *testing.T) { +func TestParseFilterStringRange(t *testing.T) { f := func(s, fieldNameExpected, minValueExpected, maxValueExpected string) { t.Helper() q, err := ParseQuery(s) if err != nil { t.Fatalf("unexpected error: %s", err) } - rf, ok := q.f.(*stringRangeFilter) + rf, ok := q.f.(*filterStringRange) if !ok { - t.Fatalf("unexpected filter type; got %T; want *stringRangeFilter; filter: %s", q.f, q.f) + t.Fatalf("unexpected filter type; got %T; want *filterStringRange; filter: %s", q.f, q.f) } if rf.fieldName != fieldNameExpected { t.Fatalf("unexpected fieldName; got %q; want %q", rf.fieldName, fieldNameExpected)