mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-31 15:06:26 +00:00
wip
This commit is contained in:
Aliaksandr Valialkin
parent
8d0eb0c43c
commit
75838f0a87
8 changed files with 1241 additions and 1218 deletions
96
lib/logstorage/bitmap_test.go
Normal file
96
lib/logstorage/bitmap_test.go
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
package logstorage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBitmap(t *testing.T) {
|
||||
for i := 0; i < 100; i++ {
|
||||
bm := getBitmap(i)
|
||||
if bm.bitsLen != i {
|
||||
t.Fatalf("unexpected bits length: %d; want %d", bm.bitsLen, i)
|
||||
}
|
||||
|
||||
if !bm.isZero() {
|
||||
t.Fatalf("all the bits must be zero for bitmap with %d bits", i)
|
||||
}
|
||||
if i == 0 && !bm.areAllBitsSet() {
|
||||
t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits")
|
||||
}
|
||||
if i > 0 && bm.areAllBitsSet() {
|
||||
t.Fatalf("areAllBitsSet() must return false on new bitmap with %d bits; %#v", i, bm)
|
||||
}
|
||||
|
||||
bm.setBits()
|
||||
|
||||
// Make sure that all the bits are set.
|
||||
nextIdx := 0
|
||||
bm.forEachSetBit(func(idx int) bool {
|
||||
if idx >= i {
|
||||
t.Fatalf("index must be smaller than %d", i)
|
||||
}
|
||||
if idx != nextIdx {
|
||||
t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx)
|
||||
}
|
||||
nextIdx++
|
||||
return true
|
||||
})
|
||||
|
||||
if !bm.areAllBitsSet() {
|
||||
t.Fatalf("all bits must be set for bitmap with %d bits", i)
|
||||
}
|
||||
|
||||
// Clear a part of bits
|
||||
bm.forEachSetBit(func(idx int) bool {
|
||||
return idx%2 != 0
|
||||
})
|
||||
|
||||
if i <= 1 && !bm.isZero() {
|
||||
t.Fatalf("bm.isZero() must return true for bitmap with %d bits", i)
|
||||
}
|
||||
if i > 1 && bm.isZero() {
|
||||
t.Fatalf("bm.isZero() must return false, since some bits are set for bitmap with %d bits", i)
|
||||
}
|
||||
if i == 0 && !bm.areAllBitsSet() {
|
||||
t.Fatalf("areAllBitsSet() must return true for bitmap with 0 bits")
|
||||
}
|
||||
if i > 0 && bm.areAllBitsSet() {
|
||||
t.Fatalf("some bits mustn't be set for bitmap with %d bits", i)
|
||||
}
|
||||
|
||||
nextIdx = 1
|
||||
bm.forEachSetBit(func(idx int) bool {
|
||||
if idx != nextIdx {
|
||||
t.Fatalf("unexpected idx; got %d; want %d", idx, nextIdx)
|
||||
}
|
||||
nextIdx += 2
|
||||
return true
|
||||
})
|
||||
|
||||
// Clear all the bits
|
||||
bm.forEachSetBit(func(_ int) bool {
|
||||
return false
|
||||
})
|
||||
|
||||
if !bm.isZero() {
|
||||
t.Fatalf("all the bits must be reset for bitmap with %d bits", i)
|
||||
}
|
||||
if i == 0 && !bm.areAllBitsSet() {
|
||||
t.Fatalf("allAllBitsSet() must return true for bitmap with 0 bits")
|
||||
}
|
||||
if i > 0 && bm.areAllBitsSet() {
|
||||
t.Fatalf("areAllBitsSet() must return false for bitmap with %d bits", i)
|
||||
}
|
||||
|
||||
bitsCount := 0
|
||||
bm.forEachSetBit(func(_ int) bool {
|
||||
bitsCount++
|
||||
return true
|
||||
})
|
||||
if bitsCount != 0 {
|
||||
t.Fatalf("unexpected non-zero number of set bits remained: %d", bitsCount)
|
||||
}
|
||||
|
||||
putBitmap(bm)
|
||||
}
|
||||
}
|
||||
|
|
@ -72,73 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
|
|||
}
|
||||
}
|
||||
|
||||
// stringRangeFilter matches tie given string range [minValue..maxValue)
|
||||
//
|
||||
// Note that the minValue is included in the range, while the maxValue isn't included in the range.
|
||||
// This simplifies querying distincts log sets with string_range(A, B), string_range(B, C), etc.
|
||||
//
|
||||
// Example LogsQL: `fieldName:string_range(minValue, maxValue)`
|
||||
type stringRangeFilter struct {
|
||||
fieldName string
|
||||
minValue string
|
||||
maxValue string
|
||||
}
|
||||
|
||||
func (fr *stringRangeFilter) String() string {
|
||||
return fmt.Sprintf("%sstring_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.minValue), quoteTokenIfNeeded(fr.maxValue))
|
||||
}
|
||||
|
||||
func (fr *stringRangeFilter) apply(bs *blockSearch, bm *bitmap) {
|
||||
fieldName := fr.fieldName
|
||||
minValue := fr.minValue
|
||||
maxValue := fr.maxValue
|
||||
|
||||
if minValue > maxValue {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
v := bs.csh.getConstColumnValue(fieldName)
|
||||
if v != "" {
|
||||
if !matchStringRange(v, minValue, maxValue) {
|
||||
bm.resetBits()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify whether filter matches other columns
|
||||
ch := bs.csh.getColumnHeader(fieldName)
|
||||
if ch == nil {
|
||||
if !matchStringRange("", minValue, maxValue) {
|
||||
bm.resetBits()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
switch ch.valueType {
|
||||
case valueTypeString:
|
||||
matchStringByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeDict:
|
||||
matchValuesDictByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint8:
|
||||
matchUint8ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint16:
|
||||
matchUint16ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint32:
|
||||
matchUint32ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint64:
|
||||
matchUint64ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeFloat64:
|
||||
matchFloat64ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeIPv4:
|
||||
matchIPv4ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeTimestampISO8601:
|
||||
matchTimestampISO8601ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
default:
|
||||
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
|
||||
}
|
||||
}
|
||||
|
||||
// lenRangeFilter matches field values with the length in the given range [minLen, maxLen].
|
||||
//
|
||||
// Example LogsQL: `fieldName:len_range(10, 20)`
|
||||
|
|
@ -670,20 +603,6 @@ func matchTimestampISO8601ByLenRange(bm *bitmap, minLen, maxLen uint64) {
|
|||
}
|
||||
}
|
||||
|
||||
func matchTimestampISO8601ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
|
|
@ -736,20 +655,6 @@ func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap
|
|||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchIPv4ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toIPv4StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchIPv4ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) {
|
||||
if minLen > uint64(len("255.255.255.255")) || maxLen < uint64(len("0.0.0.0")) {
|
||||
bm.resetBits()
|
||||
|
|
@ -834,20 +739,6 @@ func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase str
|
|||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchFloat64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "+" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toFloat64StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchFloat64ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) {
|
||||
if minLen > 24 || maxLen == 0 {
|
||||
bm.resetBits()
|
||||
|
|
@ -945,17 +836,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
|
|||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchValuesDictByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
bb := bbPool.Get()
|
||||
for i, v := range ch.valuesDict.values {
|
||||
if matchStringRange(v, minValue, maxValue) {
|
||||
bb.B = append(bb.B, byte(i))
|
||||
}
|
||||
}
|
||||
matchEncodedValuesDict(bs, ch, bm, bb.B)
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchValuesDictByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) {
|
||||
bb := bbPool.Get()
|
||||
for i, v := range ch.valuesDict.values {
|
||||
|
|
@ -1060,12 +940,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod
|
|||
})
|
||||
}
|
||||
|
||||
func matchStringByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
return matchStringRange(v, minValue, maxValue)
|
||||
})
|
||||
}
|
||||
|
||||
func matchStringByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen, maxLen uint64) {
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
return matchLenRange(v, minLen, maxLen)
|
||||
|
|
@ -1116,58 +990,6 @@ func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase s
|
|||
})
|
||||
}
|
||||
|
||||
func matchUint8ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint8String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint16ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint16String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint32ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint32String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint64String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchMinMaxValueLen(ch *columnHeader, minLen, maxLen uint64) bool {
|
||||
bb := bbPool.Get()
|
||||
defer bbPool.Put(bb)
|
||||
|
|
@ -1526,10 +1348,6 @@ func matchPrefix(s, prefix string) bool {
|
|||
}
|
||||
}
|
||||
|
||||
func matchStringRange(s, minValue, maxValue string) bool {
|
||||
return s >= minValue && s < maxValue
|
||||
}
|
||||
|
||||
func matchLenRange(s string, minLen, maxLen uint64) bool {
|
||||
sLen := uint64(utf8.RuneCountInString(s))
|
||||
return sLen >= minLen && sLen <= maxLen
|
||||
|
|
|
|||
402
lib/logstorage/filter_ipv4_range_test.go
Normal file
402
lib/logstorage/filter_ipv4_range_test.go
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
package logstorage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMatchIPv4Range(t *testing.T) {
|
||||
f := func(s string, minValue, maxValue uint32, resultExpected bool) {
|
||||
t.Helper()
|
||||
result := matchIPv4Range(s, minValue, maxValue)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// Invalid IP
|
||||
f("", 0, 1000, false)
|
||||
f("123", 0, 1000, false)
|
||||
|
||||
// range mismatch
|
||||
f("0.0.0.1", 2, 100, false)
|
||||
f("127.0.0.1", 0x6f000000, 0x7f000000, false)
|
||||
|
||||
// range match
|
||||
f("0.0.0.1", 1, 1, true)
|
||||
f("0.0.0.1", 0, 100, true)
|
||||
f("127.0.0.1", 0x7f000000, 0x7f000001, true)
|
||||
}
|
||||
|
||||
func TestFilterIPv4Range(t *testing.T) {
|
||||
t.Run("const-column", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0x80000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x7f000001,
|
||||
maxValue: 0x7f000001,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
|
||||
|
||||
// mismatch
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0x7f000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "non-existing-column",
|
||||
minValue: 0,
|
||||
maxValue: 20000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x80000000,
|
||||
maxValue: 0,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("dict", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"",
|
||||
"127.0.0.1",
|
||||
"Abc",
|
||||
"127.255.255.255",
|
||||
"10.4",
|
||||
"foo 127.0.0.1",
|
||||
"127.0.0.1 bar",
|
||||
"127.0.0.1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x7f000000,
|
||||
maxValue: 0x80000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 7})
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0x7f000001,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7})
|
||||
|
||||
// mismatch
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 1000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x7f000002,
|
||||
maxValue: 0x7f7f0000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x80000000,
|
||||
maxValue: 0x7f000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("strings", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"A FOO",
|
||||
"a 10",
|
||||
"127.0.0.1",
|
||||
"20",
|
||||
"15.5",
|
||||
"-5",
|
||||
"a fooBaR",
|
||||
"a 127.0.0.1 dfff",
|
||||
"a ТЕСТЙЦУК НГКШ ",
|
||||
"a !!,23.(!1)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x7f000000,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{2})
|
||||
|
||||
// mismatch
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 10000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0xffffffff,
|
||||
maxValue: 0x7f000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint8", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"12",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint16", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"65535",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint32", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"65536",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint64", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"12345678901",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("float64", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"123456.78901",
|
||||
"-0.2",
|
||||
"2",
|
||||
"-334",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("ipv4", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"1.2.3.4",
|
||||
"0.0.0.0",
|
||||
"127.0.0.1",
|
||||
"254.255.255.255",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
"127.0.4.2",
|
||||
"127.0.0.1",
|
||||
"12.0.127.6",
|
||||
"55.55.12.55",
|
||||
"66.66.66.66",
|
||||
"7.7.7.7",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0,
|
||||
maxValue: 0x08000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 11})
|
||||
|
||||
// mismatch
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x80000000,
|
||||
maxValue: 0x90000000,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0xff000000,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterIPv4Range{
|
||||
fieldName: "foo",
|
||||
minValue: 0x08000000,
|
||||
maxValue: 0,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("timestamp-iso8601", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "_msg",
|
||||
values: []string{
|
||||
"2006-01-02T15:04:05.001Z",
|
||||
"2006-01-02T15:04:05.002Z",
|
||||
"2006-01-02T15:04:05.003Z",
|
||||
"2006-01-02T15:04:05.004Z",
|
||||
"2006-01-02T15:04:05.005Z",
|
||||
"2006-01-02T15:04:05.006Z",
|
||||
"2006-01-02T15:04:05.007Z",
|
||||
"2006-01-02T15:04:05.008Z",
|
||||
"2006-01-02T15:04:05.009Z",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// mismatch
|
||||
fr := &filterIPv4Range{
|
||||
fieldName: "_msg",
|
||||
minValue: 0,
|
||||
maxValue: 0xffffffff,
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
|
||||
})
|
||||
}
|
||||
189
lib/logstorage/filter_string_range.go
Normal file
189
lib/logstorage/filter_string_range.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package logstorage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
)
|
||||
|
||||
// filterStringRange matches tie given string range [minValue..maxValue)
|
||||
//
|
||||
// Note that the minValue is included in the range, while the maxValue isn't included in the range.
|
||||
// This simplifies querying distincts log sets with string_range(A, B), string_range(B, C), etc.
|
||||
//
|
||||
// Example LogsQL: `fieldName:string_range(minValue, maxValue)`
|
||||
type filterStringRange struct {
|
||||
fieldName string
|
||||
minValue string
|
||||
maxValue string
|
||||
}
|
||||
|
||||
func (fr *filterStringRange) String() string {
|
||||
return fmt.Sprintf("%sstring_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), quoteTokenIfNeeded(fr.minValue), quoteTokenIfNeeded(fr.maxValue))
|
||||
}
|
||||
|
||||
func (fr *filterStringRange) apply(bs *blockSearch, bm *bitmap) {
|
||||
fieldName := fr.fieldName
|
||||
minValue := fr.minValue
|
||||
maxValue := fr.maxValue
|
||||
|
||||
if minValue > maxValue {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
v := bs.csh.getConstColumnValue(fieldName)
|
||||
if v != "" {
|
||||
if !matchStringRange(v, minValue, maxValue) {
|
||||
bm.resetBits()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify whether filter matches other columns
|
||||
ch := bs.csh.getColumnHeader(fieldName)
|
||||
if ch == nil {
|
||||
if !matchStringRange("", minValue, maxValue) {
|
||||
bm.resetBits()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
switch ch.valueType {
|
||||
case valueTypeString:
|
||||
matchStringByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeDict:
|
||||
matchValuesDictByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint8:
|
||||
matchUint8ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint16:
|
||||
matchUint16ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint32:
|
||||
matchUint32ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeUint64:
|
||||
matchUint64ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeFloat64:
|
||||
matchFloat64ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeIPv4:
|
||||
matchIPv4ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
case valueTypeTimestampISO8601:
|
||||
matchTimestampISO8601ByStringRange(bs, ch, bm, minValue, maxValue)
|
||||
default:
|
||||
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
|
||||
}
|
||||
}
|
||||
|
||||
func matchTimestampISO8601ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchIPv4ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toIPv4StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchFloat64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "+" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toFloat64StringExt(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchValuesDictByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
bb := bbPool.Get()
|
||||
for i, v := range ch.valuesDict.values {
|
||||
if matchStringRange(v, minValue, maxValue) {
|
||||
bb.B = append(bb.B, byte(i))
|
||||
}
|
||||
}
|
||||
matchEncodedValuesDict(bs, ch, bm, bb.B)
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchStringByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
return matchStringRange(v, minValue, maxValue)
|
||||
})
|
||||
}
|
||||
|
||||
func matchUint8ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint8String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint16ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint16String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint32ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint32String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchUint64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
|
||||
if minValue > "9" || maxValue < "0" {
|
||||
bm.resetBits()
|
||||
return
|
||||
}
|
||||
bb := bbPool.Get()
|
||||
visitValues(bs, ch, bm, func(v string) bool {
|
||||
s := toUint64String(bs, bb, v)
|
||||
return matchStringRange(s, minValue, maxValue)
|
||||
})
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func matchStringRange(s, minValue, maxValue string) bool {
|
||||
return s >= minValue && s < maxValue
|
||||
}
|
||||
548
lib/logstorage/filter_string_range_test.go
Normal file
548
lib/logstorage/filter_string_range_test.go
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
package logstorage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMatchStringRange(t *testing.T) {
|
||||
f := func(s, minValue, maxValue string, resultExpected bool) {
|
||||
t.Helper()
|
||||
result := matchStringRange(s, minValue, maxValue)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("foo", "a", "b", false)
|
||||
f("foo", "a", "foa", false)
|
||||
f("foo", "a", "foz", true)
|
||||
f("foo", "foo", "foo", false)
|
||||
f("foo", "foo", "fooa", true)
|
||||
f("foo", "fooa", "foo", false)
|
||||
}
|
||||
|
||||
func TestFilterStringRange(t *testing.T) {
|
||||
t.Run("const-column", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.1",
|
||||
maxValue: "255.",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.1",
|
||||
maxValue: "127.0.0.1",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "",
|
||||
maxValue: "127.0.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "non-existing-column",
|
||||
minValue: "1",
|
||||
maxValue: "2",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.2",
|
||||
maxValue: "",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("dict", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"",
|
||||
"127.0.0.1",
|
||||
"Abc",
|
||||
"127.255.255.255",
|
||||
"10.4",
|
||||
"foo 127.0.0.1",
|
||||
"127.0.0.1 bar",
|
||||
"127.0.0.1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.0",
|
||||
maxValue: "128.0.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 6, 7})
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127",
|
||||
maxValue: "127.0.0.1",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "0",
|
||||
maxValue: "10",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.2",
|
||||
maxValue: "127.127.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "128.0.0.0",
|
||||
maxValue: "127.0.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("strings", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"A FOO",
|
||||
"a 10",
|
||||
"127.0.0.1",
|
||||
"20",
|
||||
"15.5",
|
||||
"-5",
|
||||
"a fooBaR",
|
||||
"a 127.0.0.1 dfff",
|
||||
"a ТЕСТЙЦУК НГКШ ",
|
||||
"a !!,23.(!1)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0.1",
|
||||
maxValue: "255.255.255.255",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "0",
|
||||
maxValue: "10",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "255.255.255.255",
|
||||
maxValue: "127.0.0.1",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint8", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"12",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "33",
|
||||
maxValue: "5",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "100",
|
||||
maxValue: "101",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "5",
|
||||
maxValue: "33",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint16", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"65535",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "33",
|
||||
maxValue: "5",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "100",
|
||||
maxValue: "101",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "5",
|
||||
maxValue: "33",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint32", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"65536",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "33",
|
||||
maxValue: "5",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "100",
|
||||
maxValue: "101",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "5",
|
||||
maxValue: "33",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("uint64", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"12345678901",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "33",
|
||||
maxValue: "5",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "100",
|
||||
maxValue: "101",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "5",
|
||||
maxValue: "33",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("float64", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"123",
|
||||
"12",
|
||||
"32",
|
||||
"0",
|
||||
"0",
|
||||
"123456.78901",
|
||||
"-0.2",
|
||||
"2",
|
||||
"-334",
|
||||
"4",
|
||||
"5",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "33",
|
||||
maxValue: "5",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{9, 10})
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "-0",
|
||||
maxValue: "-1",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{6})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "100",
|
||||
maxValue: "101",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "5",
|
||||
maxValue: "33",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("ipv4", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "foo",
|
||||
values: []string{
|
||||
"1.2.3.4",
|
||||
"0.0.0.0",
|
||||
"127.0.0.1",
|
||||
"254.255.255.255",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1",
|
||||
"127.0.4.2",
|
||||
"127.0.0.1",
|
||||
"12.0.127.6",
|
||||
"55.55.12.55",
|
||||
"66.66.66.66",
|
||||
"7.7.7.7",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "127.0.0",
|
||||
maxValue: "128.0.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 4, 5, 6, 7})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "128.0.0.0",
|
||||
maxValue: "129.0.0.0",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "255.0.0.0",
|
||||
maxValue: "255.255.255.255",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "foo",
|
||||
minValue: "128.0.0.0",
|
||||
maxValue: "",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "foo", nil)
|
||||
})
|
||||
|
||||
t.Run("timestamp-iso8601", func(t *testing.T) {
|
||||
columns := []column{
|
||||
{
|
||||
name: "_msg",
|
||||
values: []string{
|
||||
"2005-01-02T15:04:05.001Z",
|
||||
"2006-02-02T15:04:05.002Z",
|
||||
"2006-01-02T15:04:05.003Z",
|
||||
"2006-01-02T15:04:05.004Z",
|
||||
"2026-01-02T15:04:05.005Z",
|
||||
"2026-01-02T15:04:05.006Z",
|
||||
"2026-01-02T15:04:05.007Z",
|
||||
"2026-01-02T15:04:05.008Z",
|
||||
"2026-01-02T15:04:05.009Z",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// match
|
||||
fr := &filterStringRange{
|
||||
fieldName: "_msg",
|
||||
minValue: "2006-01-02",
|
||||
maxValue: "2006-01-03",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", []int{2, 3})
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "_msg",
|
||||
minValue: "",
|
||||
maxValue: "2006",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", []int{0})
|
||||
|
||||
// mismatch
|
||||
fr = &filterStringRange{
|
||||
fieldName: "_msg",
|
||||
minValue: "3",
|
||||
maxValue: "4",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "_msg",
|
||||
minValue: "a",
|
||||
maxValue: "b",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
|
||||
|
||||
fr = &filterStringRange{
|
||||
fieldName: "_msg",
|
||||
minValue: "2006-01-03",
|
||||
maxValue: "2006-01-02",
|
||||
}
|
||||
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
|
||||
})
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -338,7 +338,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
|
|||
case lex.isKeyword("seq"):
|
||||
return parseFilterSequence(lex, fieldName)
|
||||
case lex.isKeyword("string_range"):
|
||||
return parseStringRangeFilter(lex, fieldName)
|
||||
return parseFilterStringRange(lex, fieldName)
|
||||
case lex.isKeyword(`"`, "'", "`"):
|
||||
return nil, fmt.Errorf("improperly quoted string")
|
||||
case lex.isKeyword(",", ")", "[", "]"):
|
||||
|
|
@ -542,13 +542,13 @@ func parseLenRangeFilter(lex *lexer, fieldName string) (filter, error) {
|
|||
})
|
||||
}
|
||||
|
||||
func parseStringRangeFilter(lex *lexer, fieldName string) (filter, error) {
|
||||
func parseFilterStringRange(lex *lexer, fieldName string) (filter, error) {
|
||||
funcName := lex.token
|
||||
return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) {
|
||||
if len(args) != 2 {
|
||||
return nil, fmt.Errorf("unexpected number of args for %s(); got %d; want 2", funcName, len(args))
|
||||
}
|
||||
fr := &stringRangeFilter{
|
||||
fr := &filterStringRange{
|
||||
fieldName: fieldName,
|
||||
minValue: args[0],
|
||||
maxValue: args[1],
|
||||
|
|
|
|||
|
|
@ -356,16 +356,16 @@ func TestParseFilterIPv4Range(t *testing.T) {
|
|||
f(`ipv4_range(1.2.3.34/0)`, ``, 0, 0xffffffff)
|
||||
}
|
||||
|
||||
func TestParseStringRangeFilter(t *testing.T) {
|
||||
func TestParseFilterStringRange(t *testing.T) {
|
||||
f := func(s, fieldNameExpected, minValueExpected, maxValueExpected string) {
|
||||
t.Helper()
|
||||
q, err := ParseQuery(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
rf, ok := q.f.(*stringRangeFilter)
|
||||
rf, ok := q.f.(*filterStringRange)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected filter type; got %T; want *stringRangeFilter; filter: %s", q.f, q.f)
|
||||
t.Fatalf("unexpected filter type; got %T; want *filterStringRange; filter: %s", q.f, q.f)
|
||||
}
|
||||
if rf.fieldName != fieldNameExpected {
|
||||
t.Fatalf("unexpected fieldName; got %q; want %q", rf.fieldName, fieldNameExpected)
|
||||
|
|
|
|||
Loading…
Reference in a new issue