github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 15:16:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 06:42:18 +02:00
parent ecef86c641
commit f49352ece4
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
4 changed files with 796 additions and 782 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

178
lib/logstorage/filter.go
View file

@ -72,71 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
}
}
// filterRange matches the given range [minValue..maxValue].
//
// Example LogsQL: `fieldName:range(minValue, maxValue]`
type filterRange struct {
fieldName string
minValue float64
maxValue float64
stringRepr string
}
func (fr *filterRange) String() string {
return quoteFieldNameIfNeeded(fr.fieldName) + "range" + fr.stringRepr
}
func (fr *filterRange) apply(bs *blockSearch, bm *bitmap) {
fieldName := fr.fieldName
minValue := fr.minValue
maxValue := fr.maxValue
if minValue > maxValue {
bm.resetBits()
return
}
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchRange(v, minValue, maxValue) {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()
return
}
switch ch.valueType {
case valueTypeString:
matchStringByRange(bs, ch, bm, minValue, maxValue)
case valueTypeDict:
matchValuesDictByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint8:
matchUint8ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint16:
matchUint16ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint32:
matchUint32ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint64:
matchUint64ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeFloat64:
matchFloat64ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeIPv4:
bm.resetBits()
case valueTypeTimestampISO8601:
bm.resetBits()
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
// regexpFilter matches the given regexp
//
// Example LogsQL: `fieldName:re("regexp")`
@ -635,23 +570,6 @@ func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase str
bbPool.Put(bb)
}
func matchFloat64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
if minValue > math.Float64frombits(ch.maxValue) || maxValue < math.Float64frombits(ch.minValue) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 8 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := encoding.UnmarshalUint64(b)
f := math.Float64frombits(n)
return f >= minValue && f <= maxValue
})
}
func matchFloat64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
@ -718,17 +636,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
bbPool.Put(bb)
}
func matchValuesDictByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchRange(v, minValue, maxValue) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchValuesDictByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
@ -811,12 +718,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod
})
}
func matchStringByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
visitValues(bs, ch, bm, func(v string) bool {
return matchRange(v, minValue, maxValue)
})
}
func matchStringByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
visitValues(bs, ch, bm, func(v string) bool {
return re.MatchString(v)
@ -869,77 +770,6 @@ func matchMinMaxValueLen(ch *columnHeader, minLen, maxLen uint64) bool {
return minLen <= uint64(len(s))
}
func matchUint8ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 1 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
}
n := uint64(v[0])
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint16ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 2 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := uint64(encoding.UnmarshalUint16(b))
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint32ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 4 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 4", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := uint64(encoding.UnmarshalUint32(b))
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 8 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 8", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := encoding.UnmarshalUint64(b)
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint8ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
@ -1157,14 +987,6 @@ func matchPrefix(s, prefix string) bool {
}
}
func matchRange(s string, minValue, maxValue float64) bool {
f, ok := tryParseFloat64(s)
if !ok {
return false
}
return f >= minValue && f <= maxValue
}
func matchAnyCasePhrase(s, phraseLowercase string) bool {
if len(phraseLowercase) == 0 {
// Special case - empty phrase matches only empty string.

187
lib/logstorage/filter_range.go Normal file
View file

@ -0,0 +1,187 @@
package logstorage
import (
"math"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// filterRange matches the given range [minValue..maxValue].
//
// Example LogsQL: `fieldName:range(minValue, maxValue]`
type filterRange struct {
fieldName string
minValue float64
maxValue float64
stringRepr string
}
func (fr *filterRange) String() string {
return quoteFieldNameIfNeeded(fr.fieldName) + "range" + fr.stringRepr
}
func (fr *filterRange) apply(bs *blockSearch, bm *bitmap) {
fieldName := fr.fieldName
minValue := fr.minValue
maxValue := fr.maxValue
if minValue > maxValue {
bm.resetBits()
return
}
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchRange(v, minValue, maxValue) {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()
return
}
switch ch.valueType {
case valueTypeString:
matchStringByRange(bs, ch, bm, minValue, maxValue)
case valueTypeDict:
matchValuesDictByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint8:
matchUint8ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint16:
matchUint16ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint32:
matchUint32ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeUint64:
matchUint64ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeFloat64:
matchFloat64ByRange(bs, ch, bm, minValue, maxValue)
case valueTypeIPv4:
bm.resetBits()
case valueTypeTimestampISO8601:
bm.resetBits()
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchFloat64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
if minValue > math.Float64frombits(ch.maxValue) || maxValue < math.Float64frombits(ch.minValue) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 8 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := encoding.UnmarshalUint64(b)
f := math.Float64frombits(n)
return f >= minValue && f <= maxValue
})
}
func matchValuesDictByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchRange(v, minValue, maxValue) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchStringByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
visitValues(bs, ch, bm, func(v string) bool {
return matchRange(v, minValue, maxValue)
})
}
func matchUint8ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 1 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
}
n := uint64(v[0])
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint16ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 2 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := uint64(encoding.UnmarshalUint16(b))
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint32ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 4 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 4", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := uint64(encoding.UnmarshalUint32(b))
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchUint64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue float64) {
minValueUint, maxValueUint := toUint64Range(minValue, maxValue)
if maxValue < 0 || minValueUint > ch.maxValue || maxValueUint < ch.minValue {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
if len(v) != 8 {
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 8", bs.partPath(), len(v))
}
b := bytesutil.ToUnsafeBytes(v)
n := encoding.UnmarshalUint64(b)
return n >= minValueUint && n <= maxValueUint
})
bbPool.Put(bb)
}
func matchRange(s string, minValue, maxValue float64) bool {
f, ok := tryParseFloat64(s)
if !ok {
return false
}
return f >= minValue && f <= maxValue
}

609
lib/logstorage/filter_range_test.go Normal file
View file

@ -0,0 +1,609 @@
package logstorage
import (
"math"
"testing"
)
func TestFilterRange(t *testing.T) {
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"10",
"10",
"10",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 9.99,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: -10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 10.1,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "non-existing-column",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 11,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"10",
"Abc",
"20",
"10.5",
"10 AFoobarbaz",
"foobar",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: 10.1,
maxValue: 19.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{4})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -11,
maxValue: 0,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 11,
maxValue: 19,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20.1,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"A FOO",
"a 10",
"10",
"20",
"15.5",
"-5",
"a fooBaR",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4, 5})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: -5,
maxValue: -5,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: -5.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20.1,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"65535",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"65536",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12345678901",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: math.Inf(-1),
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: 1000,
maxValue: math.Inf(1),
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"123456.78901",
"-0.2",
"2",
"-334",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: math.Inf(-1),
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 1.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 1000,
maxValue: math.Inf(1),
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -334.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.12.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// range filter always mismatches ipv4
fr := &filterRange{
fieldName: "foo",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// range filter always mismatches timestmap
fr := &filterRange{
fieldName: "_msg",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
})
}

604
lib/logstorage/filter_test.go
View file

@ -2,7 +2,6 @@ package logstorage
import (
"fmt"
"math"
"reflect"
"regexp"
"testing"
@ -697,609 +696,6 @@ func TestRegexpFilter(t *testing.T) {
})
}
func TestFilterRange(t *testing.T) {
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"10",
"10",
"10",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 9.99,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: -10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 10.1,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "non-existing-column",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 11,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"10",
"Abc",
"20",
"10.5",
"10 AFoobarbaz",
"foobar",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: 10.1,
maxValue: 19.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{4})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -11,
maxValue: 0,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 11,
maxValue: 19,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20.1,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"A FOO",
"a 10",
"10",
"20",
"15.5",
"-5",
"a fooBaR",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4, 5})
fr = &filterRange{
fieldName: "foo",
minValue: 10,
maxValue: 20,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{2, 3, 4})
fr = &filterRange{
fieldName: "foo",
minValue: -5,
maxValue: -5,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -10,
maxValue: -5.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20.1,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 20,
maxValue: 10,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"65535",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"65536",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: 0,
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12345678901",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: math.Inf(-1),
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7})
fr = &filterRange{
fieldName: "foo",
minValue: 1000,
maxValue: math.Inf(1),
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"123456.78901",
"-0.2",
"2",
"-334",
"4",
"5",
},
},
}
// match
fr := &filterRange{
fieldName: "foo",
minValue: math.Inf(-1),
maxValue: 3,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 7, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 2.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{7})
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: 1.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{3, 4, 6, 8})
fr = &filterRange{
fieldName: "foo",
minValue: 1000,
maxValue: math.Inf(1),
}
testFilterMatchForColumns(t, columns, fr, "foo", []int{5})
// mismatch
fr = &filterRange{
fieldName: "foo",
minValue: -1e18,
maxValue: -334.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 0.1,
maxValue: 0.9,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
fr = &filterRange{
fieldName: "foo",
minValue: 2.9,
maxValue: 0.1,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.12.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// range filter always mismatches ipv4
fr := &filterRange{
fieldName: "foo",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// range filter always mismatches timestmap
fr := &filterRange{
fieldName: "_msg",
minValue: -100,
maxValue: 100,
}
testFilterMatchForColumns(t, columns, fr, "_msg", nil)
})
}
func TestAnyCasePrefixFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{