github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 06:01:06 +02:00
parent a968561182
commit 2a1cede9f7
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
6 changed files with 1039 additions and 1022 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

331
lib/logstorage/filter.go
View file

@ -72,308 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
} }
} }
// inFilter matches any exact value from the values map.
//
// Example LogsQL: `fieldName:in("foo", "bar baz")`
type inFilter struct {
fieldName string
values []string
tokenSetsOnce sync.Once
tokenSets [][]string
stringValuesOnce sync.Once
stringValues map[string]struct{}
uint8ValuesOnce sync.Once
uint8Values map[string]struct{}
uint16ValuesOnce sync.Once
uint16Values map[string]struct{}
uint32ValuesOnce sync.Once
uint32Values map[string]struct{}
uint64ValuesOnce sync.Once
uint64Values map[string]struct{}
float64ValuesOnce sync.Once
float64Values map[string]struct{}
ipv4ValuesOnce sync.Once
ipv4Values map[string]struct{}
timestampISO8601ValuesOnce sync.Once
timestampISO8601Values map[string]struct{}
}
func (fi *inFilter) String() string {
values := fi.values
a := make([]string, len(values))
for i, value := range values {
a[i] = quoteTokenIfNeeded(value)
}
return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ","))
}
func (fi *inFilter) getTokenSets() [][]string {
fi.tokenSetsOnce.Do(fi.initTokenSets)
return fi.tokenSets
}
// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter.
const maxTokenSetsToInit = 1000
func (fi *inFilter) initTokenSets() {
values := fi.values
tokenSetsLen := len(values)
if tokenSetsLen > maxTokenSetsToInit {
tokenSetsLen = maxTokenSetsToInit
}
tokenSets := make([][]string, 0, tokenSetsLen+1)
for _, v := range values {
tokens := tokenizeStrings(nil, []string{v})
tokenSets = append(tokenSets, tokens)
if len(tokens) > maxTokenSetsToInit {
break
}
}
fi.tokenSets = tokenSets
}
func (fi *inFilter) getStringValues() map[string]struct{} {
fi.stringValuesOnce.Do(fi.initStringValues)
return fi.stringValues
}
func (fi *inFilter) initStringValues() {
values := fi.values
m := make(map[string]struct{}, len(values))
for _, v := range values {
m[v] = struct{}{}
}
fi.stringValues = m
}
func (fi *inFilter) getUint8Values() map[string]struct{} {
fi.uint8ValuesOnce.Do(fi.initUint8Values)
return fi.uint8Values
}
func (fi *inFilter) initUint8Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*1)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<8) {
continue
}
bufLen := len(buf)
buf = append(buf, byte(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint8Values = m
}
func (fi *inFilter) getUint16Values() map[string]struct{} {
fi.uint16ValuesOnce.Do(fi.initUint16Values)
return fi.uint16Values
}
func (fi *inFilter) initUint16Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*2)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<16) {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint16(buf, uint16(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint16Values = m
}
func (fi *inFilter) getUint32Values() map[string]struct{} {
fi.uint32ValuesOnce.Do(fi.initUint32Values)
return fi.uint32Values
}
func (fi *inFilter) initUint32Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<32) {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint32(buf, uint32(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint32Values = m
}
func (fi *inFilter) getUint64Values() map[string]struct{} {
fi.uint64ValuesOnce.Do(fi.initUint64Values)
return fi.uint64Values
}
func (fi *inFilter) initUint64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint64Values = m
}
func (fi *inFilter) getFloat64Values() map[string]struct{} {
fi.float64ValuesOnce.Do(fi.initFloat64Values)
return fi.float64Values
}
func (fi *inFilter) initFloat64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
f, ok := tryParseFloat64(v)
if !ok {
continue
}
n := math.Float64bits(f)
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.float64Values = m
}
func (fi *inFilter) getIPv4Values() map[string]struct{} {
fi.ipv4ValuesOnce.Do(fi.initIPv4Values)
return fi.ipv4Values
}
func (fi *inFilter) initIPv4Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
n, ok := tryParseIPv4(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint32(buf, uint32(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.ipv4Values = m
}
func (fi *inFilter) getTimestampISO8601Values() map[string]struct{} {
fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values)
return fi.timestampISO8601Values
}
func (fi *inFilter) initTimestampISO8601Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
n, ok := tryParseTimestampISO8601(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.timestampISO8601Values = m
}
func (fi *inFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := fi.fieldName
if len(fi.values) == 0 {
bm.resetBits()
return
}
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
stringValues := fi.getStringValues()
if _, ok := stringValues[""]; !ok {
bm.resetBits()
}
return
}
tokenSets := fi.getTokenSets()
switch ch.valueType {
case valueTypeString:
stringValues := fi.getStringValues()
matchAnyValue(bs, ch, bm, stringValues, tokenSets)
case valueTypeDict:
stringValues := fi.getStringValues()
matchValuesDictByAnyValue(bs, ch, bm, stringValues)
case valueTypeUint8:
binValues := fi.getUint8Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint16:
binValues := fi.getUint16Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint32:
binValues := fi.getUint32Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint64:
binValues := fi.getUint64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeFloat64:
binValues := fi.getFloat64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeIPv4:
binValues := fi.getIPv4Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeTimestampISO8601:
binValues := fi.getTimestampISO8601Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
// ipv4RangeFilter matches the given ipv4 range [minValue..maxValue]. // ipv4RangeFilter matches the given ipv4 range [minValue..maxValue].
// //
// Example LogsQL: `fieldName:ipv4_range(127.0.0.1, 127.0.0.255)` // Example LogsQL: `fieldName:ipv4_range(127.0.0.1, 127.0.0.255)`
@ -1821,35 +1519,6 @@ func matchUint64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, tokenSets [][]string) {
if !matchBloomFilterAnyTokenSet(bs, ch, tokenSets) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
_, ok := values[v]
return ok
})
}
func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, tokenSets [][]string) bool {
if len(tokenSets) == 0 {
return false
}
if len(tokenSets) > maxTokenSetsToInit || uint64(len(tokenSets)) > 10*bs.bsw.bh.rowsCount {
// It is faster to match every row in the block against all the values
// instead of using bloom filter for too big number of tokenSets.
return true
}
bf := bs.getBloomFilterForColumn(ch)
for _, tokens := range tokenSets {
if bf.containsAll(tokens) {
return true
}
}
return false
}
func matchBloomFilterAllTokens(bs *blockSearch, ch *columnHeader, tokens []string) bool { func matchBloomFilterAllTokens(bs *blockSearch, ch *columnHeader, tokens []string) bool {
if len(tokens) == 0 { if len(tokens) == 0 {
return true return true

343
lib/logstorage/filter_in.go Normal file
View file

@ -0,0 +1,343 @@
package logstorage
import (
"fmt"
"math"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// filterIn matches any exact value from the values map.
//
// Example LogsQL: `fieldName:in("foo", "bar baz")`
type filterIn struct {
fieldName string
values []string
tokenSetsOnce sync.Once
tokenSets [][]string
stringValuesOnce sync.Once
stringValues map[string]struct{}
uint8ValuesOnce sync.Once
uint8Values map[string]struct{}
uint16ValuesOnce sync.Once
uint16Values map[string]struct{}
uint32ValuesOnce sync.Once
uint32Values map[string]struct{}
uint64ValuesOnce sync.Once
uint64Values map[string]struct{}
float64ValuesOnce sync.Once
float64Values map[string]struct{}
ipv4ValuesOnce sync.Once
ipv4Values map[string]struct{}
timestampISO8601ValuesOnce sync.Once
timestampISO8601Values map[string]struct{}
}
func (fi *filterIn) String() string {
values := fi.values
a := make([]string, len(values))
for i, value := range values {
a[i] = quoteTokenIfNeeded(value)
}
return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ","))
}
func (fi *filterIn) getTokenSets() [][]string {
fi.tokenSetsOnce.Do(fi.initTokenSets)
return fi.tokenSets
}
// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter.
const maxTokenSetsToInit = 1000
func (fi *filterIn) initTokenSets() {
values := fi.values
tokenSetsLen := len(values)
if tokenSetsLen > maxTokenSetsToInit {
tokenSetsLen = maxTokenSetsToInit
}
tokenSets := make([][]string, 0, tokenSetsLen+1)
for _, v := range values {
tokens := tokenizeStrings(nil, []string{v})
tokenSets = append(tokenSets, tokens)
if len(tokens) > maxTokenSetsToInit {
break
}
}
fi.tokenSets = tokenSets
}
func (fi *filterIn) getStringValues() map[string]struct{} {
fi.stringValuesOnce.Do(fi.initStringValues)
return fi.stringValues
}
func (fi *filterIn) initStringValues() {
values := fi.values
m := make(map[string]struct{}, len(values))
for _, v := range values {
m[v] = struct{}{}
}
fi.stringValues = m
}
func (fi *filterIn) getUint8Values() map[string]struct{} {
fi.uint8ValuesOnce.Do(fi.initUint8Values)
return fi.uint8Values
}
func (fi *filterIn) initUint8Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*1)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<8) {
continue
}
bufLen := len(buf)
buf = append(buf, byte(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint8Values = m
}
func (fi *filterIn) getUint16Values() map[string]struct{} {
fi.uint16ValuesOnce.Do(fi.initUint16Values)
return fi.uint16Values
}
func (fi *filterIn) initUint16Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*2)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<16) {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint16(buf, uint16(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint16Values = m
}
func (fi *filterIn) getUint32Values() map[string]struct{} {
fi.uint32ValuesOnce.Do(fi.initUint32Values)
return fi.uint32Values
}
func (fi *filterIn) initUint32Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok || n >= (1<<32) {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint32(buf, uint32(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint32Values = m
}
func (fi *filterIn) getUint64Values() map[string]struct{} {
fi.uint64ValuesOnce.Do(fi.initUint64Values)
return fi.uint64Values
}
func (fi *filterIn) initUint64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
n, ok := tryParseUint64(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.uint64Values = m
}
func (fi *filterIn) getFloat64Values() map[string]struct{} {
fi.float64ValuesOnce.Do(fi.initFloat64Values)
return fi.float64Values
}
func (fi *filterIn) initFloat64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
f, ok := tryParseFloat64(v)
if !ok {
continue
}
n := math.Float64bits(f)
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.float64Values = m
}
func (fi *filterIn) getIPv4Values() map[string]struct{} {
fi.ipv4ValuesOnce.Do(fi.initIPv4Values)
return fi.ipv4Values
}
func (fi *filterIn) initIPv4Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
n, ok := tryParseIPv4(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint32(buf, uint32(n))
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.ipv4Values = m
}
func (fi *filterIn) getTimestampISO8601Values() map[string]struct{} {
fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values)
return fi.timestampISO8601Values
}
func (fi *filterIn) initTimestampISO8601Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
n, ok := tryParseTimestampISO8601(v)
if !ok {
continue
}
bufLen := len(buf)
buf = encoding.MarshalUint64(buf, n)
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
fi.timestampISO8601Values = m
}
func (fi *filterIn) apply(bs *blockSearch, bm *bitmap) {
fieldName := fi.fieldName
if len(fi.values) == 0 {
bm.resetBits()
return
}
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
stringValues := fi.getStringValues()
if _, ok := stringValues[""]; !ok {
bm.resetBits()
}
return
}
tokenSets := fi.getTokenSets()
switch ch.valueType {
case valueTypeString:
stringValues := fi.getStringValues()
matchAnyValue(bs, ch, bm, stringValues, tokenSets)
case valueTypeDict:
stringValues := fi.getStringValues()
matchValuesDictByAnyValue(bs, ch, bm, stringValues)
case valueTypeUint8:
binValues := fi.getUint8Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint16:
binValues := fi.getUint16Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint32:
binValues := fi.getUint32Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint64:
binValues := fi.getUint64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeFloat64:
binValues := fi.getFloat64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeIPv4:
binValues := fi.getIPv4Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeTimestampISO8601:
binValues := fi.getTimestampISO8601Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, tokenSets [][]string) {
if !matchBloomFilterAnyTokenSet(bs, ch, tokenSets) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
_, ok := values[v]
return ok
})
}
func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, tokenSets [][]string) bool {
if len(tokenSets) == 0 {
return false
}
if len(tokenSets) > maxTokenSetsToInit || uint64(len(tokenSets)) > 10*bs.bsw.bh.rowsCount {
// It is faster to match every row in the block against all the values
// instead of using bloom filter for too big number of tokenSets.
return true
}
bf := bs.getBloomFilterForColumn(ch)
for _, tokens := range tokenSets {
if bf.containsAll(tokens) {
return true
}
}
return false
}

690
lib/logstorage/filter_in_test.go Normal file
View file

@ -0,0 +1,690 @@
package logstorage
import (
"testing"
)
func TestFilterIn(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
},
},
{
name: "other column",
values: []string{
"asdfdsf",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"abc def", "abc", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &filterIn{
fieldName: "other column",
values: []string{"asdfdsf", ""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{"", "foo"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "other column",
values: []string{"sd"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "non-existing column",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
"abc def",
"abc def",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"aaaa", "abc def", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"abc def ", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "non-existing column",
values: []string{"x"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"foobar",
"abc",
"afdf foobar baz",
"fddf foobarbaz",
"afoobarbaz",
"foobar",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"foobar", "aaaa", "abc", "baz"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
fi = &filterIn{
fieldName: "foo",
values: []string{"bbbb", "", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "non-existing column",
values: []string{"foobar", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"a foobar",
"aa abc a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a afoobarbaz",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"a foobar", "aa abc a"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"aa a"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &filterIn{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"1234"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"256",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &filterIn{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"123456"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"65536",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &filterIn{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"12345678901"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"12345678901",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &filterIn{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"2",
"3",
"4",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"1234", "1", "foobar", "123211"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5})
fi = &filterIn{
fieldName: "foo",
values: []string{"1234.5678901"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{4})
fi = &filterIn{
fieldName: "foo",
values: []string{"-65536"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"65536"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"123"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"12345678901234567890"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.55.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
fi := &filterIn{
fieldName: "foo",
values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
fi = &filterIn{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"5"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &filterIn{
fieldName: "foo",
values: []string{"255.255.255.255"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
fi := &filterIn{
fieldName: "_msg",
values: []string{"2006-01-02T15:04:05.005Z", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", []int{4})
fi = &filterIn{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
fi = &filterIn{
fieldName: "_msg",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &filterIn{
fieldName: "_msg",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &filterIn{
fieldName: "_msg",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &filterIn{
fieldName: "_msg",
values: []string{"2006-04-02T15:04:05.005Z"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
})
}

685
lib/logstorage/filter_test.go
View file

@ -491,691 +491,6 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil) testFilterMatchForColumns(t, columns, f, "foo", nil)
} }
func TestInFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
},
},
{
name: "other column",
values: []string{
"asdfdsf",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"abc def", "abc", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &inFilter{
fieldName: "other column",
values: []string{"asdfdsf", ""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{"", "foo"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "other column",
values: []string{"sd"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
"abc def",
"abc def",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"aaaa", "abc def", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"abc def ", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"x"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"foobar",
"abc",
"afdf foobar baz",
"fddf foobarbaz",
"afoobarbaz",
"foobar",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"foobar", "aaaa", "abc", "baz"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
fi = &inFilter{
fieldName: "foo",
values: []string{"bbbb", "", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"foobar", "aaaa"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"a foobar",
"aa abc a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a afoobarbaz",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"a foobar", "aa abc a"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"aa a"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"1234"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"256",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"123456"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"65536",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"12345678901"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"12345678901",
"2",
"3",
"4",
"5",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"2",
"3",
"4",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"1234", "1", "foobar", "123211"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5})
fi = &inFilter{
fieldName: "foo",
values: []string{"1234.5678901"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{4})
fi = &inFilter{
fieldName: "foo",
values: []string{"-65536"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{3})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"65536"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"123"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"12345678901234567890"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.55.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
fi := &inFilter{
fieldName: "foo",
values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"5"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
fi = &inFilter{
fieldName: "foo",
values: []string{"255.255.255.255"},
}
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
fi := &inFilter{
fieldName: "_msg",
values: []string{"2006-01-02T15:04:05.005Z", "foobar"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", []int{4})
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
fi = &inFilter{
fieldName: "_msg",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &inFilter{
fieldName: "_msg",
values: []string{},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &inFilter{
fieldName: "_msg",
values: []string{""},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
fi = &inFilter{
fieldName: "_msg",
values: []string{"2006-04-02T15:04:05.005Z"},
}
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
})
}
func TestRegexpFilter(t *testing.T) { func TestRegexpFilter(t *testing.T) {
t.Run("const-column", func(t *testing.T) { t.Run("const-column", func(t *testing.T) {
columns := []column{ columns := []column{

6
lib/logstorage/parser.go
View file

@ -326,7 +326,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
case lex.isKeyword("i"): case lex.isKeyword("i"):
return parseAnyCaseFilter(lex, fieldName) return parseAnyCaseFilter(lex, fieldName)
case lex.isKeyword("in"): case lex.isKeyword("in"):
return parseInFilter(lex, fieldName) return parseFilterIn(lex, fieldName)
case lex.isKeyword("ipv4_range"): case lex.isKeyword("ipv4_range"):
return parseIPv4RangeFilter(lex, fieldName) return parseIPv4RangeFilter(lex, fieldName)
case lex.isKeyword("len_range"): case lex.isKeyword("len_range"):
@ -612,9 +612,9 @@ func tryParseIPv4CIDR(s string) (uint32, uint32, bool) {
return minValue, maxValue, true return minValue, maxValue, true
} }
func parseInFilter(lex *lexer, fieldName string) (filter, error) { func parseFilterIn(lex *lexer, fieldName string) (filter, error) {
return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) { return parseFuncArgs(lex, fieldName, func(args []string) (filter, error) {
f := &inFilter{ f := &filterIn{
fieldName: fieldName, fieldName: fieldName,
values: args, values: args,
} }

6
lib/logstorage/parser_test.go
View file

@ -299,16 +299,16 @@ func TestParseFilterSequence(t *testing.T) {
f(`seq(foo,bar-baz.aa"bb","c,)d")`, ``, []string{"foo", `bar-baz.aa"bb"`, "c,)d"}) f(`seq(foo,bar-baz.aa"bb","c,)d")`, ``, []string{"foo", `bar-baz.aa"bb"`, "c,)d"})
} }
func TestParseInFilter(t *testing.T) { func TestParseFilterIn(t *testing.T) {
f := func(s, fieldNameExpected string, valuesExpected []string) { f := func(s, fieldNameExpected string, valuesExpected []string) {
t.Helper() t.Helper()
q, err := ParseQuery(s) q, err := ParseQuery(s)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
f, ok := q.f.(*inFilter) f, ok := q.f.(*filterIn)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *inFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *filterIn; filter: %s", q.f, q.f)
} }
if f.fieldName != fieldNameExpected { if f.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected)