github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 05:42:32 +02:00
parent 37329d0604
commit 1528b379cf
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
6 changed files with 922 additions and 452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

242
lib/logstorage/filter.go
View file

@ -72,78 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
} }
} }
// exactPrefixFilter matches the exact prefix.
//
// Example LogsQL: `fieldName:exact("foo bar"*)
type exactPrefixFilter struct {
fieldName string
prefix string
tokensOnce sync.Once
tokens []string
}
func (ef *exactPrefixFilter) String() string {
return fmt.Sprintf("%sexact(%s*)", quoteFieldNameIfNeeded(ef.fieldName), quoteTokenIfNeeded(ef.prefix))
}
func (ef *exactPrefixFilter) getTokens() []string {
ef.tokensOnce.Do(ef.initTokens)
return ef.tokens
}
func (ef *exactPrefixFilter) initTokens() {
ef.tokens = getTokensSkipLast(ef.prefix)
}
func (ef *exactPrefixFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := ef.fieldName
prefix := ef.prefix
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchExactPrefix(v, prefix) {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
if !matchExactPrefix("", prefix) {
bm.resetBits()
}
return
}
tokens := ef.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeDict:
matchValuesDictByExactPrefix(bs, ch, bm, prefix)
case valueTypeUint8:
matchUint8ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint16:
matchUint16ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint32:
matchUint32ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint64:
matchUint64ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeFloat64:
matchFloat64ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeIPv4:
matchIPv4ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeTimestampISO8601:
matchTimestampISO8601ByExactPrefix(bs, ch, bm, prefix, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
// exactFilter matches the exact value. // exactFilter matches the exact value.
// //
// Example LogsQL: `fieldName:exact("foo bar")` // Example LogsQL: `fieldName:exact("foo bar")`
@ -155,22 +83,22 @@ type exactFilter struct {
tokens []string tokens []string
} }
func (ef *exactFilter) String() string { func (fe *exactFilter) String() string {
return fmt.Sprintf("%sexact(%s)", quoteFieldNameIfNeeded(ef.fieldName), quoteTokenIfNeeded(ef.value)) return fmt.Sprintf("%sexact(%s)", quoteFieldNameIfNeeded(fe.fieldName), quoteTokenIfNeeded(fe.value))
} }
func (ef *exactFilter) getTokens() []string { func (fe *exactFilter) getTokens() []string {
ef.tokensOnce.Do(ef.initTokens) fe.tokensOnce.Do(fe.initTokens)
return ef.tokens return fe.tokens
} }
func (ef *exactFilter) initTokens() { func (fe *exactFilter) initTokens() {
ef.tokens = tokenizeStrings(nil, []string{ef.value}) fe.tokens = tokenizeStrings(nil, []string{fe.value})
} }
func (ef *exactFilter) apply(bs *blockSearch, bm *bitmap) { func (fe *exactFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := ef.fieldName fieldName := fe.fieldName
value := ef.value value := fe.value
v := bs.csh.getConstColumnValue(fieldName) v := bs.csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
@ -191,7 +119,7 @@ func (ef *exactFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
tokens := ef.getTokens() tokens := fe.getTokens()
switch ch.valueType { switch ch.valueType {
case valueTypeString: case valueTypeString:
@ -1226,23 +1154,6 @@ func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
return
}
if prefix < "0" || prefix > "9" || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toTimestampISO8601StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchTimestampISO8601ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) { func matchTimestampISO8601ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
n, ok := tryParseTimestampISO8601(value) n, ok := tryParseTimestampISO8601(value)
if !ok || n < ch.minValue || n > ch.maxValue { if !ok || n < ch.minValue || n > ch.maxValue {
@ -1351,23 +1262,6 @@ func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix str
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
return
}
if prefix < "0" || prefix > "9" || len(tokens) > 3 || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toIPv4StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchIPv4ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) { func matchIPv4ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
n, ok := tryParseIPv4(value) n, ok := tryParseIPv4(value)
if !ok || uint64(n) < ch.minValue || uint64(n) > ch.maxValue { if !ok || uint64(n) < ch.minValue || uint64(n) > ch.maxValue {
@ -1486,24 +1380,6 @@ func matchFloat64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
// An empty prefix matches all the values
return
}
if len(tokens) > 2 || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toFloat64StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchFloat64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) { func matchFloat64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
f, ok := tryParseFloat64(value) f, ok := tryParseFloat64(value)
if !ok || f < math.Float64frombits(ch.minValue) || f > math.Float64frombits(ch.maxValue) { if !ok || f < math.Float64frombits(ch.minValue) || f > math.Float64frombits(ch.maxValue) {
@ -1634,17 +1510,6 @@ func matchValuesDictByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pref
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchValuesDictByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchExactPrefix(v, prefix) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchValuesDictByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string) { func matchValuesDictByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string) {
bb := bbPool.Get() bb := bbPool.Get()
for i, v := range ch.valuesDict.values { for i, v := range ch.valuesDict.values {
@ -1746,16 +1611,6 @@ func matchStringByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s
}) })
} }
func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
return matchExactPrefix(v, prefix)
})
}
func matchStringByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) { func matchStringByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
if !matchBloomFilterAllTokens(bs, ch, tokens) { if !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits() bm.resetBits()
@ -2097,77 +1952,6 @@ func matchUint64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix s
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint8String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint16ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint16String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint32ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint32String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint64String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchMinMaxExactPrefix(ch *columnHeader, bm *bitmap, prefix string, tokens []string) bool {
if prefix == "" {
// An empty prefix matches all the values
return false
}
if len(tokens) > 0 {
// Non-empty tokens means that the prefix contains at least two tokens.
// Multiple tokens cannot match any uint value.
bm.resetBits()
return false
}
n, ok := tryParseUint64(prefix)
if !ok || n > ch.maxValue {
bm.resetBits()
return false
}
return true
}
func matchUint8ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) { func matchUint8ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
n, ok := tryParseUint64(phrase) n, ok := tryParseUint64(phrase)
if !ok || n < ch.minValue || n > ch.maxValue { if !ok || n < ch.minValue || n > ch.maxValue {
@ -2393,10 +2177,6 @@ func matchAnyCasePhrase(s, phraseLowercase string) bool {
return ok return ok
} }
func matchExactPrefix(s, prefix string) bool {
return strings.HasPrefix(s, prefix)
}
func matchPhrase(s, phrase string) bool { func matchPhrase(s, phrase string) bool {
if len(phrase) == 0 { if len(phrase) == 0 {
// Special case - empty phrase matches only empty string. // Special case - empty phrase matches only empty string.

2
lib/logstorage/filter_and.go
View file

@ -77,7 +77,7 @@ func (fa *filterAnd) initMsgTokens() {
if isMsgFieldName(t.fieldName) { if isMsgFieldName(t.fieldName) {
a = append(a, t.getTokens()...) a = append(a, t.getTokens()...)
} }
case *exactPrefixFilter: case *filterExactPrefix:
if isMsgFieldName(t.fieldName) { if isMsgFieldName(t.fieldName) {
a = append(a, t.getTokens()...) a = append(a, t.getTokens()...)
} }

229
lib/logstorage/filter_exact_prefix.go Normal file
View file

@ -0,0 +1,229 @@
package logstorage
import (
"fmt"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// filterExactPrefix matches the exact prefix.
//
// Example LogsQL: `fieldName:exact("foo bar"*)
type filterExactPrefix struct {
fieldName string
prefix string
tokensOnce sync.Once
tokens []string
}
func (fep *filterExactPrefix) String() string {
return fmt.Sprintf("%sexact(%s*)", quoteFieldNameIfNeeded(fep.fieldName), quoteTokenIfNeeded(fep.prefix))
}
func (fep *filterExactPrefix) getTokens() []string {
fep.tokensOnce.Do(fep.initTokens)
return fep.tokens
}
func (fep *filterExactPrefix) initTokens() {
fep.tokens = getTokensSkipLast(fep.prefix)
}
func (fep *filterExactPrefix) apply(bs *blockSearch, bm *bitmap) {
fieldName := fep.fieldName
prefix := fep.prefix
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchExactPrefix(v, prefix) {
bm.resetBits()
}
return
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
if !matchExactPrefix("", prefix) {
bm.resetBits()
}
return
}
tokens := fep.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeDict:
matchValuesDictByExactPrefix(bs, ch, bm, prefix)
case valueTypeUint8:
matchUint8ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint16:
matchUint16ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint32:
matchUint32ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeUint64:
matchUint64ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeFloat64:
matchFloat64ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeIPv4:
matchIPv4ByExactPrefix(bs, ch, bm, prefix, tokens)
case valueTypeTimestampISO8601:
matchTimestampISO8601ByExactPrefix(bs, ch, bm, prefix, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
return
}
if prefix < "0" || prefix > "9" || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toTimestampISO8601StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
return
}
if prefix < "0" || prefix > "9" || len(tokens) > 3 || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toIPv4StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if prefix == "" {
// An empty prefix matches all the values
return
}
if len(tokens) > 2 || !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toFloat64StringExt(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchValuesDictByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchExactPrefix(v, prefix) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
return
}
visitValues(bs, ch, bm, func(v string) bool {
return matchExactPrefix(v, prefix)
})
}
func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint8String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint16ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint16String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint32ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint32String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchUint64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
return
}
bb := bbPool.Get()
visitValues(bs, ch, bm, func(v string) bool {
s := toUint64String(bs, bb, v)
return matchExactPrefix(s, prefix)
})
bbPool.Put(bb)
}
func matchMinMaxExactPrefix(ch *columnHeader, bm *bitmap, prefix string, tokens []string) bool {
if prefix == "" {
// An empty prefix matches all the values
return false
}
if len(tokens) > 0 {
// Non-empty tokens means that the prefix contains at least two tokens.
// Multiple tokens cannot match any uint value.
bm.resetBits()
return false
}
n, ok := tryParseUint64(prefix)
if !ok || n > ch.maxValue {
bm.resetBits()
return false
}
return true
}
func matchExactPrefix(s, prefix string) bool {
return strings.HasPrefix(s, prefix)
}

564
lib/logstorage/filter_exact_prefix_test.go Normal file
View file

@ -0,0 +1,564 @@
package logstorage
import (
"testing"
)
func TestExactPrefixFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "abc d",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0})
fep = &filterExactPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "xabc",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "non-existing column",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
"abc def",
"abc def",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2})
fep = &filterExactPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "non-existing column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"foobar",
"abc",
"afdf foobar baz",
"fddf foobarbaz",
"foobarbaz",
"foobar",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{1, 5, 6})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "baz",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "non-existing column",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"a foobar",
"aa abc a",
"ca afdf a,foobar baz",
"aa fddf foobarbaz",
"a afoobarbaz",
"a foobar baz",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "aa ",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{2, 4})
fep = &filterExactPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "aa b",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "fobar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "non-existing-column",
prefix: "aa",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 5})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "999",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "7",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"467",
"5",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 5})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "999",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "7",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"65536",
"5",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 5})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "99999",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "7",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"123456789012",
"5",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 5})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "1234567890123",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "7",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"2",
"3",
"4",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 4})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "1234.567",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{4})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "-65536",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{3})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "6511",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.2",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.55.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "foo",
prefix: "127.0.",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{2, 4, 5, 7})
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
fep = &filterExactPrefix{
fieldName: "foo",
prefix: "255",
}
testFilterMatchForColumns(t, columns, fep, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:06.004Z",
"2006-01-02T15:04:06.005Z",
"2006-01-02T15:04:07.006Z",
"2006-01-02T15:04:10.007Z",
"2006-01-02T15:04:12.008Z",
"2006-01-02T15:04:15.009Z",
},
},
}
// match
fep := &filterExactPrefix{
fieldName: "_msg",
prefix: "2006-01-02T15:04:05",
}
testFilterMatchForColumns(t, columns, fep, "_msg", []int{0, 1, 2})
fep = &filterExactPrefix{
fieldName: "_msg",
prefix: "",
}
testFilterMatchForColumns(t, columns, fep, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
fep = &filterExactPrefix{
fieldName: "_msg",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fep, "_msg", nil)
fep = &filterExactPrefix{
fieldName: "_msg",
prefix: "0",
}
testFilterMatchForColumns(t, columns, fep, "_msg", nil)
})
}

335
lib/logstorage/filter_test.go
View file

@ -491,109 +491,6 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil) testFilterMatchForColumns(t, columns, f, "foo", nil)
} }
func TestExactPrefixFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
},
},
}
// match
ef := &exactPrefixFilter{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0})
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "abc d",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0})
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0})
ef = &exactPrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0})
// mismatch
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "xabc",
}
testFilterMatchForColumns(t, columns, ef, "foo", nil)
ef = &exactPrefixFilter{
fieldName: "non-existing column",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, ef, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"abc def",
"abc def",
"abc def",
},
},
}
// match
ef := &exactPrefixFilter{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2})
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2})
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2})
ef = &exactPrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2})
// mismatch
ef = &exactPrefixFilter{
fieldName: "foo",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, ef, "foo", nil)
ef = &exactPrefixFilter{
fieldName: "non-existing column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, ef, "foo", nil)
})
}
func TestExactFilter(t *testing.T) { func TestExactFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) { t.Run("single-row", func(t *testing.T) {
columns := []column{ columns := []column{
@ -606,30 +503,30 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "abc def", value: "abc def",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "abc", value: "abc",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
value: "abc", value: "abc",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("const-column", func(t *testing.T) { t.Run("const-column", func(t *testing.T) {
@ -645,36 +542,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "abc def", value: "abc def",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "foobar", value: "foobar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
value: "x", value: "x",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("dict", func(t *testing.T) { t.Run("dict", func(t *testing.T) {
@ -694,30 +591,30 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "foobar", value: "foobar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 6}) testFilterMatchForColumns(t, columns, fe, "foo", []int{1, 6})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "baz", value: "baz",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
value: "foobar", value: "foobar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("strings", func(t *testing.T) { t.Run("strings", func(t *testing.T) {
@ -740,36 +637,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "aa abc a", value: "aa abc a",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{2}) testFilterMatchForColumns(t, columns, fe, "foo", []int{2})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "aa a", value: "aa a",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "fooaaazz a", value: "fooaaazz a",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("uint8", func(t *testing.T) { t.Run("uint8", func(t *testing.T) {
@ -793,36 +690,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "12", value: "12",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 5}) testFilterMatchForColumns(t, columns, fe, "foo", []int{1, 5})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "33", value: "33",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("uint16", func(t *testing.T) { t.Run("uint16", func(t *testing.T) {
@ -846,36 +743,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "12", value: "12",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 5}) testFilterMatchForColumns(t, columns, fe, "foo", []int{1, 5})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "33", value: "33",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("uint32", func(t *testing.T) { t.Run("uint32", func(t *testing.T) {
@ -899,36 +796,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "12", value: "12",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 5}) testFilterMatchForColumns(t, columns, fe, "foo", []int{1, 5})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "33", value: "33",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("uint64", func(t *testing.T) { t.Run("uint64", func(t *testing.T) {
@ -952,36 +849,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "12", value: "12",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{1, 5}) testFilterMatchForColumns(t, columns, fe, "foo", []int{1, 5})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "33", value: "33",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("float64", func(t *testing.T) { t.Run("float64", func(t *testing.T) {
@ -1003,60 +900,60 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "1234", value: "1234",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "1234.5678901", value: "1234.5678901",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{4}) testFilterMatchForColumns(t, columns, fe, "foo", []int{4})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "-65536", value: "-65536",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{3}) testFilterMatchForColumns(t, columns, fe, "foo", []int{3})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "65536", value: "65536",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "123", value: "123",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "12345678901234567890", value: "12345678901234567890",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("ipv4", func(t *testing.T) { t.Run("ipv4", func(t *testing.T) {
@ -1081,42 +978,42 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "127.0.0.1", value: "127.0.0.1",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{2, 4, 5, 7}) testFilterMatchForColumns(t, columns, fe, "foo", []int{2, 4, 5, 7})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) testFilterMatchForColumns(t, columns, fe, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch // mismatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "127.0", value: "127.0",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "foo", fieldName: "foo",
value: "255.255.255.255", value: "255.255.255.255",
} }
testFilterMatchForColumns(t, columns, ef, "foo", nil) testFilterMatchForColumns(t, columns, fe, "foo", nil)
}) })
t.Run("timestamp-iso8601", func(t *testing.T) { t.Run("timestamp-iso8601", func(t *testing.T) {
@ -1138,36 +1035,36 @@ func TestExactFilter(t *testing.T) {
} }
// match // match
ef := &exactFilter{ fe := &exactFilter{
fieldName: "_msg", fieldName: "_msg",
value: "2006-01-02T15:04:05.005Z", value: "2006-01-02T15:04:05.005Z",
} }
testFilterMatchForColumns(t, columns, ef, "_msg", []int{4}) testFilterMatchForColumns(t, columns, fe, "_msg", []int{4})
ef = &exactFilter{ fe = &exactFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, fe, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch // mimatch
ef = &exactFilter{ fe = &exactFilter{
fieldName: "_msg", fieldName: "_msg",
value: "bar", value: "bar",
} }
testFilterMatchForColumns(t, columns, ef, "_msg", nil) testFilterMatchForColumns(t, columns, fe, "_msg", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "_msg", fieldName: "_msg",
value: "", value: "",
} }
testFilterMatchForColumns(t, columns, ef, "_msg", nil) testFilterMatchForColumns(t, columns, fe, "_msg", nil)
ef = &exactFilter{ fe = &exactFilter{
fieldName: "_msg", fieldName: "_msg",
value: "2006-03-02T15:04:05.005Z", value: "2006-03-02T15:04:05.005Z",
} }
testFilterMatchForColumns(t, columns, ef, "_msg", nil) testFilterMatchForColumns(t, columns, fe, "_msg", nil)
}) })
} }

2
lib/logstorage/parser.go
View file

@ -635,7 +635,7 @@ func parseFilterSequence(lex *lexer, fieldName string) (filter, error) {
func parseExactFilter(lex *lexer, fieldName string) (filter, error) { func parseExactFilter(lex *lexer, fieldName string) (filter, error) {
return parseFuncArgMaybePrefix(lex, "exact", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) { return parseFuncArgMaybePrefix(lex, "exact", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter { if isPrefixFilter {
f := &exactPrefixFilter{ f := &filterExactPrefix{
fieldName: fieldName, fieldName: fieldName,
prefix: phrase, prefix: phrase,
} }