github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 07:06:26 +02:00
parent b370a785e9
commit 9b5dd883b4
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
6 changed files with 1140 additions and 1124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

224
lib/logstorage/filter.go
View file

@ -71,95 +71,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
} }
} }
// anyCasePrefixFilter matches the given prefix in lower, upper and mixed case.
//
// Example LogsQL: `fieldName:i(prefix*)` or `fieldName:i("some prefix"*)`
//
// A special case `fieldName:i(*)` equals to `fieldName:*` and matches non-emtpy value for the given `fieldName` field.
type anyCasePrefixFilter struct {
fieldName string
prefix string
prefixLowercaseOnce sync.Once
prefixLowercase string
tokensOnce sync.Once
tokens []string
}
func (pf *anyCasePrefixFilter) String() string {
if pf.prefix == "" {
return quoteFieldNameIfNeeded(pf.fieldName) + "i(*)"
}
return fmt.Sprintf("%si(%s*)", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.prefix))
}
func (pf *anyCasePrefixFilter) getTokens() []string {
pf.tokensOnce.Do(pf.initTokens)
return pf.tokens
}
func (pf *anyCasePrefixFilter) initTokens() {
pf.tokens = getTokensSkipLast(pf.prefix)
}
func (pf *anyCasePrefixFilter) getPrefixLowercase() string {
pf.prefixLowercaseOnce.Do(pf.initPrefixLowercase)
return pf.prefixLowercase
}
func (pf *anyCasePrefixFilter) initPrefixLowercase() {
pf.prefixLowercase = strings.ToLower(pf.prefix)
}
func (pf *anyCasePrefixFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := pf.fieldName
prefixLowercase := pf.getPrefixLowercase()
// Verify whether pf matches const column
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePrefix(v, prefixLowercase) {
bm.resetBits()
}
return
}
// Verify whether pf matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()
return
}
tokens := pf.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByAnyCasePrefix(bs, ch, bm, prefixLowercase)
case valueTypeDict:
matchValuesDictByAnyCasePrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint8:
matchUint8ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint16:
matchUint16ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint32:
matchUint32ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint64:
matchUint64ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeFloat64:
matchFloat64ByPrefix(bs, ch, bm, prefixLowercase, tokens)
case valueTypeIPv4:
matchIPv4ByPrefix(bs, ch, bm, prefixLowercase, tokens)
case valueTypeTimestampISO8601:
prefixUppercase := strings.ToUpper(pf.prefix)
matchTimestampISO8601ByPrefix(bs, ch, bm, prefixUppercase, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
// prefixFilter matches the given prefix. // prefixFilter matches the given prefix.
// //
// Example LogsQL: `fieldName:prefix*` or `fieldName:"some prefix"*` // Example LogsQL: `fieldName:prefix*` or `fieldName:"some prefix"*`
@ -173,27 +84,27 @@ type prefixFilter struct {
tokens []string tokens []string
} }
func (pf *prefixFilter) String() string { func (fp *prefixFilter) String() string {
if pf.prefix == "" { if fp.prefix == "" {
return quoteFieldNameIfNeeded(pf.fieldName) + "*" return quoteFieldNameIfNeeded(fp.fieldName) + "*"
} }
return fmt.Sprintf("%s%s*", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.prefix)) return fmt.Sprintf("%s%s*", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.prefix))
} }
func (pf *prefixFilter) getTokens() []string { func (fp *prefixFilter) getTokens() []string {
pf.tokensOnce.Do(pf.initTokens) fp.tokensOnce.Do(fp.initTokens)
return pf.tokens return fp.tokens
} }
func (pf *prefixFilter) initTokens() { func (fp *prefixFilter) initTokens() {
pf.tokens = getTokensSkipLast(pf.prefix) fp.tokens = getTokensSkipLast(fp.prefix)
} }
func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) { func (fp *prefixFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := pf.fieldName fieldName := fp.fieldName
prefix := pf.prefix prefix := fp.prefix
// Verify whether pf matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) v := bs.csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchPrefix(v, prefix) { if !matchPrefix(v, prefix) {
@ -202,7 +113,7 @@ func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
// Verify whether pf matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := bs.csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
@ -210,7 +121,7 @@ func (pf *prefixFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
tokens := pf.getTokens() tokens := fp.getTokens()
switch ch.valueType { switch ch.valueType {
case valueTypeString: case valueTypeString:
@ -250,33 +161,33 @@ type anyCasePhraseFilter struct {
tokens []string tokens []string
} }
func (pf *anyCasePhraseFilter) String() string { func (fp *anyCasePhraseFilter) String() string {
return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(pf.fieldName), quoteTokenIfNeeded(pf.phrase)) return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase))
} }
func (pf *anyCasePhraseFilter) getTokens() []string { func (fp *anyCasePhraseFilter) getTokens() []string {
pf.tokensOnce.Do(pf.initTokens) fp.tokensOnce.Do(fp.initTokens)
return pf.tokens return fp.tokens
} }
func (pf *anyCasePhraseFilter) initTokens() { func (fp *anyCasePhraseFilter) initTokens() {
pf.tokens = tokenizeStrings(nil, []string{pf.phrase}) fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
} }
func (pf *anyCasePhraseFilter) getPhraseLowercase() string { func (fp *anyCasePhraseFilter) getPhraseLowercase() string {
pf.phraseLowercaseOnce.Do(pf.initPhraseLowercase) fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase)
return pf.phraseLowercase return fp.phraseLowercase
} }
func (pf *anyCasePhraseFilter) initPhraseLowercase() { func (fp *anyCasePhraseFilter) initPhraseLowercase() {
pf.phraseLowercase = strings.ToLower(pf.phrase) fp.phraseLowercase = strings.ToLower(fp.phrase)
} }
func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) { func (fp *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := pf.fieldName fieldName := fp.fieldName
phraseLowercase := pf.getPhraseLowercase() phraseLowercase := fp.getPhraseLowercase()
// Verify whether pf matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) v := bs.csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) { if !matchAnyCasePhrase(v, phraseLowercase) {
@ -285,7 +196,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
// Verify whether pf matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := bs.csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
@ -296,7 +207,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
tokens := pf.getTokens() tokens := fp.getTokens()
switch ch.valueType { switch ch.valueType {
case valueTypeString: case valueTypeString:
@ -316,7 +227,7 @@ func (pf *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) {
case valueTypeIPv4: case valueTypeIPv4:
matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens) matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeTimestampISO8601: case valueTypeTimestampISO8601:
phraseUppercase := strings.ToUpper(pf.phrase) phraseUppercase := strings.ToUpper(fp.phrase)
matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens) matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens)
default: default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
@ -341,24 +252,24 @@ type phraseFilter struct {
tokens []string tokens []string
} }
func (pf *phraseFilter) String() string { func (fp *phraseFilter) String() string {
return quoteFieldNameIfNeeded(pf.fieldName) + quoteTokenIfNeeded(pf.phrase) return quoteFieldNameIfNeeded(fp.fieldName) + quoteTokenIfNeeded(fp.phrase)
} }
func (pf *phraseFilter) getTokens() []string { func (fp *phraseFilter) getTokens() []string {
pf.tokensOnce.Do(pf.initTokens) fp.tokensOnce.Do(fp.initTokens)
return pf.tokens return fp.tokens
} }
func (pf *phraseFilter) initTokens() { func (fp *phraseFilter) initTokens() {
pf.tokens = tokenizeStrings(nil, []string{pf.phrase}) fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
} }
func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) { func (fp *phraseFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := pf.fieldName fieldName := fp.fieldName
phrase := pf.phrase phrase := fp.phrase
// Verify whether pf matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) v := bs.csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchPhrase(v, phrase) { if !matchPhrase(v, phrase) {
@ -367,7 +278,7 @@ func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
// Verify whether pf matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := bs.csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
@ -378,7 +289,7 @@ func (pf *phraseFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
tokens := pf.getTokens() tokens := fp.getTokens()
switch ch.valueType { switch ch.valueType {
case valueTypeString: case valueTypeString:
@ -549,17 +460,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
bbPool.Put(bb) bbPool.Put(bb)
} }
func matchValuesDictByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchAnyCasePrefix(v, prefixLowercase) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
bb := bbPool.Get() bb := bbPool.Get()
for i, v := range ch.valuesDict.values { for i, v := range ch.valuesDict.values {
@ -620,12 +520,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod
}) })
} }
func matchStringByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) {
visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePrefix(v, prefixLowercase)
})
}
func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) { func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
visitValues(bs, ch, bm, func(v string) bool { visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePhrase(v, phraseLowercase) return matchAnyCasePhrase(v, phraseLowercase)
@ -777,30 +671,6 @@ func visitValues(bs *blockSearch, ch *columnHeader, bm *bitmap, f func(value str
}) })
} }
func matchAnyCasePrefix(s, prefixLowercase string) bool {
if len(prefixLowercase) == 0 {
// Special case - empty prefix matches any non-empty string.
return len(s) > 0
}
if len(prefixLowercase) > len(s) {
return false
}
if isASCIILowercase(s) {
// Fast path - s is in lowercase
return matchPrefix(s, prefixLowercase)
}
// Slow path - convert s to lowercase before matching
bb := bbPool.Get()
bb.B = stringsutil.AppendLowercase(bb.B, s)
sLowercase := bytesutil.ToUnsafeString(bb.B)
ok := matchPrefix(sLowercase, prefixLowercase)
bbPool.Put(bb)
return ok
}
func isASCIILowercase(s string) bool { func isASCIILowercase(s string) bool {
for i := 0; i < len(s); i++ { for i := 0; i < len(s); i++ {
c := s[i] c := s[i]

141
lib/logstorage/filter_any_case_prefix.go Normal file
View file

@ -0,0 +1,141 @@
package logstorage
import (
"fmt"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
// filterAnyCasePrefix matches the given prefix in lower, upper and mixed case.
//
// Example LogsQL: `fieldName:i(prefix*)` or `fieldName:i("some prefix"*)`
//
// A special case `fieldName:i(*)` equals to `fieldName:*` and matches non-emtpy value for the given `fieldName` field.
type filterAnyCasePrefix struct {
fieldName string
prefix string
prefixLowercaseOnce sync.Once
prefixLowercase string
tokensOnce sync.Once
tokens []string
}
func (fp *filterAnyCasePrefix) String() string {
if fp.prefix == "" {
return quoteFieldNameIfNeeded(fp.fieldName) + "i(*)"
}
return fmt.Sprintf("%si(%s*)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.prefix))
}
func (fp *filterAnyCasePrefix) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens)
return fp.tokens
}
func (fp *filterAnyCasePrefix) initTokens() {
fp.tokens = getTokensSkipLast(fp.prefix)
}
func (fp *filterAnyCasePrefix) getPrefixLowercase() string {
fp.prefixLowercaseOnce.Do(fp.initPrefixLowercase)
return fp.prefixLowercase
}
func (fp *filterAnyCasePrefix) initPrefixLowercase() {
fp.prefixLowercase = strings.ToLower(fp.prefix)
}
func (fp *filterAnyCasePrefix) apply(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName
prefixLowercase := fp.getPrefixLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePrefix(v, prefixLowercase) {
bm.resetBits()
}
return
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()
return
}
tokens := fp.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByAnyCasePrefix(bs, ch, bm, prefixLowercase)
case valueTypeDict:
matchValuesDictByAnyCasePrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint8:
matchUint8ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint16:
matchUint16ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint32:
matchUint32ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeUint64:
matchUint64ByPrefix(bs, ch, bm, prefixLowercase)
case valueTypeFloat64:
matchFloat64ByPrefix(bs, ch, bm, prefixLowercase, tokens)
case valueTypeIPv4:
matchIPv4ByPrefix(bs, ch, bm, prefixLowercase, tokens)
case valueTypeTimestampISO8601:
prefixUppercase := strings.ToUpper(fp.prefix)
matchTimestampISO8601ByPrefix(bs, ch, bm, prefixUppercase, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchValuesDictByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchAnyCasePrefix(v, prefixLowercase) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchStringByAnyCasePrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefixLowercase string) {
visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePrefix(v, prefixLowercase)
})
}
func matchAnyCasePrefix(s, prefixLowercase string) bool {
if len(prefixLowercase) == 0 {
// Special case - empty prefix matches any non-empty string.
return len(s) > 0
}
if len(prefixLowercase) > len(s) {
return false
}
if isASCIILowercase(s) {
// Fast path - s is in lowercase
return matchPrefix(s, prefixLowercase)
}
// Slow path - convert s to lowercase before matching
bb := bbPool.Get()
bb.B = stringsutil.AppendLowercase(bb.B, s)
sLowercase := bytesutil.ToUnsafeString(bb.B)
ok := matchPrefix(sLowercase, prefixLowercase)
bbPool.Put(bb)
return ok
}

930
lib/logstorage/filter_any_case_prefix_test.go Normal file
View file

@ -0,0 +1,930 @@
package logstorage
import (
"testing"
)
func TestMatchAnyCasePrefix(t *testing.T) {
f := func(s, prefixLowercase string, resultExpected bool) {
t.Helper()
result := matchAnyCasePrefix(s, prefixLowercase)
if result != resultExpected {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
// empty prefix matches non-empty strings
f("", "", false)
f("foo", "", true)
f("тест", "", true)
// empty string doesn't match non-empty prefix
f("", "foo", false)
f("", "тест", false)
// full match
f("foo", "foo", true)
f("FOo", "foo", true)
f("Test ТЕСт 123", "test тест 123", true)
// prefix match
f("foo", "f", true)
f("foo тест bar", "те", true)
f("foo ТЕСТ bar", "те", true)
// mismatch
f("foo", "o", false)
f("тест", "foo", false)
f("Тест", "ест", false)
}
func TestFilterAnyCasePrefix(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"aBc DEf",
},
},
{
name: "other column",
values: []string{
"aSDfdsf",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "ABC",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "def",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "other column",
prefix: "asdfdSF",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bc",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "other column",
prefix: "sd",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "other-column",
values: []string{
"x",
"X",
"X",
},
},
{
name: "foo",
values: []string{
"abc def",
"ABC DEF",
"AbC Def",
},
},
{
name: "_msg",
values: []string{
"1 2 3",
"1 2 3",
"1 2 3",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "Abc",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "AB",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "abc de",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: " de",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "other-column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: " 2 ",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "abc def ",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "x",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "other-column",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"fOObar",
"Abc",
"aFDf FooBar baz",
"fddf FOObarBAZ",
"AFoobarbaz",
"foobar",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "FooBar",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 3, 4, 6})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 3, 4, 5, 6})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "ba",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"A FOO",
"a fOoBar",
"aA aBC A",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a afoobarbaz",
"a fooBaR",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "a",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "нГк",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{8})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "aa a",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{2})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "!,",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{9})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "aa ax",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "qwe rty abc",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "@",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 5})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{3, 4})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65535",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{1})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "123456",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"12345678901",
"1",
"2",
"3",
"4",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"0.0002",
"-320001",
"4",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "1234.5678901",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "56789",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "-6553",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "7344.8943",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "-1234",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "+1234",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "23",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "678",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.12.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "foo",
prefix: "127.0.0.1",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 6, 7, 8, 9})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "127.0.0",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "2.3.",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 4, 5, 6, 7, 8})
// mismatch
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "8",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "127.1",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "27.0",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "foo",
prefix: "255.255.255.255",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
fp := &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "2006-01-02t15:04:05.005z",
}
testFilterMatchForColumns(t, columns, fp, "_msg", []int{4})
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "2006-01-0",
}
testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "002",
}
testFilterMatchForColumns(t, columns, fp, "_msg", []int{1})
// mimatch
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "2006-03-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "06",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "2006-01-02T16:04:05.005+01:00",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part
fp = &filterAnyCasePrefix{
fieldName: "_msg",
prefix: "2006-01-02T15:04:05.00500Z",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
fp = &filterAnyCasePrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
})
}

925
lib/logstorage/filter_test.go
View file

@ -8,40 +8,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
) )
func TestMatchAnyCasePrefix(t *testing.T) {
f := func(s, prefixLowercase string, resultExpected bool) {
t.Helper()
result := matchAnyCasePrefix(s, prefixLowercase)
if result != resultExpected {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
// empty prefix matches non-empty strings
f("", "", false)
f("foo", "", true)
f("тест", "", true)
// empty string doesn't match non-empty prefix
f("", "foo", false)
f("", "тест", false)
// full match
f("foo", "foo", true)
f("FOo", "foo", true)
f("Test ТЕСт 123", "test тест 123", true)
// prefix match
f("foo", "f", true)
f("foo тест bar", "те", true)
f("foo ТЕСТ bar", "те", true)
// mismatch
f("foo", "o", false)
f("тест", "foo", false)
f("Тест", "ест", false)
}
func TestMatchAnyCasePhrase(t *testing.T) { func TestMatchAnyCasePhrase(t *testing.T) {
f := func(s, phraseLowercase string, resultExpected bool) { f := func(s, phraseLowercase string, resultExpected bool) {
t.Helper() t.Helper()
@ -332,897 +298,6 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil) testFilterMatchForColumns(t, columns, f, "foo", nil)
} }
func TestAnyCasePrefixFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"aBc DEf",
},
},
{
name: "other column",
values: []string{
"aSDfdsf",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "ABC",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "other column",
prefix: "asdfdSF",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "other column",
prefix: "sd",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "other-column",
values: []string{
"x",
"X",
"X",
},
},
{
name: "foo",
values: []string{
"abc def",
"ABC DEF",
"AbC Def",
},
},
{
name: "_msg",
values: []string{
"1 2 3",
"1 2 3",
"1 2 3",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "Abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "AB",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "abc de",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: " de",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "other-column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: " 2 ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "abc def ",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "other-column",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"fOObar",
"Abc",
"aFDf FooBar baz",
"fddf FOObarBAZ",
"AFoobarbaz",
"foobar",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "FooBar",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 4, 6})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 3, 4, 5, 6})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "ba",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"A FOO",
"a fOoBar",
"aA aBC A",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a afoobarbaz",
"a fooBaR",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "a",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "нГк",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{8})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "aa a",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "!,",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{9})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "aa ax",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "qwe rty abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "@",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 5})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65535",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "123456",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"12345678901",
"1",
"2",
"3",
"4",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"0.0002",
"-320001",
"4",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "1234.5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "56789",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "-6553",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "7344.8943",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "-1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "+1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "23",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "678",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.12.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "foo",
prefix: "127.0.0.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8, 9})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "127.0.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "2.3.",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8})
// mismatch
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "8",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "127.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "27.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "foo",
prefix: "255.255.255.255",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
pf := &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "2006-01-02t15:04:05.005z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{4})
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "2006-01-0",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "002",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{1})
// mimatch
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "2006-03-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "06",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "2006-01-02T16:04:05.005+01:00",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part
pf = &anyCasePrefixFilter{
fieldName: "_msg",
prefix: "2006-01-02T15:04:05.00500Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePrefixFilter{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
})
}
func TestPrefixFilter(t *testing.T) { func TestPrefixFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) { t.Run("single-row", func(t *testing.T) {
columns := []column{ columns := []column{

2
lib/logstorage/parser.go
View file

@ -477,7 +477,7 @@ func parseFilterNot(lex *lexer, fieldName string) (filter, error) {
func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) { func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {
return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) { return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter { if isPrefixFilter {
f := &anyCasePrefixFilter{ f := &filterAnyCasePrefix{
fieldName: fieldName, fieldName: fieldName,
prefix: phrase, prefix: phrase,
} }

42
lib/logstorage/parser_test.go
View file

@ -411,15 +411,15 @@ func TestParseAnyCasePhraseFilter(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
pf, ok := q.f.(*anyCasePhraseFilter) fp, ok := q.f.(*anyCasePhraseFilter)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *anyCasePhraseFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *anyCasePhraseFilter; filter: %s", q.f, q.f)
} }
if pf.fieldName != fieldNameExpected { if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
} }
if pf.phrase != phraseExpected { if fp.phrase != phraseExpected {
t.Fatalf("unexpected phrase; got %q; want %q", pf.phrase, phraseExpected) t.Fatalf("unexpected phrase; got %q; want %q", fp.phrase, phraseExpected)
} }
} }
@ -436,15 +436,15 @@ func TestParseAnyCasePrefixFilter(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
pf, ok := q.f.(*anyCasePrefixFilter) fp, ok := q.f.(*filterAnyCasePrefix)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *anyCasePrefixFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *filterAnyCasePrefix; filter: %s", q.f, q.f)
} }
if pf.fieldName != fieldNameExpected { if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
} }
if pf.prefix != prefixExpected { if fp.prefix != prefixExpected {
t.Fatalf("unexpected prefix; got %q; want %q", pf.prefix, prefixExpected) t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected)
} }
} }
@ -463,15 +463,15 @@ func TestParsePhraseFilter(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
pf, ok := q.f.(*phraseFilter) fp, ok := q.f.(*phraseFilter)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *phraseFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *phraseFilter; filter: %s", q.f, q.f)
} }
if pf.fieldName != fieldNameExpected { if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
} }
if pf.phrase != phraseExpected { if fp.phrase != phraseExpected {
t.Fatalf("unexpected prefix; got %q; want %q", pf.phrase, phraseExpected) t.Fatalf("unexpected prefix; got %q; want %q", fp.phrase, phraseExpected)
} }
} }
@ -490,15 +490,15 @@ func TestParsePrefixFilter(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
pf, ok := q.f.(*prefixFilter) fp, ok := q.f.(*prefixFilter)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *prefixFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *prefixFilter; filter: %s", q.f, q.f)
} }
if pf.fieldName != fieldNameExpected { if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", pf.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
} }
if pf.prefix != prefixExpected { if fp.prefix != prefixExpected {
t.Fatalf("unexpected prefix; got %q; want %q", pf.prefix, prefixExpected) t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected)
} }
} }