github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 15:16:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 07:27:45 +02:00
parent bb89151ae8
commit 292d7fcfc6
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
6 changed files with 1030 additions and 1016 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
lib/logstorage/filter.go
View file

@ -2,7 +2,6 @@ package logstorage
import (
"bytes"
"fmt"
"math"
"strconv"
"strings"
@ -12,7 +11,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
type filter interface {
@ -71,93 +69,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
}
}
// anyCasePhraseFilter filters field entries by case-insensitive phrase match.
//
// An example LogsQL query: `fieldName:i(word)` or `fieldName:i("word1 ... wordN")`
type anyCasePhraseFilter struct {
fieldName string
phrase string
phraseLowercaseOnce sync.Once
phraseLowercase string
tokensOnce sync.Once
tokens []string
}
func (fp *anyCasePhraseFilter) String() string {
return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase))
}
func (fp *anyCasePhraseFilter) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens)
return fp.tokens
}
func (fp *anyCasePhraseFilter) initTokens() {
fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
}
func (fp *anyCasePhraseFilter) getPhraseLowercase() string {
fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase)
return fp.phraseLowercase
}
func (fp *anyCasePhraseFilter) initPhraseLowercase() {
fp.phraseLowercase = strings.ToLower(fp.phrase)
}
func (fp *anyCasePhraseFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName
phraseLowercase := fp.getPhraseLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) {
bm.resetBits()
}
return
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
if len(phraseLowercase) > 0 {
bm.resetBits()
}
return
}
tokens := fp.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeDict:
matchValuesDictByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeUint8:
matchUint8ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint16:
matchUint16ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint32:
matchUint32ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint64:
matchUint64ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeFloat64:
matchFloat64ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeIPv4:
matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeTimestampISO8601:
phraseUppercase := strings.ToUpper(fp.phrase)
matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
// phraseFilter filters field entries by phrase match (aka full text search).
//
// A phrase consists of any number of words with delimiters between them.
@ -314,17 +225,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
bbPool.Put(bb)
}
func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchAnyCasePhrase(v, phraseLowercase) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchValuesDictByAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
@ -363,12 +263,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod
})
}
func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePhrase(v, phraseLowercase)
})
}
func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
if !matchBloomFilterAllTokens(bs, ch, tokens) {
bm.resetBits()
@ -422,30 +316,6 @@ func isASCIILowercase(s string) bool {
return true
}
func matchAnyCasePhrase(s, phraseLowercase string) bool {
if len(phraseLowercase) == 0 {
// Special case - empty phrase matches only empty string.
return len(s) == 0
}
if len(phraseLowercase) > len(s) {
return false
}
if isASCIILowercase(s) {
// Fast path - s is in lowercase
return matchPhrase(s, phraseLowercase)
}
// Slow path - convert s to lowercase before matching
bb := bbPool.Get()
bb.B = stringsutil.AppendLowercase(bb.B, s)
sLowercase := bytesutil.ToUnsafeString(bb.B)
ok := matchPhrase(sLowercase, phraseLowercase)
bbPool.Put(bb)
return ok
}
func matchPhrase(s, phrase string) bool {
if len(phrase) == 0 {
// Special case - empty phrase matches only empty string.

139
lib/logstorage/filter_any_case_phrase.go Normal file
View file

@ -0,0 +1,139 @@
package logstorage
import (
"fmt"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
// filterAnyCasePhrase filters field entries by case-insensitive phrase match.
//
// An example LogsQL query: `fieldName:i(word)` or `fieldName:i("word1 ... wordN")`
type filterAnyCasePhrase struct {
fieldName string
phrase string
phraseLowercaseOnce sync.Once
phraseLowercase string
tokensOnce sync.Once
tokens []string
}
func (fp *filterAnyCasePhrase) String() string {
return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase))
}
func (fp *filterAnyCasePhrase) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens)
return fp.tokens
}
func (fp *filterAnyCasePhrase) initTokens() {
fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
}
func (fp *filterAnyCasePhrase) getPhraseLowercase() string {
fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase)
return fp.phraseLowercase
}
func (fp *filterAnyCasePhrase) initPhraseLowercase() {
fp.phraseLowercase = strings.ToLower(fp.phrase)
}
func (fp *filterAnyCasePhrase) apply(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName
phraseLowercase := fp.getPhraseLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) {
bm.resetBits()
}
return
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
if len(phraseLowercase) > 0 {
bm.resetBits()
}
return
}
tokens := fp.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeDict:
matchValuesDictByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeUint8:
matchUint8ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint16:
matchUint16ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint32:
matchUint32ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint64:
matchUint64ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeFloat64:
matchFloat64ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeIPv4:
matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeTimestampISO8601:
phraseUppercase := strings.ToUpper(fp.phrase)
matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
bb := bbPool.Get()
for i, v := range ch.valuesDict.values {
if matchAnyCasePhrase(v, phraseLowercase) {
bb.B = append(bb.B, byte(i))
}
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePhrase(v, phraseLowercase)
})
}
func matchAnyCasePhrase(s, phraseLowercase string) bool {
if len(phraseLowercase) == 0 {
// Special case - empty phrase matches only empty string.
return len(s) == 0
}
if len(phraseLowercase) > len(s) {
return false
}
if isASCIILowercase(s) {
// Fast path - s is in lowercase
return matchPhrase(s, phraseLowercase)
}
// Slow path - convert s to lowercase before matching
bb := bbPool.Get()
bb.B = stringsutil.AppendLowercase(bb.B, s)
sLowercase := bytesutil.ToUnsafeString(bb.B)
ok := matchPhrase(sLowercase, phraseLowercase)
bbPool.Put(bb)
return ok
}

888
lib/logstorage/filter_any_case_phrase_test.go Normal file
View file

@ -0,0 +1,888 @@
package logstorage
import (
"testing"
)
func TestMatchAnyCasePhrase(t *testing.T) {
f := func(s, phraseLowercase string, resultExpected bool) {
t.Helper()
result := matchAnyCasePhrase(s, phraseLowercase)
if result != resultExpected {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
// empty phrase matches only empty string
f("", "", true)
f("foo", "", false)
f("тест", "", false)
// empty string doesn't match non-empty phrase
f("", "foo", false)
f("", "тест", false)
// full match
f("foo", "foo", true)
f("FOo", "foo", true)
f("Test ТЕСт 123", "test тест 123", true)
// phrase match
f("a foo", "foo", true)
f("foo тест bar", "тест", true)
f("foo ТЕСТ bar", "тест bar", true)
// mismatch
f("foo", "fo", false)
f("тест", "foo", false)
f("Тест", "ест", false)
}
func TestFilterAnyCasePhrase(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"aBc DEf",
},
},
{
name: "other column",
values: []string{
"aSDfdsF",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "Abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "other column",
phrase: "ASdfdsf",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "ab",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "other column",
phrase: "sd",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "non-existing column",
phrase: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "other-column",
values: []string{
"X",
"x",
"x",
},
},
{
name: "foo",
values: []string{
"aBC def",
"abc DEf",
"Abc deF",
},
},
{
name: "_msg",
values: []string{
"1 2 3",
"1 2 3",
"1 2 3",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: " def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "other-column",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: " 2 ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "abc def ",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "other-column",
phrase: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "non-existing column",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"fooBar",
"ABc",
"afdf foobar BAz",
"fddf fOObARbaz",
"AfooBarbaz",
"foobar",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "FoobAr",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "baZ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "non-existing column",
phrase: "foobar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"A Foobar",
"aA aBC a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a aFOObarbaz",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "A",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "НгкШ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{8})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "!,",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{9})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "aa a",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "@",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65535",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "123456",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"12345678901",
"1",
"2",
"3",
"4",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"2",
"3",
"4",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "1234.5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "-65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "-1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "+1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "5678",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.55.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "foo",
phrase: "127.0.0.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "127",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "127.0.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "2.3",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "5",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "127.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "27.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &filterAnyCasePhrase{
fieldName: "foo",
phrase: "255.255.255.255",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
pf := &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "2006-01-02t15:04:05.005z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{4})
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "2006-01",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "002Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{1})
pf = &filterAnyCasePhrase{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "2006-03-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "06",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "2006-01-02T16:04:05.005+01:00",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part
pf = &filterAnyCasePhrase{
fieldName: "_msg",
phrase: "2006-01-02T15:04:05.00500Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
})
}

883
lib/logstorage/filter_test.go
View file

@ -8,40 +8,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
)
func TestMatchAnyCasePhrase(t *testing.T) {
f := func(s, phraseLowercase string, resultExpected bool) {
t.Helper()
result := matchAnyCasePhrase(s, phraseLowercase)
if result != resultExpected {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
// empty phrase matches only empty string
f("", "", true)
f("foo", "", false)
f("тест", "", false)
// empty string doesn't match non-empty phrase
f("", "foo", false)
f("", "тест", false)
// full match
f("foo", "foo", true)
f("FOo", "foo", true)
f("Test ТЕСт 123", "test тест 123", true)
// phrase match
f("a foo", "foo", true)
f("foo тест bar", "тест", true)
f("foo ТЕСТ bar", "тест bar", true)
// mismatch
f("foo", "fo", false)
f("тест", "foo", false)
f("Тест", "ест", false)
}
func TestMatchPhrase(t *testing.T) {
f := func(s, phrase string, resultExpected bool) {
t.Helper()
@ -259,855 +225,6 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil)
}
func TestAnyCasePhraseFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"aBc DEf",
},
},
{
name: "other column",
values: []string{
"aSDfdsF",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "Abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "other column",
phrase: "ASdfdsf",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "ab",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "other column",
phrase: "sd",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "non-existing column",
phrase: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
columns := []column{
{
name: "other-column",
values: []string{
"X",
"x",
"x",
},
},
{
name: "foo",
values: []string{
"aBC def",
"abc DEf",
"Abc deF",
},
},
{
name: "_msg",
values: []string{
"1 2 3",
"1 2 3",
"1 2 3",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: " def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "other-column",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: " 2 ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "abc def ",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "other-column",
phrase: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "non-existing column",
phrase: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"",
"fooBar",
"ABc",
"afdf foobar BAz",
"fddf fOObARbaz",
"AfooBarbaz",
"foobar",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "FoobAr",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "baZ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "non-existing column",
phrase: "foobar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"A Foobar",
"aA aBC a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"a aFOObarbaz",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "A",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "НгкШ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{8})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "!,",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{9})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "aa a",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "@",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"123",
"12",
"32",
"0",
"0",
"12",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65535",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "123456",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"1234",
"1",
"2",
"3",
"4",
"5",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"65536",
"12345678901",
"1",
"2",
"3",
"4",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1234",
"0",
"3454",
"-65536",
"1234.5678901",
"1",
"2",
"3",
"4",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "1234.5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "-65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "-1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "+1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "5678",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"1.2.3.4",
"0.0.0.0",
"127.0.0.1",
"254.255.255.255",
"127.0.0.1",
"127.0.0.1",
"127.0.4.2",
"127.0.0.1",
"12.0.127.6",
"55.55.55.55",
"66.66.66.66",
"7.7.7.7",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "foo",
phrase: "127.0.0.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "127",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "127.0.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "2.3",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "5",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "127.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "27.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &anyCasePhraseFilter{
fieldName: "foo",
phrase: "255.255.255.255",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
columns := []column{
{
name: "_msg",
values: []string{
"2006-01-02T15:04:05.001Z",
"2006-01-02T15:04:05.002Z",
"2006-01-02T15:04:05.003Z",
"2006-01-02T15:04:05.004Z",
"2006-01-02T15:04:05.005Z",
"2006-01-02T15:04:05.006Z",
"2006-01-02T15:04:05.007Z",
"2006-01-02T15:04:05.008Z",
"2006-01-02T15:04:05.009Z",
},
},
}
// match
pf := &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "2006-01-02t15:04:05.005z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{4})
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "2006-01",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "002Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{1})
pf = &anyCasePhraseFilter{
fieldName: "non-existing-column",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "bar",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "2006-03-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "06",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "2006-01-02T16:04:05.005+01:00",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part
pf = &anyCasePhraseFilter{
fieldName: "_msg",
phrase: "2006-01-02T15:04:05.00500Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
})
}
func TestPhraseFilter(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{

2
lib/logstorage/parser.go
View file

@ -483,7 +483,7 @@ func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {
}
return f, nil
}
f := &anyCasePhraseFilter{
f := &filterAnyCasePhrase{
fieldName: fieldName,
phrase: phrase,
}

4
lib/logstorage/parser_test.go
View file

@ -411,9 +411,9 @@ func TestParseAnyCasePhraseFilter(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
fp, ok := q.f.(*anyCasePhraseFilter)
fp, ok := q.f.(*filterAnyCasePhrase)
if !ok {
t.Fatalf("unexpected filter type; got %T; want *anyCasePhraseFilter; filter: %s", q.f, q.f)
t.Fatalf("unexpected filter type; got %T; want *filterAnyCasePhrase; filter: %s", q.f, q.f)
}
if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)