github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 07:30:34 +02:00
parent 292d7fcfc6
commit 4e1f20338f
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
8 changed files with 148 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
lib/logstorage/filter.go
View file

@ -69,7 +69,7 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
} }
} }
// phraseFilter filters field entries by phrase match (aka full text search). // filterPhrase filters field entries by phrase match (aka full text search).
// //
// A phrase consists of any number of words with delimiters between them. // A phrase consists of any number of words with delimiters between them.
// //
@ -79,7 +79,7 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
// Multi-word phrase is expressed as `fieldName:"word1 ... wordN"` in LogsQL. // Multi-word phrase is expressed as `fieldName:"word1 ... wordN"` in LogsQL.
// //
// A special case `fieldName:""` matches any value without `fieldName` field. // A special case `fieldName:""` matches any value without `fieldName` field.
type phraseFilter struct { type filterPhrase struct {
fieldName string fieldName string
phrase string phrase string
@ -87,20 +87,20 @@ type phraseFilter struct {
tokens []string tokens []string
} }
func (fp *phraseFilter) String() string { func (fp *filterPhrase) String() string {
return quoteFieldNameIfNeeded(fp.fieldName) + quoteTokenIfNeeded(fp.phrase) return quoteFieldNameIfNeeded(fp.fieldName) + quoteTokenIfNeeded(fp.phrase)
} }
func (fp *phraseFilter) getTokens() []string { func (fp *filterPhrase) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens) fp.tokensOnce.Do(fp.initTokens)
return fp.tokens return fp.tokens
} }
func (fp *phraseFilter) initTokens() { func (fp *filterPhrase) initTokens() {
fp.tokens = tokenizeStrings(nil, []string{fp.phrase}) fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
} }
func (fp *phraseFilter) apply(bs *blockSearch, bm *bitmap) { func (fp *filterPhrase) apply(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName fieldName := fp.fieldName
phrase := fp.phrase phrase := fp.phrase

2
lib/logstorage/filter_and.go
View file

@ -65,7 +65,7 @@ func (fa *filterAnd) initMsgTokens() {
var a []string var a []string
for _, f := range fa.filters { for _, f := range fa.filters {
switch t := f.(type) { switch t := f.(type) {
case *phraseFilter: case *filterPhrase:
if isMsgFieldName(t.fieldName) { if isMsgFieldName(t.fieldName) {
a = append(a, t.getTokens()...) a = append(a, t.getTokens()...)
} }

12
lib/logstorage/filter_and_test.go
View file

@ -26,7 +26,7 @@ func TestFilterAnd(t *testing.T) {
// non-empty intersection // non-empty intersection
fa := &filterAnd{ fa := &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -45,7 +45,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "abc", prefix: "abc",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -60,7 +60,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "bc", prefix: "bc",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -71,7 +71,7 @@ func TestFilterAnd(t *testing.T) {
// the last filter mismatch // the last filter mismatch
fa = &filterAnd{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc", phrase: "abc",
}, },
@ -86,7 +86,7 @@ func TestFilterAnd(t *testing.T) {
// empty intersection // empty intersection
fa = &filterAnd{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foo", phrase: "foo",
}, },
@ -105,7 +105,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "abc", prefix: "abc",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foo", phrase: "foo",
}, },

8
lib/logstorage/filter_not_test.go
View file

@ -25,7 +25,7 @@ func TestFilterNot(t *testing.T) {
// match // match
fn := &filterNot{ fn := &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
}, },
@ -33,7 +33,7 @@ func TestFilterNot(t *testing.T) {
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9})
fn = &filterNot{ fn = &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -41,7 +41,7 @@ func TestFilterNot(t *testing.T) {
testFilterMatchForColumns(t, columns, fn, "foo", []int{5}) testFilterMatchForColumns(t, columns, fn, "foo", []int{5})
fn = &filterNot{ fn = &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "non-existing-field", fieldName: "non-existing-field",
phrase: "foobar", phrase: "foobar",
}, },
@ -66,7 +66,7 @@ func TestFilterNot(t *testing.T) {
// mismatch // mismatch
fn = &filterNot{ fn = &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "non-existing-field", fieldName: "non-existing-field",
phrase: "", phrase: "",
}, },

14
lib/logstorage/filter_or_test.go
View file

@ -26,7 +26,7 @@ func TestFilterOr(t *testing.T) {
// non-empty union // non-empty union
fo := &filterOr{ fo := &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "23", phrase: "23",
}, },
@ -45,7 +45,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "abc", prefix: "abc",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "23", phrase: "23",
}, },
@ -60,7 +60,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "xabc", prefix: "xabc",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "23", phrase: "23",
}, },
@ -71,7 +71,7 @@ func TestFilterOr(t *testing.T) {
// first non-empty result, second empty result // first non-empty result, second empty result
fo = &filterOr{ fo = &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "23", phrase: "23",
}, },
@ -86,7 +86,7 @@ func TestFilterOr(t *testing.T) {
// first match all // first match all
fo = &filterOr{ fo = &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -105,7 +105,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo", fieldName: "foo",
prefix: "23", prefix: "23",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -116,7 +116,7 @@ func TestFilterOr(t *testing.T) {
// both empty results // both empty results
fo = &filterOr{ fo = &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "x23", phrase: "x23",
}, },

238
lib/logstorage/filter_test.go
View file

@ -69,23 +69,23 @@ func TestComplexFilters(t *testing.T) {
// (foobar AND NOT baz AND (abcdef OR xyz)) // (foobar AND NOT baz AND (abcdef OR xyz))
f := &filterAnd{ f := &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foobar", phrase: "foobar",
}, },
&filterNot{ &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "baz", phrase: "baz",
}, },
}, },
&filterOr{ &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abcdef", phrase: "abcdef",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "xyz", phrase: "xyz",
}, },
@ -98,23 +98,23 @@ func TestComplexFilters(t *testing.T) {
// (foobaz AND NOT baz AND (abcdef OR xyz)) // (foobaz AND NOT baz AND (abcdef OR xyz))
f = &filterAnd{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foobaz", phrase: "foobaz",
}, },
&filterNot{ &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "baz", phrase: "baz",
}, },
}, },
&filterOr{ &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abcdef", phrase: "abcdef",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "xyz", phrase: "xyz",
}, },
@ -127,27 +127,27 @@ func TestComplexFilters(t *testing.T) {
// (foobaz AND NOT baz AND (abcdef OR xyz OR a)) // (foobaz AND NOT baz AND (abcdef OR xyz OR a))
f = &filterAnd{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foobar", phrase: "foobar",
}, },
&filterNot{ &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "baz", phrase: "baz",
}, },
}, },
&filterOr{ &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abcdef", phrase: "abcdef",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "xyz", phrase: "xyz",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -160,27 +160,27 @@ func TestComplexFilters(t *testing.T) {
// (foobaz AND NOT qwert AND (abcdef OR xyz OR a)) // (foobaz AND NOT qwert AND (abcdef OR xyz OR a))
f = &filterAnd{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foobar", phrase: "foobar",
}, },
&filterNot{ &filterNot{
f: &phraseFilter{ f: &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "qwert", phrase: "qwert",
}, },
}, },
&filterOr{ &filterOr{
filters: []filter{ filters: []filter{
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abcdef", phrase: "abcdef",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "xyz", phrase: "xyz",
}, },
&phraseFilter{ &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
}, },
@ -225,7 +225,7 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil) testFilterMatchForColumns(t, columns, f, "foo", nil)
} }
func TestPhraseFilter(t *testing.T) { func TestFilterPhrase(t *testing.T) {
t.Run("single-row", func(t *testing.T) { t.Run("single-row", func(t *testing.T) {
columns := []column{ columns := []column{
{ {
@ -243,56 +243,56 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc", phrase: "abc",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc def", phrase: "abc def",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "def", phrase: "def",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "other column", fieldName: "other column",
phrase: "asdfdsf", phrase: "asdfdsf",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "ab", phrase: "ab",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "other column", fieldName: "other column",
phrase: "sd", phrase: "sd",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing column", fieldName: "non-existing column",
phrase: "abc", phrase: "abc",
} }
@ -328,80 +328,80 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc", phrase: "abc",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "def", phrase: "def",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: " def", phrase: " def",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc def", phrase: "abc def",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "other-column", fieldName: "other-column",
phrase: "x", phrase: "x",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: " 2 ", phrase: " 2 ",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "abc def ", phrase: "abc def ",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "x", phrase: "x",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "other-column", fieldName: "other-column",
phrase: "foo", phrase: "foo",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing column", fieldName: "non-existing column",
phrase: "x", phrase: "x",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "foo", phrase: "foo",
} }
@ -425,38 +425,38 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "foobar", phrase: "foobar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6}) testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 6})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "baz", phrase: "baz",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing column", fieldName: "non-existing column",
phrase: "foobar", phrase: "foobar",
} }
@ -483,50 +483,50 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "a", phrase: "a",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "НГКШ", phrase: "НГКШ",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{8}) testFilterMatchForColumns(t, columns, pf, "foo", []int{8})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "!,", phrase: "!,",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{9}) testFilterMatchForColumns(t, columns, pf, "foo", []int{9})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "aa a", phrase: "aa a",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "@", phrase: "@",
} }
@ -554,44 +554,44 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "12", phrase: "12",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5}) testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 5})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "0", phrase: "0",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "33", phrase: "33",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234", phrase: "1234",
} }
@ -618,44 +618,44 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234", phrase: "1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "0", phrase: "0",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{1}) testFilterMatchForColumns(t, columns, pf, "foo", []int{1})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "33", phrase: "33",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "123456", phrase: "123456",
} }
@ -682,44 +682,44 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234", phrase: "1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "65536", phrase: "65536",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "33", phrase: "33",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "12345678901", phrase: "12345678901",
} }
@ -745,44 +745,44 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234", phrase: "1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "12345678901", phrase: "12345678901",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "33", phrase: "33",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "12345678901234567890", phrase: "12345678901234567890",
} }
@ -808,86 +808,86 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234", phrase: "1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "1234.5678901", phrase: "1234.5678901",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "5678901", phrase: "5678901",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{4}) testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "-65536", phrase: "-65536",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "65536", phrase: "65536",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{3}) testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "-1234", phrase: "-1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "+1234", phrase: "+1234",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "123", phrase: "123",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "5678", phrase: "5678",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "33", phrase: "33",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "12345678901234567890", phrase: "12345678901234567890",
} }
@ -916,74 +916,74 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "127.0.0.1", phrase: "127.0.0.1",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "127", phrase: "127",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "127.0.0", phrase: "127.0.0",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7}) testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "2.3", phrase: "2.3",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "0", phrase: "0",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch // mismatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "5", phrase: "5",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "127.1", phrase: "127.1",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "27.0", phrase: "27.0",
} }
testFilterMatchForColumns(t, columns, pf, "foo", nil) testFilterMatchForColumns(t, columns, pf, "foo", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "foo", fieldName: "foo",
phrase: "255.255.255.255", phrase: "255.255.255.255",
} }
@ -1009,64 +1009,64 @@ func TestPhraseFilter(t *testing.T) {
} }
// match // match
pf := &phraseFilter{ pf := &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "2006-01-02T15:04:05.005Z", phrase: "2006-01-02T15:04:05.005Z",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", []int{4}) testFilterMatchForColumns(t, columns, pf, "_msg", []int{4})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "2006-01", phrase: "2006-01",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "002Z", phrase: "002Z",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", []int{1}) testFilterMatchForColumns(t, columns, pf, "_msg", []int{1})
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "non-existing-column", fieldName: "non-existing-column",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch // mimatch
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "bar", phrase: "bar",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", nil) testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "", phrase: "",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", nil) testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "2006-03-02T15:04:05.005Z", phrase: "2006-03-02T15:04:05.005Z",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", nil) testFilterMatchForColumns(t, columns, pf, "_msg", nil)
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "06", phrase: "06",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", nil) testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp // This filter shouldn't match row=4, since it has different string representation of the timestamp
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "2006-01-02T16:04:05.005+01:00", phrase: "2006-01-02T16:04:05.005+01:00",
} }
testFilterMatchForColumns(t, columns, pf, "_msg", nil) testFilterMatchForColumns(t, columns, pf, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part // This filter shouldn't match row=4, since it contains too many digits for millisecond part
pf = &phraseFilter{ pf = &filterPhrase{
fieldName: "_msg", fieldName: "_msg",
phrase: "2006-01-02T15:04:05.00500Z", phrase: "2006-01-02T15:04:05.00500Z",
} }

2
lib/logstorage/parser.go
View file

@ -418,7 +418,7 @@ func parseFilterForPhrase(lex *lexer, phrase, fieldName string) (filter, error)
return f, nil return f, nil
} }
// The phrase is a search phrase. // The phrase is a search phrase.
f := &phraseFilter{ f := &filterPhrase{
fieldName: fieldName, fieldName: fieldName,
phrase: phrase, phrase: phrase,
} }

8
lib/logstorage/parser_test.go
View file

@ -404,7 +404,7 @@ func TestParseFilterRegexp(t *testing.T) {
f(`"foo(bar|baz),x[y]"`, `foo(bar|baz),x[y]`) f(`"foo(bar|baz),x[y]"`, `foo(bar|baz),x[y]`)
} }
func TestParseAnyCasePhraseFilter(t *testing.T) { func TestParseAnyCaseFilterPhrase(t *testing.T) {
f := func(s, fieldNameExpected, phraseExpected string) { f := func(s, fieldNameExpected, phraseExpected string) {
t.Helper() t.Helper()
q, err := ParseQuery(s) q, err := ParseQuery(s)
@ -456,16 +456,16 @@ func TestParseAnyCaseFilterPrefix(t *testing.T) {
f(`"abc-de.fg":i("foo-bar*baz *"*)`, `abc-de.fg`, `foo-bar*baz *`) f(`"abc-de.fg":i("foo-bar*baz *"*)`, `abc-de.fg`, `foo-bar*baz *`)
} }
func TestParsePhraseFilter(t *testing.T) { func TestParseFilterPhrase(t *testing.T) {
f := func(s, fieldNameExpected, phraseExpected string) { f := func(s, fieldNameExpected, phraseExpected string) {
t.Helper() t.Helper()
q, err := ParseQuery(s) q, err := ParseQuery(s)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
fp, ok := q.f.(*phraseFilter) fp, ok := q.f.(*filterPhrase)
if !ok { if !ok {
t.Fatalf("unexpected filter type; got %T; want *phraseFilter; filter: %s", q.f, q.f) t.Fatalf("unexpected filter type; got %T; want *filterPhrase; filter: %s", q.f, q.f)
} }
if fp.fieldName != fieldNameExpected { if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)