github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 07:15:54 +02:00
parent 9b5dd883b4
commit 7591f72919
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
8 changed files with 248 additions and 248 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
lib/logstorage/filter.go
View file

@ -71,12 +71,12 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) {
}
}
// prefixFilter matches the given prefix.
// filterPrefix matches the given prefix.
//
// Example LogsQL: `fieldName:prefix*` or `fieldName:"some prefix"*`
//
// A special case `fieldName:*` matches non-empty value for the given `fieldName` field
type prefixFilter struct {
type filterPrefix struct {
fieldName string
prefix string
@ -84,23 +84,23 @@ type prefixFilter struct {
tokens []string
}
func (fp *prefixFilter) String() string {
func (fp *filterPrefix) String() string {
if fp.prefix == "" {
return quoteFieldNameIfNeeded(fp.fieldName) + "*"
}
return fmt.Sprintf("%s%s*", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.prefix))
}
func (fp *prefixFilter) getTokens() []string {
func (fp *filterPrefix) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens)
return fp.tokens
}
func (fp *prefixFilter) initTokens() {
func (fp *filterPrefix) initTokens() {
fp.tokens = getTokensSkipLast(fp.prefix)
}
func (fp *prefixFilter) apply(bs *blockSearch, bm *bitmap) {
func (fp *filterPrefix) apply(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName
prefix := fp.prefix

2
lib/logstorage/filter_and.go
View file

@ -81,7 +81,7 @@ func (fa *filterAnd) initMsgTokens() {
if isMsgFieldName(t.fieldName) {
a = append(a, t.getTokens()...)
}
case *prefixFilter:
case *filterPrefix:
if isMsgFieldName(t.fieldName) {
a = append(a, t.getTokens()...)
}

12
lib/logstorage/filter_and_test.go
View file

@ -30,7 +30,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo",
phrase: "a",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},
@ -41,7 +41,7 @@ func TestFilterAnd(t *testing.T) {
// reverse non-empty intersection
fa = &filterAnd{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},
@ -56,7 +56,7 @@ func TestFilterAnd(t *testing.T) {
// the first filter mismatch
fa = &filterAnd{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "bc",
},
@ -75,7 +75,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo",
phrase: "abc",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "foo",
},
@ -90,7 +90,7 @@ func TestFilterAnd(t *testing.T) {
fieldName: "foo",
phrase: "foo",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},
@ -101,7 +101,7 @@ func TestFilterAnd(t *testing.T) {
// reverse empty intersection
fa = &filterAnd{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},

4
lib/logstorage/filter_not_test.go
View file

@ -49,7 +49,7 @@ func TestFilterNot(t *testing.T) {
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fn = &filterNot{
f: &prefixFilter{
f: &filterPrefix{
fieldName: "non-existing-field",
prefix: "",
},
@ -57,7 +57,7 @@ func TestFilterNot(t *testing.T) {
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fn = &filterNot{
f: &prefixFilter{
f: &filterPrefix{
fieldName: "foo",
prefix: "",
},

14
lib/logstorage/filter_or_test.go
View file

@ -30,7 +30,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo",
phrase: "23",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},
@ -41,7 +41,7 @@ func TestFilterOr(t *testing.T) {
// reverse non-empty union
fo = &filterOr{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "abc",
},
@ -56,7 +56,7 @@ func TestFilterOr(t *testing.T) {
// first empty result, second non-empty result
fo = &filterOr{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "xabc",
},
@ -75,7 +75,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo",
phrase: "23",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "xabc",
},
@ -90,7 +90,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo",
phrase: "a",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "23",
},
@ -101,7 +101,7 @@ func TestFilterOr(t *testing.T) {
// second match all
fo = &filterOr{
filters: []filter{
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "23",
},
@ -120,7 +120,7 @@ func TestFilterOr(t *testing.T) {
fieldName: "foo",
phrase: "x23",
},
&prefixFilter{
&filterPrefix{
fieldName: "foo",
prefix: "xabc",
},

426
lib/logstorage/filter_test.go
View file

@ -298,7 +298,7 @@ func TestStreamFilter(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil)
}
func TestPrefixFilter(t *testing.T) {
func TestFilterPrefix(t *testing.T) {
t.Run("single-row", func(t *testing.T) {
columns := []column{
{
@ -316,72 +316,72 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "other column",
prefix: "asdfdsf",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "other column",
prefix: "sd",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
@ -413,90 +413,90 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "ab",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "abc de",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: " de",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "abc def",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "other-column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: " 2 ",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "abc def ",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "other-column",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "x",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "foo",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
@ -516,42 +516,42 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 3, 4, 6})
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 3, 4, 6})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 3, 4, 5, 6})
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 3, 4, 5, 6})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "ba",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "foobar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
@ -574,66 +574,66 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "a",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "НГК",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{8})
testFilterMatchForColumns(t, columns, fp, "foo", []int{8})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "aa a",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2})
testFilterMatchForColumns(t, columns, fp, "foo", []int{2})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "!,",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{9})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "aa ax",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "qwe rty abc",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "@",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
@ -657,48 +657,48 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 5})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 5})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3, 4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{3, 4})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
@ -721,48 +721,48 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1})
testFilterMatchForColumns(t, columns, fp, "foo", []int{1})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "123456",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
@ -785,48 +785,48 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
@ -848,48 +848,48 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "12345678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
@ -911,96 +911,96 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "123",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "1234.5678901",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "56789",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{4})
testFilterMatchForColumns(t, columns, fp, "foo", []int{4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "-6553",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "65536",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{3})
testFilterMatchForColumns(t, columns, fp, "foo", []int{3})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "7344.8943",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "-1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "+1234",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "23",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "678",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "33",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "12345678901234567890",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
@ -1025,78 +1025,78 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "foo",
prefix: "127.0.0.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "12",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 6, 7, 8, 9})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "127.0.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{2, 4, 5, 7})
testFilterMatchForColumns(t, columns, fp, "foo", []int{2, 4, 5, 7})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "2.3.",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{0})
testFilterMatchForColumns(t, columns, fp, "foo", []int{0})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "0",
}
testFilterMatchForColumns(t, columns, pf, "foo", []int{1, 2, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fp, "foo", []int{1, 2, 4, 5, 6, 7, 8})
// mismatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "8",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "127.1",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "27.0",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "foo",
prefix: "255.255.255.255",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "foo", nil)
testFilterMatchForColumns(t, columns, fp, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
@ -1118,68 +1118,68 @@ func TestPrefixFilter(t *testing.T) {
}
// match
pf := &prefixFilter{
fp := &filterPrefix{
fieldName: "_msg",
prefix: "2006-01-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{4})
testFilterMatchForColumns(t, columns, fp, "_msg", []int{4})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "2006-01-0",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fp, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "002",
}
testFilterMatchForColumns(t, columns, pf, "_msg", []int{1})
testFilterMatchForColumns(t, columns, fp, "_msg", []int{1})
// mimatch
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "bar",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "2006-03-02T15:04:05.005Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "06",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
// This filter shouldn't match row=4, since it has different string representation of the timestamp
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "2006-01-02T16:04:05.005+01:00",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
// This filter shouldn't match row=4, since it contains too many digits for millisecond part
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "_msg",
prefix: "2006-01-02T15:04:05.00500Z",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
pf = &prefixFilter{
fp = &filterPrefix{
fieldName: "non-existing-column",
prefix: "",
}
testFilterMatchForColumns(t, columns, pf, "_msg", nil)
testFilterMatchForColumns(t, columns, fp, "_msg", nil)
})
}

18
lib/logstorage/parser.go
View file

@ -309,7 +309,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
return parseGenericFilter(lex, fieldName)
case lex.isKeyword("*"):
lex.nextToken()
f := &prefixFilter{
f := &filterPrefix{
fieldName: fieldName,
prefix: "",
}
@ -411,7 +411,7 @@ func parseFilterForPhrase(lex *lexer, phrase, fieldName string) (filter, error)
if lex.isKeyword("*") && !lex.isSkippedSpace {
// The phrase is a search prefix in the form `foo*`.
lex.nextToken()
f := &prefixFilter{
f := &filterPrefix{
fieldName: fieldName,
prefix: phrase,
}
@ -475,8 +475,8 @@ func parseFilterNot(lex *lexer, fieldName string) (filter, error) {
}
func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {
return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter {
return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isFilterPrefix bool) (filter, error) {
if isFilterPrefix {
f := &filterAnyCasePrefix{
fieldName: fieldName,
prefix: phrase,
@ -502,9 +502,9 @@ func parseFuncArgMaybePrefix(lex *lexer, funcName, fieldName string, callback fu
return nil, fmt.Errorf("missing arg for %s()", funcName)
}
phrase = getCompoundFuncArg(lex)
isPrefixFilter := false
isFilterPrefix := false
if lex.isKeyword("*") && !lex.isSkippedSpace {
isPrefixFilter = true
isFilterPrefix = true
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing ')' after %s()", funcName)
}
@ -513,7 +513,7 @@ func parseFuncArgMaybePrefix(lex *lexer, funcName, fieldName string, callback fu
return nil, fmt.Errorf("unexpected token %q instead of ')' in %s()", lex.token, funcName)
}
lex.nextToken()
return callback(phrase, isPrefixFilter)
return callback(phrase, isFilterPrefix)
}
func parseFilterLenRange(lex *lexer, fieldName string) (filter, error) {
@ -633,8 +633,8 @@ func parseFilterSequence(lex *lexer, fieldName string) (filter, error) {
}
func parseFilterExact(lex *lexer, fieldName string) (filter, error) {
return parseFuncArgMaybePrefix(lex, "exact", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter {
return parseFuncArgMaybePrefix(lex, "exact", fieldName, func(phrase string, isFilterPrefix bool) (filter, error) {
if isFilterPrefix {
f := &filterExactPrefix{
fieldName: fieldName,
prefix: phrase,

8
lib/logstorage/parser_test.go
View file

@ -429,7 +429,7 @@ func TestParseAnyCasePhraseFilter(t *testing.T) {
f(`"abc-de.fg":i("foo-bar+baz")`, `abc-de.fg`, `foo-bar+baz`)
}
func TestParseAnyCasePrefixFilter(t *testing.T) {
func TestParseAnyCaseFilterPrefix(t *testing.T) {
f := func(s, fieldNameExpected, prefixExpected string) {
t.Helper()
q, err := ParseQuery(s)
@ -483,16 +483,16 @@ func TestParsePhraseFilter(t *testing.T) {
f(`"foo:bar*,( baz"`, ``, `foo:bar*,( baz`)
}
func TestParsePrefixFilter(t *testing.T) {
func TestParseFilterPrefix(t *testing.T) {
f := func(s, fieldNameExpected, prefixExpected string) {
t.Helper()
q, err := ParseQuery(s)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
fp, ok := q.f.(*prefixFilter)
fp, ok := q.f.(*filterPrefix)
if !ok {
t.Fatalf("unexpected filter type; got %T; want *prefixFilter; filter: %s", q.f, q.f)
t.Fatalf("unexpected filter type; got %T; want *filterPrefix; filter: %s", q.f, q.f)
}
if fp.fieldName != fieldNameExpected {
t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)