github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 04:20:42 +02:00
parent 93c5f2f9bc
commit 89235a3489
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
6 changed files with 120 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
lib/logstorage/filter.go
View file

@ -24,32 +24,6 @@ type filter interface {
apply(bs *blockSearch, bm *bitmap)
}
// notFilter negates the filter.
//
// It is expressed as `NOT f` or `!f` in LogsQL.
type notFilter struct {
f filter
}
func (fn *notFilter) String() string {
s := fn.f.String()
switch fn.f.(type) {
case *filterAnd, *filterOr:
s = "(" + s + ")"
}
return "!" + s
}
func (fn *notFilter) apply(bs *blockSearch, bm *bitmap) {
// Minimize the number of rows to check by the filter by applying it
// only to the rows, which match the bm, e.g. they may change the bm result.
bmTmp := getBitmap(bm.bitsLen)
bmTmp.copyFrom(bm)
fn.f.apply(bs, bmTmp)
bm.andNot(bmTmp)
putBitmap(bmTmp)
}
// streamFilter is the filter for `_stream:{...}`
type streamFilter struct {
// f is the filter to apply

27
lib/logstorage/filter_not.go Normal file
View file

@ -0,0 +1,27 @@
package logstorage
// filterNot negates the filter.
//
// It is expressed as `NOT f` or `!f` in LogsQL.
type filterNot struct {
f filter
}
func (fn *filterNot) String() string {
s := fn.f.String()
switch fn.f.(type) {
case *filterAnd, *filterOr:
s = "(" + s + ")"
}
return "!" + s
}
func (fn *filterNot) apply(bs *blockSearch, bm *bitmap) {
// Minimize the number of rows to check by the filter by applying it
// only to the rows, which match the bm, e.g. they may change the bm result.
bmTmp := getBitmap(bm.bitsLen)
bmTmp.copyFrom(bm)
fn.f.apply(bs, bmTmp)
bm.andNot(bmTmp)
putBitmap(bmTmp)
}

75
lib/logstorage/filter_not_test.go Normal file
View file

@ -0,0 +1,75 @@
package logstorage
import (
"testing"
)
func TestFilterNot(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"a foobar",
"aa abc a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
fn := &filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9})
fn = &filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "a",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", []int{5})
fn = &filterNot{
f: &phraseFilter{
fieldName: "non-existing-field",
phrase: "foobar",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fn = &filterNot{
f: &prefixFilter{
fieldName: "non-existing-field",
prefix: "",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
fn = &filterNot{
f: &prefixFilter{
fieldName: "foo",
prefix: "",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", []int{5})
// mismatch
fn = &filterNot{
f: &phraseFilter{
fieldName: "non-existing-field",
phrase: "",
},
}
testFilterMatchForColumns(t, columns, fn, "foo", nil)
}

78
lib/logstorage/filter_test.go
View file

@ -362,7 +362,7 @@ func TestComplexFilters(t *testing.T) {
fieldName: "foo",
phrase: "foobar",
},
&notFilter{
&filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "baz",
@ -391,7 +391,7 @@ func TestComplexFilters(t *testing.T) {
fieldName: "foo",
phrase: "foobaz",
},
&notFilter{
&filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "baz",
@ -420,7 +420,7 @@ func TestComplexFilters(t *testing.T) {
fieldName: "foo",
phrase: "foobar",
},
&notFilter{
&filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "baz",
@ -453,7 +453,7 @@ func TestComplexFilters(t *testing.T) {
fieldName: "foo",
phrase: "foobar",
},
&notFilter{
&filterNot{
f: &phraseFilter{
fieldName: "foo",
phrase: "qwert",
@ -480,76 +480,6 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", []int{1, 3, 6})
}
func TestNotFilter(t *testing.T) {
columns := []column{
{
name: "foo",
values: []string{
"a foo",
"a foobar",
"aa abc a",
"ca afdf a,foobar baz",
"a fddf foobarbaz",
"",
"a foobar",
"a kjlkjf dfff",
"a ТЕСТЙЦУК НГКШ ",
"a !!,23.(!1)",
},
},
}
// match
nf := &notFilter{
f: &phraseFilter{
fieldName: "foo",
phrase: "",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 6, 7, 8, 9})
nf = &notFilter{
f: &phraseFilter{
fieldName: "foo",
phrase: "a",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", []int{5})
nf = &notFilter{
f: &phraseFilter{
fieldName: "non-existing-field",
phrase: "foobar",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
nf = &notFilter{
f: &prefixFilter{
fieldName: "non-existing-field",
prefix: "",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
nf = &notFilter{
f: &prefixFilter{
fieldName: "foo",
prefix: "",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", []int{5})
// mismatch
nf = &notFilter{
f: &phraseFilter{
fieldName: "non-existing-field",
phrase: "",
},
}
testFilterMatchForColumns(t, columns, nf, "foo", nil)
}
func TestTimeFilter(t *testing.T) {
timestamps := []int64{
1,

22
lib/logstorage/parser.go
View file

@ -242,17 +242,17 @@ func parseFilter(lex *lexer) (filter, error) {
if !lex.mustNextToken() || lex.isKeyword("|") {
return nil, fmt.Errorf("missing query")
}
fo, err := parseOrFilter(lex, "")
fo, err := parseFilterOr(lex, "")
if err != nil {
return nil, err
}
return fo, nil
}
func parseOrFilter(lex *lexer, fieldName string) (filter, error) {
func parseFilterOr(lex *lexer, fieldName string) (filter, error) {
var filters []filter
for {
f, err := parseAndFilter(lex, fieldName)
f, err := parseFilterAnd(lex, fieldName)
if err != nil {
return nil, err
}
@ -274,7 +274,7 @@ func parseOrFilter(lex *lexer, fieldName string) (filter, error) {
}
}
func parseAndFilter(lex *lexer, fieldName string) (filter, error) {
func parseFilterAnd(lex *lexer, fieldName string) (filter, error) {
var filters []filter
for {
f, err := parseGenericFilter(lex, fieldName)
@ -320,7 +320,7 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
}
return parseParensFilter(lex, fieldName)
case lex.isKeyword("not", "!"):
return parseNotFilter(lex, fieldName)
return parseFilterNot(lex, fieldName)
case lex.isKeyword("exact"):
return parseExactFilter(lex, fieldName)
case lex.isKeyword("i"):
@ -444,7 +444,7 @@ func parseParensFilter(lex *lexer, fieldName string) (filter, error) {
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing filter after '('")
}
f, err := parseOrFilter(lex, fieldName)
f, err := parseFilterOr(lex, fieldName)
if err != nil {
return nil, err
}
@ -455,7 +455,7 @@ func parseParensFilter(lex *lexer, fieldName string) (filter, error) {
return f, nil
}
func parseNotFilter(lex *lexer, fieldName string) (filter, error) {
func parseFilterNot(lex *lexer, fieldName string) (filter, error) {
notKeyword := lex.token
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing filters after '%s'", notKeyword)
@ -464,14 +464,14 @@ func parseNotFilter(lex *lexer, fieldName string) (filter, error) {
if err != nil {
return nil, err
}
nf, ok := f.(*notFilter)
fn, ok := f.(*filterNot)
if ok {
return nf.f, nil
return fn.f, nil
}
nf = &notFilter{
fn = &filterNot{
f: f,
}
return nf, nil
return fn, nil
}
func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {

6
lib/logstorage/storage_search.go
View file

@ -343,7 +343,7 @@ func hasStreamFilters(f filter) bool {
return hasStreamFiltersInList(t.filters)
case *filterOr:
return hasStreamFiltersInList(t.filters)
case *notFilter:
case *filterNot:
return hasStreamFilters(t.f)
case *streamFilter:
return true
@ -371,8 +371,8 @@ func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter {
return &filterOr{
filters: initStreamFiltersList(tenantIDs, idb, t.filters),
}
case *notFilter:
return &notFilter{
case *filterNot:
return &filterNot{
f: initStreamFilters(tenantIDs, idb, t.f),
}
case *streamFilter: