github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 04:07:51 +02:00
parent 78938aaaae
commit 70baaace98
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
5 changed files with 267 additions and 267 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
lib/logstorage/filter.go
View file

@ -76,18 +76,18 @@ func (fo *filterOr) apply(bs *blockSearch, bm *bitmap) {
putBitmap(bmResult) putBitmap(bmResult)
} }
// andFilter contains filters joined by AND opertor. // filterAnd contains filters joined by AND opertor.
// //
// It is expressed as `f1 AND f2 ... AND fN` in LogsQL. // It is expressed as `f1 AND f2 ... AND fN` in LogsQL.
type andFilter struct { type filterAnd struct {
filters []filter filters []filter
msgTokensOnce sync.Once msgTokensOnce sync.Once
msgTokens []string msgTokens []string
} }
func (af *andFilter) String() string { func (fa *filterAnd) String() string {
filters := af.filters filters := fa.filters
a := make([]string, len(filters)) a := make([]string, len(filters))
for i, f := range filters { for i, f := range filters {
s := f.String() s := f.String()
@ -100,9 +100,9 @@ func (af *andFilter) String() string {
return strings.Join(a, " ") return strings.Join(a, " ")
} }
func (af *andFilter) apply(bs *blockSearch, bm *bitmap) { func (fa *filterAnd) apply(bs *blockSearch, bm *bitmap) {
if tokens := af.getMsgTokens(); len(tokens) > 0 { if tokens := fa.getMsgTokens(); len(tokens) > 0 {
// Verify whether af tokens for the _msg field match bloom filter. // Verify whether fa tokens for the _msg field match bloom filter.
ch := bs.csh.getColumnHeader("_msg") ch := bs.csh.getColumnHeader("_msg")
if ch == nil { if ch == nil {
// Fast path - there is no _msg field in the block. // Fast path - there is no _msg field in the block.
@ -110,14 +110,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) {
return return
} }
if !matchBloomFilterAllTokens(bs, ch, tokens) { if !matchBloomFilterAllTokens(bs, ch, tokens) {
// Fast path - af tokens for the _msg field do not match bloom filter. // Fast path - fa tokens for the _msg field do not match bloom filter.
bm.resetBits() bm.resetBits()
return return
} }
} }
// Slow path - verify every filter separately. // Slow path - verify every filter separately.
for _, f := range af.filters { for _, f := range fa.filters {
f.apply(bs, bm) f.apply(bs, bm)
if bm.isZero() { if bm.isZero() {
// Shortcut - there is no need in applying the remaining filters, // Shortcut - there is no need in applying the remaining filters,
@ -127,14 +127,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) {
} }
} }
func (af *andFilter) getMsgTokens() []string { func (fa *filterAnd) getMsgTokens() []string {
af.msgTokensOnce.Do(af.initMsgTokens) fa.msgTokensOnce.Do(fa.initMsgTokens)
return af.msgTokens return fa.msgTokens
} }
func (af *andFilter) initMsgTokens() { func (fa *filterAnd) initMsgTokens() {
var a []string var a []string
for _, f := range af.filters { for _, f := range fa.filters {
switch t := f.(type) { switch t := f.(type) {
case *phraseFilter: case *phraseFilter:
if isMsgFieldName(t.fieldName) { if isMsgFieldName(t.fieldName) {
@ -158,7 +158,7 @@ func (af *andFilter) initMsgTokens() {
} }
} }
} }
af.msgTokens = a fa.msgTokens = a
} }
// notFilter negates the filter. // notFilter negates the filter.
@ -171,7 +171,7 @@ type notFilter struct {
func (fn *notFilter) String() string { func (fn *notFilter) String() string {
s := fn.f.String() s := fn.f.String()
switch fn.f.(type) { switch fn.f.(type) {
case *andFilter, *filterOr: case *filterAnd, *filterOr:
s = "(" + s + ")" s = "(" + s + ")"
} }
return "!" + s return "!" + s
@ -557,25 +557,25 @@ type inFilter struct {
timestampISO8601Values map[string]struct{} timestampISO8601Values map[string]struct{}
} }
func (af *inFilter) String() string { func (fi *inFilter) String() string {
values := af.values values := fi.values
a := make([]string, len(values)) a := make([]string, len(values))
for i, value := range values { for i, value := range values {
a[i] = quoteTokenIfNeeded(value) a[i] = quoteTokenIfNeeded(value)
} }
return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(af.fieldName), strings.Join(a, ",")) return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ","))
} }
func (af *inFilter) getTokenSets() [][]string { func (fi *inFilter) getTokenSets() [][]string {
af.tokenSetsOnce.Do(af.initTokenSets) fi.tokenSetsOnce.Do(fi.initTokenSets)
return af.tokenSets return fi.tokenSets
} }
// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter. // It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter.
const maxTokenSetsToInit = 1000 const maxTokenSetsToInit = 1000
func (af *inFilter) initTokenSets() { func (fi *inFilter) initTokenSets() {
values := af.values values := fi.values
tokenSetsLen := len(values) tokenSetsLen := len(values)
if tokenSetsLen > maxTokenSetsToInit { if tokenSetsLen > maxTokenSetsToInit {
tokenSetsLen = maxTokenSetsToInit tokenSetsLen = maxTokenSetsToInit
@ -588,30 +588,30 @@ func (af *inFilter) initTokenSets() {
break break
} }
} }
af.tokenSets = tokenSets fi.tokenSets = tokenSets
} }
func (af *inFilter) getStringValues() map[string]struct{} { func (fi *inFilter) getStringValues() map[string]struct{} {
af.stringValuesOnce.Do(af.initStringValues) fi.stringValuesOnce.Do(fi.initStringValues)
return af.stringValues return fi.stringValues
} }
func (af *inFilter) initStringValues() { func (fi *inFilter) initStringValues() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
for _, v := range values { for _, v := range values {
m[v] = struct{}{} m[v] = struct{}{}
} }
af.stringValues = m fi.stringValues = m
} }
func (af *inFilter) getUint8Values() map[string]struct{} { func (fi *inFilter) getUint8Values() map[string]struct{} {
af.uint8ValuesOnce.Do(af.initUint8Values) fi.uint8ValuesOnce.Do(fi.initUint8Values)
return af.uint8Values return fi.uint8Values
} }
func (af *inFilter) initUint8Values() { func (fi *inFilter) initUint8Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*1) buf := make([]byte, 0, len(values)*1)
for _, v := range values { for _, v := range values {
@ -624,16 +624,16 @@ func (af *inFilter) initUint8Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.uint8Values = m fi.uint8Values = m
} }
func (af *inFilter) getUint16Values() map[string]struct{} { func (fi *inFilter) getUint16Values() map[string]struct{} {
af.uint16ValuesOnce.Do(af.initUint16Values) fi.uint16ValuesOnce.Do(fi.initUint16Values)
return af.uint16Values return fi.uint16Values
} }
func (af *inFilter) initUint16Values() { func (fi *inFilter) initUint16Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*2) buf := make([]byte, 0, len(values)*2)
for _, v := range values { for _, v := range values {
@ -646,16 +646,16 @@ func (af *inFilter) initUint16Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.uint16Values = m fi.uint16Values = m
} }
func (af *inFilter) getUint32Values() map[string]struct{} { func (fi *inFilter) getUint32Values() map[string]struct{} {
af.uint32ValuesOnce.Do(af.initUint32Values) fi.uint32ValuesOnce.Do(fi.initUint32Values)
return af.uint32Values return fi.uint32Values
} }
func (af *inFilter) initUint32Values() { func (fi *inFilter) initUint32Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4) buf := make([]byte, 0, len(values)*4)
for _, v := range values { for _, v := range values {
@ -668,16 +668,16 @@ func (af *inFilter) initUint32Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.uint32Values = m fi.uint32Values = m
} }
func (af *inFilter) getUint64Values() map[string]struct{} { func (fi *inFilter) getUint64Values() map[string]struct{} {
af.uint64ValuesOnce.Do(af.initUint64Values) fi.uint64ValuesOnce.Do(fi.initUint64Values)
return af.uint64Values return fi.uint64Values
} }
func (af *inFilter) initUint64Values() { func (fi *inFilter) initUint64Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8) buf := make([]byte, 0, len(values)*8)
for _, v := range values { for _, v := range values {
@ -690,16 +690,16 @@ func (af *inFilter) initUint64Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.uint64Values = m fi.uint64Values = m
} }
func (af *inFilter) getFloat64Values() map[string]struct{} { func (fi *inFilter) getFloat64Values() map[string]struct{} {
af.float64ValuesOnce.Do(af.initFloat64Values) fi.float64ValuesOnce.Do(fi.initFloat64Values)
return af.float64Values return fi.float64Values
} }
func (af *inFilter) initFloat64Values() { func (fi *inFilter) initFloat64Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8) buf := make([]byte, 0, len(values)*8)
for _, v := range values { for _, v := range values {
@ -713,16 +713,16 @@ func (af *inFilter) initFloat64Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.float64Values = m fi.float64Values = m
} }
func (af *inFilter) getIPv4Values() map[string]struct{} { func (fi *inFilter) getIPv4Values() map[string]struct{} {
af.ipv4ValuesOnce.Do(af.initIPv4Values) fi.ipv4ValuesOnce.Do(fi.initIPv4Values)
return af.ipv4Values return fi.ipv4Values
} }
func (af *inFilter) initIPv4Values() { func (fi *inFilter) initIPv4Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4) buf := make([]byte, 0, len(values)*4)
for _, v := range values { for _, v := range values {
@ -735,16 +735,16 @@ func (af *inFilter) initIPv4Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.ipv4Values = m fi.ipv4Values = m
} }
func (af *inFilter) getTimestampISO8601Values() map[string]struct{} { func (fi *inFilter) getTimestampISO8601Values() map[string]struct{} {
af.timestampISO8601ValuesOnce.Do(af.initTimestampISO8601Values) fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values)
return af.timestampISO8601Values return fi.timestampISO8601Values
} }
func (af *inFilter) initTimestampISO8601Values() { func (fi *inFilter) initTimestampISO8601Values() {
values := af.values values := fi.values
m := make(map[string]struct{}, len(values)) m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8) buf := make([]byte, 0, len(values)*8)
for _, v := range values { for _, v := range values {
@ -757,20 +757,20 @@ func (af *inFilter) initTimestampISO8601Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:]) s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{} m[s] = struct{}{}
} }
af.timestampISO8601Values = m fi.timestampISO8601Values = m
} }
func (af *inFilter) apply(bs *blockSearch, bm *bitmap) { func (fi *inFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := af.fieldName fieldName := fi.fieldName
if len(af.values) == 0 { if len(fi.values) == 0 {
bm.resetBits() bm.resetBits()
return return
} }
v := bs.csh.getConstColumnValue(fieldName) v := bs.csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
stringValues := af.getStringValues() stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok { if _, ok := stringValues[v]; !ok {
bm.resetBits() bm.resetBits()
} }
@ -782,42 +782,42 @@ func (af *inFilter) apply(bs *blockSearch, bm *bitmap) {
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty phrase. // It matches anything only for empty phrase.
stringValues := af.getStringValues() stringValues := fi.getStringValues()
if _, ok := stringValues[""]; !ok { if _, ok := stringValues[""]; !ok {
bm.resetBits() bm.resetBits()
} }
return return
} }
tokenSets := af.getTokenSets() tokenSets := fi.getTokenSets()
switch ch.valueType { switch ch.valueType {
case valueTypeString: case valueTypeString:
stringValues := af.getStringValues() stringValues := fi.getStringValues()
matchAnyValue(bs, ch, bm, stringValues, tokenSets) matchAnyValue(bs, ch, bm, stringValues, tokenSets)
case valueTypeDict: case valueTypeDict:
stringValues := af.getStringValues() stringValues := fi.getStringValues()
matchValuesDictByAnyValue(bs, ch, bm, stringValues) matchValuesDictByAnyValue(bs, ch, bm, stringValues)
case valueTypeUint8: case valueTypeUint8:
binValues := af.getUint8Values() binValues := fi.getUint8Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint16: case valueTypeUint16:
binValues := af.getUint16Values() binValues := fi.getUint16Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint32: case valueTypeUint32:
binValues := af.getUint32Values() binValues := fi.getUint32Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint64: case valueTypeUint64:
binValues := af.getUint64Values() binValues := fi.getUint64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeFloat64: case valueTypeFloat64:
binValues := af.getFloat64Values() binValues := fi.getFloat64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeIPv4: case valueTypeIPv4:
binValues := af.getIPv4Values() binValues := fi.getIPv4Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeTimestampISO8601: case valueTypeTimestampISO8601:
binValues := af.getTimestampISO8601Values() binValues := fi.getTimestampISO8601Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets) matchAnyValue(bs, ch, bm, binValues, tokenSets)
default: default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)

332
lib/logstorage/filter_test.go
View file

@ -356,7 +356,7 @@ func TestComplexFilters(t *testing.T) {
} }
// (foobar AND NOT baz AND (abcdef OR xyz)) // (foobar AND NOT baz AND (abcdef OR xyz))
f := &andFilter{ f := &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -385,7 +385,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", []int{6}) testFilterMatchForColumns(t, columns, f, "foo", []int{6})
// (foobaz AND NOT baz AND (abcdef OR xyz)) // (foobaz AND NOT baz AND (abcdef OR xyz))
f = &andFilter{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -414,7 +414,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil) testFilterMatchForColumns(t, columns, f, "foo", nil)
// (foobaz AND NOT baz AND (abcdef OR xyz OR a)) // (foobaz AND NOT baz AND (abcdef OR xyz OR a))
f = &andFilter{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -447,7 +447,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", []int{1, 6}) testFilterMatchForColumns(t, columns, f, "foo", []int{1, 6})
// (foobaz AND NOT qwert AND (abcdef OR xyz OR a)) // (foobaz AND NOT qwert AND (abcdef OR xyz OR a))
f = &andFilter{ f = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -625,7 +625,7 @@ func TestAndFilter(t *testing.T) {
} }
// non-empty intersection // non-empty intersection
af := &andFilter{ fa := &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -637,10 +637,10 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6}) testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
// reverse non-empty intersection // reverse non-empty intersection
af = &andFilter{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&prefixFilter{ &prefixFilter{
fieldName: "foo", fieldName: "foo",
@ -652,10 +652,10 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6}) testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
// the first filter mismatch // the first filter mismatch
af = &andFilter{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&prefixFilter{ &prefixFilter{
fieldName: "foo", fieldName: "foo",
@ -667,10 +667,10 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fa, "foo", nil)
// the last filter mismatch // the last filter mismatch
af = &andFilter{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -682,10 +682,10 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fa, "foo", nil)
// empty intersection // empty intersection
af = &andFilter{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&phraseFilter{ &phraseFilter{
fieldName: "foo", fieldName: "foo",
@ -697,10 +697,10 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fa, "foo", nil)
// reverse empty intersection // reverse empty intersection
af = &andFilter{ fa = &filterAnd{
filters: []filter{ filters: []filter{
&prefixFilter{ &prefixFilter{
fieldName: "foo", fieldName: "foo",
@ -712,7 +712,7 @@ func TestAndFilter(t *testing.T) {
}, },
}, },
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fa, "foo", nil)
} }
func TestNotFilter(t *testing.T) { func TestNotFilter(t *testing.T) {
@ -2845,54 +2845,54 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"abc def", "abc", "foobar"}, values: []string{"abc def", "abc", "foobar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{ fi = &inFilter{
fieldName: "other column", fieldName: "other column",
values: []string{"asdfdsf", ""}, values: []string{"asdfdsf", ""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{"", "foo"}, values: []string{"", "foo"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"abc", "def"}, values: []string{"abc", "def"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"", "abc"}, values: []string{"", "abc"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "other column", fieldName: "other column",
values: []string{"sd"}, values: []string{"sd"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
values: []string{"abc", "def"}, values: []string{"abc", "def"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("const-column", func(t *testing.T) { t.Run("const-column", func(t *testing.T) {
@ -2908,42 +2908,42 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"aaaa", "abc def", "foobar"}, values: []string{"aaaa", "abc def", "foobar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{"", "abc"}, values: []string{"", "abc"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"abc def ", "foobar"}, values: []string{"abc def ", "foobar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
values: []string{"x"}, values: []string{"x"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("dict", func(t *testing.T) { t.Run("dict", func(t *testing.T) {
@ -2963,42 +2963,42 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"foobar", "aaaa", "abc", "baz"}, values: []string{"foobar", "aaaa", "abc", "baz"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bbbb", "", "aaaa"}, values: []string{"bbbb", "", "aaaa"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar", "aaaa"}, values: []string{"bar", "aaaa"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing column", fieldName: "non-existing column",
values: []string{"foobar", "aaaa"}, values: []string{"foobar", "aaaa"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("strings", func(t *testing.T) { t.Run("strings", func(t *testing.T) {
@ -3021,36 +3021,36 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"a foobar", "aa abc a"}, values: []string{"a foobar", "aa abc a"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"aa a"}, values: []string{"aa a"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("uint8", func(t *testing.T) { t.Run("uint8", func(t *testing.T) {
@ -3074,48 +3074,48 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12", "32"}, values: []string{"12", "32"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"0"}, values: []string{"0"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"33"}, values: []string{"33"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"1234"}, values: []string{"1234"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("uint16", func(t *testing.T) { t.Run("uint16", func(t *testing.T) {
@ -3139,48 +3139,48 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12", "32"}, values: []string{"12", "32"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"0"}, values: []string{"0"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"33"}, values: []string{"33"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"123456"}, values: []string{"123456"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("uint32", func(t *testing.T) { t.Run("uint32", func(t *testing.T) {
@ -3204,48 +3204,48 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12", "32"}, values: []string{"12", "32"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"0"}, values: []string{"0"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"33"}, values: []string{"33"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12345678901"}, values: []string{"12345678901"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("uint64", func(t *testing.T) { t.Run("uint64", func(t *testing.T) {
@ -3269,42 +3269,42 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12", "32"}, values: []string{"12", "32"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5}) testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"0"}, values: []string{"0"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4}) testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"33"}, values: []string{"33"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("float64", func(t *testing.T) { t.Run("float64", func(t *testing.T) {
@ -3326,66 +3326,66 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"1234", "1", "foobar", "123211"}, values: []string{"1234", "1", "foobar", "123211"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 5}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"1234.5678901"}, values: []string{"1234.5678901"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{4}) testFilterMatchForColumns(t, columns, fi, "foo", []int{4})
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"-65536"}, values: []string{"-65536"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{3}) testFilterMatchForColumns(t, columns, fi, "foo", []int{3})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"65536"}, values: []string{"65536"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"123"}, values: []string{"123"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"12345678901234567890"}, values: []string{"12345678901234567890"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("ipv4", func(t *testing.T) { t.Run("ipv4", func(t *testing.T) {
@ -3410,48 +3410,48 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"}, values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 4, 5, 6, 7}) testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch // mismatch
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"5"}, values: []string{"5"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "foo", fieldName: "foo",
values: []string{"255.255.255.255"}, values: []string{"255.255.255.255"},
} }
testFilterMatchForColumns(t, columns, af, "foo", nil) testFilterMatchForColumns(t, columns, fi, "foo", nil)
}) })
t.Run("timestamp-iso8601", func(t *testing.T) { t.Run("timestamp-iso8601", func(t *testing.T) {
@ -3473,42 +3473,42 @@ func TestInFilter(t *testing.T) {
} }
// match // match
af := &inFilter{ fi := &inFilter{
fieldName: "_msg", fieldName: "_msg",
values: []string{"2006-01-02T15:04:05.005Z", "foobar"}, values: []string{"2006-01-02T15:04:05.005Z", "foobar"},
} }
testFilterMatchForColumns(t, columns, af, "_msg", []int{4}) testFilterMatchForColumns(t, columns, fi, "_msg", []int{4})
af = &inFilter{ fi = &inFilter{
fieldName: "non-existing-column", fieldName: "non-existing-column",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8}) testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch // mimatch
af = &inFilter{ fi = &inFilter{
fieldName: "_msg", fieldName: "_msg",
values: []string{"bar"}, values: []string{"bar"},
} }
testFilterMatchForColumns(t, columns, af, "_msg", nil) testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "_msg", fieldName: "_msg",
values: []string{}, values: []string{},
} }
testFilterMatchForColumns(t, columns, af, "_msg", nil) testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "_msg", fieldName: "_msg",
values: []string{""}, values: []string{""},
} }
testFilterMatchForColumns(t, columns, af, "_msg", nil) testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{ fi = &inFilter{
fieldName: "_msg", fieldName: "_msg",
values: []string{"2006-04-02T15:04:05.005Z"}, values: []string{"2006-04-02T15:04:05.005Z"},
} }
testFilterMatchForColumns(t, columns, af, "_msg", nil) testFilterMatchForColumns(t, columns, fi, "_msg", nil)
}) })
} }

8
lib/logstorage/parser.go
View file

@ -242,11 +242,11 @@ func parseFilter(lex *lexer) (filter, error) {
if !lex.mustNextToken() || lex.isKeyword("|") { if !lex.mustNextToken() || lex.isKeyword("|") {
return nil, fmt.Errorf("missing query") return nil, fmt.Errorf("missing query")
} }
af, err := parseOrFilter(lex, "") fo, err := parseOrFilter(lex, "")
if err != nil { if err != nil {
return nil, err return nil, err
} }
return af, nil return fo, nil
} }
func parseOrFilter(lex *lexer, fieldName string) (filter, error) { func parseOrFilter(lex *lexer, fieldName string) (filter, error) {
@ -287,10 +287,10 @@ func parseAndFilter(lex *lexer, fieldName string) (filter, error) {
if len(filters) == 1 { if len(filters) == 1 {
return filters[0], nil return filters[0], nil
} }
af := &andFilter{ fa := &filterAnd{
filters: filters, filters: filters,
} }
return af, nil return fa, nil
case lex.isKeyword("and"): case lex.isKeyword("and"):
if !lex.mustNextToken() { if !lex.mustNextToken() {
return nil, fmt.Errorf("missing filter after 'and'") return nil, fmt.Errorf("missing filter after 'and'")

14
lib/logstorage/storage_search.go
View file

@ -339,7 +339,7 @@ func (pt *partition) search(tf *timeFilter, sf *StreamFilter, f filter, so *gene
func hasStreamFilters(f filter) bool { func hasStreamFilters(f filter) bool {
switch t := f.(type) { switch t := f.(type) {
case *andFilter: case *filterAnd:
return hasStreamFiltersInList(t.filters) return hasStreamFiltersInList(t.filters)
case *filterOr: case *filterOr:
return hasStreamFiltersInList(t.filters) return hasStreamFiltersInList(t.filters)
@ -363,8 +363,8 @@ func hasStreamFiltersInList(filters []filter) bool {
func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter { func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter {
switch t := f.(type) { switch t := f.(type) {
case *andFilter: case *filterAnd:
return &andFilter{ return &filterAnd{
filters: initStreamFiltersList(tenantIDs, idb, t.filters), filters: initStreamFiltersList(tenantIDs, idb, t.filters),
} }
case *filterOr: case *filterOr:
@ -695,16 +695,16 @@ func appendPartsInTimeRange(dst, src []*partWrapper, minTimestamp, maxTimestamp
func getCommonStreamFilter(f filter) (*StreamFilter, filter) { func getCommonStreamFilter(f filter) (*StreamFilter, filter) {
switch t := f.(type) { switch t := f.(type) {
case *andFilter: case *filterAnd:
filters := t.filters filters := t.filters
for i, filter := range filters { for i, filter := range filters {
sf, ok := filter.(*streamFilter) sf, ok := filter.(*streamFilter)
if ok && !sf.f.isEmpty() { if ok && !sf.f.isEmpty() {
// Remove sf from filters, since it doesn't filter out anything then. // Remove sf from filters, since it doesn't filter out anything then.
af := &andFilter{ fa := &filterAnd{
filters: append(filters[:i:i], filters[i+1:]...), filters: append(filters[:i:i], filters[i+1:]...),
} }
return sf.f, af return sf.f, fa
} }
} }
case *streamFilter: case *streamFilter:
@ -715,7 +715,7 @@ func getCommonStreamFilter(f filter) (*StreamFilter, filter) {
func getCommonTimeFilter(f filter) (*timeFilter, filter) { func getCommonTimeFilter(f filter) (*timeFilter, filter) {
switch t := f.(type) { switch t := f.(type) {
case *andFilter: case *filterAnd:
for _, filter := range t.filters { for _, filter := range t.filters {
tf, ok := filter.(*timeFilter) tf, ok := filter.(*timeFilter)
if ok { if ok {

4
lib/logstorage/storage_search_test.go
View file

@ -388,7 +388,7 @@ func TestStorageSearch(t *testing.T) {
f: sf, f: sf,
}) })
} }
return &andFilter{ return &filterAnd{
filters: filters, filters: filters,
} }
} }
@ -578,7 +578,7 @@ func TestStorageSearch(t *testing.T) {
minTimestamp := baseTimestamp minTimestamp := baseTimestamp
maxTimestamp := baseTimestamp + rowsPerBlock*1e9 + blocksPerStream maxTimestamp := baseTimestamp + rowsPerBlock*1e9 + blocksPerStream
f := getBaseFilter(minTimestamp, maxTimestamp, sf) f := getBaseFilter(minTimestamp, maxTimestamp, sf)
f = &andFilter{ f = &filterAnd{
filters: []filter{ filters: []filter{
f, f,
&regexpFilter{ &regexpFilter{