github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-04-29 04:07:51 +02:00
parent 78938aaaae
commit 70baaace98
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
5 changed files with 267 additions and 267 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
lib/logstorage/filter.go
View file

@ -76,18 +76,18 @@ func (fo *filterOr) apply(bs *blockSearch, bm *bitmap) {
putBitmap(bmResult)
}
// andFilter contains filters joined by AND opertor.
// filterAnd contains filters joined by AND opertor.
//
// It is expressed as `f1 AND f2 ... AND fN` in LogsQL.
type andFilter struct {
type filterAnd struct {
filters []filter
msgTokensOnce sync.Once
msgTokens []string
}
func (af *andFilter) String() string {
filters := af.filters
func (fa *filterAnd) String() string {
filters := fa.filters
a := make([]string, len(filters))
for i, f := range filters {
s := f.String()
@ -100,9 +100,9 @@ func (af *andFilter) String() string {
return strings.Join(a, " ")
}
func (af *andFilter) apply(bs *blockSearch, bm *bitmap) {
if tokens := af.getMsgTokens(); len(tokens) > 0 {
// Verify whether af tokens for the _msg field match bloom filter.
func (fa *filterAnd) apply(bs *blockSearch, bm *bitmap) {
if tokens := fa.getMsgTokens(); len(tokens) > 0 {
// Verify whether fa tokens for the _msg field match bloom filter.
ch := bs.csh.getColumnHeader("_msg")
if ch == nil {
// Fast path - there is no _msg field in the block.
@ -110,14 +110,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) {
return
}
if !matchBloomFilterAllTokens(bs, ch, tokens) {
// Fast path - af tokens for the _msg field do not match bloom filter.
// Fast path - fa tokens for the _msg field do not match bloom filter.
bm.resetBits()
return
}
}
// Slow path - verify every filter separately.
for _, f := range af.filters {
for _, f := range fa.filters {
f.apply(bs, bm)
if bm.isZero() {
// Shortcut - there is no need in applying the remaining filters,
@ -127,14 +127,14 @@ func (af *andFilter) apply(bs *blockSearch, bm *bitmap) {
}
}
func (af *andFilter) getMsgTokens() []string {
af.msgTokensOnce.Do(af.initMsgTokens)
return af.msgTokens
func (fa *filterAnd) getMsgTokens() []string {
fa.msgTokensOnce.Do(fa.initMsgTokens)
return fa.msgTokens
}
func (af *andFilter) initMsgTokens() {
func (fa *filterAnd) initMsgTokens() {
var a []string
for _, f := range af.filters {
for _, f := range fa.filters {
switch t := f.(type) {
case *phraseFilter:
if isMsgFieldName(t.fieldName) {
@ -158,7 +158,7 @@ func (af *andFilter) initMsgTokens() {
}
}
}
af.msgTokens = a
fa.msgTokens = a
}
// notFilter negates the filter.
@ -171,7 +171,7 @@ type notFilter struct {
func (fn *notFilter) String() string {
s := fn.f.String()
switch fn.f.(type) {
case *andFilter, *filterOr:
case *filterAnd, *filterOr:
s = "(" + s + ")"
}
return "!" + s
@ -557,25 +557,25 @@ type inFilter struct {
timestampISO8601Values map[string]struct{}
}
func (af *inFilter) String() string {
values := af.values
func (fi *inFilter) String() string {
values := fi.values
a := make([]string, len(values))
for i, value := range values {
a[i] = quoteTokenIfNeeded(value)
}
return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(af.fieldName), strings.Join(a, ","))
return fmt.Sprintf("%sin(%s)", quoteFieldNameIfNeeded(fi.fieldName), strings.Join(a, ","))
}
func (af *inFilter) getTokenSets() [][]string {
af.tokenSetsOnce.Do(af.initTokenSets)
return af.tokenSets
func (fi *inFilter) getTokenSets() [][]string {
fi.tokenSetsOnce.Do(fi.initTokenSets)
return fi.tokenSets
}
// It is faster to match every row in the block instead of checking too big number of tokenSets against bloom filter.
const maxTokenSetsToInit = 1000
func (af *inFilter) initTokenSets() {
values := af.values
func (fi *inFilter) initTokenSets() {
values := fi.values
tokenSetsLen := len(values)
if tokenSetsLen > maxTokenSetsToInit {
tokenSetsLen = maxTokenSetsToInit
@ -588,30 +588,30 @@ func (af *inFilter) initTokenSets() {
break
}
}
af.tokenSets = tokenSets
fi.tokenSets = tokenSets
}
func (af *inFilter) getStringValues() map[string]struct{} {
af.stringValuesOnce.Do(af.initStringValues)
return af.stringValues
func (fi *inFilter) getStringValues() map[string]struct{} {
fi.stringValuesOnce.Do(fi.initStringValues)
return fi.stringValues
}
func (af *inFilter) initStringValues() {
values := af.values
func (fi *inFilter) initStringValues() {
values := fi.values
m := make(map[string]struct{}, len(values))
for _, v := range values {
m[v] = struct{}{}
}
af.stringValues = m
fi.stringValues = m
}
func (af *inFilter) getUint8Values() map[string]struct{} {
af.uint8ValuesOnce.Do(af.initUint8Values)
return af.uint8Values
func (fi *inFilter) getUint8Values() map[string]struct{} {
fi.uint8ValuesOnce.Do(fi.initUint8Values)
return fi.uint8Values
}
func (af *inFilter) initUint8Values() {
values := af.values
func (fi *inFilter) initUint8Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*1)
for _, v := range values {
@ -624,16 +624,16 @@ func (af *inFilter) initUint8Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.uint8Values = m
fi.uint8Values = m
}
func (af *inFilter) getUint16Values() map[string]struct{} {
af.uint16ValuesOnce.Do(af.initUint16Values)
return af.uint16Values
func (fi *inFilter) getUint16Values() map[string]struct{} {
fi.uint16ValuesOnce.Do(fi.initUint16Values)
return fi.uint16Values
}
func (af *inFilter) initUint16Values() {
values := af.values
func (fi *inFilter) initUint16Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*2)
for _, v := range values {
@ -646,16 +646,16 @@ func (af *inFilter) initUint16Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.uint16Values = m
fi.uint16Values = m
}
func (af *inFilter) getUint32Values() map[string]struct{} {
af.uint32ValuesOnce.Do(af.initUint32Values)
return af.uint32Values
func (fi *inFilter) getUint32Values() map[string]struct{} {
fi.uint32ValuesOnce.Do(fi.initUint32Values)
return fi.uint32Values
}
func (af *inFilter) initUint32Values() {
values := af.values
func (fi *inFilter) initUint32Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
@ -668,16 +668,16 @@ func (af *inFilter) initUint32Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.uint32Values = m
fi.uint32Values = m
}
func (af *inFilter) getUint64Values() map[string]struct{} {
af.uint64ValuesOnce.Do(af.initUint64Values)
return af.uint64Values
func (fi *inFilter) getUint64Values() map[string]struct{} {
fi.uint64ValuesOnce.Do(fi.initUint64Values)
return fi.uint64Values
}
func (af *inFilter) initUint64Values() {
values := af.values
func (fi *inFilter) initUint64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
@ -690,16 +690,16 @@ func (af *inFilter) initUint64Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.uint64Values = m
fi.uint64Values = m
}
func (af *inFilter) getFloat64Values() map[string]struct{} {
af.float64ValuesOnce.Do(af.initFloat64Values)
return af.float64Values
func (fi *inFilter) getFloat64Values() map[string]struct{} {
fi.float64ValuesOnce.Do(fi.initFloat64Values)
return fi.float64Values
}
func (af *inFilter) initFloat64Values() {
values := af.values
func (fi *inFilter) initFloat64Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
@ -713,16 +713,16 @@ func (af *inFilter) initFloat64Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.float64Values = m
fi.float64Values = m
}
func (af *inFilter) getIPv4Values() map[string]struct{} {
af.ipv4ValuesOnce.Do(af.initIPv4Values)
return af.ipv4Values
func (fi *inFilter) getIPv4Values() map[string]struct{} {
fi.ipv4ValuesOnce.Do(fi.initIPv4Values)
return fi.ipv4Values
}
func (af *inFilter) initIPv4Values() {
values := af.values
func (fi *inFilter) initIPv4Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*4)
for _, v := range values {
@ -735,16 +735,16 @@ func (af *inFilter) initIPv4Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.ipv4Values = m
fi.ipv4Values = m
}
func (af *inFilter) getTimestampISO8601Values() map[string]struct{} {
af.timestampISO8601ValuesOnce.Do(af.initTimestampISO8601Values)
return af.timestampISO8601Values
func (fi *inFilter) getTimestampISO8601Values() map[string]struct{} {
fi.timestampISO8601ValuesOnce.Do(fi.initTimestampISO8601Values)
return fi.timestampISO8601Values
}
func (af *inFilter) initTimestampISO8601Values() {
values := af.values
func (fi *inFilter) initTimestampISO8601Values() {
values := fi.values
m := make(map[string]struct{}, len(values))
buf := make([]byte, 0, len(values)*8)
for _, v := range values {
@ -757,20 +757,20 @@ func (af *inFilter) initTimestampISO8601Values() {
s := bytesutil.ToUnsafeString(buf[bufLen:])
m[s] = struct{}{}
}
af.timestampISO8601Values = m
fi.timestampISO8601Values = m
}
func (af *inFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := af.fieldName
func (fi *inFilter) apply(bs *blockSearch, bm *bitmap) {
fieldName := fi.fieldName
if len(af.values) == 0 {
if len(fi.values) == 0 {
bm.resetBits()
return
}
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
stringValues := af.getStringValues()
stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok {
bm.resetBits()
}
@ -782,42 +782,42 @@ func (af *inFilter) apply(bs *blockSearch, bm *bitmap) {
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
stringValues := af.getStringValues()
stringValues := fi.getStringValues()
if _, ok := stringValues[""]; !ok {
bm.resetBits()
}
return
}
tokenSets := af.getTokenSets()
tokenSets := fi.getTokenSets()
switch ch.valueType {
case valueTypeString:
stringValues := af.getStringValues()
stringValues := fi.getStringValues()
matchAnyValue(bs, ch, bm, stringValues, tokenSets)
case valueTypeDict:
stringValues := af.getStringValues()
stringValues := fi.getStringValues()
matchValuesDictByAnyValue(bs, ch, bm, stringValues)
case valueTypeUint8:
binValues := af.getUint8Values()
binValues := fi.getUint8Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint16:
binValues := af.getUint16Values()
binValues := fi.getUint16Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint32:
binValues := af.getUint32Values()
binValues := fi.getUint32Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeUint64:
binValues := af.getUint64Values()
binValues := fi.getUint64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeFloat64:
binValues := af.getFloat64Values()
binValues := fi.getFloat64Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeIPv4:
binValues := af.getIPv4Values()
binValues := fi.getIPv4Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
case valueTypeTimestampISO8601:
binValues := af.getTimestampISO8601Values()
binValues := fi.getTimestampISO8601Values()
matchAnyValue(bs, ch, bm, binValues, tokenSets)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)

332
lib/logstorage/filter_test.go
View file

@ -356,7 +356,7 @@ func TestComplexFilters(t *testing.T) {
}
// (foobar AND NOT baz AND (abcdef OR xyz))
f := &andFilter{
f := &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -385,7 +385,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", []int{6})
// (foobaz AND NOT baz AND (abcdef OR xyz))
f = &andFilter{
f = &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -414,7 +414,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", nil)
// (foobaz AND NOT baz AND (abcdef OR xyz OR a))
f = &andFilter{
f = &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -447,7 +447,7 @@ func TestComplexFilters(t *testing.T) {
testFilterMatchForColumns(t, columns, f, "foo", []int{1, 6})
// (foobaz AND NOT qwert AND (abcdef OR xyz OR a))
f = &andFilter{
f = &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -625,7 +625,7 @@ func TestAndFilter(t *testing.T) {
}
// non-empty intersection
af := &andFilter{
fa := &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -637,10 +637,10 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6})
testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
// reverse non-empty intersection
af = &andFilter{
fa = &filterAnd{
filters: []filter{
&prefixFilter{
fieldName: "foo",
@ -652,10 +652,10 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 6})
testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
// the first filter mismatch
af = &andFilter{
fa = &filterAnd{
filters: []filter{
&prefixFilter{
fieldName: "foo",
@ -667,10 +667,10 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fa, "foo", nil)
// the last filter mismatch
af = &andFilter{
fa = &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -682,10 +682,10 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fa, "foo", nil)
// empty intersection
af = &andFilter{
fa = &filterAnd{
filters: []filter{
&phraseFilter{
fieldName: "foo",
@ -697,10 +697,10 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fa, "foo", nil)
// reverse empty intersection
af = &andFilter{
fa = &filterAnd{
filters: []filter{
&prefixFilter{
fieldName: "foo",
@ -712,7 +712,7 @@ func TestAndFilter(t *testing.T) {
},
},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fa, "foo", nil)
}
func TestNotFilter(t *testing.T) {
@ -2845,54 +2845,54 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"abc def", "abc", "foobar"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{
fi = &inFilter{
fieldName: "other column",
values: []string{"asdfdsf", ""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{"", "foo"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "other column",
values: []string{"sd"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"abc", "def"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("const-column", func(t *testing.T) {
@ -2908,42 +2908,42 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"aaaa", "abc def", "foobar"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{"", "abc"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"abc def ", "foobar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"x"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("dict", func(t *testing.T) {
@ -2963,42 +2963,42 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"foobar", "aaaa", "abc", "baz"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bbbb", "", "aaaa"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar", "aaaa"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing column",
values: []string{"foobar", "aaaa"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("strings", func(t *testing.T) {
@ -3021,36 +3021,36 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"a foobar", "aa abc a"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 6})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 6})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"aa a"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint8", func(t *testing.T) {
@ -3074,48 +3074,48 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4})
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"1234"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint16", func(t *testing.T) {
@ -3139,48 +3139,48 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4})
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"123456"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint32", func(t *testing.T) {
@ -3204,48 +3204,48 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4})
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"12345678901"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("uint64", func(t *testing.T) {
@ -3269,42 +3269,42 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"12", "32"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{1, 2, 5})
testFilterMatchForColumns(t, columns, fi, "foo", []int{1, 2, 5})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"0"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{3, 4})
testFilterMatchForColumns(t, columns, fi, "foo", []int{3, 4})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"33"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("float64", func(t *testing.T) {
@ -3326,66 +3326,66 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"1234", "1", "foobar", "123211"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 5})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 5})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"1234.5678901"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{4})
testFilterMatchForColumns(t, columns, fi, "foo", []int{4})
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"-65536"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{3})
testFilterMatchForColumns(t, columns, fi, "foo", []int{3})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"65536"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"123"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"12345678901234567890"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("ipv4", func(t *testing.T) {
@ -3410,48 +3410,48 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "foo",
values: []string{"127.0.0.1", "24.54.1.2", "127.0.4.2"},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{2, 4, 5, 6, 7})
testFilterMatchForColumns(t, columns, fi, "foo", []int{2, 4, 5, 6, 7})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
testFilterMatchForColumns(t, columns, fi, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
// mismatch
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"5"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "foo",
values: []string{"255.255.255.255"},
}
testFilterMatchForColumns(t, columns, af, "foo", nil)
testFilterMatchForColumns(t, columns, fi, "foo", nil)
})
t.Run("timestamp-iso8601", func(t *testing.T) {
@ -3473,42 +3473,42 @@ func TestInFilter(t *testing.T) {
}
// match
af := &inFilter{
fi := &inFilter{
fieldName: "_msg",
values: []string{"2006-01-02T15:04:05.005Z", "foobar"},
}
testFilterMatchForColumns(t, columns, af, "_msg", []int{4})
testFilterMatchForColumns(t, columns, fi, "_msg", []int{4})
af = &inFilter{
fi = &inFilter{
fieldName: "non-existing-column",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
testFilterMatchForColumns(t, columns, fi, "_msg", []int{0, 1, 2, 3, 4, 5, 6, 7, 8})
// mimatch
af = &inFilter{
fi = &inFilter{
fieldName: "_msg",
values: []string{"bar"},
}
testFilterMatchForColumns(t, columns, af, "_msg", nil)
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "_msg",
values: []string{},
}
testFilterMatchForColumns(t, columns, af, "_msg", nil)
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "_msg",
values: []string{""},
}
testFilterMatchForColumns(t, columns, af, "_msg", nil)
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
af = &inFilter{
fi = &inFilter{
fieldName: "_msg",
values: []string{"2006-04-02T15:04:05.005Z"},
}
testFilterMatchForColumns(t, columns, af, "_msg", nil)
testFilterMatchForColumns(t, columns, fi, "_msg", nil)
})
}

8
lib/logstorage/parser.go
View file

@ -242,11 +242,11 @@ func parseFilter(lex *lexer) (filter, error) {
if !lex.mustNextToken() || lex.isKeyword("|") {
return nil, fmt.Errorf("missing query")
}
af, err := parseOrFilter(lex, "")
fo, err := parseOrFilter(lex, "")
if err != nil {
return nil, err
}
return af, nil
return fo, nil
}
func parseOrFilter(lex *lexer, fieldName string) (filter, error) {
@ -287,10 +287,10 @@ func parseAndFilter(lex *lexer, fieldName string) (filter, error) {
if len(filters) == 1 {
return filters[0], nil
}
af := &andFilter{
fa := &filterAnd{
filters: filters,
}
return af, nil
return fa, nil
case lex.isKeyword("and"):
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing filter after 'and'")

14
lib/logstorage/storage_search.go
View file

@ -339,7 +339,7 @@ func (pt *partition) search(tf *timeFilter, sf *StreamFilter, f filter, so *gene
func hasStreamFilters(f filter) bool {
switch t := f.(type) {
case *andFilter:
case *filterAnd:
return hasStreamFiltersInList(t.filters)
case *filterOr:
return hasStreamFiltersInList(t.filters)
@ -363,8 +363,8 @@ func hasStreamFiltersInList(filters []filter) bool {
func initStreamFilters(tenantIDs []TenantID, idb *indexdb, f filter) filter {
switch t := f.(type) {
case *andFilter:
return &andFilter{
case *filterAnd:
return &filterAnd{
filters: initStreamFiltersList(tenantIDs, idb, t.filters),
}
case *filterOr:
@ -695,16 +695,16 @@ func appendPartsInTimeRange(dst, src []*partWrapper, minTimestamp, maxTimestamp
func getCommonStreamFilter(f filter) (*StreamFilter, filter) {
switch t := f.(type) {
case *andFilter:
case *filterAnd:
filters := t.filters
for i, filter := range filters {
sf, ok := filter.(*streamFilter)
if ok && !sf.f.isEmpty() {
// Remove sf from filters, since it doesn't filter out anything then.
af := &andFilter{
fa := &filterAnd{
filters: append(filters[:i:i], filters[i+1:]...),
}
return sf.f, af
return sf.f, fa
}
}
case *streamFilter:
@ -715,7 +715,7 @@ func getCommonStreamFilter(f filter) (*StreamFilter, filter) {
func getCommonTimeFilter(f filter) (*timeFilter, filter) {
switch t := f.(type) {
case *andFilter:
case *filterAnd:
for _, filter := range t.filters {
tf, ok := filter.(*timeFilter)
if ok {

4
lib/logstorage/storage_search_test.go
View file

@ -388,7 +388,7 @@ func TestStorageSearch(t *testing.T) {
f: sf,
})
}
return &andFilter{
return &filterAnd{
filters: filters,
}
}
@ -578,7 +578,7 @@ func TestStorageSearch(t *testing.T) {
minTimestamp := baseTimestamp
maxTimestamp := baseTimestamp + rowsPerBlock*1e9 + blocksPerStream
f := getBaseFilter(minTimestamp, maxTimestamp, sf)
f = &andFilter{
f = &filterAnd{
filters: []filter{
f,
&regexpFilter{