From 15598986e61881adc2b9c7ea79d09d74ba73b56a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 30 May 2024 13:50:12 +0200 Subject: [PATCH] wip --- lib/logstorage/filter_and.go | 73 +++++++++++++++++++----------------- lib/logstorage/filter_or.go | 63 +++++++++++++++++-------------- 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/lib/logstorage/filter_and.go b/lib/logstorage/filter_and.go index 08bf8736a..59d623b2e 100644 --- a/lib/logstorage/filter_and.go +++ b/lib/logstorage/filter_and.go @@ -117,43 +117,48 @@ func (fa *filterAnd) initByFieldTokens() { byFieldFilters := make(map[string]int) var fieldNames []string - for _, f := range fa.filters { - fieldName := "" - var tokens []string - - switch t := f.(type) { - case *filterExact: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterExactPrefix: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterPhrase: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterPrefix: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterRegexp: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterSequence: - fieldName = t.fieldName - tokens = t.getTokens() - } - + mergeFieldTokens := func(fieldName string, tokens []string) { fieldName = getCanonicalColumnName(fieldName) byFieldFilters[fieldName]++ + if len(tokens) == 0 { + return + } - if len(tokens) > 0 { - mTokens, ok := m[fieldName] - if !ok { - fieldNames = append(fieldNames, fieldName) - mTokens = make(map[string]struct{}) - m[fieldName] = mTokens - } - for _, token := range tokens { - mTokens[token] = struct{}{} + mTokens, ok := m[fieldName] + if !ok { + fieldNames = append(fieldNames, fieldName) + mTokens = make(map[string]struct{}) + m[fieldName] = mTokens + } + for _, token := range tokens { + mTokens[token] = struct{}{} + } + } + + for _, f := range fa.filters { + switch t := f.(type) { + case *filterExact: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterExactPrefix: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterPhrase: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterPrefix: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterRegexp: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterSequence: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterOr: + bfts := t.getByFieldTokens() + for _, bft := range bfts { + mergeFieldTokens(bft.field, bft.tokens) } } } diff --git a/lib/logstorage/filter_or.go b/lib/logstorage/filter_or.go index 256337a0c..040c74680 100644 --- a/lib/logstorage/filter_or.go +++ b/lib/logstorage/filter_or.go @@ -130,39 +130,44 @@ func (fo *filterOr) initByFieldTokens() { byFieldFilters := make(map[string]int) var fieldNames []string - for _, f := range fo.filters { - fieldName := "" - var tokens []string - - switch t := f.(type) { - case *filterExact: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterExactPrefix: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterPhrase: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterPrefix: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterRegexp: - fieldName = t.fieldName - tokens = t.getTokens() - case *filterSequence: - fieldName = t.fieldName - tokens = t.getTokens() - } - + mergeFieldTokens := func(fieldName string, tokens []string) { fieldName = getCanonicalColumnName(fieldName) byFieldFilters[fieldName]++ + if len(tokens) == 0 { + return + } - if len(tokens) > 0 { - if _, ok := m[fieldName]; !ok { - fieldNames = append(fieldNames, fieldName) + if _, ok := m[fieldName]; !ok { + fieldNames = append(fieldNames, fieldName) + } + m[fieldName] = append(m[fieldName], tokens) + } + + for _, f := range fo.filters { + switch t := f.(type) { + case *filterExact: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterExactPrefix: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterPhrase: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterPrefix: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterRegexp: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterSequence: + tokens := t.getTokens() + mergeFieldTokens(t.fieldName, tokens) + case *filterAnd: + bfts := t.getByFieldTokens() + for _, bft := range bfts { + mergeFieldTokens(bft.field, bft.tokens) } - m[fieldName] = append(m[fieldName], tokens) } }