diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go index a6a1cfc89..0b4092e20 100644 --- a/app/vlselect/logsql/logsql.go +++ b/app/vlselect/logsql/logsql.go @@ -982,7 +982,6 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, if err != nil { return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err) } - q.Optimize() // Parse optional start and end args start, okStart, err := getTimeNsec(r, "start") diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 4b8024bff..19917a3bf 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -49,6 +49,17 @@ func visitFilters(filters []filter, visitFunc func(f filter) bool) bool { // // It doesn't copy other filters by returning them as is. func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) { + f, err := copyFilterInternal(f, visitFunc, copyFunc) + if err != nil { + return nil, err + } + if !visitFunc(f) { + return f, nil + } + return copyFunc(f) +} + +func copyFilterInternal(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) { switch t := f.(type) { case *filterAnd: filters, err := copyFilters(t.filters, visitFunc, copyFunc) @@ -78,11 +89,7 @@ func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) } return fn, nil default: - if !visitFunc(t) { - // Nothing to copy - return t, nil - } - return copyFunc(t) + return f, nil } } diff --git a/lib/logstorage/filter_and_test.go b/lib/logstorage/filter_and_test.go index 223fffb7a..8e25dd588 100644 --- a/lib/logstorage/filter_and_test.go +++ b/lib/logstorage/filter_and_test.go @@ -85,6 +85,13 @@ func TestGetCommonTokensForAndFilters(t *testing.T) { if err != nil { t.Fatalf("unexpected error in ParseQuery: %s", err) } + if _, ok := q.f.(*filterNoop); ok { + if len(tokensExpected) != 0 { + t.Fatalf("expecting non-empty tokens %q", tokensExpected) + } + return + } + fa, ok := q.f.(*filterAnd) if !ok { t.Fatalf("unexpected filter type: %T; want *filterAnd", q.f) @@ -134,6 +141,7 @@ func TestGetCommonTokensForAndFilters(t *testing.T) { tokens: []string{"foo", "bar"}, }, }) + f(`*`, nil) f(`* *`, nil) // empty filter must be skipped diff --git a/lib/logstorage/if_filter.go b/lib/logstorage/if_filter.go index f6852e604..9cfb6e4fd 100644 --- a/lib/logstorage/if_filter.go +++ b/lib/logstorage/if_filter.go @@ -50,26 +50,3 @@ func parseIfFilter(lex *lexer) (*ifFilter, error) { return iff, nil } - -func (iff *ifFilter) optimizeFilterIn() { - if iff == nil { - return - } - - optimizeFilterIn(iff.f) -} - -func optimizeFilterIn(f filter) { - if f == nil { - return - } - - visitFunc := func(f filter) bool { - fi, ok := f.(*filterIn) - if ok && fi.q != nil { - fi.q.Optimize() - } - return false - } - _ = visitFilter(f, visitFunc) -} diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 1c47b7d25..b89b511f5 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -349,9 +349,6 @@ func (q *Query) Clone(timestamp int64) *Query { func (q *Query) CloneWithTimeFilter(timestamp, start, end int64) *Query { q = q.Clone(timestamp) q.AddTimeFilter(start, end) - // q.Optimize() call is needed for converting '*' into filterNoop. - // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733 - q.Optimize() return q } @@ -534,8 +531,8 @@ func (q *Query) AddPipeLimit(n uint64) { }) } -// Optimize tries optimizing the query. -func (q *Query) Optimize() { +// optimize tries optimizing the query. +func (q *Query) optimize() { q.pipes = optimizeSortOffsetPipes(q.pipes) q.pipes = optimizeSortLimitPipes(q.pipes) q.pipes = optimizeUniqLimitPipes(q.pipes) @@ -560,14 +557,6 @@ func (q *Query) Optimize() { // Substitute '*' prefixFilter with filterNoop in order to avoid reading _msg data. q.f = removeStarFilters(q.f) - - // Call Optimize for queries from 'in(query)' filters. - optimizeFilterIn(q.f) - - // Optimize individual pipes. - for _, p := range q.pipes { - p.optimize() - } } // GetStatsByFields returns `by (...)` fields from the last `stats` pipe at q. @@ -746,6 +735,7 @@ func addByTimeField(byFields []*byStatsField, step int64) []*byStatsField { } func removeStarFilters(f filter) filter { + // Substitute `*` filterPrefix with filterNoop visitFunc := func(f filter) bool { fp, ok := f.(*filterPrefix) return ok && isMsgFieldName(fp.fieldName) && fp.prefix == "" @@ -758,6 +748,43 @@ func removeStarFilters(f filter) filter { if err != nil { logger.Fatalf("BUG: unexpected error: %s", err) } + + // Drop filterNoop inside filterAnd + visitFunc = func(f filter) bool { + fa, ok := f.(*filterAnd) + if !ok { + return false + } + for _, f := range fa.filters { + if _, ok := f.(*filterNoop); ok { + return true + } + } + return false + } + copyFunc = func(f filter) (filter, error) { + fa := f.(*filterAnd) + var resultFilters []filter + for _, f := range fa.filters { + if _, ok := f.(*filterNoop); !ok { + resultFilters = append(resultFilters, f) + } + } + if len(resultFilters) == 0 { + return &filterNoop{}, nil + } + if len(resultFilters) == 1 { + return resultFilters[0], nil + } + return &filterAnd{ + filters: resultFilters, + }, nil + } + f, err = copyFilter(f, visitFunc, copyFunc) + if err != nil { + logger.Fatalf("BUG: unexpected error: %s", err) + } + return f } @@ -946,6 +973,7 @@ func ParseQueryAtTimestamp(s string, timestamp int64) (*Query, error) { return nil, fmt.Errorf("unexpected unparsed tail after [%s]; context: [%s]; tail: [%s]", q, lex.context(), lex.s) } q.timestamp = timestamp + q.optimize() return q, nil } diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 615a80a40..7f99ffde3 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -534,19 +534,28 @@ func TestParseFilterPrefix(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - fp, ok := q.f.(*filterPrefix) - if !ok { - t.Fatalf("unexpected filter type; got %T; want *filterPrefix; filter: %s", q.f, q.f) - } - if fp.fieldName != fieldNameExpected { - t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected) - } - if fp.prefix != prefixExpected { - t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected) + switch f := q.f.(type) { + case *filterPrefix: + if f.fieldName != fieldNameExpected { + t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected) + } + if f.prefix != prefixExpected { + t.Fatalf("unexpected prefix; got %q; want %q", f.prefix, prefixExpected) + } + case *filterNoop: + if fieldNameExpected != "" { + t.Fatalf("expecting non-empty fieldName %q", fieldNameExpected) + } + if prefixExpected != "" { + t.Fatalf("expecting non-empty prefix %q", prefixExpected) + } + default: + t.Fatalf("unexpected filter type; got %T; want *filterPrefix or *filterNoop; filter: %s", q.f, q.f) } } f(`*`, ``, ``) + f(`f:*`, `f`, ``) f(`""*`, ``, ``) f(`foo*`, ``, `foo`) f(`abc-de.fg:foo-bar+baz*`, `abc-de.fg`, `foo-bar+baz`) @@ -695,8 +704,8 @@ func TestParseQuerySuccess(t *testing.T) { f(`'foo'* and (a:x* and x:* or y:i(""*)) and i("abc def"*)`, `foo* (a:x* x:* or y:i(*)) i("abc def"*)`) // This isn't a prefix search - it equals to `foo AND *` - f(`foo *`, `foo *`) - f(`"foo" *`, `foo *`) + f(`foo *`, `foo`) + f(`"foo" *`, `foo`) // empty filter f(`"" or foo:"" and not bar:""`, `"" or foo:"" !bar:""`) @@ -1197,13 +1206,13 @@ func TestParseQuerySuccess(t *testing.T) { f(`* | uniq limit 10`, `* | uniq limit 10`) // filter pipe - f(`* | filter error ip:12.3.4.5 or warn`, `* | filter error ip:12.3.4.5 or warn`) - f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) | limit 10`) - f(`* | error`, `* | filter error`) - f(`* | "by"`, `* | filter "by"`) - f(`* | "stats"`, `* | filter "stats"`) - f(`* | "count"`, `* | filter "count"`) - f(`* | foo:bar AND baz:<10`, `* | filter foo:bar baz:<10`) + f(`* | filter error ip:12.3.4.5 or warn`, `error ip:12.3.4.5 or warn`) + f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) limit 10`) + f(`* | error`, `error`) + f(`* | "by"`, `"by"`) + f(`* | "stats" *`, `"stats"`) + f(`* | * "count"`, `"count"`) + f(`* | foo:bar AND baz:<10`, `foo:bar baz:<10`) // extract pipe f(`* | extract "foobaz"`, `* | extract "foobaz"`) @@ -1734,7 +1743,6 @@ func TestQueryGetNeededColumns(t *testing.T) { if err != nil { t.Fatalf("cannot parse query [%s]: %s", s, err) } - q.Optimize() needed, unneeded := q.getNeededColumns() neededColumns := strings.Join(needed, ",") @@ -2200,7 +2208,6 @@ func TestQueryDropAllPipes(t *testing.T) { if err != nil { t.Fatalf("cannot parse [%s]: %s", qStr, err) } - q.Optimize() q.DropAllPipes() result := q.String() if result != resultExpected { diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go index 00ac68e42..f4b49e910 100644 --- a/lib/logstorage/pipe.go +++ b/lib/logstorage/pipe.go @@ -26,9 +26,6 @@ type pipe interface { // The returned pipeProcessor may call cancel() at any time in order to notify the caller to stop sending new data to it. newPipeProcessor(workersCount int, stopCh <-chan struct{}, cancel func(), ppNext pipeProcessor) pipeProcessor - // optimize must optimize the pipe - optimize() - // hasFilterInWithQuery must return true of pipe contains 'in(subquery)' filter (recursively). hasFilterInWithQuery() bool diff --git a/lib/logstorage/pipe_block_stats.go b/lib/logstorage/pipe_block_stats.go index 8e6fc0b47..40e74201c 100644 --- a/lib/logstorage/pipe_block_stats.go +++ b/lib/logstorage/pipe_block_stats.go @@ -22,10 +22,6 @@ func (ps *pipeBlockStats) canLiveTail() bool { return false } -func (ps *pipeBlockStats) optimize() { - // nothing to do -} - func (ps *pipeBlockStats) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_blocks_count.go b/lib/logstorage/pipe_blocks_count.go index f1381b241..6c109147c 100644 --- a/lib/logstorage/pipe_blocks_count.go +++ b/lib/logstorage/pipe_blocks_count.go @@ -31,10 +31,6 @@ func (pc *pipeBlocksCount) updateNeededFields(neededFields, unneededFields field unneededFields.reset() } -func (pc *pipeBlocksCount) optimize() { - // nothing to do -} - func (pc *pipeBlocksCount) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_copy.go b/lib/logstorage/pipe_copy.go index 997023167..2c43920fb 100644 --- a/lib/logstorage/pipe_copy.go +++ b/lib/logstorage/pipe_copy.go @@ -54,10 +54,6 @@ func (pc *pipeCopy) updateNeededFields(neededFields, unneededFields fieldsSet) { } } -func (pc *pipeCopy) optimize() { - // Nothing to do -} - func (pc *pipeCopy) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_delete.go b/lib/logstorage/pipe_delete.go index e61fcfcc6..15429d15d 100644 --- a/lib/logstorage/pipe_delete.go +++ b/lib/logstorage/pipe_delete.go @@ -34,10 +34,6 @@ func (pd *pipeDelete) updateNeededFields(neededFields, unneededFields fieldsSet) } } -func (pd *pipeDelete) optimize() { - // nothing to do -} - func (pd *pipeDelete) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_drop_empty_fields.go b/lib/logstorage/pipe_drop_empty_fields.go index ba18ad65b..292842b66 100644 --- a/lib/logstorage/pipe_drop_empty_fields.go +++ b/lib/logstorage/pipe_drop_empty_fields.go @@ -21,10 +21,6 @@ func (pd *pipeDropEmptyFields) canLiveTail() bool { return true } -func (pd *pipeDropEmptyFields) optimize() { - // nothing to do -} - func (pd *pipeDropEmptyFields) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_extract.go b/lib/logstorage/pipe_extract.go index ec89b15f7..4cf90897c 100644 --- a/lib/logstorage/pipe_extract.go +++ b/lib/logstorage/pipe_extract.go @@ -45,10 +45,6 @@ func (pe *pipeExtract) canLiveTail() bool { return true } -func (pe *pipeExtract) optimize() { - pe.iff.optimizeFilterIn() -} - func (pe *pipeExtract) hasFilterInWithQuery() bool { return pe.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_extract_regexp.go b/lib/logstorage/pipe_extract_regexp.go index 2f40d36ad..c5abe5dc3 100644 --- a/lib/logstorage/pipe_extract_regexp.go +++ b/lib/logstorage/pipe_extract_regexp.go @@ -47,10 +47,6 @@ func (pe *pipeExtractRegexp) canLiveTail() bool { return true } -func (pe *pipeExtractRegexp) optimize() { - pe.iff.optimizeFilterIn() -} - func (pe *pipeExtractRegexp) hasFilterInWithQuery() bool { return pe.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_field_names.go b/lib/logstorage/pipe_field_names.go index 283cb7675..c5257e108 100644 --- a/lib/logstorage/pipe_field_names.go +++ b/lib/logstorage/pipe_field_names.go @@ -39,10 +39,6 @@ func (pf *pipeFieldNames) updateNeededFields(neededFields, unneededFields fields unneededFields.reset() } -func (pf *pipeFieldNames) optimize() { - // nothing to do -} - func (pf *pipeFieldNames) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_field_values.go b/lib/logstorage/pipe_field_values.go index 4270f9e31..80c8408ca 100644 --- a/lib/logstorage/pipe_field_values.go +++ b/lib/logstorage/pipe_field_values.go @@ -46,10 +46,6 @@ func (pf *pipeFieldValues) updateNeededFields(neededFields, unneededFields field } } -func (pf *pipeFieldValues) optimize() { - // nothing to do -} - func (pf *pipeFieldValues) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_fields.go b/lib/logstorage/pipe_fields.go index f08e5c608..f2dd03ba5 100644 --- a/lib/logstorage/pipe_fields.go +++ b/lib/logstorage/pipe_fields.go @@ -54,10 +54,6 @@ func (pf *pipeFields) updateNeededFields(neededFields, unneededFields fieldsSet) unneededFields.reset() } -func (pf *pipeFields) optimize() { - // nothing to do -} - func (pf *pipeFields) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_filter.go b/lib/logstorage/pipe_filter.go index 7b7308342..2c153d26f 100644 --- a/lib/logstorage/pipe_filter.go +++ b/lib/logstorage/pipe_filter.go @@ -33,10 +33,6 @@ func (pf *pipeFilter) updateNeededFields(neededFields, unneededFields fieldsSet) } } -func (pf *pipeFilter) optimize() { - optimizeFilterIn(pf.f) -} - func (pf *pipeFilter) hasFilterInWithQuery() bool { return hasFilterInWithQueryForFilter(pf.f) } diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go index 765810186..6d83312b5 100644 --- a/lib/logstorage/pipe_format.go +++ b/lib/logstorage/pipe_format.go @@ -87,10 +87,6 @@ func (pf *pipeFormat) updateNeededFields(neededFields, unneededFields fieldsSet) } } -func (pf *pipeFormat) optimize() { - pf.iff.optimizeFilterIn() -} - func (pf *pipeFormat) hasFilterInWithQuery() bool { return pf.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_join.go b/lib/logstorage/pipe_join.go index 606e64e40..89c7556e0 100644 --- a/lib/logstorage/pipe_join.go +++ b/lib/logstorage/pipe_join.go @@ -37,10 +37,6 @@ func (pj *pipeJoin) canLiveTail() bool { return true } -func (pj *pipeJoin) optimize() { - pj.q.Optimize() -} - func (pj *pipeJoin) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_len.go b/lib/logstorage/pipe_len.go index 907257e17..4d902a7e4 100644 --- a/lib/logstorage/pipe_len.go +++ b/lib/logstorage/pipe_len.go @@ -41,10 +41,6 @@ func (pl *pipeLen) updateNeededFields(neededFields, unneededFields fieldsSet) { } } -func (pl *pipeLen) optimize() { - // Nothing to do -} - func (pl *pipeLen) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_limit.go b/lib/logstorage/pipe_limit.go index 6dec31bae..99a2e65ca 100644 --- a/lib/logstorage/pipe_limit.go +++ b/lib/logstorage/pipe_limit.go @@ -24,10 +24,6 @@ func (pl *pipeLimit) updateNeededFields(_, _ fieldsSet) { // nothing to do } -func (pl *pipeLimit) optimize() { - // nothing to do -} - func (pl *pipeLimit) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go index c3999351f..1bc93f656 100644 --- a/lib/logstorage/pipe_math.go +++ b/lib/logstorage/pipe_math.go @@ -221,10 +221,6 @@ func (me *mathExpr) updateNeededFields(neededFields fieldsSet) { } } -func (pm *pipeMath) optimize() { - // nothing to do -} - func (pm *pipeMath) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_offset.go b/lib/logstorage/pipe_offset.go index 5f3f92314..458858669 100644 --- a/lib/logstorage/pipe_offset.go +++ b/lib/logstorage/pipe_offset.go @@ -24,10 +24,6 @@ func (po *pipeOffset) updateNeededFields(_, _ fieldsSet) { // nothing to do } -func (po *pipeOffset) optimize() { - // nothing to do -} - func (po *pipeOffset) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_pack_json.go b/lib/logstorage/pipe_pack_json.go index a7c2a3fec..01c7f9c7b 100644 --- a/lib/logstorage/pipe_pack_json.go +++ b/lib/logstorage/pipe_pack_json.go @@ -33,10 +33,6 @@ func (pp *pipePackJSON) updateNeededFields(neededFields, unneededFields fieldsSe updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields) } -func (pp *pipePackJSON) optimize() { - // nothing to do -} - func (pp *pipePackJSON) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_pack_logfmt.go b/lib/logstorage/pipe_pack_logfmt.go index c01c888a6..3edc4ce22 100644 --- a/lib/logstorage/pipe_pack_logfmt.go +++ b/lib/logstorage/pipe_pack_logfmt.go @@ -33,10 +33,6 @@ func (pp *pipePackLogfmt) updateNeededFields(neededFields, unneededFields fields updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields) } -func (pp *pipePackLogfmt) optimize() { - // nothing to do -} - func (pp *pipePackLogfmt) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_rename.go b/lib/logstorage/pipe_rename.go index c35c409ee..e4dbb6ac3 100644 --- a/lib/logstorage/pipe_rename.go +++ b/lib/logstorage/pipe_rename.go @@ -58,10 +58,6 @@ func (pr *pipeRename) updateNeededFields(neededFields, unneededFields fieldsSet) } } -func (pr *pipeRename) optimize() { - // nothing to do -} - func (pr *pipeRename) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_replace.go b/lib/logstorage/pipe_replace.go index 31aaff9da..fa0ab7c1f 100644 --- a/lib/logstorage/pipe_replace.go +++ b/lib/logstorage/pipe_replace.go @@ -45,10 +45,6 @@ func (pr *pipeReplace) updateNeededFields(neededFields, unneededFields fieldsSet updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff) } -func (pr *pipeReplace) optimize() { - pr.iff.optimizeFilterIn() -} - func (pr *pipeReplace) hasFilterInWithQuery() bool { return pr.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_replace_regexp.go b/lib/logstorage/pipe_replace_regexp.go index 1265da996..4f11f3cfd 100644 --- a/lib/logstorage/pipe_replace_regexp.go +++ b/lib/logstorage/pipe_replace_regexp.go @@ -45,10 +45,6 @@ func (pr *pipeReplaceRegexp) updateNeededFields(neededFields, unneededFields fie updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff) } -func (pr *pipeReplaceRegexp) optimize() { - pr.iff.optimizeFilterIn() -} - func (pr *pipeReplaceRegexp) hasFilterInWithQuery() bool { return pr.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go index 7f9a7e779..6901cbd21 100644 --- a/lib/logstorage/pipe_sort.go +++ b/lib/logstorage/pipe_sort.go @@ -89,10 +89,6 @@ func (ps *pipeSort) updateNeededFields(neededFields, unneededFields fieldsSet) { } } -func (ps *pipeSort) optimize() { - // nothing to do -} - func (ps *pipeSort) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index 7cf723a1b..b2d85e28c 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -123,12 +123,6 @@ func (ps *pipeStats) updateNeededFields(neededFields, unneededFields fieldsSet) unneededFields.reset() } -func (ps *pipeStats) optimize() { - for _, f := range ps.funcs { - f.iff.optimizeFilterIn() - } -} - func (ps *pipeStats) hasFilterInWithQuery() bool { for _, f := range ps.funcs { if f.iff.hasFilterInWithQuery() { diff --git a/lib/logstorage/pipe_stream_context.go b/lib/logstorage/pipe_stream_context.go index 09400bdf5..0fe0ba95d 100644 --- a/lib/logstorage/pipe_stream_context.go +++ b/lib/logstorage/pipe_stream_context.go @@ -55,10 +55,6 @@ func (pc *pipeStreamContext) updateNeededFields(neededFields, unneededFields fie unneededFields.removeFields(neededFieldsForStreamContext) } -func (pc *pipeStreamContext) optimize() { - // nothing to do -} - func (pc *pipeStreamContext) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_top.go b/lib/logstorage/pipe_top.go index e33a57840..ad7ff4465 100644 --- a/lib/logstorage/pipe_top.go +++ b/lib/logstorage/pipe_top.go @@ -71,10 +71,6 @@ func (pt *pipeTop) updateNeededFields(neededFields, unneededFields fieldsSet) { } } -func (pt *pipeTop) optimize() { - // nothing to do -} - func (pt *pipeTop) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go index ddf043c58..087dffe7a 100644 --- a/lib/logstorage/pipe_uniq.go +++ b/lib/logstorage/pipe_uniq.go @@ -58,10 +58,6 @@ func (pu *pipeUniq) updateNeededFields(neededFields, unneededFields fieldsSet) { } } -func (pu *pipeUniq) optimize() { - // nothing to do -} - func (pu *pipeUniq) hasFilterInWithQuery() bool { return false } diff --git a/lib/logstorage/pipe_unpack_json.go b/lib/logstorage/pipe_unpack_json.go index 635c92511..f2855ff46 100644 --- a/lib/logstorage/pipe_unpack_json.go +++ b/lib/logstorage/pipe_unpack_json.go @@ -60,10 +60,6 @@ func (pu *pipeUnpackJSON) updateNeededFields(neededFields, unneededFields fields updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields) } -func (pu *pipeUnpackJSON) optimize() { - pu.iff.optimizeFilterIn() -} - func (pu *pipeUnpackJSON) hasFilterInWithQuery() bool { return pu.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_unpack_logfmt.go b/lib/logstorage/pipe_unpack_logfmt.go index ab1d4d61d..d45343c62 100644 --- a/lib/logstorage/pipe_unpack_logfmt.go +++ b/lib/logstorage/pipe_unpack_logfmt.go @@ -58,10 +58,6 @@ func (pu *pipeUnpackLogfmt) updateNeededFields(neededFields, unneededFields fiel updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields) } -func (pu *pipeUnpackLogfmt) optimize() { - pu.iff.optimizeFilterIn() -} - func (pu *pipeUnpackLogfmt) hasFilterInWithQuery() bool { return pu.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_unpack_syslog.go b/lib/logstorage/pipe_unpack_syslog.go index 6717c2dcc..6eea365db 100644 --- a/lib/logstorage/pipe_unpack_syslog.go +++ b/lib/logstorage/pipe_unpack_syslog.go @@ -54,10 +54,6 @@ func (pu *pipeUnpackSyslog) updateNeededFields(neededFields, unneededFields fiel updateNeededFieldsForUnpackPipe(pu.fromField, nil, pu.keepOriginalFields, false, pu.iff, neededFields, unneededFields) } -func (pu *pipeUnpackSyslog) optimize() { - pu.iff.optimizeFilterIn() -} - func (pu *pipeUnpackSyslog) hasFilterInWithQuery() bool { return pu.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/pipe_unroll.go b/lib/logstorage/pipe_unroll.go index fb9202caa..0a406d64a 100644 --- a/lib/logstorage/pipe_unroll.go +++ b/lib/logstorage/pipe_unroll.go @@ -36,10 +36,6 @@ func (pu *pipeUnroll) canLiveTail() bool { return true } -func (pu *pipeUnroll) optimize() { - pu.iff.optimizeFilterIn() -} - func (pu *pipeUnroll) hasFilterInWithQuery() bool { return pu.iff.hasFilterInWithQuery() } diff --git a/lib/logstorage/storage_search.go b/lib/logstorage/storage_search.go index 54fe066e8..221e0450c 100644 --- a/lib/logstorage/storage_search.go +++ b/lib/logstorage/storage_search.go @@ -220,8 +220,6 @@ func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []TenantID, q *Qu func (s *Storage) getJoinMap(ctx context.Context, tenantIDs []TenantID, q *Query, byFields []string, prefix string) (map[string][][]Field, error) { // TODO: track memory usage - logger.Infof("DEBUG: byFields=%q, prefix=%q", byFields, prefix) - m := make(map[string][][]Field) var mLock sync.Mutex writeBlockResult := func(_ uint, br *blockResult) {