From 66b2987f49dc5270e45f8a07694b3635c2eb7c81 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 8 Nov 2024 14:23:02 +0100
Subject: [PATCH] lib/logstorage: optimize query imeediately after its parsing

This eliminates possible bugs related to forgotten Query.Optimize() calls.

This also allows removing optimize() function from pipe interface.

While at it, drop filterNoop inside filterAnd.
---
 app/vlselect/logsql/logsql.go            |  1 -
 lib/logstorage/filter.go                 | 17 +++++---
 lib/logstorage/filter_and_test.go        |  8 ++++
 lib/logstorage/if_filter.go              | 23 ----------
 lib/logstorage/parser.go                 | 54 ++++++++++++++++++------
 lib/logstorage/parser_test.go            | 47 ++++++++++++---------
 lib/logstorage/pipe.go                   |  3 --
 lib/logstorage/pipe_block_stats.go       |  4 --
 lib/logstorage/pipe_blocks_count.go      |  4 --
 lib/logstorage/pipe_copy.go              |  4 --
 lib/logstorage/pipe_delete.go            |  4 --
 lib/logstorage/pipe_drop_empty_fields.go |  4 --
 lib/logstorage/pipe_extract.go           |  4 --
 lib/logstorage/pipe_extract_regexp.go    |  4 --
 lib/logstorage/pipe_field_names.go       |  4 --
 lib/logstorage/pipe_field_values.go      |  4 --
 lib/logstorage/pipe_fields.go            |  4 --
 lib/logstorage/pipe_filter.go            |  4 --
 lib/logstorage/pipe_format.go            |  4 --
 lib/logstorage/pipe_join.go              |  4 --
 lib/logstorage/pipe_len.go               |  4 --
 lib/logstorage/pipe_limit.go             |  4 --
 lib/logstorage/pipe_math.go              |  4 --
 lib/logstorage/pipe_offset.go            |  4 --
 lib/logstorage/pipe_pack_json.go         |  4 --
 lib/logstorage/pipe_pack_logfmt.go       |  4 --
 lib/logstorage/pipe_rename.go            |  4 --
 lib/logstorage/pipe_replace.go           |  4 --
 lib/logstorage/pipe_replace_regexp.go    |  4 --
 lib/logstorage/pipe_sort.go              |  4 --
 lib/logstorage/pipe_stats.go             |  6 ---
 lib/logstorage/pipe_stream_context.go    |  4 --
 lib/logstorage/pipe_top.go               |  4 --
 lib/logstorage/pipe_uniq.go              |  4 --
 lib/logstorage/pipe_unpack_json.go       |  4 --
 lib/logstorage/pipe_unpack_logfmt.go     |  4 --
 lib/logstorage/pipe_unpack_syslog.go     |  4 --
 lib/logstorage/pipe_unroll.go            |  4 --
 lib/logstorage/storage_search.go         |  2 -
 39 files changed, 88 insertions(+), 193 deletions(-)

diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go
index a6a1cfc89..0b4092e20 100644
--- a/app/vlselect/logsql/logsql.go
+++ b/app/vlselect/logsql/logsql.go
@@ -982,7 +982,6 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
 	}
-	q.Optimize()
 
 	// Parse optional start and end args
 	start, okStart, err := getTimeNsec(r, "start")
diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go
index 4b8024bff..19917a3bf 100644
--- a/lib/logstorage/filter.go
+++ b/lib/logstorage/filter.go
@@ -49,6 +49,17 @@ func visitFilters(filters []filter, visitFunc func(f filter) bool) bool {
 //
 // It doesn't copy other filters by returning them as is.
 func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) {
+	f, err := copyFilterInternal(f, visitFunc, copyFunc)
+	if err != nil {
+		return nil, err
+	}
+	if !visitFunc(f) {
+		return f, nil
+	}
+	return copyFunc(f)
+}
+
+func copyFilterInternal(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) {
 	switch t := f.(type) {
 	case *filterAnd:
 		filters, err := copyFilters(t.filters, visitFunc, copyFunc)
@@ -78,11 +89,7 @@ func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter)
 		}
 		return fn, nil
 	default:
-		if !visitFunc(t) {
-			// Nothing to copy
-			return t, nil
-		}
-		return copyFunc(t)
+		return f, nil
 	}
 }
 
diff --git a/lib/logstorage/filter_and_test.go b/lib/logstorage/filter_and_test.go
index 223fffb7a..8e25dd588 100644
--- a/lib/logstorage/filter_and_test.go
+++ b/lib/logstorage/filter_and_test.go
@@ -85,6 +85,13 @@ func TestGetCommonTokensForAndFilters(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error in ParseQuery: %s", err)
 		}
+		if _, ok := q.f.(*filterNoop); ok {
+			if len(tokensExpected) != 0 {
+				t.Fatalf("expecting non-empty tokens %q", tokensExpected)
+			}
+			return
+		}
+
 		fa, ok := q.f.(*filterAnd)
 		if !ok {
 			t.Fatalf("unexpected filter type: %T; want *filterAnd", q.f)
@@ -134,6 +141,7 @@ func TestGetCommonTokensForAndFilters(t *testing.T) {
 			tokens: []string{"foo", "bar"},
 		},
 	})
+	f(`*`, nil)
 	f(`* *`, nil)
 
 	// empty filter must be skipped
diff --git a/lib/logstorage/if_filter.go b/lib/logstorage/if_filter.go
index f6852e604..9cfb6e4fd 100644
--- a/lib/logstorage/if_filter.go
+++ b/lib/logstorage/if_filter.go
@@ -50,26 +50,3 @@ func parseIfFilter(lex *lexer) (*ifFilter, error) {
 
 	return iff, nil
 }
-
-func (iff *ifFilter) optimizeFilterIn() {
-	if iff == nil {
-		return
-	}
-
-	optimizeFilterIn(iff.f)
-}
-
-func optimizeFilterIn(f filter) {
-	if f == nil {
-		return
-	}
-
-	visitFunc := func(f filter) bool {
-		fi, ok := f.(*filterIn)
-		if ok && fi.q != nil {
-			fi.q.Optimize()
-		}
-		return false
-	}
-	_ = visitFilter(f, visitFunc)
-}
diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go
index 1c47b7d25..b89b511f5 100644
--- a/lib/logstorage/parser.go
+++ b/lib/logstorage/parser.go
@@ -349,9 +349,6 @@ func (q *Query) Clone(timestamp int64) *Query {
 func (q *Query) CloneWithTimeFilter(timestamp, start, end int64) *Query {
 	q = q.Clone(timestamp)
 	q.AddTimeFilter(start, end)
-	// q.Optimize() call is needed for converting '*' into filterNoop.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-	q.Optimize()
 	return q
 }
 
@@ -534,8 +531,8 @@ func (q *Query) AddPipeLimit(n uint64) {
 	})
 }
 
-// Optimize tries optimizing the query.
-func (q *Query) Optimize() {
+// optimize tries optimizing the query.
+func (q *Query) optimize() {
 	q.pipes = optimizeSortOffsetPipes(q.pipes)
 	q.pipes = optimizeSortLimitPipes(q.pipes)
 	q.pipes = optimizeUniqLimitPipes(q.pipes)
@@ -560,14 +557,6 @@ func (q *Query) Optimize() {
 
 	// Substitute '*' prefixFilter with filterNoop in order to avoid reading _msg data.
 	q.f = removeStarFilters(q.f)
-
-	// Call Optimize for queries from 'in(query)' filters.
-	optimizeFilterIn(q.f)
-
-	// Optimize individual pipes.
-	for _, p := range q.pipes {
-		p.optimize()
-	}
 }
 
 // GetStatsByFields returns `by (...)` fields from the last `stats` pipe at q.
@@ -746,6 +735,7 @@ func addByTimeField(byFields []*byStatsField, step int64) []*byStatsField {
 }
 
 func removeStarFilters(f filter) filter {
+	// Substitute `*` filterPrefix with filterNoop
 	visitFunc := func(f filter) bool {
 		fp, ok := f.(*filterPrefix)
 		return ok && isMsgFieldName(fp.fieldName) && fp.prefix == ""
@@ -758,6 +748,43 @@ func removeStarFilters(f filter) filter {
 	if err != nil {
 		logger.Fatalf("BUG: unexpected error: %s", err)
 	}
+
+	// Drop filterNoop inside filterAnd
+	visitFunc = func(f filter) bool {
+		fa, ok := f.(*filterAnd)
+		if !ok {
+			return false
+		}
+		for _, f := range fa.filters {
+			if _, ok := f.(*filterNoop); ok {
+				return true
+			}
+		}
+		return false
+	}
+	copyFunc = func(f filter) (filter, error) {
+		fa := f.(*filterAnd)
+		var resultFilters []filter
+		for _, f := range fa.filters {
+			if _, ok := f.(*filterNoop); !ok {
+				resultFilters = append(resultFilters, f)
+			}
+		}
+		if len(resultFilters) == 0 {
+			return &filterNoop{}, nil
+		}
+		if len(resultFilters) == 1 {
+			return resultFilters[0], nil
+		}
+		return &filterAnd{
+			filters: resultFilters,
+		}, nil
+	}
+	f, err = copyFilter(f, visitFunc, copyFunc)
+	if err != nil {
+		logger.Fatalf("BUG: unexpected error: %s", err)
+	}
+
 	return f
 }
 
@@ -946,6 +973,7 @@ func ParseQueryAtTimestamp(s string, timestamp int64) (*Query, error) {
 		return nil, fmt.Errorf("unexpected unparsed tail after [%s]; context: [%s]; tail: [%s]", q, lex.context(), lex.s)
 	}
 	q.timestamp = timestamp
+	q.optimize()
 	return q, nil
 }
 
diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go
index 615a80a40..7f99ffde3 100644
--- a/lib/logstorage/parser_test.go
+++ b/lib/logstorage/parser_test.go
@@ -534,19 +534,28 @@ func TestParseFilterPrefix(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		fp, ok := q.f.(*filterPrefix)
-		if !ok {
-			t.Fatalf("unexpected filter type; got %T; want *filterPrefix; filter: %s", q.f, q.f)
-		}
-		if fp.fieldName != fieldNameExpected {
-			t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
-		}
-		if fp.prefix != prefixExpected {
-			t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected)
+		switch f := q.f.(type) {
+		case *filterPrefix:
+			if f.fieldName != fieldNameExpected {
+				t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected)
+			}
+			if f.prefix != prefixExpected {
+				t.Fatalf("unexpected prefix; got %q; want %q", f.prefix, prefixExpected)
+			}
+		case *filterNoop:
+			if fieldNameExpected != "" {
+				t.Fatalf("expecting non-empty fieldName %q", fieldNameExpected)
+			}
+			if prefixExpected != "" {
+				t.Fatalf("expecting non-empty prefix %q", prefixExpected)
+			}
+		default:
+			t.Fatalf("unexpected filter type; got %T; want *filterPrefix or *filterNoop; filter: %s", q.f, q.f)
 		}
 	}
 
 	f(`*`, ``, ``)
+	f(`f:*`, `f`, ``)
 	f(`""*`, ``, ``)
 	f(`foo*`, ``, `foo`)
 	f(`abc-de.fg:foo-bar+baz*`, `abc-de.fg`, `foo-bar+baz`)
@@ -695,8 +704,8 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`'foo'* and (a:x* and x:* or y:i(""*)) and i("abc def"*)`, `foo* (a:x* x:* or y:i(*)) i("abc def"*)`)
 
 	// This isn't a prefix search - it equals to `foo AND *`
-	f(`foo *`, `foo *`)
-	f(`"foo" *`, `foo *`)
+	f(`foo *`, `foo`)
+	f(`"foo" *`, `foo`)
 
 	// empty filter
 	f(`"" or foo:"" and not bar:""`, `"" or foo:"" !bar:""`)
@@ -1197,13 +1206,13 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`* | uniq limit 10`, `* | uniq limit 10`)
 
 	// filter pipe
-	f(`* | filter error ip:12.3.4.5 or warn`, `* | filter error ip:12.3.4.5 or warn`)
-	f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) | limit 10`)
-	f(`* | error`, `* | filter error`)
-	f(`* | "by"`, `* | filter "by"`)
-	f(`* | "stats"`, `* | filter "stats"`)
-	f(`* | "count"`, `* | filter "count"`)
-	f(`* | foo:bar AND baz:<10`, `* | filter foo:bar baz:<10`)
+	f(`* | filter error ip:12.3.4.5 or warn`, `error ip:12.3.4.5 or warn`)
+	f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) limit 10`)
+	f(`* | error`, `error`)
+	f(`* | "by"`, `"by"`)
+	f(`* | "stats" *`, `"stats"`)
+	f(`* | * "count"`, `"count"`)
+	f(`* | foo:bar AND baz:<10`, `foo:bar baz:<10`)
 
 	// extract pipe
 	f(`* | extract "foo<bar>baz"`, `* | extract "foo<bar>baz"`)
@@ -1734,7 +1743,6 @@ func TestQueryGetNeededColumns(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse query [%s]: %s", s, err)
 		}
-		q.Optimize()
 
 		needed, unneeded := q.getNeededColumns()
 		neededColumns := strings.Join(needed, ",")
@@ -2200,7 +2208,6 @@ func TestQueryDropAllPipes(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse [%s]: %s", qStr, err)
 		}
-		q.Optimize()
 		q.DropAllPipes()
 		result := q.String()
 		if result != resultExpected {
diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go
index 00ac68e42..f4b49e910 100644
--- a/lib/logstorage/pipe.go
+++ b/lib/logstorage/pipe.go
@@ -26,9 +26,6 @@ type pipe interface {
 	// The returned pipeProcessor may call cancel() at any time in order to notify the caller to stop sending new data to it.
 	newPipeProcessor(workersCount int, stopCh <-chan struct{}, cancel func(), ppNext pipeProcessor) pipeProcessor
 
-	// optimize must optimize the pipe
-	optimize()
-
 	// hasFilterInWithQuery must return true of pipe contains 'in(subquery)' filter (recursively).
 	hasFilterInWithQuery() bool
 
diff --git a/lib/logstorage/pipe_block_stats.go b/lib/logstorage/pipe_block_stats.go
index 8e6fc0b47..40e74201c 100644
--- a/lib/logstorage/pipe_block_stats.go
+++ b/lib/logstorage/pipe_block_stats.go
@@ -22,10 +22,6 @@ func (ps *pipeBlockStats) canLiveTail() bool {
 	return false
 }
 
-func (ps *pipeBlockStats) optimize() {
-	// nothing to do
-}
-
 func (ps *pipeBlockStats) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_blocks_count.go b/lib/logstorage/pipe_blocks_count.go
index f1381b241..6c109147c 100644
--- a/lib/logstorage/pipe_blocks_count.go
+++ b/lib/logstorage/pipe_blocks_count.go
@@ -31,10 +31,6 @@ func (pc *pipeBlocksCount) updateNeededFields(neededFields, unneededFields field
 	unneededFields.reset()
 }
 
-func (pc *pipeBlocksCount) optimize() {
-	// nothing to do
-}
-
 func (pc *pipeBlocksCount) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_copy.go b/lib/logstorage/pipe_copy.go
index 997023167..2c43920fb 100644
--- a/lib/logstorage/pipe_copy.go
+++ b/lib/logstorage/pipe_copy.go
@@ -54,10 +54,6 @@ func (pc *pipeCopy) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pc *pipeCopy) optimize() {
-	// Nothing to do
-}
-
 func (pc *pipeCopy) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_delete.go b/lib/logstorage/pipe_delete.go
index e61fcfcc6..15429d15d 100644
--- a/lib/logstorage/pipe_delete.go
+++ b/lib/logstorage/pipe_delete.go
@@ -34,10 +34,6 @@ func (pd *pipeDelete) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pd *pipeDelete) optimize() {
-	// nothing to do
-}
-
 func (pd *pipeDelete) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_drop_empty_fields.go b/lib/logstorage/pipe_drop_empty_fields.go
index ba18ad65b..292842b66 100644
--- a/lib/logstorage/pipe_drop_empty_fields.go
+++ b/lib/logstorage/pipe_drop_empty_fields.go
@@ -21,10 +21,6 @@ func (pd *pipeDropEmptyFields) canLiveTail() bool {
 	return true
 }
 
-func (pd *pipeDropEmptyFields) optimize() {
-	// nothing to do
-}
-
 func (pd *pipeDropEmptyFields) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_extract.go b/lib/logstorage/pipe_extract.go
index ec89b15f7..4cf90897c 100644
--- a/lib/logstorage/pipe_extract.go
+++ b/lib/logstorage/pipe_extract.go
@@ -45,10 +45,6 @@ func (pe *pipeExtract) canLiveTail() bool {
 	return true
 }
 
-func (pe *pipeExtract) optimize() {
-	pe.iff.optimizeFilterIn()
-}
-
 func (pe *pipeExtract) hasFilterInWithQuery() bool {
 	return pe.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_extract_regexp.go b/lib/logstorage/pipe_extract_regexp.go
index 2f40d36ad..c5abe5dc3 100644
--- a/lib/logstorage/pipe_extract_regexp.go
+++ b/lib/logstorage/pipe_extract_regexp.go
@@ -47,10 +47,6 @@ func (pe *pipeExtractRegexp) canLiveTail() bool {
 	return true
 }
 
-func (pe *pipeExtractRegexp) optimize() {
-	pe.iff.optimizeFilterIn()
-}
-
 func (pe *pipeExtractRegexp) hasFilterInWithQuery() bool {
 	return pe.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_field_names.go b/lib/logstorage/pipe_field_names.go
index 283cb7675..c5257e108 100644
--- a/lib/logstorage/pipe_field_names.go
+++ b/lib/logstorage/pipe_field_names.go
@@ -39,10 +39,6 @@ func (pf *pipeFieldNames) updateNeededFields(neededFields, unneededFields fields
 	unneededFields.reset()
 }
 
-func (pf *pipeFieldNames) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFieldNames) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_field_values.go b/lib/logstorage/pipe_field_values.go
index 4270f9e31..80c8408ca 100644
--- a/lib/logstorage/pipe_field_values.go
+++ b/lib/logstorage/pipe_field_values.go
@@ -46,10 +46,6 @@ func (pf *pipeFieldValues) updateNeededFields(neededFields, unneededFields field
 	}
 }
 
-func (pf *pipeFieldValues) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFieldValues) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_fields.go b/lib/logstorage/pipe_fields.go
index f08e5c608..f2dd03ba5 100644
--- a/lib/logstorage/pipe_fields.go
+++ b/lib/logstorage/pipe_fields.go
@@ -54,10 +54,6 @@ func (pf *pipeFields) updateNeededFields(neededFields, unneededFields fieldsSet)
 	unneededFields.reset()
 }
 
-func (pf *pipeFields) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFields) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_filter.go b/lib/logstorage/pipe_filter.go
index 7b7308342..2c153d26f 100644
--- a/lib/logstorage/pipe_filter.go
+++ b/lib/logstorage/pipe_filter.go
@@ -33,10 +33,6 @@ func (pf *pipeFilter) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pf *pipeFilter) optimize() {
-	optimizeFilterIn(pf.f)
-}
-
 func (pf *pipeFilter) hasFilterInWithQuery() bool {
 	return hasFilterInWithQueryForFilter(pf.f)
 }
diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go
index 765810186..6d83312b5 100644
--- a/lib/logstorage/pipe_format.go
+++ b/lib/logstorage/pipe_format.go
@@ -87,10 +87,6 @@ func (pf *pipeFormat) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pf *pipeFormat) optimize() {
-	pf.iff.optimizeFilterIn()
-}
-
 func (pf *pipeFormat) hasFilterInWithQuery() bool {
 	return pf.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_join.go b/lib/logstorage/pipe_join.go
index 606e64e40..89c7556e0 100644
--- a/lib/logstorage/pipe_join.go
+++ b/lib/logstorage/pipe_join.go
@@ -37,10 +37,6 @@ func (pj *pipeJoin) canLiveTail() bool {
 	return true
 }
 
-func (pj *pipeJoin) optimize() {
-	pj.q.Optimize()
-}
-
 func (pj *pipeJoin) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_len.go b/lib/logstorage/pipe_len.go
index 907257e17..4d902a7e4 100644
--- a/lib/logstorage/pipe_len.go
+++ b/lib/logstorage/pipe_len.go
@@ -41,10 +41,6 @@ func (pl *pipeLen) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pl *pipeLen) optimize() {
-	// Nothing to do
-}
-
 func (pl *pipeLen) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_limit.go b/lib/logstorage/pipe_limit.go
index 6dec31bae..99a2e65ca 100644
--- a/lib/logstorage/pipe_limit.go
+++ b/lib/logstorage/pipe_limit.go
@@ -24,10 +24,6 @@ func (pl *pipeLimit) updateNeededFields(_, _ fieldsSet) {
 	// nothing to do
 }
 
-func (pl *pipeLimit) optimize() {
-	// nothing to do
-}
-
 func (pl *pipeLimit) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go
index c3999351f..1bc93f656 100644
--- a/lib/logstorage/pipe_math.go
+++ b/lib/logstorage/pipe_math.go
@@ -221,10 +221,6 @@ func (me *mathExpr) updateNeededFields(neededFields fieldsSet) {
 	}
 }
 
-func (pm *pipeMath) optimize() {
-	// nothing to do
-}
-
 func (pm *pipeMath) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_offset.go b/lib/logstorage/pipe_offset.go
index 5f3f92314..458858669 100644
--- a/lib/logstorage/pipe_offset.go
+++ b/lib/logstorage/pipe_offset.go
@@ -24,10 +24,6 @@ func (po *pipeOffset) updateNeededFields(_, _ fieldsSet) {
 	// nothing to do
 }
 
-func (po *pipeOffset) optimize() {
-	// nothing to do
-}
-
 func (po *pipeOffset) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_pack_json.go b/lib/logstorage/pipe_pack_json.go
index a7c2a3fec..01c7f9c7b 100644
--- a/lib/logstorage/pipe_pack_json.go
+++ b/lib/logstorage/pipe_pack_json.go
@@ -33,10 +33,6 @@ func (pp *pipePackJSON) updateNeededFields(neededFields, unneededFields fieldsSe
 	updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields)
 }
 
-func (pp *pipePackJSON) optimize() {
-	// nothing to do
-}
-
 func (pp *pipePackJSON) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_pack_logfmt.go b/lib/logstorage/pipe_pack_logfmt.go
index c01c888a6..3edc4ce22 100644
--- a/lib/logstorage/pipe_pack_logfmt.go
+++ b/lib/logstorage/pipe_pack_logfmt.go
@@ -33,10 +33,6 @@ func (pp *pipePackLogfmt) updateNeededFields(neededFields, unneededFields fields
 	updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields)
 }
 
-func (pp *pipePackLogfmt) optimize() {
-	// nothing to do
-}
-
 func (pp *pipePackLogfmt) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_rename.go b/lib/logstorage/pipe_rename.go
index c35c409ee..e4dbb6ac3 100644
--- a/lib/logstorage/pipe_rename.go
+++ b/lib/logstorage/pipe_rename.go
@@ -58,10 +58,6 @@ func (pr *pipeRename) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pr *pipeRename) optimize() {
-	// nothing to do
-}
-
 func (pr *pipeRename) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_replace.go b/lib/logstorage/pipe_replace.go
index 31aaff9da..fa0ab7c1f 100644
--- a/lib/logstorage/pipe_replace.go
+++ b/lib/logstorage/pipe_replace.go
@@ -45,10 +45,6 @@ func (pr *pipeReplace) updateNeededFields(neededFields, unneededFields fieldsSet
 	updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff)
 }
 
-func (pr *pipeReplace) optimize() {
-	pr.iff.optimizeFilterIn()
-}
-
 func (pr *pipeReplace) hasFilterInWithQuery() bool {
 	return pr.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_replace_regexp.go b/lib/logstorage/pipe_replace_regexp.go
index 1265da996..4f11f3cfd 100644
--- a/lib/logstorage/pipe_replace_regexp.go
+++ b/lib/logstorage/pipe_replace_regexp.go
@@ -45,10 +45,6 @@ func (pr *pipeReplaceRegexp) updateNeededFields(neededFields, unneededFields fie
 	updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff)
 }
 
-func (pr *pipeReplaceRegexp) optimize() {
-	pr.iff.optimizeFilterIn()
-}
-
 func (pr *pipeReplaceRegexp) hasFilterInWithQuery() bool {
 	return pr.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go
index 7f9a7e779..6901cbd21 100644
--- a/lib/logstorage/pipe_sort.go
+++ b/lib/logstorage/pipe_sort.go
@@ -89,10 +89,6 @@ func (ps *pipeSort) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (ps *pipeSort) optimize() {
-	// nothing to do
-}
-
 func (ps *pipeSort) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go
index 7cf723a1b..b2d85e28c 100644
--- a/lib/logstorage/pipe_stats.go
+++ b/lib/logstorage/pipe_stats.go
@@ -123,12 +123,6 @@ func (ps *pipeStats) updateNeededFields(neededFields, unneededFields fieldsSet)
 	unneededFields.reset()
 }
 
-func (ps *pipeStats) optimize() {
-	for _, f := range ps.funcs {
-		f.iff.optimizeFilterIn()
-	}
-}
-
 func (ps *pipeStats) hasFilterInWithQuery() bool {
 	for _, f := range ps.funcs {
 		if f.iff.hasFilterInWithQuery() {
diff --git a/lib/logstorage/pipe_stream_context.go b/lib/logstorage/pipe_stream_context.go
index 09400bdf5..0fe0ba95d 100644
--- a/lib/logstorage/pipe_stream_context.go
+++ b/lib/logstorage/pipe_stream_context.go
@@ -55,10 +55,6 @@ func (pc *pipeStreamContext) updateNeededFields(neededFields, unneededFields fie
 	unneededFields.removeFields(neededFieldsForStreamContext)
 }
 
-func (pc *pipeStreamContext) optimize() {
-	// nothing to do
-}
-
 func (pc *pipeStreamContext) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_top.go b/lib/logstorage/pipe_top.go
index e33a57840..ad7ff4465 100644
--- a/lib/logstorage/pipe_top.go
+++ b/lib/logstorage/pipe_top.go
@@ -71,10 +71,6 @@ func (pt *pipeTop) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pt *pipeTop) optimize() {
-	// nothing to do
-}
-
 func (pt *pipeTop) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go
index ddf043c58..087dffe7a 100644
--- a/lib/logstorage/pipe_uniq.go
+++ b/lib/logstorage/pipe_uniq.go
@@ -58,10 +58,6 @@ func (pu *pipeUniq) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pu *pipeUniq) optimize() {
-	// nothing to do
-}
-
 func (pu *pipeUniq) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_unpack_json.go b/lib/logstorage/pipe_unpack_json.go
index 635c92511..f2855ff46 100644
--- a/lib/logstorage/pipe_unpack_json.go
+++ b/lib/logstorage/pipe_unpack_json.go
@@ -60,10 +60,6 @@ func (pu *pipeUnpackJSON) updateNeededFields(neededFields, unneededFields fields
 	updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackJSON) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackJSON) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unpack_logfmt.go b/lib/logstorage/pipe_unpack_logfmt.go
index ab1d4d61d..d45343c62 100644
--- a/lib/logstorage/pipe_unpack_logfmt.go
+++ b/lib/logstorage/pipe_unpack_logfmt.go
@@ -58,10 +58,6 @@ func (pu *pipeUnpackLogfmt) updateNeededFields(neededFields, unneededFields fiel
 	updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackLogfmt) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackLogfmt) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unpack_syslog.go b/lib/logstorage/pipe_unpack_syslog.go
index 6717c2dcc..6eea365db 100644
--- a/lib/logstorage/pipe_unpack_syslog.go
+++ b/lib/logstorage/pipe_unpack_syslog.go
@@ -54,10 +54,6 @@ func (pu *pipeUnpackSyslog) updateNeededFields(neededFields, unneededFields fiel
 	updateNeededFieldsForUnpackPipe(pu.fromField, nil, pu.keepOriginalFields, false, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackSyslog) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackSyslog) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unroll.go b/lib/logstorage/pipe_unroll.go
index fb9202caa..0a406d64a 100644
--- a/lib/logstorage/pipe_unroll.go
+++ b/lib/logstorage/pipe_unroll.go
@@ -36,10 +36,6 @@ func (pu *pipeUnroll) canLiveTail() bool {
 	return true
 }
 
-func (pu *pipeUnroll) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnroll) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/storage_search.go b/lib/logstorage/storage_search.go
index 54fe066e8..221e0450c 100644
--- a/lib/logstorage/storage_search.go
+++ b/lib/logstorage/storage_search.go
@@ -220,8 +220,6 @@ func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []TenantID, q *Qu
 func (s *Storage) getJoinMap(ctx context.Context, tenantIDs []TenantID, q *Query, byFields []string, prefix string) (map[string][][]Field, error) {
 	// TODO: track memory usage
 
-	logger.Infof("DEBUG: byFields=%q, prefix=%q", byFields, prefix)
-
 	m := make(map[string][][]Field)
 	var mLock sync.Mutex
 	writeBlockResult := func(_ uint, br *blockResult) {