From a4ea3b87d756b9e52d4bc644e3693a01a84bf9da Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 8 Nov 2024 14:23:02 +0100
Subject: [PATCH] lib/logstorage: optimize query imeediately after its parsing

This eliminates possible bugs related to forgotten Query.Optimize() calls.

This also allows removing optimize() function from pipe interface.

While at it, drop filterNoop inside filterAnd.

(cherry picked from commit 66b2987f49dc5270e45f8a07694b3635c2eb7c81)
---
 app/vlselect/logsql/logsql.go            |  1 -
 lib/logstorage/filter.go                 | 17 +++++---
 lib/logstorage/filter_and_test.go        |  8 ++++
 lib/logstorage/if_filter.go              | 23 ----------
 lib/logstorage/parser.go                 | 54 ++++++++++++++++++------
 lib/logstorage/parser_test.go            | 47 ++++++++++++---------
 lib/logstorage/pipe.go                   |  3 --
 lib/logstorage/pipe_block_stats.go       |  4 --
 lib/logstorage/pipe_blocks_count.go      |  4 --
 lib/logstorage/pipe_copy.go              |  4 --
 lib/logstorage/pipe_delete.go            |  4 --
 lib/logstorage/pipe_drop_empty_fields.go |  4 --
 lib/logstorage/pipe_extract.go           |  4 --
 lib/logstorage/pipe_extract_regexp.go    |  4 --
 lib/logstorage/pipe_field_names.go       |  4 --
 lib/logstorage/pipe_field_values.go      |  4 --
 lib/logstorage/pipe_fields.go            |  4 --
 lib/logstorage/pipe_filter.go            |  4 --
 lib/logstorage/pipe_format.go            |  4 --
 lib/logstorage/pipe_join.go              |  4 --
 lib/logstorage/pipe_len.go               |  4 --
 lib/logstorage/pipe_limit.go             |  4 --
 lib/logstorage/pipe_math.go              |  4 --
 lib/logstorage/pipe_offset.go            |  4 --
 lib/logstorage/pipe_pack_json.go         |  4 --
 lib/logstorage/pipe_pack_logfmt.go       |  4 --
 lib/logstorage/pipe_rename.go            |  4 --
 lib/logstorage/pipe_replace.go           |  4 --
 lib/logstorage/pipe_replace_regexp.go    |  4 --
 lib/logstorage/pipe_sort.go              |  4 --
 lib/logstorage/pipe_stats.go             |  6 ---
 lib/logstorage/pipe_stream_context.go    |  4 --
 lib/logstorage/pipe_top.go               |  4 --
 lib/logstorage/pipe_uniq.go              |  4 --
 lib/logstorage/pipe_unpack_json.go       |  4 --
 lib/logstorage/pipe_unpack_logfmt.go     |  4 --
 lib/logstorage/pipe_unpack_syslog.go     |  4 --
 lib/logstorage/pipe_unroll.go            |  4 --
 lib/logstorage/storage_search.go         |  2 -
 39 files changed, 88 insertions(+), 193 deletions(-)

diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go
index a6a1cfc891..0b4092e202 100644
--- a/app/vlselect/logsql/logsql.go
+++ b/app/vlselect/logsql/logsql.go
@@ -982,7 +982,6 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
 	}
-	q.Optimize()
 
 	// Parse optional start and end args
 	start, okStart, err := getTimeNsec(r, "start")
diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go
index 4b8024bff3..19917a3bf9 100644
--- a/lib/logstorage/filter.go
+++ b/lib/logstorage/filter.go
@@ -49,6 +49,17 @@ func visitFilters(filters []filter, visitFunc func(f filter) bool) bool {
 //
 // It doesn't copy other filters by returning them as is.
 func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) {
+	f, err := copyFilterInternal(f, visitFunc, copyFunc)
+	if err != nil {
+		return nil, err
+	}
+	if !visitFunc(f) {
+		return f, nil
+	}
+	return copyFunc(f)
+}
+
+func copyFilterInternal(f filter, visitFunc func(f filter) bool, copyFunc func(f filter) (filter, error)) (filter, error) {
 	switch t := f.(type) {
 	case *filterAnd:
 		filters, err := copyFilters(t.filters, visitFunc, copyFunc)
@@ -78,11 +89,7 @@ func copyFilter(f filter, visitFunc func(f filter) bool, copyFunc func(f filter)
 		}
 		return fn, nil
 	default:
-		if !visitFunc(t) {
-			// Nothing to copy
-			return t, nil
-		}
-		return copyFunc(t)
+		return f, nil
 	}
 }
 
diff --git a/lib/logstorage/filter_and_test.go b/lib/logstorage/filter_and_test.go
index 223fffb7ac..8e25dd588b 100644
--- a/lib/logstorage/filter_and_test.go
+++ b/lib/logstorage/filter_and_test.go
@@ -85,6 +85,13 @@ func TestGetCommonTokensForAndFilters(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error in ParseQuery: %s", err)
 		}
+		if _, ok := q.f.(*filterNoop); ok {
+			if len(tokensExpected) != 0 {
+				t.Fatalf("expecting non-empty tokens %q", tokensExpected)
+			}
+			return
+		}
+
 		fa, ok := q.f.(*filterAnd)
 		if !ok {
 			t.Fatalf("unexpected filter type: %T; want *filterAnd", q.f)
@@ -134,6 +141,7 @@ func TestGetCommonTokensForAndFilters(t *testing.T) {
 			tokens: []string{"foo", "bar"},
 		},
 	})
+	f(`*`, nil)
 	f(`* *`, nil)
 
 	// empty filter must be skipped
diff --git a/lib/logstorage/if_filter.go b/lib/logstorage/if_filter.go
index f6852e6045..9cfb6e4fd5 100644
--- a/lib/logstorage/if_filter.go
+++ b/lib/logstorage/if_filter.go
@@ -50,26 +50,3 @@ func parseIfFilter(lex *lexer) (*ifFilter, error) {
 
 	return iff, nil
 }
-
-func (iff *ifFilter) optimizeFilterIn() {
-	if iff == nil {
-		return
-	}
-
-	optimizeFilterIn(iff.f)
-}
-
-func optimizeFilterIn(f filter) {
-	if f == nil {
-		return
-	}
-
-	visitFunc := func(f filter) bool {
-		fi, ok := f.(*filterIn)
-		if ok && fi.q != nil {
-			fi.q.Optimize()
-		}
-		return false
-	}
-	_ = visitFilter(f, visitFunc)
-}
diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go
index 6282334cda..63f07b5a0c 100644
--- a/lib/logstorage/parser.go
+++ b/lib/logstorage/parser.go
@@ -349,9 +349,6 @@ func (q *Query) Clone(timestamp int64) *Query {
 func (q *Query) CloneWithTimeFilter(timestamp, start, end int64) *Query {
 	q = q.Clone(timestamp)
 	q.AddTimeFilter(start, end)
-	// q.Optimize() call is needed for converting '*' into filterNoop.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-	q.Optimize()
 	return q
 }
 
@@ -534,8 +531,8 @@ func (q *Query) AddPipeLimit(n uint64) {
 	})
 }
 
-// Optimize tries optimizing the query.
-func (q *Query) Optimize() {
+// optimize tries optimizing the query.
+func (q *Query) optimize() {
 	q.pipes = optimizeSortOffsetPipes(q.pipes)
 	q.pipes = optimizeSortLimitPipes(q.pipes)
 	q.pipes = optimizeUniqLimitPipes(q.pipes)
@@ -560,14 +557,6 @@ func (q *Query) Optimize() {
 
 	// Substitute '*' prefixFilter with filterNoop in order to avoid reading _msg data.
 	q.f = removeStarFilters(q.f)
-
-	// Call Optimize for queries from 'in(query)' filters.
-	optimizeFilterIn(q.f)
-
-	// Optimize individual pipes.
-	for _, p := range q.pipes {
-		p.optimize()
-	}
 }
 
 // GetStatsByFields returns `by (...)` fields from the last `stats` pipe at q.
@@ -746,6 +735,7 @@ func addByTimeField(byFields []*byStatsField, step int64) []*byStatsField {
 }
 
 func removeStarFilters(f filter) filter {
+	// Substitute `*` filterPrefix with filterNoop
 	visitFunc := func(f filter) bool {
 		fp, ok := f.(*filterPrefix)
 		return ok && isMsgFieldName(fp.fieldName) && fp.prefix == ""
@@ -758,6 +748,43 @@ func removeStarFilters(f filter) filter {
 	if err != nil {
 		logger.Fatalf("BUG: unexpected error: %s", err)
 	}
+
+	// Drop filterNoop inside filterAnd
+	visitFunc = func(f filter) bool {
+		fa, ok := f.(*filterAnd)
+		if !ok {
+			return false
+		}
+		for _, f := range fa.filters {
+			if _, ok := f.(*filterNoop); ok {
+				return true
+			}
+		}
+		return false
+	}
+	copyFunc = func(f filter) (filter, error) {
+		fa := f.(*filterAnd)
+		var resultFilters []filter
+		for _, f := range fa.filters {
+			if _, ok := f.(*filterNoop); !ok {
+				resultFilters = append(resultFilters, f)
+			}
+		}
+		if len(resultFilters) == 0 {
+			return &filterNoop{}, nil
+		}
+		if len(resultFilters) == 1 {
+			return resultFilters[0], nil
+		}
+		return &filterAnd{
+			filters: resultFilters,
+		}, nil
+	}
+	f, err = copyFilter(f, visitFunc, copyFunc)
+	if err != nil {
+		logger.Fatalf("BUG: unexpected error: %s", err)
+	}
+
 	return f
 }
 
@@ -946,6 +973,7 @@ func ParseQueryAtTimestamp(s string, timestamp int64) (*Query, error) {
 		return nil, fmt.Errorf("unexpected unparsed tail after [%s]; context: [%s]; tail: [%s]", q, lex.context(), lex.s)
 	}
 	q.timestamp = timestamp
+	q.optimize()
 	return q, nil
 }
 
diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go
index 615a80a405..7f99ffde3e 100644
--- a/lib/logstorage/parser_test.go
+++ b/lib/logstorage/parser_test.go
@@ -534,19 +534,28 @@ func TestParseFilterPrefix(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		fp, ok := q.f.(*filterPrefix)
-		if !ok {
-			t.Fatalf("unexpected filter type; got %T; want *filterPrefix; filter: %s", q.f, q.f)
-		}
-		if fp.fieldName != fieldNameExpected {
-			t.Fatalf("unexpected fieldName; got %q; want %q", fp.fieldName, fieldNameExpected)
-		}
-		if fp.prefix != prefixExpected {
-			t.Fatalf("unexpected prefix; got %q; want %q", fp.prefix, prefixExpected)
+		switch f := q.f.(type) {
+		case *filterPrefix:
+			if f.fieldName != fieldNameExpected {
+				t.Fatalf("unexpected fieldName; got %q; want %q", f.fieldName, fieldNameExpected)
+			}
+			if f.prefix != prefixExpected {
+				t.Fatalf("unexpected prefix; got %q; want %q", f.prefix, prefixExpected)
+			}
+		case *filterNoop:
+			if fieldNameExpected != "" {
+				t.Fatalf("expecting non-empty fieldName %q", fieldNameExpected)
+			}
+			if prefixExpected != "" {
+				t.Fatalf("expecting non-empty prefix %q", prefixExpected)
+			}
+		default:
+			t.Fatalf("unexpected filter type; got %T; want *filterPrefix or *filterNoop; filter: %s", q.f, q.f)
 		}
 	}
 
 	f(`*`, ``, ``)
+	f(`f:*`, `f`, ``)
 	f(`""*`, ``, ``)
 	f(`foo*`, ``, `foo`)
 	f(`abc-de.fg:foo-bar+baz*`, `abc-de.fg`, `foo-bar+baz`)
@@ -695,8 +704,8 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`'foo'* and (a:x* and x:* or y:i(""*)) and i("abc def"*)`, `foo* (a:x* x:* or y:i(*)) i("abc def"*)`)
 
 	// This isn't a prefix search - it equals to `foo AND *`
-	f(`foo *`, `foo *`)
-	f(`"foo" *`, `foo *`)
+	f(`foo *`, `foo`)
+	f(`"foo" *`, `foo`)
 
 	// empty filter
 	f(`"" or foo:"" and not bar:""`, `"" or foo:"" !bar:""`)
@@ -1197,13 +1206,13 @@ func TestParseQuerySuccess(t *testing.T) {
 	f(`* | uniq limit 10`, `* | uniq limit 10`)
 
 	// filter pipe
-	f(`* | filter error ip:12.3.4.5 or warn`, `* | filter error ip:12.3.4.5 or warn`)
-	f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) | limit 10`)
-	f(`* | error`, `* | filter error`)
-	f(`* | "by"`, `* | filter "by"`)
-	f(`* | "stats"`, `* | filter "stats"`)
-	f(`* | "count"`, `* | filter "count"`)
-	f(`* | foo:bar AND baz:<10`, `* | filter foo:bar baz:<10`)
+	f(`* | filter error ip:12.3.4.5 or warn`, `error ip:12.3.4.5 or warn`)
+	f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) limit 10`)
+	f(`* | error`, `error`)
+	f(`* | "by"`, `"by"`)
+	f(`* | "stats" *`, `"stats"`)
+	f(`* | * "count"`, `"count"`)
+	f(`* | foo:bar AND baz:<10`, `foo:bar baz:<10`)
 
 	// extract pipe
 	f(`* | extract "foo<bar>baz"`, `* | extract "foo<bar>baz"`)
@@ -1734,7 +1743,6 @@ func TestQueryGetNeededColumns(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse query [%s]: %s", s, err)
 		}
-		q.Optimize()
 
 		needed, unneeded := q.getNeededColumns()
 		neededColumns := strings.Join(needed, ",")
@@ -2200,7 +2208,6 @@ func TestQueryDropAllPipes(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse [%s]: %s", qStr, err)
 		}
-		q.Optimize()
 		q.DropAllPipes()
 		result := q.String()
 		if result != resultExpected {
diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go
index 00ac68e424..f4b49e9105 100644
--- a/lib/logstorage/pipe.go
+++ b/lib/logstorage/pipe.go
@@ -26,9 +26,6 @@ type pipe interface {
 	// The returned pipeProcessor may call cancel() at any time in order to notify the caller to stop sending new data to it.
 	newPipeProcessor(workersCount int, stopCh <-chan struct{}, cancel func(), ppNext pipeProcessor) pipeProcessor
 
-	// optimize must optimize the pipe
-	optimize()
-
 	// hasFilterInWithQuery must return true of pipe contains 'in(subquery)' filter (recursively).
 	hasFilterInWithQuery() bool
 
diff --git a/lib/logstorage/pipe_block_stats.go b/lib/logstorage/pipe_block_stats.go
index 8e6fc0b479..40e74201cc 100644
--- a/lib/logstorage/pipe_block_stats.go
+++ b/lib/logstorage/pipe_block_stats.go
@@ -22,10 +22,6 @@ func (ps *pipeBlockStats) canLiveTail() bool {
 	return false
 }
 
-func (ps *pipeBlockStats) optimize() {
-	// nothing to do
-}
-
 func (ps *pipeBlockStats) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_blocks_count.go b/lib/logstorage/pipe_blocks_count.go
index f1381b2414..6c109147c6 100644
--- a/lib/logstorage/pipe_blocks_count.go
+++ b/lib/logstorage/pipe_blocks_count.go
@@ -31,10 +31,6 @@ func (pc *pipeBlocksCount) updateNeededFields(neededFields, unneededFields field
 	unneededFields.reset()
 }
 
-func (pc *pipeBlocksCount) optimize() {
-	// nothing to do
-}
-
 func (pc *pipeBlocksCount) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_copy.go b/lib/logstorage/pipe_copy.go
index 9970231673..2c43920fb1 100644
--- a/lib/logstorage/pipe_copy.go
+++ b/lib/logstorage/pipe_copy.go
@@ -54,10 +54,6 @@ func (pc *pipeCopy) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pc *pipeCopy) optimize() {
-	// Nothing to do
-}
-
 func (pc *pipeCopy) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_delete.go b/lib/logstorage/pipe_delete.go
index e61fcfcc61..15429d15dd 100644
--- a/lib/logstorage/pipe_delete.go
+++ b/lib/logstorage/pipe_delete.go
@@ -34,10 +34,6 @@ func (pd *pipeDelete) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pd *pipeDelete) optimize() {
-	// nothing to do
-}
-
 func (pd *pipeDelete) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_drop_empty_fields.go b/lib/logstorage/pipe_drop_empty_fields.go
index ba18ad65bc..292842b663 100644
--- a/lib/logstorage/pipe_drop_empty_fields.go
+++ b/lib/logstorage/pipe_drop_empty_fields.go
@@ -21,10 +21,6 @@ func (pd *pipeDropEmptyFields) canLiveTail() bool {
 	return true
 }
 
-func (pd *pipeDropEmptyFields) optimize() {
-	// nothing to do
-}
-
 func (pd *pipeDropEmptyFields) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_extract.go b/lib/logstorage/pipe_extract.go
index ec89b15f76..4cf90897c4 100644
--- a/lib/logstorage/pipe_extract.go
+++ b/lib/logstorage/pipe_extract.go
@@ -45,10 +45,6 @@ func (pe *pipeExtract) canLiveTail() bool {
 	return true
 }
 
-func (pe *pipeExtract) optimize() {
-	pe.iff.optimizeFilterIn()
-}
-
 func (pe *pipeExtract) hasFilterInWithQuery() bool {
 	return pe.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_extract_regexp.go b/lib/logstorage/pipe_extract_regexp.go
index 2f40d36adc..c5abe5dc3f 100644
--- a/lib/logstorage/pipe_extract_regexp.go
+++ b/lib/logstorage/pipe_extract_regexp.go
@@ -47,10 +47,6 @@ func (pe *pipeExtractRegexp) canLiveTail() bool {
 	return true
 }
 
-func (pe *pipeExtractRegexp) optimize() {
-	pe.iff.optimizeFilterIn()
-}
-
 func (pe *pipeExtractRegexp) hasFilterInWithQuery() bool {
 	return pe.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_field_names.go b/lib/logstorage/pipe_field_names.go
index 283cb7675d..c5257e108f 100644
--- a/lib/logstorage/pipe_field_names.go
+++ b/lib/logstorage/pipe_field_names.go
@@ -39,10 +39,6 @@ func (pf *pipeFieldNames) updateNeededFields(neededFields, unneededFields fields
 	unneededFields.reset()
 }
 
-func (pf *pipeFieldNames) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFieldNames) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_field_values.go b/lib/logstorage/pipe_field_values.go
index 4270f9e31a..80c8408ca0 100644
--- a/lib/logstorage/pipe_field_values.go
+++ b/lib/logstorage/pipe_field_values.go
@@ -46,10 +46,6 @@ func (pf *pipeFieldValues) updateNeededFields(neededFields, unneededFields field
 	}
 }
 
-func (pf *pipeFieldValues) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFieldValues) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_fields.go b/lib/logstorage/pipe_fields.go
index f08e5c608a..f2dd03ba54 100644
--- a/lib/logstorage/pipe_fields.go
+++ b/lib/logstorage/pipe_fields.go
@@ -54,10 +54,6 @@ func (pf *pipeFields) updateNeededFields(neededFields, unneededFields fieldsSet)
 	unneededFields.reset()
 }
 
-func (pf *pipeFields) optimize() {
-	// nothing to do
-}
-
 func (pf *pipeFields) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_filter.go b/lib/logstorage/pipe_filter.go
index 7b7308342a..2c153d26fb 100644
--- a/lib/logstorage/pipe_filter.go
+++ b/lib/logstorage/pipe_filter.go
@@ -33,10 +33,6 @@ func (pf *pipeFilter) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pf *pipeFilter) optimize() {
-	optimizeFilterIn(pf.f)
-}
-
 func (pf *pipeFilter) hasFilterInWithQuery() bool {
 	return hasFilterInWithQueryForFilter(pf.f)
 }
diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go
index 7658101866..6d83312b5c 100644
--- a/lib/logstorage/pipe_format.go
+++ b/lib/logstorage/pipe_format.go
@@ -87,10 +87,6 @@ func (pf *pipeFormat) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pf *pipeFormat) optimize() {
-	pf.iff.optimizeFilterIn()
-}
-
 func (pf *pipeFormat) hasFilterInWithQuery() bool {
 	return pf.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_join.go b/lib/logstorage/pipe_join.go
index 606e64e406..89c7556e03 100644
--- a/lib/logstorage/pipe_join.go
+++ b/lib/logstorage/pipe_join.go
@@ -37,10 +37,6 @@ func (pj *pipeJoin) canLiveTail() bool {
 	return true
 }
 
-func (pj *pipeJoin) optimize() {
-	pj.q.Optimize()
-}
-
 func (pj *pipeJoin) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_len.go b/lib/logstorage/pipe_len.go
index 907257e17e..4d902a7e43 100644
--- a/lib/logstorage/pipe_len.go
+++ b/lib/logstorage/pipe_len.go
@@ -41,10 +41,6 @@ func (pl *pipeLen) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pl *pipeLen) optimize() {
-	// Nothing to do
-}
-
 func (pl *pipeLen) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_limit.go b/lib/logstorage/pipe_limit.go
index 6dec31baef..99a2e65ca1 100644
--- a/lib/logstorage/pipe_limit.go
+++ b/lib/logstorage/pipe_limit.go
@@ -24,10 +24,6 @@ func (pl *pipeLimit) updateNeededFields(_, _ fieldsSet) {
 	// nothing to do
 }
 
-func (pl *pipeLimit) optimize() {
-	// nothing to do
-}
-
 func (pl *pipeLimit) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go
index c3999351fb..1bc93f656e 100644
--- a/lib/logstorage/pipe_math.go
+++ b/lib/logstorage/pipe_math.go
@@ -221,10 +221,6 @@ func (me *mathExpr) updateNeededFields(neededFields fieldsSet) {
 	}
 }
 
-func (pm *pipeMath) optimize() {
-	// nothing to do
-}
-
 func (pm *pipeMath) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_offset.go b/lib/logstorage/pipe_offset.go
index 5f3f923142..4588586692 100644
--- a/lib/logstorage/pipe_offset.go
+++ b/lib/logstorage/pipe_offset.go
@@ -24,10 +24,6 @@ func (po *pipeOffset) updateNeededFields(_, _ fieldsSet) {
 	// nothing to do
 }
 
-func (po *pipeOffset) optimize() {
-	// nothing to do
-}
-
 func (po *pipeOffset) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_pack_json.go b/lib/logstorage/pipe_pack_json.go
index a7c2a3fec4..01c7f9c7bc 100644
--- a/lib/logstorage/pipe_pack_json.go
+++ b/lib/logstorage/pipe_pack_json.go
@@ -33,10 +33,6 @@ func (pp *pipePackJSON) updateNeededFields(neededFields, unneededFields fieldsSe
 	updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields)
 }
 
-func (pp *pipePackJSON) optimize() {
-	// nothing to do
-}
-
 func (pp *pipePackJSON) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_pack_logfmt.go b/lib/logstorage/pipe_pack_logfmt.go
index c01c888a6e..3edc4ce22b 100644
--- a/lib/logstorage/pipe_pack_logfmt.go
+++ b/lib/logstorage/pipe_pack_logfmt.go
@@ -33,10 +33,6 @@ func (pp *pipePackLogfmt) updateNeededFields(neededFields, unneededFields fields
 	updateNeededFieldsForPipePack(neededFields, unneededFields, pp.resultField, pp.fields)
 }
 
-func (pp *pipePackLogfmt) optimize() {
-	// nothing to do
-}
-
 func (pp *pipePackLogfmt) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_rename.go b/lib/logstorage/pipe_rename.go
index c35c409ee7..e4dbb6ac3b 100644
--- a/lib/logstorage/pipe_rename.go
+++ b/lib/logstorage/pipe_rename.go
@@ -58,10 +58,6 @@ func (pr *pipeRename) updateNeededFields(neededFields, unneededFields fieldsSet)
 	}
 }
 
-func (pr *pipeRename) optimize() {
-	// nothing to do
-}
-
 func (pr *pipeRename) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_replace.go b/lib/logstorage/pipe_replace.go
index 31aaff9da0..fa0ab7c1ff 100644
--- a/lib/logstorage/pipe_replace.go
+++ b/lib/logstorage/pipe_replace.go
@@ -45,10 +45,6 @@ func (pr *pipeReplace) updateNeededFields(neededFields, unneededFields fieldsSet
 	updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff)
 }
 
-func (pr *pipeReplace) optimize() {
-	pr.iff.optimizeFilterIn()
-}
-
 func (pr *pipeReplace) hasFilterInWithQuery() bool {
 	return pr.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_replace_regexp.go b/lib/logstorage/pipe_replace_regexp.go
index 1265da996b..4f11f3cfde 100644
--- a/lib/logstorage/pipe_replace_regexp.go
+++ b/lib/logstorage/pipe_replace_regexp.go
@@ -45,10 +45,6 @@ func (pr *pipeReplaceRegexp) updateNeededFields(neededFields, unneededFields fie
 	updateNeededFieldsForUpdatePipe(neededFields, unneededFields, pr.field, pr.iff)
 }
 
-func (pr *pipeReplaceRegexp) optimize() {
-	pr.iff.optimizeFilterIn()
-}
-
 func (pr *pipeReplaceRegexp) hasFilterInWithQuery() bool {
 	return pr.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go
index 7f9a7e7797..6901cbd212 100644
--- a/lib/logstorage/pipe_sort.go
+++ b/lib/logstorage/pipe_sort.go
@@ -89,10 +89,6 @@ func (ps *pipeSort) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (ps *pipeSort) optimize() {
-	// nothing to do
-}
-
 func (ps *pipeSort) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go
index 7cf723a1b8..b2d85e28cd 100644
--- a/lib/logstorage/pipe_stats.go
+++ b/lib/logstorage/pipe_stats.go
@@ -123,12 +123,6 @@ func (ps *pipeStats) updateNeededFields(neededFields, unneededFields fieldsSet)
 	unneededFields.reset()
 }
 
-func (ps *pipeStats) optimize() {
-	for _, f := range ps.funcs {
-		f.iff.optimizeFilterIn()
-	}
-}
-
 func (ps *pipeStats) hasFilterInWithQuery() bool {
 	for _, f := range ps.funcs {
 		if f.iff.hasFilterInWithQuery() {
diff --git a/lib/logstorage/pipe_stream_context.go b/lib/logstorage/pipe_stream_context.go
index 09400bdf56..0fe0ba95d2 100644
--- a/lib/logstorage/pipe_stream_context.go
+++ b/lib/logstorage/pipe_stream_context.go
@@ -55,10 +55,6 @@ func (pc *pipeStreamContext) updateNeededFields(neededFields, unneededFields fie
 	unneededFields.removeFields(neededFieldsForStreamContext)
 }
 
-func (pc *pipeStreamContext) optimize() {
-	// nothing to do
-}
-
 func (pc *pipeStreamContext) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_top.go b/lib/logstorage/pipe_top.go
index e33a578405..ad7ff44658 100644
--- a/lib/logstorage/pipe_top.go
+++ b/lib/logstorage/pipe_top.go
@@ -71,10 +71,6 @@ func (pt *pipeTop) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pt *pipeTop) optimize() {
-	// nothing to do
-}
-
 func (pt *pipeTop) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go
index ddf043c58b..087dffe7ab 100644
--- a/lib/logstorage/pipe_uniq.go
+++ b/lib/logstorage/pipe_uniq.go
@@ -58,10 +58,6 @@ func (pu *pipeUniq) updateNeededFields(neededFields, unneededFields fieldsSet) {
 	}
 }
 
-func (pu *pipeUniq) optimize() {
-	// nothing to do
-}
-
 func (pu *pipeUniq) hasFilterInWithQuery() bool {
 	return false
 }
diff --git a/lib/logstorage/pipe_unpack_json.go b/lib/logstorage/pipe_unpack_json.go
index 635c925114..f2855ff467 100644
--- a/lib/logstorage/pipe_unpack_json.go
+++ b/lib/logstorage/pipe_unpack_json.go
@@ -60,10 +60,6 @@ func (pu *pipeUnpackJSON) updateNeededFields(neededFields, unneededFields fields
 	updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackJSON) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackJSON) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unpack_logfmt.go b/lib/logstorage/pipe_unpack_logfmt.go
index ab1d4d61d7..d45343c62c 100644
--- a/lib/logstorage/pipe_unpack_logfmt.go
+++ b/lib/logstorage/pipe_unpack_logfmt.go
@@ -58,10 +58,6 @@ func (pu *pipeUnpackLogfmt) updateNeededFields(neededFields, unneededFields fiel
 	updateNeededFieldsForUnpackPipe(pu.fromField, pu.fields, pu.keepOriginalFields, pu.skipEmptyResults, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackLogfmt) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackLogfmt) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unpack_syslog.go b/lib/logstorage/pipe_unpack_syslog.go
index 6717c2dcc0..6eea365dbd 100644
--- a/lib/logstorage/pipe_unpack_syslog.go
+++ b/lib/logstorage/pipe_unpack_syslog.go
@@ -54,10 +54,6 @@ func (pu *pipeUnpackSyslog) updateNeededFields(neededFields, unneededFields fiel
 	updateNeededFieldsForUnpackPipe(pu.fromField, nil, pu.keepOriginalFields, false, pu.iff, neededFields, unneededFields)
 }
 
-func (pu *pipeUnpackSyslog) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnpackSyslog) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/pipe_unroll.go b/lib/logstorage/pipe_unroll.go
index fb9202caa1..0a406d64af 100644
--- a/lib/logstorage/pipe_unroll.go
+++ b/lib/logstorage/pipe_unroll.go
@@ -36,10 +36,6 @@ func (pu *pipeUnroll) canLiveTail() bool {
 	return true
 }
 
-func (pu *pipeUnroll) optimize() {
-	pu.iff.optimizeFilterIn()
-}
-
 func (pu *pipeUnroll) hasFilterInWithQuery() bool {
 	return pu.iff.hasFilterInWithQuery()
 }
diff --git a/lib/logstorage/storage_search.go b/lib/logstorage/storage_search.go
index 54fe066e85..221e0450ce 100644
--- a/lib/logstorage/storage_search.go
+++ b/lib/logstorage/storage_search.go
@@ -220,8 +220,6 @@ func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []TenantID, q *Qu
 func (s *Storage) getJoinMap(ctx context.Context, tenantIDs []TenantID, q *Query, byFields []string, prefix string) (map[string][][]Field, error) {
 	// TODO: track memory usage
 
-	logger.Infof("DEBUG: byFields=%q, prefix=%q", byFields, prefix)
-
 	m := make(map[string][][]Field)
 	var mLock sync.Mutex
 	writeBlockResult := func(_ uint, br *blockResult) {