From ba40c1a2d5f02dbaf6a126817b4ddce25a96e7d5 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Tue, 28 May 2024 15:58:36 +0200
Subject: [PATCH] wip

---
 lib/logstorage/filter_and.go | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/lib/logstorage/filter_and.go b/lib/logstorage/filter_and.go
index 5d46151db..08bf8736a 100644
--- a/lib/logstorage/filter_and.go
+++ b/lib/logstorage/filter_and.go
@@ -113,7 +113,7 @@ func (fa *filterAnd) getByFieldTokens() []fieldTokens {
 }
 
 func (fa *filterAnd) initByFieldTokens() {
-	m := make(map[string][]string)
+	m := make(map[string]map[string]struct{})
 	byFieldFilters := make(map[string]int)
 	var fieldNames []string
 
@@ -146,11 +146,15 @@ func (fa *filterAnd) initByFieldTokens() {
 		byFieldFilters[fieldName]++
 
 		if len(tokens) > 0 {
-			a, ok := m[fieldName]
+			mTokens, ok := m[fieldName]
 			if !ok {
 				fieldNames = append(fieldNames, fieldName)
+				mTokens = make(map[string]struct{})
+				m[fieldName] = mTokens
+			}
+			for _, token := range tokens {
+				mTokens[token] = struct{}{}
 			}
-			m[fieldName] = append(a, tokens...)
 		}
 	}
 
@@ -160,9 +164,16 @@ func (fa *filterAnd) initByFieldTokens() {
 			// It is faster to perform bloom filter match inline when visiting the corresponding column
 			continue
 		}
+
+		mTokens := m[fieldName]
+		tokens := make([]string, 0, len(mTokens))
+		for token := range mTokens {
+			tokens = append(tokens, token)
+		}
+
 		byFieldTokens = append(byFieldTokens, fieldTokens{
 			field:  fieldName,
-			tokens: m[fieldName],
+			tokens: tokens,
 		})
 	}