From 833e413cc0f293dc9972f20895e3eb3e00bc4f2e Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 30 May 2024 12:51:27 +0200
Subject: [PATCH] wip

---
 lib/logstorage/filter_in.go      | 23 ++++++++++++++++++++---
 lib/logstorage/filter_in_test.go |  2 +-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/lib/logstorage/filter_in.go b/lib/logstorage/filter_in.go
index 742a0fb40..eb39f41f6 100644
--- a/lib/logstorage/filter_in.go
+++ b/lib/logstorage/filter_in.go
@@ -426,10 +426,12 @@ func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[str
 
 func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, commonTokens []string, tokenSets [][]string) bool {
 	if len(commonTokens) > 0 {
-		return matchBloomFilterAllTokens(bs, ch, commonTokens)
+		if !matchBloomFilterAllTokens(bs, ch, commonTokens) {
+			return false
+		}
 	}
 	if len(tokenSets) == 0 {
-		return false
+		return len(commonTokens) > 0
 	}
 	if len(tokenSets) > maxTokenSetsToInit || uint64(len(tokenSets)) > 10*bs.bsw.bh.rowsCount {
 		// It is faster to match every row in the block against all the values
@@ -471,7 +473,22 @@ func getCommonTokensAndTokenSets(values []string) ([]string, [][]string) {
 	if len(commonTokens) == 0 {
 		return nil, tokenSets
 	}
-	return commonTokens, nil
+
+	// remove commonTokens from tokenSets
+	for i, tokens := range tokenSets {
+		dstTokens := tokens[:0]
+		for _, token := range tokens {
+			if !slices.Contains(commonTokens, token) {
+				dstTokens = append(dstTokens, token)
+			}
+		}
+		if len(dstTokens) == 0 {
+			return commonTokens, nil
+		}
+		tokenSets[i] = dstTokens
+	}
+
+	return commonTokens, tokenSets
 }
 
 func getCommonTokens(tokenSets [][]string) []string {
diff --git a/lib/logstorage/filter_in_test.go b/lib/logstorage/filter_in_test.go
index 29f1e35ff..80c76c616 100644
--- a/lib/logstorage/filter_in_test.go
+++ b/lib/logstorage/filter_in_test.go
@@ -716,6 +716,6 @@ func TestGetCommonTokensAndTokenSets(t *testing.T) {
 	f([]string{"foo", "foo"}, []string{"foo"}, nil)
 	f([]string{"foo", "bar", "bar", "foo"}, nil, [][]string{{"foo"}, {"bar"}, {"bar"}, {"foo"}})
 	f([]string{"foo", "foo bar", "bar foo"}, []string{"foo"}, nil)
-	f([]string{"a foo bar", "bar abc foo", "foo abc a bar"}, []string{"bar", "foo"}, nil)
+	f([]string{"a foo bar", "bar abc foo", "foo abc a bar"}, []string{"bar", "foo"}, [][]string{{"a"}, {"abc"}, {"a", "abc"}})
 	f([]string{"a xfoo bar", "xbar abc foo", "foo abc a bar"}, nil, [][]string{{"a", "bar", "xfoo"}, {"abc", "foo", "xbar"}, {"a", "abc", "bar", "foo"}})
 }