From 41754e12f89051689ae6d6a68e145432c708bbdb Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 7 Jul 2021 15:21:24 +0300
Subject: [PATCH] lib/mergeset: cache indexBlock items only on the second
 request

This should reduce the indexdb/indexBlocks cache size, since it won't contain one-time-wonders items.
---
 docs/CHANGELOG.md    |  2 +-
 lib/mergeset/part.go | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 51a24c20b..47988d05e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -7,7 +7,7 @@ sort: 15
 ## tip
 
 * FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
-* FEATURE: reduce memory usage when performing heavy queries over big number of time series.
+* FEATURE: reduce memory usage by up to 30% on production workloads.
 * FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
 * FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution for the served requests.
 
diff --git a/lib/mergeset/part.go b/lib/mergeset/part.go
index fb4584c39..eb5daace5 100644
--- a/lib/mergeset/part.go
+++ b/lib/mergeset/part.go
@@ -156,6 +156,9 @@ type indexBlockCache struct {
 	m  map[uint64]*indexBlockCacheEntry
 	mu sync.RWMutex
 
+	perKeyMisses     map[uint64]int
+	perKeyMissesLock sync.Mutex
+
 	cleanerStopCh chan struct{}
 	cleanerWG     sync.WaitGroup
 }
@@ -172,6 +175,7 @@ type indexBlockCacheEntry struct {
 func newIndexBlockCache() *indexBlockCache {
 	var idxbc indexBlockCache
 	idxbc.m = make(map[uint64]*indexBlockCacheEntry)
+	idxbc.perKeyMisses = make(map[uint64]int)
 	idxbc.cleanerStopCh = make(chan struct{})
 	idxbc.cleanerWG.Add(1)
 	go func() {
@@ -185,6 +189,7 @@ func (idxbc *indexBlockCache) MustClose() {
 	close(idxbc.cleanerStopCh)
 	idxbc.cleanerWG.Wait()
 	idxbc.m = nil
+	idxbc.perKeyMisses = nil
 }
 
 // cleaner periodically cleans least recently used items.
@@ -212,6 +217,10 @@ func (idxbc *indexBlockCache) cleanByTimeout() {
 		}
 	}
 	idxbc.mu.Unlock()
+
+	idxbc.perKeyMissesLock.Lock()
+	idxbc.perKeyMisses = make(map[uint64]int, len(idxbc.perKeyMisses))
+	idxbc.perKeyMissesLock.Unlock()
 }
 
 func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
@@ -227,12 +236,24 @@ func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
 		}
 		return idxbe.idxb
 	}
+	idxbc.perKeyMissesLock.Lock()
+	idxbc.perKeyMisses[k]++
+	idxbc.perKeyMissesLock.Unlock()
 	atomic.AddUint64(&idxbc.misses, 1)
 	return nil
 }
 
 // Put puts idxb under the key k into idxbc.
 func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) {
+	idxbc.perKeyMissesLock.Lock()
+	doNotCache := idxbc.perKeyMisses[k] == 1
+	idxbc.perKeyMissesLock.Unlock()
+	if doNotCache {
+		// Do not cache ib if it has been requested only once (aka one-time-wonders items).
+		// This should reduce memory usage for the ibc cache.
+		return
+	}
+
 	idxbc.mu.Lock()
 	// Remove superfluous entries.
 	if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {