From 9826f7c1bec662c17a50fe71b209a32c2293d133 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 7 Jul 2021 10:57:38 +0300
Subject: [PATCH] lib/storage: do not cache inmemoryBlock entries requested
 only once (aka one-time-wonder items)

This should reduce the cache size and memory usage for the indexdb/dataBlocks cache
---
 lib/mergeset/part.go | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/lib/mergeset/part.go b/lib/mergeset/part.go
index e8b78543eb..fb4584c392 100644
--- a/lib/mergeset/part.go
+++ b/lib/mergeset/part.go
@@ -234,7 +234,6 @@ func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
 // Put puts idxb under the key k into idxbc.
 func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) {
 	idxbc.mu.Lock()
-
 	// Remove superfluous entries.
 	if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {
 		// Remove 10% of items from the cache.
@@ -292,6 +291,9 @@ type inmemoryBlockCache struct {
 	m  map[inmemoryBlockCacheKey]*inmemoryBlockCacheEntry
 	mu sync.RWMutex
 
+	perKeyMisses     map[inmemoryBlockCacheKey]int
+	perKeyMissesLock sync.Mutex
+
 	cleanerStopCh chan struct{}
 	cleanerWG     sync.WaitGroup
 }
@@ -316,7 +318,7 @@ type inmemoryBlockCacheEntry struct {
 func newInmemoryBlockCache() *inmemoryBlockCache {
 	var ibc inmemoryBlockCache
 	ibc.m = make(map[inmemoryBlockCacheKey]*inmemoryBlockCacheEntry)
-
+	ibc.perKeyMisses = make(map[inmemoryBlockCacheKey]int)
 	ibc.cleanerStopCh = make(chan struct{})
 	ibc.cleanerWG.Add(1)
 	go func() {
@@ -330,6 +332,7 @@ func (ibc *inmemoryBlockCache) MustClose() {
 	close(ibc.cleanerStopCh)
 	ibc.cleanerWG.Wait()
 	ibc.m = nil
+	ibc.perKeyMisses = nil
 }
 
 // cleaner periodically cleans least recently used items.
@@ -357,6 +360,10 @@ func (ibc *inmemoryBlockCache) cleanByTimeout() {
 		}
 	}
 	ibc.mu.Unlock()
+
+	ibc.perKeyMissesLock.Lock()
+	ibc.perKeyMisses = make(map[inmemoryBlockCacheKey]int, len(ibc.perKeyMisses))
+	ibc.perKeyMissesLock.Unlock()
 }
 
 func (ibc *inmemoryBlockCache) Get(k inmemoryBlockCacheKey) *inmemoryBlock {
@@ -373,14 +380,25 @@ func (ibc *inmemoryBlockCache) Get(k inmemoryBlockCacheKey) *inmemoryBlock {
 		}
 		return ibe.ib
 	}
+	ibc.perKeyMissesLock.Lock()
+	ibc.perKeyMisses[k]++
+	ibc.perKeyMissesLock.Unlock()
 	atomic.AddUint64(&ibc.misses, 1)
 	return nil
 }
 
 // Put puts ib under key k into ibc.
 func (ibc *inmemoryBlockCache) Put(k inmemoryBlockCacheKey, ib *inmemoryBlock) {
-	ibc.mu.Lock()
+	ibc.perKeyMissesLock.Lock()
+	doNotCache := ibc.perKeyMisses[k] == 1
+	ibc.perKeyMissesLock.Unlock()
+	if doNotCache {
+		// Do not cache ib if it has been requested only once (aka one-time-wonders items).
+		// This should reduce memory usage for the ibc cache.
+		return
+	}
 
+	ibc.mu.Lock()
 	// Clean superfluous entries in cache.
 	if overflow := len(ibc.m) - getMaxCachedInmemoryBlocksPerPart(); overflow > 0 {
 		// Remove 10% of items from the cache.