{app/vmstorage,lib/blockcache}: add flags to configure number of shards to use for indexdb/dataBlocks and indexdb/indexBlocks caches

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

app/vmstorage/main.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/servers"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -22,7 +24,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -68,8 +69,12 @@ var (
 		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+
 		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
+	cacheShardsIndexDBIndexBlocks = flag.Int("storage.cacheShardsIndexDBIndexBlocks", 0, "Overrides number of shards for indexdb/indexBlocks cache. "+
+		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+
 		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
+	cacheShardsIndexDBDataBlocks = flag.Int("storage.cacheShardsIndexDBDataBlocks", 0, "Overrides number of shards for indexdb/dataBlocks cache. "+
+		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
 		"See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 )
@@ -92,7 +97,9 @@ func main() {
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
+	mergeset.SetIndexBlocksCacheShardsCount(*cacheShardsIndexDBIndexBlocks)
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
+	mergeset.SetDataBlocksCacheShardsCount(*cacheShardsIndexDBDataBlocks)
 
 	if retentionPeriod.Msecs < 24*3600*1000 {
 		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)

lib/blockcache/blockcache.go

@@ -7,9 +7,10 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/cespare/xxhash/v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/cespare/xxhash/v2"
 )
 
 // Cache caches Block entries.
@@ -26,16 +27,18 @@ type Cache struct {
 //
 // Cache size in bytes is limited by the value returned by getMaxSizeBytes() callback.
 // Call MustStop() in order to free up resources occupied by Cache.
-func NewCache(getMaxSizeBytes func() int) *Cache {
+func NewCache(getMaxSizeBytes func() int, shardsCount int) *Cache {
-	cpusCount := cgroup.AvailableCPUs()
+	if shardsCount <= 0 {
-	shardsCount := cgroup.AvailableCPUs()
+		cpusCount := cgroup.AvailableCPUs()
-	// Increase the number of shards with the increased number of available CPU cores.
+		shardsCount = cgroup.AvailableCPUs()
-	// This should reduce contention on per-shard mutexes.
+		// Increase the number of shards with the increased number of available CPU cores.
-	multiplier := cpusCount
+		// This should reduce contention on per-shard mutexes.
-	if multiplier > 16 {
+		multiplier := cpusCount
-		multiplier = 16
+		if multiplier > 16 {
+			multiplier = 16
+		}
+		shardsCount *= multiplier
 	}
-	shardsCount *= multiplier
 	shards := make([]*cache, shardsCount)
 	getMaxShardBytes := func() int {
 		n := getMaxSizeBytes()

lib/mergeset/part.go

@@ -12,8 +12,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 )
 
-var idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize)
+var idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, 0)
-var ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize)
+var ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize, 0)
 
 // SetIndexBlocksCacheSize overrides the default size of indexdb/indexBlock cache
 func SetIndexBlocksCacheSize(size int) {
@@ -29,6 +29,20 @@ func getMaxIndexBlocksCacheSize() int {
 	return maxIndexBlockCacheSize
 }
 
+func SetIndexBlocksCacheShardsCount(size int) {
+	if size <= 0 {
+		return
+	}
+	idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, size)
+}
+
+func SetDataBlocksCacheShardsCount(size int) {
+	if size <= 0 {
+		return
+	}
+	ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize, size)
+}
+
 var (
 	maxIndexBlockCacheSize     int
 	maxIndexBlockCacheSizeOnce sync.Once

lib/storage/part.go

@@ -12,7 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 )
 
-var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize)
+var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, 0)
 
 func getMaxIndexBlocksCacheSize() int {
 	maxIndexBlockCacheSizeOnce.Do(func() {