From 45d7cc0d33d668b54a9b77698f6ee7c3bad9a92e Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Fri, 1 Dec 2023 16:30:28 +0400 Subject: [PATCH] {app/vmstorage,lib/blockcache}: add flags to configure number of shards to use for indexdb/dataBlocks and indexdb/indexBlocks caches Signed-off-by: Zakhar Bessarab --- app/vmstorage/main.go | 9 ++++++++- lib/blockcache/blockcache.go | 23 +++++++++++++---------- lib/mergeset/part.go | 18 ++++++++++++++++-- lib/storage/part.go | 2 +- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 3e7cf66a0..c27f14d50 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "github.com/VictoriaMetrics/metrics" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/servers" "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo" "github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag" @@ -22,7 +24,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" "github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" - "github.com/VictoriaMetrics/metrics" ) var ( @@ -68,8 +69,12 @@ var ( "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+ "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") + cacheShardsIndexDBIndexBlocks = flag.Int("storage.cacheShardsIndexDBIndexBlocks", 0, "Overrides number of shards for indexdb/indexBlocks cache. "+ + "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+ "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") + cacheShardsIndexDBDataBlocks = flag.Int("storage.cacheShardsIndexDBDataBlocks", 0, "Overrides number of shards for indexdb/dataBlocks cache. "+ + "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+ "See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning") ) @@ -92,7 +97,9 @@ func main() { storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN()) storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN()) mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN()) + mergeset.SetIndexBlocksCacheShardsCount(*cacheShardsIndexDBIndexBlocks) mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN()) + mergeset.SetDataBlocksCacheShardsCount(*cacheShardsIndexDBDataBlocks) if retentionPeriod.Msecs < 24*3600*1000 { logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod) diff --git a/lib/blockcache/blockcache.go b/lib/blockcache/blockcache.go index ce2aaab0e..17310d7dc 100644 --- a/lib/blockcache/blockcache.go +++ b/lib/blockcache/blockcache.go @@ -7,9 +7,10 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash/v2" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - "github.com/cespare/xxhash/v2" ) // Cache caches Block entries. @@ -26,16 +27,18 @@ type Cache struct { // // Cache size in bytes is limited by the value returned by getMaxSizeBytes() callback. // Call MustStop() in order to free up resources occupied by Cache. -func NewCache(getMaxSizeBytes func() int) *Cache { - cpusCount := cgroup.AvailableCPUs() - shardsCount := cgroup.AvailableCPUs() - // Increase the number of shards with the increased number of available CPU cores. - // This should reduce contention on per-shard mutexes. - multiplier := cpusCount - if multiplier > 16 { - multiplier = 16 +func NewCache(getMaxSizeBytes func() int, shardsCount int) *Cache { + if shardsCount <= 0 { + cpusCount := cgroup.AvailableCPUs() + shardsCount = cgroup.AvailableCPUs() + // Increase the number of shards with the increased number of available CPU cores. + // This should reduce contention on per-shard mutexes. + multiplier := cpusCount + if multiplier > 16 { + multiplier = 16 + } + shardsCount *= multiplier } - shardsCount *= multiplier shards := make([]*cache, shardsCount) getMaxShardBytes := func() int { n := getMaxSizeBytes() diff --git a/lib/mergeset/part.go b/lib/mergeset/part.go index 13e8c705b..e412f9a76 100644 --- a/lib/mergeset/part.go +++ b/lib/mergeset/part.go @@ -12,8 +12,8 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" ) -var idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize) -var ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize) +var idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, 0) +var ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize, 0) // SetIndexBlocksCacheSize overrides the default size of indexdb/indexBlock cache func SetIndexBlocksCacheSize(size int) { @@ -29,6 +29,20 @@ func getMaxIndexBlocksCacheSize() int { return maxIndexBlockCacheSize } +func SetIndexBlocksCacheShardsCount(size int) { + if size <= 0 { + return + } + idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, size) +} + +func SetDataBlocksCacheShardsCount(size int) { + if size <= 0 { + return + } + ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize, size) +} + var ( maxIndexBlockCacheSize int maxIndexBlockCacheSizeOnce sync.Once diff --git a/lib/storage/part.go b/lib/storage/part.go index 4efd74382..ffab3db17 100644 --- a/lib/storage/part.go +++ b/lib/storage/part.go @@ -12,7 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" ) -var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize) +var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize, 0) func getMaxIndexBlocksCacheSize() int { maxIndexBlockCacheSizeOnce.Do(func() {