From 62b46007c5745354b0cf468c2a6a495a9ff15676 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 23 Feb 2022 13:39:11 +0200 Subject: [PATCH] lib/workingsetcache: reduce the default cache rotation period from hour to 20 minutes This should reduce memory usage under high time series churn rate --- app/vmselect/promql/rollup_result_cache.go | 6 ++--- docs/CHANGELOG.md | 2 ++ lib/storage/index_db.go | 4 +-- lib/storage/index_db_test.go | 6 ++--- lib/storage/storage.go | 2 +- lib/workingsetcache/cache.go | 29 ++++++++++++++++++---- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/app/vmselect/promql/rollup_result_cache.go b/app/vmselect/promql/rollup_result_cache.go index d251fb0029..9ffd38a92f 100644 --- a/app/vmselect/promql/rollup_result_cache.go +++ b/app/vmselect/promql/rollup_result_cache.go @@ -76,7 +76,7 @@ var checkRollupResultCacheResetOnce sync.Once var rollupResultResetMetricRowSample atomic.Value var rollupResultCacheV = &rollupResultCache{ - c: workingsetcache.New(1024*1024, time.Hour), // This is a cache for testing. + c: workingsetcache.New(1024 * 1024), // This is a cache for testing. } var rollupResultCachePath string @@ -104,9 +104,9 @@ func InitRollupResultCache(cachePath string) { var c *workingsetcache.Cache if len(rollupResultCachePath) > 0 { logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath) - c = workingsetcache.Load(rollupResultCachePath, cacheSize, time.Hour) + c = workingsetcache.Load(rollupResultCachePath, cacheSize) } else { - c = workingsetcache.New(cacheSize, time.Hour) + c = workingsetcache.New(cacheSize) } if *disableCache { c.Reset() diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fb942c346f..4b6d5d3f4d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -14,6 +14,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* FEATURE: reduce memory usage for various caches under [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). + ## [v1.73.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.73.1) diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 35ff9be3a0..83fd399b12 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -142,9 +142,9 @@ func openIndexDB(path string, s *Storage, rotationTimestamp uint64) (*indexDB, e tb: tb, name: name, - tagFiltersCache: workingsetcache.New(mem/32, time.Hour), + tagFiltersCache: workingsetcache.New(mem / 32), s: s, - loopsPerDateTagFilterCache: workingsetcache.New(mem/128, time.Hour), + loopsPerDateTagFilterCache: workingsetcache.New(mem / 128), } return db, nil } diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index 61d5692f8c..1f79360f04 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -1851,9 +1851,9 @@ func newTestStorage() *Storage { s := &Storage{ cachePath: "test-storage-cache", - metricIDCache: workingsetcache.New(1234, time.Hour), - metricNameCache: workingsetcache.New(1234, time.Hour), - tsidCache: workingsetcache.New(1234, time.Hour), + metricIDCache: workingsetcache.New(1234), + metricNameCache: workingsetcache.New(1234), + tsidCache: workingsetcache.New(1234), } s.setDeletedMetricIDs(&uint64set.Set{}) return s diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 18e194e616..6189ebfea7 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -993,7 +993,7 @@ func (s *Storage) mustLoadCache(info, name string, sizeBytes int) *workingsetcac path := s.cachePath + "/" + name logger.Infof("loading %s cache from %q...", info, path) startTime := time.Now() - c := workingsetcache.Load(path, sizeBytes, time.Hour) + c := workingsetcache.Load(path, sizeBytes) var cs fastcache.Stats c.UpdateStats(&cs) logger.Infof("loaded %s cache from %q in %.3f seconds; entriesCount: %d; sizeBytes: %d", diff --git a/lib/workingsetcache/cache.go b/lib/workingsetcache/cache.go index c249693f3b..aab534b4d6 100644 --- a/lib/workingsetcache/cache.go +++ b/lib/workingsetcache/cache.go @@ -17,6 +17,8 @@ const ( whole = 2 ) +const defaultExpireDuration = 20 * time.Minute + // Cache is a cache for working set entries. // // The cache evicts inactive entries after the given expireDuration. @@ -48,10 +50,18 @@ type Cache struct { } // Load loads the cache from filePath and limits its size to maxBytes +// and evicts inactive entries in 20 minutes. +// +// Stop must be called on the returned cache when it is no longer needed. +func Load(filePath string, maxBytes int) *Cache { + return LoadWithExpire(filePath, maxBytes, defaultExpireDuration) +} + +// LoadWithExpire loads the cache from filePath and limits its size to maxBytes // and evicts inactive entires after expireDuration. // // Stop must be called on the returned cache when it is no longer needed. -func Load(filePath string, maxBytes int, expireDuration time.Duration) *Cache { +func LoadWithExpire(filePath string, maxBytes int, expireDuration time.Duration) *Cache { curr := fastcache.LoadFromFileOrNew(filePath, maxBytes) var cs fastcache.Stats curr.UpdateStats(&cs) @@ -60,8 +70,10 @@ func Load(filePath string, maxBytes int, expireDuration time.Duration) *Cache { // The cache couldn't be loaded with maxBytes size. // This may mean that the cache is split into curr and prev caches. // Try loading it again with maxBytes / 2 size. - curr := fastcache.LoadFromFileOrNew(filePath, maxBytes/2) - prev := fastcache.New(maxBytes / 2) + // Put the loaded cache into `prev` instead of `curr` + // in order to limit the growth of the cache for the current period of time. + prev := fastcache.LoadFromFileOrNew(filePath, maxBytes/2) + curr := fastcache.New(maxBytes / 2) c := newCacheInternal(curr, prev, split, maxBytes) c.runWatchers(expireDuration) return c @@ -74,11 +86,18 @@ func Load(filePath string, maxBytes int, expireDuration time.Duration) *Cache { return newCacheInternal(curr, prev, whole, maxBytes) } -// New creates new cache with the given maxBytes capacity and the given expireDuration +// New creates new cache with the given maxBytes capacity. +// +// Stop must be called on the returned cache when it is no longer needed. +func New(maxBytes int) *Cache { + return NewWithExpire(maxBytes, defaultExpireDuration) +} + +// NewWithExpire creates new cache with the given maxBytes capacity and the given expireDuration // for inactive entries. // // Stop must be called on the returned cache when it is no longer needed. -func New(maxBytes int, expireDuration time.Duration) *Cache { +func NewWithExpire(maxBytes int, expireDuration time.Duration) *Cache { curr := fastcache.New(maxBytes / 2) prev := fastcache.New(1024) c := newCacheInternal(curr, prev, split, maxBytes)