From fe5611d6e16a5408b64e957e4e261ae89ebc878c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 21 Oct 2022 14:59:04 +0300 Subject: [PATCH] lib/storage: free up memory occupied by Storage.pendingHourEntries after a temporary spike in its memory usage This reduces vmstorage memory usage by up to 20% in production workload --- lib/storage/storage.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 4b3ce2f0e..cf0ee599e 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -796,7 +796,7 @@ func (s *Storage) mustRotateIndexDB() { // So queries for the last 24 hours stop returning samples added at step 3. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698 s.pendingHourEntriesLock.Lock() - s.pendingHourEntries = []pendingHourMetricIDEntry{} + s.pendingHourEntries = nil s.pendingHourEntriesLock.Unlock() s.currHourMetricIDs.Store(&hourMetricIDs{}) s.prevHourMetricIDs.Store(&hourMetricIDs{}) @@ -2516,9 +2516,21 @@ func (s *Storage) updateNextDayMetricIDs() { func (s *Storage) updateCurrHourMetricIDs() { hm := s.currHourMetricIDs.Load().(*hourMetricIDs) + var newEntries []pendingHourMetricIDEntry s.pendingHourEntriesLock.Lock() - newEntries := append([]pendingHourMetricIDEntry{}, s.pendingHourEntries...) - s.pendingHourEntries = s.pendingHourEntries[:0] + if len(s.pendingHourEntries) < cap(s.pendingHourEntries)/2 { + // Free up memory occupied by s.pendingHourEntries, + // since it looks like now it needs much lower amounts of memory. + newEntries = s.pendingHourEntries + s.pendingHourEntries = nil + } else { + // Copy s.pendingHourEntries to newEntries and re-use s.pendingHourEntries capacity, + // since its memory usage is at stable state. + // This should reduce the number of additional memory re-allocations + // when adding new items to s.pendingHourEntries. + newEntries = append([]pendingHourMetricIDEntry{}, s.pendingHourEntries...) + s.pendingHourEntries = s.pendingHourEntries[:0] + } s.pendingHourEntriesLock.Unlock() hour := fasttime.UnixHour()