From f79944532b137e7b8cb0386a0534d386fcd37100 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 23 Feb 2024 04:04:03 +0200
Subject: [PATCH] lib/storage: do not drop (date, metricID) entries for the
 date older than 2 days if samples are ingested at this date

Previously the (date, metricID) entries for dates older than the last 2 days were removed.
This could lead to slow check for the (date, metricID) entry in the indexdb during ingesting historical data (aka backfilling).

The issue has been introduced in 431aa16c8de4f36ece730edced4117243f141c80
---
 lib/storage/storage.go | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index 0df55998e..808a08945 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -2342,7 +2342,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
 	byDateMutable := dmc.byDateMutable
 	byDateMutable.hotEntry.Store(&byDateMetricIDEntry{})
 
+	keepDatesMap := make(map[uint64]struct{}, len(byDateMutable.m))
 	for k, e := range byDateMutable.m {
+		keepDatesMap[k.date] = struct{}{}
 		v := byDate.get(k.generation, k.date)
 		if v == nil {
 			// Nothing to merge
@@ -2358,7 +2360,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
 	}
 
 	// Copy entries from byDate, which are missing in byDateMutable
+	allDatesMap := make(map[uint64]struct{}, len(byDate.m))
 	for k, e := range byDate.m {
+		allDatesMap[k.date] = struct{}{}
 		v := byDateMutable.get(k.generation, k.date)
 		if v != nil {
 			continue
@@ -2367,18 +2371,22 @@ func (dmc *dateMetricIDCache) syncLocked() {
 	}
 
 	if len(byDateMutable.m) > 2 {
-		// Keep only entries for the last two dates - these are usually
-		// the current date and the next date.
-		dates := make([]uint64, 0, len(byDateMutable.m))
-		for k := range byDateMutable.m {
-			dates = append(dates, k.date)
+		// Keep only entries for the last two dates from allDatesMap plus all the entries for byDateMutable.
+		dates := make([]uint64, 0, len(allDatesMap))
+		for date := range allDatesMap {
+			dates = append(dates, date)
 		}
 		sort.Slice(dates, func(i, j int) bool {
 			return dates[i] < dates[j]
 		})
-		maxDate := dates[len(dates)-2]
+		if len(dates) > 2 {
+			dates = dates[len(dates)-2:]
+		}
+		for _, date := range dates {
+			keepDatesMap[date] = struct{}{}
+		}
 		for k := range byDateMutable.m {
-			if k.date < maxDate {
+			if _, ok := keepDatesMap[k.date]; !ok {
 				delete(byDateMutable.m, k)
 			}
 		}