From 5a62415bec49615f0c3ac629bb45a2b814a908b9 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Mon, 2 Dec 2019 14:42:26 +0200
Subject: [PATCH] lib/storage: protect from time drift during indexdb rotation

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248
---
 lib/storage/storage.go      | 5 ++++-
 lib/storage/storage_test.go | 5 ++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index d1f3f2fa7..c3a02a547 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -655,7 +655,10 @@ func nextRetentionDuration(retentionMonths int) time.Duration {
 	n -= n % retentionMonths
 	y := n / 12
 	m := time.Month((n % 12) + 1)
-	deadline := time.Date(y, m, 1, 0, 0, 0, 0, time.UTC)
+	// Schedule the deadline to +4 hours from the next retention period start.
+	// This should prevent from possible double deletion of indexdb
+	// due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 .
+	deadline := time.Date(y, m, 1, 4, 0, 0, 0, time.UTC)
 	return deadline.Sub(t)
 }
 
diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go
index 3d36482f0..e31347c46 100644
--- a/lib/storage/storage_test.go
+++ b/lib/storage/storage_test.go
@@ -342,10 +342,9 @@ func TestMetricRowMarshalUnmarshal(t *testing.T) {
 
 func TestNextRetentionDuration(t *testing.T) {
 	for retentionMonths := 1; retentionMonths < 360; retentionMonths++ {
-		currTime := time.Now().UTC()
-
 		d := nextRetentionDuration(retentionMonths)
-		if d < 0 {
+		if d <= 0 {
+			currTime := time.Now().UTC()
 			nextTime := time.Now().UTC().Add(d)
 			t.Fatalf("unexected retention duration for retentionMonths=%d; got %s; must be %s + %d months", retentionMonths, nextTime, currTime, retentionMonths)
 		}