lib/storage: reduce the minimum supported retention for inverted index from one month to one day

docs/CHANGELOG.md

@@ -10,6 +10,7 @@
 * FEATURE: vmauth: add ability to route requests from a single user to multiple destinations depending on the requested paths. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1064
 * FEATURE: remove dependency on external programs such as `cat`, `grep` and `cut` when detecting cpu and memory limits inside Docker or LXC container.
 * FEATURE: vmagent: add `__meta_kubernetes_endpoints_label_*`, `__meta_kubernetes_endpoints_labelpresent_*`, `__meta_kubernetes_endpoints_annotation_*` and `__meta_kubernetes_endpoints_annotationpresent_*` labels for `role: endpoints` in Kubernetes service discovery. These labels where added in Prometheus 2.25.
+* FEATURE: reduce the minimum supported retention period for inverted index (aka `indexdb`) from one month to one day. This should reduce disk space usage for `<-storageDataPath>/indexdb` folder if `-retentionPeriod` is set to values smaller than one month.
 
 * BUGFIX: properly convert regexp tag filters containing escaped dots to non-regexp tag filters. For example, `{foo=~"bar\.baz"}` should be converted to `{foo="bar.baz"}`. Previously it was incorrectly converted to `{foo="bar\.baz"}`, which could result in missing time series for this tag filter.
 * BUGFIX: do not spam error logs when discovering Docker Swarm targets without dedicated IP. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1028 .

lib/storage/storage.go

@@ -133,6 +133,9 @@ func OpenStorage(path string, retentionMsecs int64) (*Storage, error) {
 	if retentionMsecs <= 0 {
 		retentionMsecs = maxRetentionMsecs
 	}
+	if retentionMsecs > maxRetentionMsecs {
+		retentionMsecs = maxRetentionMsecs
+	}
 	s := &Storage{
 		path:           path,
 		cachePath:      path + "/cache",
@@ -505,9 +508,8 @@ func (s *Storage) startRetentionWatcher() {
 }
 
 func (s *Storage) retentionWatcher() {
-	retentionMonths := int((s.retentionMsecs + (msecsPerMonth - 1)) / msecsPerMonth)
 	for {
-		d := nextRetentionDuration(retentionMonths)
+		d := nextRetentionDuration(s.retentionMsecs)
 		select {
 		case <-s.stop:
 			return
@@ -903,17 +905,16 @@ func (s *Storage) mustSaveAndStopCache(c *workingsetcache.Cache, info, name stri
 		info, path, time.Since(startTime).Seconds(), cs.EntriesCount, cs.BytesSize)
 }
 
-func nextRetentionDuration(retentionMonths int) time.Duration {
+func nextRetentionDuration(retentionMsecs int64) time.Duration {
-	t := time.Now().UTC()
+	// Round retentionMsecs to days. This guarantees that per-day inverted index works as expected.
-	n := t.Year()*12 + int(t.Month()) - 1 + retentionMonths
+	retentionMsecs = ((retentionMsecs+msecPerDay-1)/msecPerDay)*msecPerDay
-	n -= n % retentionMonths
+	t := time.Now().UnixNano() / 1e6
-	y := n / 12
+	deadline := ((t+retentionMsecs-1)/retentionMsecs)*retentionMsecs
-	m := time.Month((n % 12) + 1)
 	// Schedule the deadline to +4 hours from the next retention period start.
 	// This should prevent from possible double deletion of indexdb
 	// due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 .
-	deadline := time.Date(y, m, 1, 4, 0, 0, 0, time.UTC)
+	deadline += 4 * 3600 * 1000
-	return deadline.Sub(t)
+	return time.Duration(deadline-t) * time.Millisecond
 }
 
 // SearchMetricNames returns metric names matching the given tfss on the given tr.

lib/storage/storage_test.go

@@ -371,12 +371,12 @@ func TestMetricRowMarshalUnmarshal(t *testing.T) {
 }
 
 func TestNextRetentionDuration(t *testing.T) {
-	for retentionMonths := 1; retentionMonths < 360; retentionMonths++ {
+	for retentionMonths := float64(0.1); retentionMonths < 120; retentionMonths += 0.3 {
-		d := nextRetentionDuration(retentionMonths)
+		d := nextRetentionDuration(int64(retentionMonths * msecsPerMonth))
 		if d <= 0 {
 			currTime := time.Now().UTC()
 			nextTime := time.Now().UTC().Add(d)
-			t.Fatalf("unexected retention duration for retentionMonths=%d; got %s; must be %s + %d months", retentionMonths, nextTime, currTime, retentionMonths)
+			t.Fatalf("unexected retention duration for retentionMonths=%f; got %s; must be %s + %f months", retentionMonths, nextTime, currTime, retentionMonths)
 		}
 	}
 }