lib/storage: add -disableRecentHourIndex flag for disabling inmemory index for recent hour

This may be useful for saving RAM on high number of time series aka high cardinality

app/vmstorage/main.go

@@ -25,6 +25,8 @@ var (
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
+	disableRecentHourIndex = flag.Bool("disableRecentHourIndex", false, "Whether to disable inmemory inverted index for recent hour. "+
+		"This may be useful in order to reduce memory usage when working with high number of time series")
 	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
 	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
 )
@@ -43,6 +45,9 @@ func InitWithoutMetrics() {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
 
+	if *disableRecentHourIndex {
+		storage.DisableRecentHourIndex()
+	}
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
 

lib/storage/index_db.go

@@ -2194,6 +2194,10 @@ func (is *indexSearch) getMetricIDsForRecentHours(tr TimeRange, maxMetrics int)
 }
 
 func (is *indexSearch) tryUpdatingMetricIDsForRecentHour(metricIDs *uint64set.Set, tfs *TagFilters, tr TimeRange) bool {
+	if disableRecentHourIndex {
+		return false
+	}
+
 	minHour := uint64(tr.MinTimestamp) / msecPerHour
 	maxHour := uint64(tr.MaxTimestamp) / msecPerHour
 

lib/storage/storage.go

@@ -27,6 +27,17 @@ import (
 
 const maxRetentionMonths = 12 * 100
 
+var disableRecentHourIndex = false
+
+// DisableRecentHourIndex disables in-memory inverted index for recent hour.
+//
+// This may be useful in order to save RAM for high cardinality data.
+//
+// This function must be called before OpenStorage.
+func DisableRecentHourIndex() {
+	disableRecentHourIndex = true
+}
+
 // Storage represents TSDB storage.
 type Storage struct {
 	// Atomic counters must go at the top of the structure in order to properly align by 8 bytes on 32-bit archs.
@@ -547,19 +558,21 @@ func (s *Storage) mustLoadHourMetricIDs(hour uint64, name string) *hourMetricIDs
 
 	// Unmarshal hm.iidx
 	iidx := newInmemoryInvertedIndex()
-	tail, err := iidx.Unmarshal(src)
-	if err != nil {
-		logger.Errorf("discarding %s, since it has broken hm.iidx data: %s", path, err)
-		return &hourMetricIDs{
-			iidx: newInmemoryInvertedIndex(),
-			hour: hour,
+	if !disableRecentHourIndex {
+		tail, err := iidx.Unmarshal(src)
+		if err != nil {
+			logger.Errorf("discarding %s, since it has broken hm.iidx data: %s", path, err)
+			return &hourMetricIDs{
+				iidx: newInmemoryInvertedIndex(),
+				hour: hour,
+			}
 		}
-	}
-	if len(tail) > 0 {
-		logger.Errorf("discarding %s, since it contains superflouos %d bytes of data", path, len(tail))
-		return &hourMetricIDs{
-			iidx: newInmemoryInvertedIndex(),
-			hour: hour,
+		if len(tail) > 0 {
+			logger.Errorf("discarding %s, since it contains superflouos %d bytes of data", path, len(tail))
+			return &hourMetricIDs{
+				iidx: newInmemoryInvertedIndex(),
+				hour: hour,
+			}
 		}
 	}
 
@@ -592,8 +605,10 @@ func (s *Storage) mustSaveHourMetricIDs(hm *hourMetricIDs, name string) {
 		dst = encoding.MarshalUint64(dst, metricID)
 	}
 
-	// Marshal hm.iidx
-	dst = hm.iidx.Marshal(dst)
+	if !disableRecentHourIndex {
+		// Marshal hm.iidx
+		dst = hm.iidx.Marshal(dst)
+	}
 
 	if err := ioutil.WriteFile(path, dst, 0644); err != nil {
 		logger.Panicf("FATAL: cannot write %d bytes to %q: %s", len(dst), path, err)
@@ -945,7 +960,9 @@ func (s *Storage) updatePerDateData(rows []rawRow, lastError error) error {
 			s.pendingHourEntriesLock.Lock()
 			s.pendingHourEntries.Add(metricID)
 			s.pendingHourEntriesLock.Unlock()
-			hm.iidx.AddMetricID(idb, metricID)
+			if !disableRecentHourIndex {
+				hm.iidx.AddMetricID(idb, metricID)
+			}
 		}
 
 		// Slower path: check global cache for (date, metricID) entry.