lib/storage: put the (date, metricID) entry in dateMetricIDCache just after the corresponding series is registered in the per-day inverted index

Previously the time series could be put into dateMetricIDCache without
registering in the per-day inverted index if GetOrCreateTSIDByName
finds TSID entry in the global index. This could lead to missing
series in query results.

The issue has been introduced in the commit 55e7afae3a,
which has been included in VictoriaMetrics v1.78.0

docs/CHANGELOG.md

@@ -43,6 +43,7 @@ scrape_configs:
   * `vm_rows_read_per_series` - the number of raw samples read per queried series.
   * `vm_series_read_per_query` - the number of series read per query.
 
+* BUGFIX: properly register time series in per-day inverted index. Previously some series could miss registration in the per-day inverted index. This could result in missing time series during querying. The issue has been introduced in [v1.78.0](#v1780).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using `__name__` label (aka [metric name](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)) in alerting annotations. For example:
 
 {% raw %}
@@ -61,12 +62,11 @@ scrape_configs:
 
 ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)
 
-**Warning (03-07-2022):** some users report issues with incomplete data returned from queries for cluster version. 
-The problem is currently under investigation. This message will be updated as soon as the problem 
-will be localized and solved. Meanwhile, we recommend postpone updating to 1.78.0.
-
 Released at 20-06-2022
 
+**Warning (03-07-2022):** VictoriaMetrics v1.78.0 contains a bug, which may result in missing time series during queries.
+It is recommended downgrading to [v1.77.2](#v1772) until the bugfix release.
+
 **Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics because of added [query tracing](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#query-tracing), so `vmselect` and `vmstorage` nodes will experience communication errors and read requests to `vmselect` will fail until the upgrade is complete. These errors will stop after all the `vmselect` and `vmstorage` nodes are updated to the new release. It is safe to downgrade to previous releases.
 
 * SECURITY: add `-flagsAuthKey` command-line flag for protecting `/flags` endpoint from unauthorized access. Though this endpoint already hides values for command-line flags with `key` and `password` substrings in their names, other sensitive information could be exposed there. See [This issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2753).

lib/storage/index_db.go

@@ -2724,6 +2724,7 @@ func (is *indexSearch) createPerDayIndexes(date, metricID uint64, mn *MetricName
 	if err := is.db.tb.AddItems(ii.Items); err != nil {
 		return fmt.Errorf("cannot add per-day entires for metricID %d: %w", metricID, err)
 	}
+	is.db.s.dateMetricIDCache.Set(date, metricID)
 	return nil
 }
 

lib/storage/index_db_test.go

@@ -2067,9 +2067,10 @@ func newTestStorage() *Storage {
 	s := &Storage{
 		cachePath: "test-storage-cache",
 
-		metricIDCache:   workingsetcache.New(1234),
-		metricNameCache: workingsetcache.New(1234),
-		tsidCache:       workingsetcache.New(1234),
+		metricIDCache:     workingsetcache.New(1234),
+		metricNameCache:   workingsetcache.New(1234),
+		tsidCache:         workingsetcache.New(1234),
+		dateMetricIDCache: newDateMetricIDCache(),
 	}
 	s.setDeletedMetricIDs(&uint64set.Set{})
 	return s

lib/storage/storage.go

@@ -1708,7 +1708,6 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) e
 		}
 		genTSID.generation = idb.generation
 		s.putTSIDToCache(&genTSID, mr.MetricNameRaw)
-		s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID)
 	}
 	return nil
 }
@@ -1797,7 +1796,6 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 				if created {
 					genTSID.generation = idb.generation
 					s.putTSIDToCache(&genTSID, mr.MetricNameRaw)
-					s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID)
 				}
 			}
 			continue
@@ -1860,7 +1858,6 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 			genTSID.generation = idb.generation
 			genTSID.TSID = r.TSID
 			s.putTSIDToCache(&genTSID, mr.MetricNameRaw)
-			s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID)
 
 			prevTSID = r.TSID
 			prevMetricNameRaw = mr.MetricNameRaw