From edc76286ac1e8648e10ee7ab417af3e6a2f24736 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 5 Jul 2022 14:49:03 +0300 Subject: [PATCH] lib/storage: put the (date, metricID) entry in dateMetricIDCache just after the corresponding series is registered in the per-day inverted index Previously the time series could be put into dateMetricIDCache without registering in the per-day inverted index if GetOrCreateTSIDByName finds TSID entry in the global index. This could lead to missing series in query results. The issue has been introduced in the commit 55e7afae3a0c20e53e790e6c6afb02fff848f868, which has been included in VictoriaMetrics v1.78.0 --- docs/CHANGELOG.md | 8 ++++---- lib/storage/index_db.go | 1 + lib/storage/index_db_test.go | 7 ++++--- lib/storage/storage.go | 3 --- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 695dcc178..2148d77ba 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -43,6 +43,7 @@ scrape_configs: * `vm_rows_read_per_series` - the number of raw samples read per queried series. * `vm_series_read_per_query` - the number of series read per query. +* BUGFIX: properly register time series in per-day inverted index. Previously some series could miss registration in the per-day inverted index. This could result in missing time series during querying. The issue has been introduced in [v1.78.0](#v1780). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using `__name__` label (aka [metric name](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)) in alerting annotations. For example: {% raw %} @@ -61,12 +62,11 @@ scrape_configs: ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) -**Warning (03-07-2022):** some users report issues with incomplete data returned from queries for cluster version. -The problem is currently under investigation. This message will be updated as soon as the problem -will be localized and solved. Meanwhile, we recommend postpone updating to 1.78.0. - Released at 20-06-2022 +**Warning (03-07-2022):** VictoriaMetrics v1.78.0 contains a bug, which may result in missing time series during queries. +It is recommended downgrading to [v1.77.2](#v1772) until the bugfix release. + **Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics because of added [query tracing](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#query-tracing), so `vmselect` and `vmstorage` nodes will experience communication errors and read requests to `vmselect` will fail until the upgrade is complete. These errors will stop after all the `vmselect` and `vmstorage` nodes are updated to the new release. It is safe to downgrade to previous releases. * SECURITY: add `-flagsAuthKey` command-line flag for protecting `/flags` endpoint from unauthorized access. Though this endpoint already hides values for command-line flags with `key` and `password` substrings in their names, other sensitive information could be exposed there. See [This issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2753). diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 9be118f91..60e612ccf 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -2724,6 +2724,7 @@ func (is *indexSearch) createPerDayIndexes(date, metricID uint64, mn *MetricName if err := is.db.tb.AddItems(ii.Items); err != nil { return fmt.Errorf("cannot add per-day entires for metricID %d: %w", metricID, err) } + is.db.s.dateMetricIDCache.Set(date, metricID) return nil } diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index 4e947ac7a..5af5605a4 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -2067,9 +2067,10 @@ func newTestStorage() *Storage { s := &Storage{ cachePath: "test-storage-cache", - metricIDCache: workingsetcache.New(1234), - metricNameCache: workingsetcache.New(1234), - tsidCache: workingsetcache.New(1234), + metricIDCache: workingsetcache.New(1234), + metricNameCache: workingsetcache.New(1234), + tsidCache: workingsetcache.New(1234), + dateMetricIDCache: newDateMetricIDCache(), } s.setDeletedMetricIDs(&uint64set.Set{}) return s diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 3135d3bfa..27e0ee7c8 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1708,7 +1708,6 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) e } genTSID.generation = idb.generation s.putTSIDToCache(&genTSID, mr.MetricNameRaw) - s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID) } return nil } @@ -1797,7 +1796,6 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci if created { genTSID.generation = idb.generation s.putTSIDToCache(&genTSID, mr.MetricNameRaw) - s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID) } } continue @@ -1860,7 +1858,6 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci genTSID.generation = idb.generation genTSID.TSID = r.TSID s.putTSIDToCache(&genTSID, mr.MetricNameRaw) - s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID) prevTSID = r.TSID prevMetricNameRaw = mr.MetricNameRaw