diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 65f9a001e..2192951ca 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -46,6 +46,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the url with vmui state when new query is entered. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2692). * BUGFIX: [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage): properly calculate sample timestamps when `moving*()` functions such as [movingAverage()](https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.movingAverage) are applied over [summarize()](https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.summarize). * BUGFIX: limit the `end` query arg value to `+2 days` in the future at `/api/v1/*` endpoints, because VictoriaMetrics doesn't allow storing samples with timestamps bigger than +2 days in the future. This should help resolving [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2669). +* BUGFIX: properly register time series in per-day inverted index during the first hour after `indexdb` rotation. Previously this could lead to missing time series during querying if these time series stopped receiving new samples during the first hour after `indexdb` rotation. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698). ## [v1.77.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.77.2) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 22ea03790..1452ed543 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -745,6 +745,27 @@ func (s *Storage) mustRotateIndexDB() { // and slowly re-populate new idb with entries from the cache via maybeCreateIndexes(). // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401 + // Flush metric id caches for the current and the previous hour, + // since they may contain entries missing in idbNew. + // This should prevent from missing data in queries when + // the following steps are performed for short -retentionPeriod (e.g. 1 day): + // + // 1. Add samples for some series between 3-4 UTC. These series are registered in currHourMetricIDs. + // 2. The indexdb rotation is performed at 4 UTC. currHourMetricIDs is moved to prevHourMetricIDs. + // 3. Continue adding samples for series from step 1 during time range 4-5 UTC. + // These series are already registered in prevHourMetricIDs, so VM doesn't add per-day entries to the current indexdb. + // 4. Stop adding new samples for these series just before 5 UTC. + // 5. The next indexdb rotation is performed at 4 UTC next day. + // The information about the series from step 5 disappears from indexdb, since the old indexdb from step 1 is deleted, + // while the current indexdb doesn't contain information about the series. + // So queries for the last 24 hours stop returning samples added at step 3. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698 + s.pendingHourEntriesLock.Lock() + s.pendingHourEntries = &uint64set.Set{} + s.pendingHourEntriesLock.Unlock() + s.currHourMetricIDs.Store(&hourMetricIDs{}) + s.prevHourMetricIDs.Store(&hourMetricIDs{}) + // Flush dateMetricIDCache, so idbNew can be populated with fresh data. s.dateMetricIDCache.Reset() @@ -2374,6 +2395,7 @@ func (s *Storage) updateCurrHourMetricIDs() { newMetricIDs := s.pendingHourEntries s.pendingHourEntries = &uint64set.Set{} s.pendingHourEntriesLock.Unlock() + hour := fasttime.UnixHour() if newMetricIDs.Len() == 0 && hm.hour == hour { // Fast path: nothing to update.