From 71b0dfdefa4a75746b7d71afee5cae66f8f143d8 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 23 Jun 2022 10:55:14 +0300 Subject: [PATCH] lib/promscrape: always send stale markers with the real scrape timestamp This guarantees that query won't return data just after the series is disappeared. --- docs/CHANGELOG.md | 2 ++ lib/promscrape/scrapework.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b6c612c59..8227ac2c9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,8 @@ scrape_configs: - targets: ["host123:8080"] ``` +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that [stale markers](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) are generated with the actual timestamp when unsuccessful scrape occurs. This should prevent from possible time series overlap on scrape target restart in dynmaic envirnoments such as Kubernetes. + ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) Released at 20-06-2022 diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 92dda54d6..f622edfbf 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -332,7 +332,10 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{}) // Do not send staleness markers on graceful shutdown as Prometheus does. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079 default: - // Send staleness markers when the given target disappears. + // Send staleness markers to all the metrics scraped last time from the target + // when the given target disappears as Prometheus does. + // Use the current real timestamp for staleness markers, so queries + // stop returning data just after the time the target disappears. sw.sendStaleSeries(lastScrape, "", t, true) } if sw.seriesLimiter != nil { @@ -491,7 +494,9 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error } // body must be released only after wc is released, since wc refers to body. if !areIdenticalSeries { - sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false) + // Send stale markers for disappeared metrics with the real scrape timestamp + // in order to guarantee that query doesn't return data after this time for the disappeared metrics. + sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false) sw.storeLastScrape(body.B) } sw.finalizeLastScrape() @@ -599,7 +604,9 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error { wc.reset() writeRequestCtxPool.Put(wc) if !areIdenticalSeries { - sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false) + // Send stale markers for disappeared metrics with the real scrape timestamp + // in order to guarantee that query doesn't return data after this time for the disappeared metrics. + sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false) sw.storeLastScrape(sbr.body) } sw.finalizeLastScrape()