github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape: always send stale markers with the real scrape timestamp

Browse source 
This guarantees that query won't return data just after the series is disappeared.
This commit is contained in:
Aliaksandr Valialkin 2022-06-23 10:55:14 +03:00
parent 668d67a3d3
commit 52eadb729e
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
2 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/CHANGELOG.md
View file

@ -28,6 +28,8 @@ scrape_configs:
- targets: ["host123:8080"]
```
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that [stale markers](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) are generated with the actual timestamp when unsuccessful scrape occurs. This should prevent from possible time series overlap on scrape target restart in dynmaic envirnoments such as Kubernetes.
## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)
Released at 20-06-2022

13
lib/promscrape/scrapework.go
View file

@ -332,7 +332,10 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{})
// Do not send staleness markers on graceful shutdown as Prometheus does.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079
default:
// Send staleness markers when the given target disappears.
// Send staleness markers to all the metrics scraped last time from the target
// when the given target disappears as Prometheus does.
// Use the current real timestamp for staleness markers, so queries
// stop returning data just after the time the target disappears.
sw.sendStaleSeries(lastScrape, "", t, true)
}
if sw.seriesLimiter != nil {
@ -491,7 +494,9 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
}
// body must be released only after wc is released, since wc refers to body.
if !areIdenticalSeries {
sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false)
// Send stale markers for disappeared metrics with the real scrape timestamp
// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
sw.storeLastScrape(body.B)
}
sw.finalizeLastScrape()
@ -599,7 +604,9 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
wc.reset()
writeRequestCtxPool.Put(wc)
if !areIdenticalSeries {
sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false)
// Send stale markers for disappeared metrics with the real scrape timestamp
// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
sw.storeLastScrape(sbr.body)
}
sw.finalizeLastScrape()