diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go index e8540eddd..0435270dd 100644 --- a/app/victoria-metrics/self_scraper.go +++ b/app/victoria-metrics/self_scraper.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" @@ -49,16 +50,8 @@ func selfScraper(scrapeInterval time.Duration) { var mrs []storage.MetricRow var labels []prompb.Label t := time.NewTicker(scrapeInterval) - var currentTimestamp int64 - for { - select { - case <-selfScraperStopCh: - t.Stop() - logger.Infof("stopped self-scraping `/metrics` page") - return - case currentTime := <-t.C: - currentTimestamp = currentTime.UnixNano() / 1e6 - } + f := func(currentTime time.Time, sendStaleMarkers bool) { + currentTimestamp := currentTime.UnixNano() / 1e6 bb.Reset() appmetrics.WritePrometheusMetrics(&bb) s := bytesutil.ToUnsafeString(bb.B) @@ -83,12 +76,27 @@ func selfScraper(scrapeInterval time.Duration) { mr := &mrs[len(mrs)-1] mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels) mr.Timestamp = currentTimestamp - mr.Value = r.Value + if sendStaleMarkers { + mr.Value = decimal.StaleNaN + } else { + mr.Value = r.Value + } } if err := vmstorage.AddRows(mrs); err != nil { logger.Errorf("cannot store self-scraped metrics: %s", err) } } + for { + select { + case <-selfScraperStopCh: + f(time.Now(), true) + t.Stop() + logger.Infof("stopped self-scraping `/metrics` page") + return + case currentTime := <-t.C: + f(currentTime, false) + } + } } func addLabel(dst []prompb.Label, key, value string) []prompb.Label { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cf03171a3..e79ab84fd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly propagate [label filters](https://docs.victoriametrics.com/keyconcepts/#filtering) from multiple arguments passed to [aggregate functions](https://docs.victoriametrics.com/metricsql/#aggregate-functions). For example, `sum({job="foo"}, {job="bar"}) by (job) + a` was improperly optimized to `sum({job="foo"}, {job="bar"}) by (job) + a{job="foo"}` before being executed. This could lead to unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5604). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle precision errors when calculating [changes](https://docs.victoriametrics.com/metricsql/#changes), [changes_prometheus](https://docs.victoriametrics.com/metricsql/#changes_prometheus), [increases_over_time](https://docs.victoriametrics.com/metricsql/#increases_over_time) and [resets](https://docs.victoriametrics.com/metricsql/#resets) functions. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767). +* BUGFIX: properly store [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) for [self-scraped metrics](https://docs.victoriametrics.com/#monitoring) on single-node VictoriaMetrics shutdown. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/943). * BUGFIX: prevent from possible `too big indexBlockSize` panic when samples with too long label values (~64Kb) are ingested into VictoriaMetrics. ## [v1.93.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.11)