From 5269b1ad775d202ead16864782803029bed041f3 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 7 Oct 2022 23:36:11 +0300 Subject: [PATCH] lib/promscrape: allow controlling staleness tracking on a per-scrape_config basis Add support for no_stale_markers option at scrape_config section. See https://docs.victoriametrics.com/sd_configs.html#scrape_configs and https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers --- app/vmagent/README.md | 12 ++++++++---- docs/CHANGELOG.md | 2 ++ docs/sd_configs.md | 7 ++++++- docs/vmagent.md | 12 ++++++++---- lib/promscrape/config.go | 13 +++++++++++-- lib/promscrape/config_test.go | 2 ++ lib/promscrape/scrapework.go | 15 +++++++++------ 7 files changed, 46 insertions(+), 17 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 0be39b7b7..96b3b1b0f 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -382,7 +382,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me ``` `vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option - (e.g. when [staleness markers](#prometheus-staleness-markers) are disabled). + or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled. * `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter). This metric is exposed only if the series limit is set. @@ -604,9 +604,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target. Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target -in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` -command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series -per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details. +in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways: + +* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets. +* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target. + +When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape, +e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details. ## Stream parsing mode diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5bf6a0171..9ccc39169 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,8 @@ The following tip changes can be tested by building VictoriaMetrics components f - host4:1234 ``` +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow controlling staleness tracking on a per-[scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) basis by specifying `no_stale_markers: true` or `no_stale_markers: false` option in the corresponding [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). + * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): automatically update graph, legend and url after the removal of query field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3169) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3196#issuecomment-1269765205). ## [v1.82.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.0) diff --git a/docs/sd_configs.md b/docs/sd_configs.md index d90ad7bc1..8d91d88c3 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -1188,7 +1188,7 @@ scrape_configs: # disable_keepalive: # stream_parse allows enabling stream parsing mode when scraping targets. - # By default stram parsing mode is disabled for targets which return up to a few thosands samples. + # By default stream parsing mode is disabled for targets which return up to a few thosands samples. # See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode . # stream_parse: @@ -1212,6 +1212,11 @@ scrape_configs: # See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter . # series_limit: ... + # no_stale_markers allows disabling staleness tracking. + # By default staleness tracking is enabled for all the discovered scrape targets. + # See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers + # no_stale_markers: + # Additional HTTP client options for target scraping can be specified here. # See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options ``` diff --git a/docs/vmagent.md b/docs/vmagent.md index 06c2729a7..5ce78e2a4 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -386,7 +386,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me ``` `vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option - (e.g. when [staleness markers](#prometheus-staleness-markers) are disabled). + or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled. * `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter). This metric is exposed only if the series limit is set. @@ -608,9 +608,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target. Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target -in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` -command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series -per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details. +in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways: + +* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets. +* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target. + +When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape, +e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details. ## Stream parsing mode diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index f3b3481e6..6dbdf5662 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -42,8 +42,9 @@ import ( ) var ( - strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields") - dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+ + noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series") + strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields") + dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+ "Returns non-zero exit code on parsing errors and emits these errors to stderr. "+ "See also -promscrape.config.strictParse command-line flag. "+ "Pass -loggerLevel=ERROR if you don't need to see info messages in the output.") @@ -289,6 +290,7 @@ type ScrapeConfig struct { ScrapeAlignInterval *promutils.Duration `yaml:"scrape_align_interval,omitempty"` ScrapeOffset *promutils.Duration `yaml:"scrape_offset,omitempty"` SeriesLimit int `yaml:"series_limit,omitempty"` + NoStaleMarkers *bool `yaml:"no_stale_markers,omitempty"` ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` // This is set in loadConfig @@ -950,6 +952,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf return nil, fmt.Errorf("cannot use stream parsing mode when `series_limit` is set for `job_name` %q", jobName) } externalLabels := globalCfg.getExternalLabels() + noStaleTracking := *noStaleMarkers + if sc.NoStaleMarkers != nil { + noStaleTracking = *sc.NoStaleMarkers + } swc := &scrapeWorkConfig{ scrapeInterval: scrapeInterval, scrapeIntervalString: scrapeInterval.String(), @@ -975,6 +981,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf scrapeAlignInterval: sc.ScrapeAlignInterval.Duration(), scrapeOffset: sc.ScrapeOffset.Duration(), seriesLimit: sc.SeriesLimit, + noStaleMarkers: noStaleTracking, } return swc, nil } @@ -1004,6 +1011,7 @@ type scrapeWorkConfig struct { scrapeAlignInterval time.Duration scrapeOffset time.Duration seriesLimit int + noStaleMarkers bool } type targetLabelsGetter interface { @@ -1357,6 +1365,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel ScrapeAlignInterval: swc.scrapeAlignInterval, ScrapeOffset: swc.scrapeOffset, SeriesLimit: seriesLimit, + NoStaleMarkers: swc.noStaleMarkers, AuthToken: at, jobNameOriginal: swc.jobName, diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 9f3ed971c..2ea4f0f0c 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -1493,6 +1493,7 @@ scrape_configs: scrape_interval: 1w scrape_align_interval: 1d scrape_offset: 2d + no_stale_markers: true static_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ @@ -1503,6 +1504,7 @@ scrape_configs: ScrapeAlignInterval: time.Hour * 24, ScrapeOffset: time.Hour * 24 * 2, HonorTimestamps: true, + NoStaleMarkers: true, Labels: []prompbmarshal.Label{ { Name: "instance", diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 878f6d62f..f188f7ae5 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -36,7 +36,6 @@ var ( "See also -promscrape.suppressScrapeErrorsDelay") suppressScrapeErrorsDelay = flag.Duration("promscrape.suppressScrapeErrorsDelay", 0, "The delay for suppressing repeated scrape errors logging per each scrape targets. "+ "This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors") - noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series") seriesLimitPerTarget = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info") minResponseSizeForStreamParse = flagutil.NewBytes("promscrape.minResponseSizeForStreamParse", 1e6, "The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode") ) @@ -122,6 +121,10 @@ type ScrapeWork struct { // Optional limit on the number of unique series the scrape target can expose. SeriesLimit int + // Whether to process stale markers for the given target. + // See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers + NoStaleMarkers bool + //The Tenant Info AuthToken *auth.Token @@ -144,12 +147,12 @@ func (sw *ScrapeWork) key() string { key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+ "ExternalLabels=%s, "+ "ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+ - "ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d", + "ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v", sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.LabelsString(), promLabelsString(sw.ExternalLabels), sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, - sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit) + sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers) return key } @@ -438,7 +441,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error wc := writeRequestCtxPool.Get(sw.prevLabelsLen) lastScrape := sw.loadLastScrape() bodyString := bytesutil.ToUnsafeString(body.B) - areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString) + areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString) if err != nil { up = 0 scrapesFailed.Inc() @@ -590,7 +593,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error { } lastScrape := sw.loadLastScrape() bodyString := bytesutil.ToUnsafeString(sbr.body) - areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString) + areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString) scrapedSamples.Update(float64(samplesScraped)) endTimestamp := time.Now().UnixNano() / 1e6 @@ -738,7 +741,7 @@ func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int { } func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp int64, addAutoSeries bool) { - if *noStaleMarkers { + if sw.Config.NoStaleMarkers { return } bodyString := lastScrape