lib/promscrape: allow controlling staleness tracking on a per-scrape_config basis

Add support for no_stale_markers option at scrape_config section.
See https://docs.victoriametrics.com/sd_configs.html#scrape_configs and
https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers

app/vmagent/README.md

@@ -382,7 +382,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   ```
 
   `vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option
-  (e.g. when [staleness markers](#prometheus-staleness-markers) are disabled).
+  or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
 
 * `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter).
   This metric is exposed only if the series limit is set.
@@ -604,9 +604,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t
 * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
 
 Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target
-in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers`
+in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways:
-command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series
+
-per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details.
+* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets.
+* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target.
+
+When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape,
+e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details.
 
 ## Stream parsing mode
 

docs/CHANGELOG.md

@@ -33,6 +33,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
       - host4:1234
   ```
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow controlling staleness tracking on a per-[scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) basis by specifying `no_stale_markers: true` or `no_stale_markers: false` option in the corresponding [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs).
+
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): automatically update graph, legend and url after the removal of query field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3169) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3196#issuecomment-1269765205).
 
 ## [v1.82.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.0)

docs/sd_configs.md

@@ -1188,7 +1188,7 @@ scrape_configs:
   # disable_keepalive: <boolean>
 
   # stream_parse allows enabling stream parsing mode when scraping targets.
-  # By default stram parsing mode is disabled for targets which return up to a few thosands samples.
+  # By default stream parsing mode is disabled for targets which return up to a few thosands samples.
   # See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode .
   # stream_parse: <boolean>
 
@@ -1212,6 +1212,11 @@ scrape_configs:
   # See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter .
   # series_limit: ...
 
+  # no_stale_markers allows disabling staleness tracking.
+  # By default staleness tracking is enabled for all the discovered scrape targets.
+  # See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers
+  # no_stale_markers: <boolean>
+
   # Additional HTTP client options for target scraping can be specified here.
   # See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options
 ```

docs/vmagent.md

@@ -386,7 +386,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   ```
 
   `vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option
-  (e.g. when [staleness markers](#prometheus-staleness-markers) are disabled).
+  or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
 
 * `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter).
   This metric is exposed only if the series limit is set.
@@ -608,9 +608,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t
 * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
 
 Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target
-in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers`
+in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways:
-command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series
+
-per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details.
+* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets.
+* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target.
+
+When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape,
+e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details.
 
 ## Stream parsing mode
 

lib/promscrape/config.go

@@ -42,6 +42,7 @@ import (
 )
 
 var (
+	noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
 	strictParse    = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
 	dryRun         = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
 		"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
@@ -289,6 +290,7 @@ type ScrapeConfig struct {
 	ScrapeAlignInterval *promutils.Duration        `yaml:"scrape_align_interval,omitempty"`
 	ScrapeOffset        *promutils.Duration        `yaml:"scrape_offset,omitempty"`
 	SeriesLimit         int                        `yaml:"series_limit,omitempty"`
+	NoStaleMarkers      *bool                      `yaml:"no_stale_markers,omitempty"`
 	ProxyClientConfig   promauth.ProxyClientConfig `yaml:",inline"`
 
 	// This is set in loadConfig
@@ -950,6 +952,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		return nil, fmt.Errorf("cannot use stream parsing mode when `series_limit` is set for `job_name` %q", jobName)
 	}
 	externalLabels := globalCfg.getExternalLabels()
+	noStaleTracking := *noStaleMarkers
+	if sc.NoStaleMarkers != nil {
+		noStaleTracking = *sc.NoStaleMarkers
+	}
 	swc := &scrapeWorkConfig{
 		scrapeInterval:       scrapeInterval,
 		scrapeIntervalString: scrapeInterval.String(),
@@ -975,6 +981,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		scrapeAlignInterval:  sc.ScrapeAlignInterval.Duration(),
 		scrapeOffset:         sc.ScrapeOffset.Duration(),
 		seriesLimit:          sc.SeriesLimit,
+		noStaleMarkers:       noStaleTracking,
 	}
 	return swc, nil
 }
@@ -1004,6 +1011,7 @@ type scrapeWorkConfig struct {
 	scrapeAlignInterval  time.Duration
 	scrapeOffset         time.Duration
 	seriesLimit          int
+	noStaleMarkers       bool
 }
 
 type targetLabelsGetter interface {
@@ -1357,6 +1365,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		ScrapeAlignInterval:  swc.scrapeAlignInterval,
 		ScrapeOffset:         swc.scrapeOffset,
 		SeriesLimit:          seriesLimit,
+		NoStaleMarkers:       swc.noStaleMarkers,
 		AuthToken:            at,
 
 		jobNameOriginal: swc.jobName,

lib/promscrape/config_test.go

@@ -1493,6 +1493,7 @@ scrape_configs:
   scrape_interval: 1w
   scrape_align_interval: 1d
   scrape_offset: 2d
+  no_stale_markers: true
   static_configs:
   - targets: ["foo.bar:1234"]
 `, []*ScrapeWork{
@@ -1503,6 +1504,7 @@ scrape_configs:
 			ScrapeAlignInterval: time.Hour * 24,
 			ScrapeOffset:        time.Hour * 24 * 2,
 			HonorTimestamps:     true,
+			NoStaleMarkers:      true,
 			Labels: []prompbmarshal.Label{
 				{
 					Name:  "instance",

lib/promscrape/scrapework.go

@@ -36,7 +36,6 @@ var (
 		"See also -promscrape.suppressScrapeErrorsDelay")
 	suppressScrapeErrorsDelay = flag.Duration("promscrape.suppressScrapeErrorsDelay", 0, "The delay for suppressing repeated scrape errors logging per each scrape targets. "+
 		"This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors")
-	noStaleMarkers                = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
 	seriesLimitPerTarget          = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
 	minResponseSizeForStreamParse = flagutil.NewBytes("promscrape.minResponseSizeForStreamParse", 1e6, "The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode")
 )
@@ -122,6 +121,10 @@ type ScrapeWork struct {
 	// Optional limit on the number of unique series the scrape target can expose.
 	SeriesLimit int
 
+	// Whether to process stale markers for the given target.
+	// See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers
+	NoStaleMarkers bool
+
 	//The Tenant Info
 	AuthToken *auth.Token
 
@@ -144,12 +147,12 @@ func (sw *ScrapeWork) key() string {
 	key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+
 		"ExternalLabels=%s, "+
 		"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
-		"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d",
+		"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v",
 		sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.LabelsString(),
 		promLabelsString(sw.ExternalLabels),
 		sw.ProxyURL.String(), sw.ProxyAuthConfig.String(),
 		sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,
-		sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit)
+		sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers)
 	return key
 }
 
@@ -438,7 +441,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
 	wc := writeRequestCtxPool.Get(sw.prevLabelsLen)
 	lastScrape := sw.loadLastScrape()
 	bodyString := bytesutil.ToUnsafeString(body.B)
-	areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
+	areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
 	if err != nil {
 		up = 0
 		scrapesFailed.Inc()
@@ -590,7 +593,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	}
 	lastScrape := sw.loadLastScrape()
 	bodyString := bytesutil.ToUnsafeString(sbr.body)
-	areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
+	areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
 
 	scrapedSamples.Update(float64(samplesScraped))
 	endTimestamp := time.Now().UnixNano() / 1e6
@@ -738,7 +741,7 @@ func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
 }
 
 func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp int64, addAutoSeries bool) {
-	if *noStaleMarkers {
+	if sw.Config.NoStaleMarkers {
 		return
 	}
 	bodyString := lastScrape