mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/promscrape: allow controlling staleness tracking on a per-scrape_config basis
Add support for no_stale_markers option at scrape_config section. See https://docs.victoriametrics.com/sd_configs.html#scrape_configs and https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers
This commit is contained in:
Aliaksandr Valialkin
parent
93811da76d
commit
5269b1ad77
7 changed files with 46 additions and 17 deletions
|
|
@ -382,7 +382,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
|
|||
```
|
||||
|
||||
`vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option
|
||||
(e.g. when [staleness markers](#prometheus-staleness-markers) are disabled).
|
||||
or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
|
||||
|
||||
* `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter).
|
||||
This metric is exposed only if the series limit is set.
|
||||
|
|
@ -604,9 +604,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t
|
|||
* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
|
||||
|
||||
Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target
|
||||
in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers`
|
||||
command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series
|
||||
per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details.
|
||||
in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways:
|
||||
|
||||
* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets.
|
||||
* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target.
|
||||
|
||||
When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape,
|
||||
e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details.
|
||||
|
||||
## Stream parsing mode
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
- host4:1234
|
||||
```
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow controlling staleness tracking on a per-[scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) basis by specifying `no_stale_markers: true` or `no_stale_markers: false` option in the corresponding [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs).
|
||||
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): automatically update graph, legend and url after the removal of query field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3169) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3196#issuecomment-1269765205).
|
||||
|
||||
## [v1.82.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.0)
|
||||
|
|
|
|||
|
|
@ -1188,7 +1188,7 @@ scrape_configs:
|
|||
# disable_keepalive: <boolean>
|
||||
|
||||
# stream_parse allows enabling stream parsing mode when scraping targets.
|
||||
# By default stram parsing mode is disabled for targets which return up to a few thosands samples.
|
||||
# By default stream parsing mode is disabled for targets which return up to a few thosands samples.
|
||||
# See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode .
|
||||
# stream_parse: <boolean>
|
||||
|
||||
|
|
@ -1212,6 +1212,11 @@ scrape_configs:
|
|||
# See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter .
|
||||
# series_limit: ...
|
||||
|
||||
# no_stale_markers allows disabling staleness tracking.
|
||||
# By default staleness tracking is enabled for all the discovered scrape targets.
|
||||
# See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers
|
||||
# no_stale_markers: <boolean>
|
||||
|
||||
# Additional HTTP client options for target scraping can be specified here.
|
||||
# See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options
|
||||
```
|
||||
|
|
|
|||
|
|
@ -386,7 +386,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
|
|||
```
|
||||
|
||||
`vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line option
|
||||
(e.g. when [staleness markers](#prometheus-staleness-markers) are disabled).
|
||||
or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
|
||||
|
||||
* `scrape_series_limit` - the limit on the number of unique time series the given target can expose according to [these docs](#cardinality-limiter).
|
||||
This metric is exposed only if the series limit is set.
|
||||
|
|
@ -608,9 +608,13 @@ Additionally, the `action: graphite` relabeling rules usually work much faster t
|
|||
* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
|
||||
|
||||
Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target
|
||||
in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers`
|
||||
command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series
|
||||
per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](#automatically-generated-metrics) for details.
|
||||
in order to compare it to the current response body. The memory usage may be reduced by disabling staleness tracking in the following ways:
|
||||
|
||||
* By passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking across all the targets.
|
||||
* By specifying `no_stale_markers: true` option in the [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for the corresponding target.
|
||||
|
||||
When staleness tracking is disabled, then `vmagent` doesn't track the number of new time series per each scrape,
|
||||
e.g. it sets `scrape_series_added` metric to zero. See [these docs](#automatically-generated-metrics) for details.
|
||||
|
||||
## Stream parsing mode
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,9 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
|
||||
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
|
||||
noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
|
||||
strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
|
||||
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
|
||||
"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
|
||||
"See also -promscrape.config.strictParse command-line flag. "+
|
||||
"Pass -loggerLevel=ERROR if you don't need to see info messages in the output.")
|
||||
|
|
@ -289,6 +290,7 @@ type ScrapeConfig struct {
|
|||
ScrapeAlignInterval *promutils.Duration `yaml:"scrape_align_interval,omitempty"`
|
||||
ScrapeOffset *promutils.Duration `yaml:"scrape_offset,omitempty"`
|
||||
SeriesLimit int `yaml:"series_limit,omitempty"`
|
||||
NoStaleMarkers *bool `yaml:"no_stale_markers,omitempty"`
|
||||
ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
|
||||
|
||||
// This is set in loadConfig
|
||||
|
|
@ -950,6 +952,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
|
|||
return nil, fmt.Errorf("cannot use stream parsing mode when `series_limit` is set for `job_name` %q", jobName)
|
||||
}
|
||||
externalLabels := globalCfg.getExternalLabels()
|
||||
noStaleTracking := *noStaleMarkers
|
||||
if sc.NoStaleMarkers != nil {
|
||||
noStaleTracking = *sc.NoStaleMarkers
|
||||
}
|
||||
swc := &scrapeWorkConfig{
|
||||
scrapeInterval: scrapeInterval,
|
||||
scrapeIntervalString: scrapeInterval.String(),
|
||||
|
|
@ -975,6 +981,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
|
|||
scrapeAlignInterval: sc.ScrapeAlignInterval.Duration(),
|
||||
scrapeOffset: sc.ScrapeOffset.Duration(),
|
||||
seriesLimit: sc.SeriesLimit,
|
||||
noStaleMarkers: noStaleTracking,
|
||||
}
|
||||
return swc, nil
|
||||
}
|
||||
|
|
@ -1004,6 +1011,7 @@ type scrapeWorkConfig struct {
|
|||
scrapeAlignInterval time.Duration
|
||||
scrapeOffset time.Duration
|
||||
seriesLimit int
|
||||
noStaleMarkers bool
|
||||
}
|
||||
|
||||
type targetLabelsGetter interface {
|
||||
|
|
@ -1357,6 +1365,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
|
|||
ScrapeAlignInterval: swc.scrapeAlignInterval,
|
||||
ScrapeOffset: swc.scrapeOffset,
|
||||
SeriesLimit: seriesLimit,
|
||||
NoStaleMarkers: swc.noStaleMarkers,
|
||||
AuthToken: at,
|
||||
|
||||
jobNameOriginal: swc.jobName,
|
||||
|
|
|
|||
|
|
@ -1493,6 +1493,7 @@ scrape_configs:
|
|||
scrape_interval: 1w
|
||||
scrape_align_interval: 1d
|
||||
scrape_offset: 2d
|
||||
no_stale_markers: true
|
||||
static_configs:
|
||||
- targets: ["foo.bar:1234"]
|
||||
`, []*ScrapeWork{
|
||||
|
|
@ -1503,6 +1504,7 @@ scrape_configs:
|
|||
ScrapeAlignInterval: time.Hour * 24,
|
||||
ScrapeOffset: time.Hour * 24 * 2,
|
||||
HonorTimestamps: true,
|
||||
NoStaleMarkers: true,
|
||||
Labels: []prompbmarshal.Label{
|
||||
{
|
||||
Name: "instance",
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ var (
|
|||
"See also -promscrape.suppressScrapeErrorsDelay")
|
||||
suppressScrapeErrorsDelay = flag.Duration("promscrape.suppressScrapeErrorsDelay", 0, "The delay for suppressing repeated scrape errors logging per each scrape targets. "+
|
||||
"This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors")
|
||||
noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
|
||||
seriesLimitPerTarget = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
|
||||
minResponseSizeForStreamParse = flagutil.NewBytes("promscrape.minResponseSizeForStreamParse", 1e6, "The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode")
|
||||
)
|
||||
|
|
@ -122,6 +121,10 @@ type ScrapeWork struct {
|
|||
// Optional limit on the number of unique series the scrape target can expose.
|
||||
SeriesLimit int
|
||||
|
||||
// Whether to process stale markers for the given target.
|
||||
// See https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers
|
||||
NoStaleMarkers bool
|
||||
|
||||
//The Tenant Info
|
||||
AuthToken *auth.Token
|
||||
|
||||
|
|
@ -144,12 +147,12 @@ func (sw *ScrapeWork) key() string {
|
|||
key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+
|
||||
"ExternalLabels=%s, "+
|
||||
"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
|
||||
"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d",
|
||||
"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v",
|
||||
sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.LabelsString(),
|
||||
promLabelsString(sw.ExternalLabels),
|
||||
sw.ProxyURL.String(), sw.ProxyAuthConfig.String(),
|
||||
sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,
|
||||
sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit)
|
||||
sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers)
|
||||
return key
|
||||
}
|
||||
|
||||
|
|
@ -438,7 +441,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
|
|||
wc := writeRequestCtxPool.Get(sw.prevLabelsLen)
|
||||
lastScrape := sw.loadLastScrape()
|
||||
bodyString := bytesutil.ToUnsafeString(body.B)
|
||||
areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
|
||||
areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
|
||||
if err != nil {
|
||||
up = 0
|
||||
scrapesFailed.Inc()
|
||||
|
|
@ -590,7 +593,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
|
|||
}
|
||||
lastScrape := sw.loadLastScrape()
|
||||
bodyString := bytesutil.ToUnsafeString(sbr.body)
|
||||
areIdenticalSeries := *noStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
|
||||
areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
|
||||
|
||||
scrapedSamples.Update(float64(samplesScraped))
|
||||
endTimestamp := time.Now().UnixNano() / 1e6
|
||||
|
|
@ -738,7 +741,7 @@ func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
|
|||
}
|
||||
|
||||
func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp int64, addAutoSeries bool) {
|
||||
if *noStaleMarkers {
|
||||
if sw.Config.NoStaleMarkers {
|
||||
return
|
||||
}
|
||||
bodyString := lastScrape
|
||||
|
|
|
|||
Loading…
Reference in a new issue