diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73bac70fc1..812d26f60c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
# tip
+* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
+ See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-724308361 for details.
+
# [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 612a62108e..b21928d402 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -237,6 +237,9 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
+* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
+ This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+
* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 73bac70fc1..812d26f60c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,9 @@
# tip
+* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
+ See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-724308361 for details.
+
# [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 612a62108e..b21928d402 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -237,6 +237,9 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
+* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
+ This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+
* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 4961397e98..6d2f1ea630 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -32,6 +32,9 @@ var (
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
"Pass -loggerLevel=ERROR if you don't need to see info messages in the output")
+ dropOriginalLabels = flag.Bool("promscrape.dropOriginalLabels", false, "Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. "+
+ "This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. "+
+ "Note that this reduces debuggability for improper per-target relabeling configs")
)
// Config represents essential parts from Prometheus config defined at https://prometheus.io/docs/prometheus/latest/configuration/configuration/
@@ -639,8 +642,11 @@ func (stc *StaticConfig) appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfi
func appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) ([]ScrapeWork, error) {
labels := mergeLabels(swc.jobName, swc.scheme, target, swc.metricsPath, extraLabels, swc.externalLabels, metaLabels, swc.params)
- originalLabels := append([]prompbmarshal.Label{}, labels...)
- promrelabel.SortLabels(originalLabels)
+ var originalLabels []prompbmarshal.Label
+ if !*dropOriginalLabels {
+ originalLabels = append([]prompbmarshal.Label{}, labels...)
+ promrelabel.SortLabels(originalLabels)
+ }
labels = promrelabel.ApplyRelabelConfigs(labels, 0, swc.relabelConfigs, false)
labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 52f99a97eb..9e964caa25 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -319,7 +319,7 @@ func (sg *scraperGroup) update(sws []ScrapeWork) {
// Stop deleted scrapers, which are missing in sws.
for key, sc := range sg.m {
- if swsMap[key] == nil {
+ if _, ok := swsMap[key]; !ok {
close(sc.stopCh)
delete(sg.m, key)
deletionsCount++