From bba08f7846f63f8361136931d7b47767c9bfc56d Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 31 Oct 2024 19:58:22 +0100 Subject: [PATCH] lib/promscrape: add relabel configs to `global` section This commit adds `metric_relabel_configs` and `relabel_configs` fields into the `global` section of scrape configuration file. New fields are used as global relabeling rules for the scrape targets. These relabel configs are prepended to the target relabel configs. This feature is useful to: * apply global rules to __meta labels from service discovery targets. * drop noisy labels during scrapping. * mutate labels without affecting metrics ingested via any of push protocols. Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6966 --------- Signed-off-by: f41gh7 Co-authored-by: Zhu Jiekun Co-authored-by: hagen1778 --- docs/changelog/CHANGELOG.md | 1 + docs/vmagent.md | 12 +++ lib/promscrape/config.go | 24 ++++-- lib/promscrape/config_test.go | 147 ++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 5 deletions(-) diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 18047ae40..b236ee582 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -21,6 +21,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a datasource. See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support scraping from Kubernetes Native Sidecars. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/): add `metric_relabel_configs` and `relabel_configs` to the `global` section of [scrape configuration](https://docs.victoriametrics.com/vmagent/#how-to-collect-metrics-in-prometheus-format). Relabeling configuration specified in `global` section will be pre-pended to relabeling configs of all jobs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6966) and [these docs](https://docs.victoriametrics.com/vmagent/#relabeling) for details. * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add a separate cache type for storing sparse entries when performing large index scans. This significantly reduces memory usage when applying [downsampling filters](https://docs.victoriametrics.com/#downsampling) and [retention filters](https://docs.victoriametrics.com/#retention-filters) during background merge. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7182) for the details. * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for VM single-node, cluster, vmalert, vmagent, VictoriaLogs: add `Restarts` panel to show the events of process restarts. This panel should help correlate events of restart with unexpected behavior of processes. * FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml): add alerting rule `RemoteWriteDroppingData` to track number of dropped samples that weren't sent to remote write URL. diff --git a/docs/vmagent.md b/docs/vmagent.md index 5525ac4c7..596bf2d1c 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -575,6 +575,18 @@ generated metrics. But they still can be relabeled via `-remoteWrite.relabelConf VictoriaMetrics components support [Prometheus-compatible relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) with [additional enhancements](#relabeling-enhancements). The relabeling can be defined in the following places processed by `vmagent`: +* At the `global -> relabel_configs` section in `-promscrape.config` file. + This relabeling is used for modifying labels in discovered targets and for dropping unneeded targets. + Configuration from global section will be prepended to the `relabel_config` of targets from `scrape_config` section. + See [relabeling cookbook](https://docs.victoriametrics.com/relabeling/) for details. +_Available from [next](https://docs.victoriametrics.com/changelog/#tip) version._ + +* At the `global -> metric_relabel_configs` section in `-promscrape.config` file. + This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics. + Configuration from global section will be prepended to the `metric_relabel_config` of targets from `scrape_config` section. + See [relabeling cookbook](https://docs.victoriametrics.com/relabeling/) for details. +_Available from [next](https://docs.victoriametrics.com/changelog/#tip) version._ + * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is used for modifying labels in discovered targets and for dropping unneeded targets. See [relabeling cookbook](https://docs.victoriametrics.com/relabeling/) for details. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 9b1fcdbf0..c212f634b 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -262,9 +262,11 @@ func (cfg *Config) getJobNames() []string { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/ type GlobalConfig struct { - ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` - ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` - ExternalLabels *promutils.Labels `yaml:"external_labels,omitempty"` + ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` + ExternalLabels *promutils.Labels `yaml:"external_labels,omitempty"` + RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"` + MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"` } // ScrapeConfig represents essential parts for `scrape_config` section of Prometheus config. @@ -918,11 +920,23 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf if err != nil { return nil, fmt.Errorf("cannot parse proxy auth config for `job_name` %q: %w", jobName, err) } - relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs) + rcs := sc.RelabelConfigs + if len(globalCfg.RelabelConfigs) > 0 { + rcs = make([]promrelabel.RelabelConfig, 0, len(globalCfg.RelabelConfigs)+len(sc.RelabelConfigs)) + rcs = append(rcs, globalCfg.RelabelConfigs...) + rcs = append(rcs, sc.RelabelConfigs...) + } + relabelConfigs, err := promrelabel.ParseRelabelConfigs(rcs) if err != nil { return nil, fmt.Errorf("cannot parse `relabel_configs` for `job_name` %q: %w", jobName, err) } - metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs) + mrcs := sc.MetricRelabelConfigs + if len(globalCfg.MetricRelabelConfigs) > 0 { + mrcs = make([]promrelabel.RelabelConfig, 0, len(globalCfg.MetricRelabelConfigs)+len(sc.MetricRelabelConfigs)) + mrcs = append(mrcs, globalCfg.MetricRelabelConfigs...) + mrcs = append(mrcs, sc.MetricRelabelConfigs...) + } + metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(mrcs) if err != nil { return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err) } diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 84c1bd72f..b77782a89 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -107,6 +107,18 @@ scrape_configs: proxy_bearer_token_file: file.txt proxy_headers: - 'My-Auth-Header: top-secret' +`) + f(` +global: + scrape_interval: 10s + relabel_configs: + - source_labels: [job] + target_label: job + regex: (.+) + replacement: prefix-${1} + metric_relabel_configs: + - action: labeldrop + source_labels: [id] `) } @@ -1352,3 +1364,138 @@ func checkEqualScrapeWorks(t *testing.T, got, want []*ScrapeWork) { } } } + +func TestStaticConfigWithGlobalRelabelConfigs(t *testing.T) { + f := func(data string, want []*ScrapeWork) { + t.Helper() + got, err := getStaticScrapeWork([]byte(data), "non-exsiting-file") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if len(got) != len(want) { + t.Fatalf("unexpected number of ScrapeWork items; got %d; want %d", len(got), len(want)) + } + + for i := range got { + gotItem := got[i] + wantItem := want[i] + if wantItem.RelabelConfigs.String() != gotItem.RelabelConfigs.String() { + t.Fatalf("unexpected relabel_config at scrape work idx=%d, want:\n%s\ngot:\n%s", + i, wantItem.RelabelConfigs.String(), gotItem.RelabelConfigs.String()) + } + if wantItem.RelabelConfigs.String() != gotItem.RelabelConfigs.String() { + t.Fatalf("unexpected metric_relabel_config at scrape work idx=%d, want:\n%s\ngot:\n%s", + i, wantItem.MetricRelabelConfigs.String(), gotItem.MetricRelabelConfigs.String()) + } + } + } + f(` +global: + relabel_configs: + - target_label: job + replacement: bar +scrape_configs: +- job_name: foo + relabel_configs: + - target_label: bar + replacement: foo + static_configs: + - targets: ["foo.bar:1234"] +`, []*ScrapeWork{ + { + jobNameOriginal: "foo", + ScrapeURL: "foo.bar:1234", + RelabelConfigs: mustParseRelabelConfigs(` + - target_label: job + replacement: bar + - target_label: bar + replacement: foo + `), + }, + }) + f(` +global: + relabel_configs: + - target_label: job + replacement: prefix_${1} + source_labels: [job] + regex: (.+) + metric_relabel_configs: + - source_labels: [id] + action: labeldrop +scrape_configs: +- job_name: foo + relabel_configs: + - target_label: bar + replacement: foo + static_configs: + - targets: ["foo.bar:1234"] +- job_name: bar + relabel_configs: + - target_label: baz + replacement: bar + metric_relabel_configs: + - source_labels: [mount_path] + replacement: ${2} + regex: '(\/.+)?\/(.+)' + target_label: mount_path + static_configs: + - targets: ["baz.bar:1235"] +- job_name: baz + static_configs: + - targets: ["baz.bar:1235"] + +`, []*ScrapeWork{ + { + jobNameOriginal: "foo", + ScrapeURL: "foo.bar:1234", + RelabelConfigs: mustParseRelabelConfigs(` + - target_label: job + replacement: prefix_${1} + source_labels: [job] + regex: (.+) + - target_label: bar + replacement: foo + `), + MetricRelabelConfigs: mustParseRelabelConfigs(` + - source_labels: [id] + action: labeldrop + `), + }, + { + jobNameOriginal: "bar", + ScrapeURL: "baz.bar:1235", + RelabelConfigs: mustParseRelabelConfigs(` + - target_label: job + replacement: prefix_${1} + source_labels: [job] + regex: (.+) + - target_label: baz + replacement: bar + `), + MetricRelabelConfigs: mustParseRelabelConfigs(` + - source_labels: [id] + action: labeldrop + - source_labels: [mount_path] + replacement: ${2} + regex: '(\/.+)?\/(.+)' + target_label: mount_path + `), + }, + { + jobNameOriginal: "baz", + ScrapeURL: "baz.bar:1235", + RelabelConfigs: mustParseRelabelConfigs(` + - target_label: job + replacement: prefix_${1} + source_labels: [job] + regex: (.+) + `), + MetricRelabelConfigs: mustParseRelabelConfigs(` + - source_labels: [id] + action: labeldrop +`), + }, + }) + +}