From ac65c6b178302bf450ba79000d58ad80e7438a17 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 29 Nov 2023 11:00:48 +0200 Subject: [PATCH] lib/promrelabel: add `keep_if_contains` and `drop_if_contains` relabeling actions --- docs/CHANGELOG.md | 1 + docs/vmagent.md | 20 +++++++ lib/promrelabel/config.go | 20 +++++++ lib/promrelabel/config_test.go | 56 +++++++++++++++++++ lib/promrelabel/relabel.go | 37 +++++++++++++ lib/promrelabel/relabel_test.go | 98 +++++++++++++++++++++++++++++++++ 6 files changed, 232 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e3969e1fe..cbcb1b21a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for reading and writing samples via [Google PubSub](https://cloud.google.com/pubsub). See [these docs](https://docs.victoriametrics.com/vmagent.html#google-pubsub-integration). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for Datadog `/api/v2/series` and `/api/beta/sketches` ingestion protocols to vmagent/vminsert components. See this [doc](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) for examples. Thanks to @AndrewChubatiuk for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5094). * FEATURE: reduce the default value for `-import.maxLineLen` command-line flag from 100MB to 10MB in order to prevent excessive memory usage during data import via [/api/v1/import](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `keep_if_contains` and `drop_if_contains` relabeling actions. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) for details. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [day_of_year()](https://docs.victoriametrics.com/MetricsQL.html#day_of_year) function, which returns the day of the year for each of the given unix timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5345) for details. Thanks to @luckyxiaoqiang for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5368/). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/vmagent.html#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357). diff --git a/docs/vmagent.md b/docs/vmagent.md index 40b755d8c..3b41bc07f 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -639,6 +639,26 @@ The following articles contain useful information about Prometheus relabeling: source_labels: ["instance", "host"] ``` + * `keep_if_contains`: keeps the entry if `target_label` contains all the label values listed in `source_labels`, + while dropping all the other entries. For example, the following relabeling config keeps targets + if `__meta_consul_tags` contains value from the `required_consul_tag` label: + + ```yaml + - action: keep_if_contains + target_label: __meta_consul_tags + source_labels: [required_consul_tag] + ``` + + * `drop_if_contains`: drops the entry if `target_label` contains all the label values listed in `source_labels`, + while keeping all the other entries. For example, the following relabeling config drops targets + if `__meta_consul_tag` contains value from the `denied_consul_tag` label: + + ```yaml + - action: drop_if_contains + target_label: __meta_consul_tags + source_labels: [denied_consul_tag] + ``` + * `keep_metrics`: keeps all the metrics with names matching the given `regex`, while dropping all the other metrics. For example, the following relabeling config keeps metrics with `foo` and `bar` names, while dropping all the other metrics: diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 137ff4b58..fdd2f853c 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -291,6 +291,26 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { if targetLabel == "" { return nil, fmt.Errorf("missing `target_label` for `action=replace_all`") } + case "keep_if_contains": + if targetLabel == "" { + return nil, fmt.Errorf("`target_label` must be set for `action=keep_if_containes`") + } + if len(sourceLabels) == 0 { + return nil, fmt.Errorf("`source_labels` must contain at least a single entry for `action=keep_if_contains`") + } + if rc.Regex != nil { + return nil, fmt.Errorf("`regex` cannot be used for `action=keep_if_contains`") + } + case "drop_if_contains": + if targetLabel == "" { + return nil, fmt.Errorf("`target_label` must be set for `action=drop_if_containes`") + } + if len(sourceLabels) == 0 { + return nil, fmt.Errorf("`source_labels` must contain at least a single entry for `action=drop_if_contains`") + } + if rc.Regex != nil { + return nil, fmt.Errorf("`regex` cannot be used for `action=drop_if_contains`") + } case "keep_if_equal": if len(sourceLabels) < 2 { return nil, fmt.Errorf("`source_labels` must contain at least two entries for `action=keep_if_equal`; got %q", sourceLabels) diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index 41f432739..8d4f4340d 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -251,6 +251,62 @@ func TestParseRelabelConfigsFailure(t *testing.T) { }, }) }) + t.Run("keep_if_contains-missing-target-label", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_if_contains", + SourceLabels: []string{"foo"}, + }, + }) + }) + t.Run("keep_if_contains-missing-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_if_contains", + TargetLabel: "foo", + }, + }) + }) + t.Run("keep_if_contains-unused-regex", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_if_contains", + TargetLabel: "foo", + SourceLabels: []string{"bar"}, + Regex: &MultiLineRegex{ + S: "bar", + }, + }, + }) + }) + t.Run("drop_if_contains-missing-target-label", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_if_contains", + SourceLabels: []string{"foo"}, + }, + }) + }) + t.Run("drop_if_contains-missing-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_if_contains", + TargetLabel: "foo", + }, + }) + }) + t.Run("drop_if_contains-unused-regex", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_if_contains", + TargetLabel: "foo", + SourceLabels: []string{"bar"}, + Regex: &MultiLineRegex{ + S: "bar", + }, + }, + }) + }) t.Run("keep_if_equal-missing-source-labels", func(t *testing.T) { f([]RelabelConfig{ { diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index f1e0a6ce7..be3c5f6fa 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -256,6 +256,32 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr) } return labels + case "keep_if_contains": + // Keep the entry if target_label contains all the label values listed in source_labels. + // For example, the following relabeling rule would leave the entry if __meta_consul_tags + // contains values of __meta_required_tag1 and __meta_required_tag2: + // + // - action: keep_if_contains + // target_label: __meta_consul_tags + // source_labels: [__meta_required_tag1, __meta_required_tag2] + // + if containsAllLabelValues(src, prc.TargetLabel, prc.SourceLabels) { + return labels + } + return labels[:labelsOffset] + case "drop_if_contains": + // Drop the entry if target_label contains all the label values listed in source_labels. + // For example, the following relabeling rule would drop the entry if __meta_consul_tags + // contains values of __meta_required_tag1 and __meta_required_tag2: + // + // - action: drop_if_contains + // target_label: __meta_consul_tags + // source_labels: [__meta_required_tag1, __meta_required_tag2] + // + if containsAllLabelValues(src, prc.TargetLabel, prc.SourceLabels) { + return labels[:labelsOffset] + } + return labels case "keep_if_equal": // Keep the entry if all the label values in source_labels are equal. // For example: @@ -489,6 +515,17 @@ func (prc *parsedRelabelConfig) expandCaptureGroups(template, source string, mat var relabelBufPool bytesutil.ByteBufferPool +func containsAllLabelValues(labels []prompbmarshal.Label, targetLabel string, sourceLabels []string) bool { + targetLabelValue := getLabelValue(labels, targetLabel) + for _, sourceLabel := range sourceLabels { + v := getLabelValue(labels, sourceLabel) + if !strings.Contains(targetLabelValue, v) { + return false + } + } + return true +} + func areEqualLabelValues(labels []prompbmarshal.Label, labelNames []string) bool { if len(labelNames) < 2 { logger.Panicf("BUG: expecting at least 2 labelNames; got %d", len(labelNames)) diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index 0f77b2607..bd2963a51 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -383,6 +383,104 @@ func TestParsedRelabelConfigsApply(t *testing.T) { target_label: foo replacement: "foobar" `, `{}`, true, `{foo="foobar"}`) + }) + t.Run("keep_if_contains-non-existing-target-and-source", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar] +`, `{x="y"}`, true, `{x="y"}`) + }) + t.Run("keep_if_contains-non-existing-target", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa"}`, true, `{}`) + }) + t.Run("keep_if_contains-non-existing-source", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar] +`, `{foo="aaa"}`, true, `{foo="aaa"}`) + }) + t.Run("keep_if_contains-matching-source-target", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa",foo="aaa"}`, true, `{bar="aaa",foo="aaa"}`) + }) + t.Run("keep_if_contains-matching-sources-target", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar, baz] +`, `{bar="aaa",foo="aaa",baz="aaa"}`, true, `{bar="aaa",baz="aaa",foo="aaa"}`) + }) + t.Run("keep_if_contains-mismatching-source-target", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa",foo="bbb"}`, true, `{}`) + }) + t.Run("keep_if_contains-mismatching-sources-target", func(t *testing.T) { + f(` +- action: keep_if_contains + target_label: foo + source_labels: [bar, baz] +`, `{bar="aaa",foo="aaa",baz="bbb"}`, true, `{}`) + }) + t.Run("drop_if_contains-non-existing-target-and-source", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar] +`, `{x="y"}`, true, `{}`) + }) + t.Run("drop_if_contains-non-existing-target", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa"}`, true, `{bar="aaa"}`) + }) + t.Run("drop_if_contains-non-existing-source", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar] +`, `{foo="aaa"}`, true, `{}`) + }) + t.Run("drop_if_contains-matching-source-target", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa",foo="aaa"}`, true, `{}`) + }) + t.Run("drop_if_contains-matching-sources-target", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar, baz] +`, `{bar="aaa",foo="aaa",baz="aaa"}`, true, `{}`) + }) + t.Run("drop_if_contains-mismatching-source-target", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar] +`, `{bar="aaa",foo="bbb"}`, true, `{bar="aaa",foo="bbb"}`) + }) + t.Run("drop_if_contains-mismatching-sources-target", func(t *testing.T) { + f(` +- action: drop_if_contains + target_label: foo + source_labels: [bar, baz] +`, `{bar="aaa",foo="aaa",baz="bbb"}`, true, `{bar="aaa",baz="bbb",foo="aaa"}`) }) t.Run("keep_if_equal-miss", func(t *testing.T) { f(`