diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 3036b18056..f466cc4ead 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -255,6 +255,23 @@ and also provides the following actions: * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`. * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal. * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal. +* `keep_metrics`: keeps all the metrics with names matching the given `regex`. +* `drop_metrics`: drops all the metrics with names matching the given `regex`. + +The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: + +```yaml +- action: keep_metrics + regex: "metric_a|metric_b|foo_.+" +``` + +```yaml +- action: keep_metrics + regex: + - "metric_a" + - "metric_b" + - "foo_.+" +``` The relabeling can be defined in the following places: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3e404fc3c3..3d89f6eccf 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ sort: 15 ## tip * FEATURE: vmalert: add web UI with the list of alerting groups, alerts and alert statuses. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1602). +* FEATURE: add new relabeling actions: `keep_metrics` and `drop_metrics`. They simplify metrics filtering by metric names. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. +* FAETURE: allow splitting long `regex` in relabeling filters into an array of shorter regexps, which can be put into multiple lines for better readability and maintainability. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. * BUGFIX: vmselect: reset connection timeouts after each request to `vmstorage`. This should prevent from `cannot read data in 0.000 seconds: unexpected EOF` warning in logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1562). Thanks to @mxlxm . diff --git a/docs/vmagent.md b/docs/vmagent.md index f0961fd6bd..03fa98e054 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -259,6 +259,23 @@ and also provides the following actions: * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`. * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal. * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal. +* `keep_metrics`: keeps all the metrics with names matching the given `regex`. +* `drop_metrics`: drops all the metrics with names matching the given `regex`. + +The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: + +```yaml +- action: keep_metrics + regex: "metric_a|metric_b|foo_.+" +``` + +```yaml +- action: keep_metrics + regex: + - "metric_a" + - "metric_b" + - "foo_.+" +``` The relabeling can be defined in the following places: diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 3c9e13660e..f3564abf42 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -14,13 +14,61 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config type RelabelConfig struct { - SourceLabels []string `yaml:"source_labels,flow,omitempty"` - Separator *string `yaml:"separator,omitempty"` - TargetLabel string `yaml:"target_label,omitempty"` - Regex *string `yaml:"regex,omitempty"` - Modulus uint64 `yaml:"modulus,omitempty"` - Replacement *string `yaml:"replacement,omitempty"` - Action string `yaml:"action,omitempty"` + SourceLabels []string `yaml:"source_labels,flow,omitempty"` + Separator *string `yaml:"separator,omitempty"` + TargetLabel string `yaml:"target_label,omitempty"` + Regex *MultiLineRegex `yaml:"regex,omitempty"` + Modulus uint64 `yaml:"modulus,omitempty"` + Replacement *string `yaml:"replacement,omitempty"` + Action string `yaml:"action,omitempty"` +} + +// MultiLineRegex contains a regex, which can be split into multiple lines. +// +// These lines are joined with "|" then. +// For example: +// +// regex: +// - foo +// - bar +// +// is equivalent to: +// +// regex: "foo|bar" +type MultiLineRegex struct { + s string +} + +// UnmarshalYAML unmarshals mlr from YAML passed to f. +func (mlr *MultiLineRegex) UnmarshalYAML(f func(interface{}) error) error { + var v interface{} + if err := f(&v); err != nil { + return err + } + var a []string + switch x := v.(type) { + case string: + a = []string{x} + case []interface{}: + a = make([]string, len(x)) + for i, xx := range x { + s, ok := xx.(string) + if !ok { + return fmt.Errorf("`regex` must contain array of strings; got %T", xx) + } + a[i] = s + } + default: + return fmt.Errorf("unexpected type for `regex`: %T; want string or []string", v) + } + mlr.s = strings.Join(a, "|") + return nil +} + +// MarshalYAML marshals mlr to YAML. +func (mlr *MultiLineRegex) MarshalYAML() (interface{}, error) { + a := strings.Split(mlr.s, "|") + return a, nil } // ParsedConfigs represents parsed relabel configs. @@ -107,18 +155,19 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { regexCompiled := defaultRegexForRelabelConfig regexOriginalCompiled := defaultOriginalRegexForRelabelConfig if rc.Regex != nil { - regex := *rc.Regex + regex := rc.Regex.s + regexOrig := regex if rc.Action != "replace_all" && rc.Action != "labelmap_all" { - regex = "^(?:" + *rc.Regex + ")$" + regex = "^(?:" + regex + ")$" } re, err := regexp.Compile(regex) if err != nil { return nil, fmt.Errorf("cannot parse `regex` %q: %w", regex, err) } regexCompiled = re - reOriginal, err := regexp.Compile(*rc.Regex) + reOriginal, err := regexp.Compile(regexOrig) if err != nil { - return nil, fmt.Errorf("cannot parse `regex` %q: %w", *rc.Regex, err) + return nil, fmt.Errorf("cannot parse `regex` %q: %w", regexOrig, err) } regexOriginalCompiled = reOriginal } @@ -169,6 +218,24 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { if modulus < 1 { return nil, fmt.Errorf("unexpected `modulus` for `action=hashmod`: %d; must be greater than 0", modulus) } + case "keep_metrics": + if rc.Regex == nil || rc.Regex.s == "" { + return nil, fmt.Errorf("`regex` must be non-empty for `action=keep_metrics`") + } + if len(sourceLabels) > 0 { + return nil, fmt.Errorf("`source_labels` must be empty for `action=keep_metrics`; got %q", sourceLabels) + } + sourceLabels = []string{"__name__"} + action = "keep" + case "drop_metrics": + if rc.Regex == nil || rc.Regex.s == "" { + return nil, fmt.Errorf("`regex` must be non-empty for `action=drop_metrics`") + } + if len(sourceLabels) > 0 { + return nil, fmt.Errorf("`source_labels` must be empty for `action=drop_metrics`; got %q", sourceLabels) + } + sourceLabels = []string{"__name__"} + action = "drop" case "labelmap": case "labelmap_all": case "labeldrop": diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index a590c3f1ef..2f5b52a323 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -3,16 +3,46 @@ package promrelabel import ( "reflect" "testing" + + "gopkg.in/yaml.v2" ) +func TestRelabelConfigMarshalUnmarshal(t *testing.T) { + f := func(data, resultExpected string) { + t.Helper() + var rcs []RelabelConfig + if err := yaml.UnmarshalStrict([]byte(data), &rcs); err != nil { + t.Fatalf("cannot unmarshal %q: %s", data, err) + } + result, err := yaml.Marshal(&rcs) + if err != nil { + t.Fatalf("cannot marshal %q: %s", data, err) + } + if string(result) != resultExpected { + t.Fatalf("unexpected marshaled data; got\n%q\nwant\n%q", result, resultExpected) + } + } + f(``, "[]\n") + f(` +- action: keep + regex: foobar +`, "- regex:\n - foobar\n action: keep\n") + f(` +- regex: + - 'fo.+' + - '.*ba[r-z]a' +`, "- regex:\n - fo.+\n - .*ba[r-z]a\n") + f(`- regex: foo|bar`, "- regex:\n - foo\n - bar\n") +} + func TestLoadRelabelConfigsSuccess(t *testing.T) { path := "testdata/relabel_configs_valid.yml" pcs, err := LoadRelabelConfigs(path, false) if err != nil { t.Fatalf("cannot load relabel configs from %q: %s", path, err) } - if n := pcs.Len(); n != 9 { - t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, n, 9) + if n := pcs.Len(); n != 12 { + t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, n, 12) } } @@ -85,7 +115,9 @@ func TestParseRelabelConfigsFailure(t *testing.T) { { SourceLabels: []string{"aaa"}, TargetLabel: "xxx", - Regex: strPtr("foo[bar"), + Regex: &MultiLineRegex{ + s: "foo[bar", + }, }, }) }) @@ -191,8 +223,40 @@ func TestParseRelabelConfigsFailure(t *testing.T) { }, }) }) -} - -func strPtr(s string) *string { - return &s + t.Run("drop_metrics-missing-regex", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_metrics", + }, + }) + }) + t.Run("drop_metrics-non-empty-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_metrics", + SourceLabels: []string{"foo"}, + Regex: &MultiLineRegex{ + s: "bar", + }, + }, + }) + }) + t.Run("keep_metrics-missing-regex", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_metrics", + }, + }) + }) + t.Run("keep_metrics-non-empty-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_metrics", + SourceLabels: []string{"foo"}, + Regex: &MultiLineRegex{ + s: "bar", + }, + }, + }) + }) } diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index c0abb8b9a0..ee9a7b967e 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -564,6 +564,37 @@ func TestApplyRelabelConfigs(t *testing.T) { }, }) }) + t.Run("keep_metrics-miss", func(t *testing.T) { + f(` +- action: keep_metrics + regex: + - foo + - bar +`, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "xxx", + }, + }, true, []prompbmarshal.Label{}) + }) + t.Run("keep_metrics-hit", func(t *testing.T) { + f(` +- action: keep_metrics + regex: + - foo + - bar +`, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "foo", + }, + }, true, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "foo", + }, + }) + }) t.Run("drop-miss", func(t *testing.T) { f(` - action: drop @@ -610,6 +641,37 @@ func TestApplyRelabelConfigs(t *testing.T) { }, }, true, []prompbmarshal.Label{}) }) + t.Run("drop_metrics-miss", func(t *testing.T) { + f(` +- action: drop_metrics + regex: + - foo + - bar +`, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "xxx", + }, + }, true, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "xxx", + }, + }) + }) + t.Run("drop_metrics-hit", func(t *testing.T) { + f(` +- action: drop_metrics + regex: + - foo + - bar +`, []prompbmarshal.Label{ + { + Name: "__name__", + Value: "foo", + }, + }, true, []prompbmarshal.Label{}) + }) t.Run("hashmod-miss", func(t *testing.T) { f(` - action: hashmod diff --git a/lib/promrelabel/testdata/relabel_configs_valid.yml b/lib/promrelabel/testdata/relabel_configs_valid.yml index 5b4a8828ec..35bb6107f8 100644 --- a/lib/promrelabel/testdata/relabel_configs_valid.yml +++ b/lib/promrelabel/testdata/relabel_configs_valid.yml @@ -22,3 +22,13 @@ source_labels: [foo, bar] - action: drop_if_equal source_labels: [foo, bar] +- action: keep + source_labels: [__name__] + regex: + - 'fo.*o' + - 'bar' + - 'baz.+' +- action: keep_metrics + regex: [foo bar baz] +- action: drop_metrics + regex: "foo|bar|baz"