lib/promrelabel: add keep_metrics and drop_metrics actions to relabeling rules

These actions simlify metrics filtering. For example,

- action: keep_metrics
  regex: 'foo|bar|baz'

would leave only metrics with `foo`, `bar` and `baz` names, while the rest of metrics will be deleted.

The commit also makes possible to split long regexps into multiple lines. For example, the following config is equivalent to the config above:

- action: keep_metrics
  regex:
  - foo
  - bar
  - baz

app/vmagent/README.md

@@ -255,6 +255,23 @@ and also provides the following actions:
 * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
 * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
 * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
+* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+
+The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
+
+```yaml
+- action: keep_metrics
+  regex: "metric_a|metric_b|foo_.+"
+```
+
+```yaml
+- action: keep_metrics
+  regex:
+  - "metric_a"
+  - "metric_b"
+  - "foo_.+"
+```
 
 The relabeling can be defined in the following places:
 

docs/CHANGELOG.md

@@ -7,6 +7,8 @@ sort: 15
 ## tip
 
 * FEATURE: vmalert: add web UI with the list of alerting groups, alerts and alert statuses. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1602).
+* FEATURE: add new relabeling actions: `keep_metrics` and `drop_metrics`. They simplify metrics filtering by metric names. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
+* FAETURE: allow splitting long `regex` in relabeling filters into an array of shorter regexps, which can be put into multiple lines for better readability and maintainability. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
 
 * BUGFIX: vmselect: reset connection timeouts after each request to `vmstorage`. This should prevent from `cannot read data in 0.000 seconds: unexpected EOF` warning in logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1562). Thanks to @mxlxm .
 

docs/vmagent.md

@@ -259,6 +259,23 @@ and also provides the following actions:
 * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
 * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
 * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
+* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+
+The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
+
+```yaml
+- action: keep_metrics
+  regex: "metric_a|metric_b|foo_.+"
+```
+
+```yaml
+- action: keep_metrics
+  regex:
+  - "metric_a"
+  - "metric_b"
+  - "foo_.+"
+```
 
 The relabeling can be defined in the following places:
 

lib/promrelabel/config.go

@@ -14,13 +14,61 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
 type RelabelConfig struct {
-	SourceLabels []string `yaml:"source_labels,flow,omitempty"`
-	Separator    *string  `yaml:"separator,omitempty"`
-	TargetLabel  string   `yaml:"target_label,omitempty"`
-	Regex        *string  `yaml:"regex,omitempty"`
-	Modulus      uint64   `yaml:"modulus,omitempty"`
-	Replacement  *string  `yaml:"replacement,omitempty"`
-	Action       string   `yaml:"action,omitempty"`
+	SourceLabels []string        `yaml:"source_labels,flow,omitempty"`
+	Separator    *string         `yaml:"separator,omitempty"`
+	TargetLabel  string          `yaml:"target_label,omitempty"`
+	Regex        *MultiLineRegex `yaml:"regex,omitempty"`
+	Modulus      uint64          `yaml:"modulus,omitempty"`
+	Replacement  *string         `yaml:"replacement,omitempty"`
+	Action       string          `yaml:"action,omitempty"`
+}
+
+// MultiLineRegex contains a regex, which can be split into multiple lines.
+//
+// These lines are joined with "|" then.
+// For example:
+//
+// regex:
+// - foo
+// - bar
+//
+// is equivalent to:
+//
+// regex: "foo|bar"
+type MultiLineRegex struct {
+	s string
+}
+
+// UnmarshalYAML unmarshals mlr from YAML passed to f.
+func (mlr *MultiLineRegex) UnmarshalYAML(f func(interface{}) error) error {
+	var v interface{}
+	if err := f(&v); err != nil {
+		return err
+	}
+	var a []string
+	switch x := v.(type) {
+	case string:
+		a = []string{x}
+	case []interface{}:
+		a = make([]string, len(x))
+		for i, xx := range x {
+			s, ok := xx.(string)
+			if !ok {
+				return fmt.Errorf("`regex` must contain array of strings; got %T", xx)
+			}
+			a[i] = s
+		}
+	default:
+		return fmt.Errorf("unexpected type for `regex`: %T; want string or []string", v)
+	}
+	mlr.s = strings.Join(a, "|")
+	return nil
+}
+
+// MarshalYAML marshals mlr to YAML.
+func (mlr *MultiLineRegex) MarshalYAML() (interface{}, error) {
+	a := strings.Split(mlr.s, "|")
+	return a, nil
 }
 
 // ParsedConfigs represents parsed relabel configs.
@@ -107,18 +155,19 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 	regexCompiled := defaultRegexForRelabelConfig
 	regexOriginalCompiled := defaultOriginalRegexForRelabelConfig
 	if rc.Regex != nil {
-		regex := *rc.Regex
+		regex := rc.Regex.s
+		regexOrig := regex
 		if rc.Action != "replace_all" && rc.Action != "labelmap_all" {
-			regex = "^(?:" + *rc.Regex + ")$"
+			regex = "^(?:" + regex + ")$"
 		}
 		re, err := regexp.Compile(regex)
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse `regex` %q: %w", regex, err)
 		}
 		regexCompiled = re
-		reOriginal, err := regexp.Compile(*rc.Regex)
+		reOriginal, err := regexp.Compile(regexOrig)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse `regex` %q: %w", *rc.Regex, err)
+			return nil, fmt.Errorf("cannot parse `regex` %q: %w", regexOrig, err)
 		}
 		regexOriginalCompiled = reOriginal
 	}
@@ -169,6 +218,24 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 		if modulus < 1 {
 			return nil, fmt.Errorf("unexpected `modulus` for `action=hashmod`: %d; must be greater than 0", modulus)
 		}
+	case "keep_metrics":
+		if rc.Regex == nil || rc.Regex.s == "" {
+			return nil, fmt.Errorf("`regex` must be non-empty for `action=keep_metrics`")
+		}
+		if len(sourceLabels) > 0 {
+			return nil, fmt.Errorf("`source_labels` must be empty for `action=keep_metrics`; got %q", sourceLabels)
+		}
+		sourceLabels = []string{"__name__"}
+		action = "keep"
+	case "drop_metrics":
+		if rc.Regex == nil || rc.Regex.s == "" {
+			return nil, fmt.Errorf("`regex` must be non-empty for `action=drop_metrics`")
+		}
+		if len(sourceLabels) > 0 {
+			return nil, fmt.Errorf("`source_labels` must be empty for `action=drop_metrics`; got %q", sourceLabels)
+		}
+		sourceLabels = []string{"__name__"}
+		action = "drop"
 	case "labelmap":
 	case "labelmap_all":
 	case "labeldrop":

lib/promrelabel/config_test.go

@@ -3,16 +3,46 @@ package promrelabel
 import (
 	"reflect"
 	"testing"
+
+	"gopkg.in/yaml.v2"
 )
 
+func TestRelabelConfigMarshalUnmarshal(t *testing.T) {
+	f := func(data, resultExpected string) {
+		t.Helper()
+		var rcs []RelabelConfig
+		if err := yaml.UnmarshalStrict([]byte(data), &rcs); err != nil {
+			t.Fatalf("cannot unmarshal %q: %s", data, err)
+		}
+		result, err := yaml.Marshal(&rcs)
+		if err != nil {
+			t.Fatalf("cannot marshal %q: %s", data, err)
+		}
+		if string(result) != resultExpected {
+			t.Fatalf("unexpected marshaled data; got\n%q\nwant\n%q", result, resultExpected)
+		}
+	}
+	f(``, "[]\n")
+	f(`
+- action: keep
+  regex: foobar
+`, "- regex:\n  - foobar\n  action: keep\n")
+	f(`
+- regex:
+  - 'fo.+'
+  - '.*ba[r-z]a'
+`, "- regex:\n  - fo.+\n  - .*ba[r-z]a\n")
+	f(`- regex: foo|bar`, "- regex:\n  - foo\n  - bar\n")
+}
+
 func TestLoadRelabelConfigsSuccess(t *testing.T) {
 	path := "testdata/relabel_configs_valid.yml"
 	pcs, err := LoadRelabelConfigs(path, false)
 	if err != nil {
 		t.Fatalf("cannot load relabel configs from %q: %s", path, err)
 	}
-	if n := pcs.Len(); n != 9 {
-		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, n, 9)
+	if n := pcs.Len(); n != 12 {
+		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, n, 12)
 	}
 }
 
@@ -85,7 +115,9 @@ func TestParseRelabelConfigsFailure(t *testing.T) {
 			{
 				SourceLabels: []string{"aaa"},
 				TargetLabel:  "xxx",
-				Regex:        strPtr("foo[bar"),
+				Regex: &MultiLineRegex{
+					s: "foo[bar",
+				},
 			},
 		})
 	})
@@ -191,8 +223,40 @@ func TestParseRelabelConfigsFailure(t *testing.T) {
 			},
 		})
 	})
-}
-
-func strPtr(s string) *string {
-	return &s
+	t.Run("drop_metrics-missing-regex", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action: "drop_metrics",
+			},
+		})
+	})
+	t.Run("drop_metrics-non-empty-source-labels", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action:       "drop_metrics",
+				SourceLabels: []string{"foo"},
+				Regex: &MultiLineRegex{
+					s: "bar",
+				},
+			},
+		})
+	})
+	t.Run("keep_metrics-missing-regex", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action: "keep_metrics",
+			},
+		})
+	})
+	t.Run("keep_metrics-non-empty-source-labels", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action:       "keep_metrics",
+				SourceLabels: []string{"foo"},
+				Regex: &MultiLineRegex{
+					s: "bar",
+				},
+			},
+		})
+	})
 }

lib/promrelabel/relabel_test.go

@@ -564,6 +564,37 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		})
 	})
+	t.Run("keep_metrics-miss", func(t *testing.T) {
+		f(`
+- action: keep_metrics
+  regex:
+  - foo
+  - bar
+`, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "xxx",
+			},
+		}, true, []prompbmarshal.Label{})
+	})
+	t.Run("keep_metrics-hit", func(t *testing.T) {
+		f(`
+- action: keep_metrics
+  regex:
+  - foo
+  - bar
+`, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foo",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foo",
+			},
+		})
+	})
 	t.Run("drop-miss", func(t *testing.T) {
 		f(`
 - action: drop
@@ -610,6 +641,37 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		}, true, []prompbmarshal.Label{})
 	})
+	t.Run("drop_metrics-miss", func(t *testing.T) {
+		f(`
+- action: drop_metrics
+  regex:
+  - foo
+  - bar
+`, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "xxx",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "xxx",
+			},
+		})
+	})
+	t.Run("drop_metrics-hit", func(t *testing.T) {
+		f(`
+- action: drop_metrics
+  regex:
+  - foo
+  - bar
+`, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foo",
+			},
+		}, true, []prompbmarshal.Label{})
+	})
 	t.Run("hashmod-miss", func(t *testing.T) {
 		f(`
 - action: hashmod

lib/promrelabel/testdata/relabel_configs_valid.yml

@@ -22,3 +22,13 @@
   source_labels: [foo, bar]
 - action: drop_if_equal
   source_labels: [foo, bar]
+- action: keep
+  source_labels: [__name__]
+  regex:
+  - 'fo.*o'
+  - 'bar'
+  - 'baz.+'
+- action: keep_metrics
+  regex: [foo bar baz]
+- action: drop_metrics
+  regex: "foo|bar|baz"