vmalert: support relabeling for alert labels sent via notifier

Before, relabeling for notifier configured via file was supported
only for target labels discovered via SD.
With this change, new config field `alert_relabel_configs` is introduced
for applying relabeling to labels of sent alerts.

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vmalert/README.md

@@ -911,12 +911,17 @@ static_configs:
 consul_sd_configs:
   [ - <consul_sd_config> ... ]
 
-# List of relabel configurations.
+# List of relabel configurations for entities discovere via service discovery.
 # Supports the same relabeling features as the rest of VictoriaMetrics components.
 # See https://docs.victoriametrics.com/vmagent.html#relabeling
 relabel_configs:
   [ - <relabel_config> ... ]
 
+# List of relabel configurations for alert labels sent via Notifier.
+# Supports the same relabeling features as the rest of VictoriaMetrics components.
+# See https://docs.victoriametrics.com/vmagent.html#relabeling
+alert_relabel_configs:
+  [ - <relabel_config> ... ]
 ```
 
 The configuration file can be [hot-reloaded](#hot-config-reload).

app/vmalert/notifier/alert.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // Alert the triggered alert
@@ -147,3 +149,18 @@ func templateAnnotation(dst io.Writer, text string, data tplData, funcs template
 	}
 	return nil
 }
+
+func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
+	for k, v := range a.Labels {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	promrelabel.SortLabels(labels)
+	if relabelCfg != nil {
+		return relabelCfg.Apply(labels, 0, false)
+	}
+	return labels
+}

app/vmalert/notifier/alert_test.go

@@ -2,9 +2,12 @@ package notifier
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
@@ -146,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestAlert_toPromLabels(t *testing.T) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
+		t.Helper()
+		a := Alert{Labels: labels}
+		got := a.toPromLabels(relabel)
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
+				exp, got)
+		}
+	}
+
+	fn(nil, nil, nil)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		nil,
+	)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	fn(
+		map[string]string{"a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		pcs,
+	)
+}

app/vmalert/notifier/alertmanager.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
@@ -22,6 +23,8 @@ type AlertManager struct {
 	timeout time.Duration
 
 	authCfg *promauth.Config
+	// stores already parsed RelabelConfigs object
+	relabelConfigs *promrelabel.ParsedConfigs
 
 	metrics *metrics
 }
@@ -59,7 +62,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 
 func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
-	writeamRequest(b, alerts, am.argFunc)
+	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
 	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
@@ -103,7 +106,8 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
+	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
@@ -131,11 +135,12 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 
 	return &AlertManager{
-		addr:    alertManagerURL,
+		addr:           alertManagerURL,
-		argFunc: fn,
+		argFunc:        fn,
-		authCfg: aCfg,
+		authCfg:        aCfg,
-		client:  &http.Client{Transport: tr},
+		relabelConfigs: relabelCfg,
-		timeout: timeout,
+		client:         &http.Client{Transport: tr},
-		metrics: newMetrics(alertManagerURL),
+		timeout:        timeout,
+		metrics:        newMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,9 +1,11 @@
 {% import (
     "time"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
 {
@@ -14,8 +16,9 @@
     {% endif %}
     "labels": {
         "alertname":{%q= alert.Name %}
-        {% for k,v := range alert.Labels %}
+        {% code lbls := alert.toPromLabels(relabelCfg) %}
-            ,{%q= k %}:{%q= v %}
+        {% for _, l := range lbls %}
+            ,{%q= l.Name %}:{%q= l.Value %}
         {% endfor %}
     },
     "annotations": {

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -7,124 +7,129 @@ package notifier
 //line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		qw422016.N().S(`"labels": {"alertname":`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-		for k, v := range alert.Labels {
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().Q(alert.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:19
-		qw422016.N().S(`},"annotations": {`)
+		lbls := alert.toPromLabels(relabelCfg)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		for _, l := range lbls {
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Value)
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 		c := len(alert.Annotations)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		for k, v := range alert.Annotations {
-//line app/vmalert/notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
 			c = c - 1
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
-	streamamRequest(qw422016, alerts, generatorURL)
+	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
-func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
+func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
-	writeamRequest(qb422016, alerts, generatorURL)
+	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

app/vmalert/notifier/alertmanager_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -89,7 +89,7 @@ func TestAlertManager_Send(t *testing.T) {
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, aCfg, 0)
+	}, aCfg, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}

app/vmalert/notifier/config.go

@@ -34,9 +34,10 @@ type Config struct {
 
 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
-	// RelabelConfigs contains list of relabeling rules
+	// RelabelConfigs contains list of relabeling rules for entities discovered via SD
 	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
-
+	// AlertRelabelConfigs contains list of relabeling rules alert labels
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
 	// The timeout used when sending alerts.
 	Timeout promutils.Duration `yaml:"timeout,omitempty"`
 
@@ -52,6 +53,8 @@ type Config struct {
 
 	// stores already parsed RelabelConfigs object
 	parsedRelabelConfigs *promrelabel.ParsedConfigs
+	// stores already parsed AlertRelabelConfigs object
+	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }
 
 // StaticConfig contains list of static targets in the following form:
@@ -78,6 +81,11 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("failed to parse relabeling config: %w", err)
 	}
 	cfg.parsedRelabelConfigs = rCfg
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
+	}
+	cfg.parsedAlertRelabelConfigs = arCfg
 
 	b, err := yaml.Marshal(cfg)
 	if err != nil {

app/vmalert/notifier/config_watcher.go

@@ -141,7 +141,7 @@ func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator)
 		}
 		duplicates[u] = struct{}{}
 
-		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			errors = append(errors, err)
 			continue
@@ -165,7 +165,7 @@ func (cw *configWatcher) start() error {
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}

app/vmalert/notifier/init.go

@@ -138,7 +138,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}
 
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
 		if err != nil {
 			return nil, err
 		}

app/vmalert/notifier/testdata/consul.good.yaml

@@ -10,4 +10,7 @@ relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/static.good.yaml

@@ -2,3 +2,6 @@ static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"