vmalert: support relabeling for alert labels sent via notifier

Before, relabeling for notifier configured via file was supported only for target labels discovered via SD. With this change, new config field `alert_relabel_configs` is introduced for applying relabeling to labels of sent alerts. Signed-off-by: hagen1778 <roman@victoriametrics.com>
2025-01-10 15:14:09 +00:00 · 2022-04-09 08:21:16 +02:00 · 2022-04-09 08:21:16 +02:00 · fa9601a0f1
commit fa9601a0f1
parent e1b274d26c
12 changed files with 181 additions and 84 deletions
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@ -911,12 +911,17 @@ static_configs:
 consul_sd_configs:
  [ - <consul_sd_config> ... ]

-# List of relabel configurations.
+# List of relabel configurations for entities discovere via service discovery.
 # Supports the same relabeling features as the rest of VictoriaMetrics components.
 # See https://docs.victoriametrics.com/vmagent.html#relabeling
 relabel_configs:
  [ - <relabel_config> ... ]

+# List of relabel configurations for alert labels sent via Notifier.
+# Supports the same relabeling features as the rest of VictoriaMetrics components.
+# See https://docs.victoriametrics.com/vmagent.html#relabeling
+alert_relabel_configs:
+  [ - <relabel_config> ... ]
 ```

 The configuration file can be [hot-reloaded](#hot-config-reload).
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@ -9,6 +9,8 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )

 // Alert the triggered alert
@ -147,3 +149,18 @@ func templateAnnotation(dst io.Writer, text string, data tplData, funcs template
 	}
 	return nil
 }
+
+func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
+	for k, v := range a.Labels {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	promrelabel.SortLabels(labels)
+	if relabelCfg != nil {
+		return relabelCfg.Apply(labels, 0, false)
+	}
+	return labels
+}
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@ -2,9 +2,12 @@ package notifier

 import (
 	"fmt"
+	"reflect"
 	"testing"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )

 func TestAlert_ExecTemplate(t *testing.T) {
@ -146,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestAlert_toPromLabels(t *testing.T) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
+		t.Helper()
+		a := Alert{Labels: labels}
+		got := a.toPromLabels(relabel)
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
+				exp, got)
+		}
+	}
+
+	fn(nil, nil, nil)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		nil,
+	)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	fn(
+		map[string]string{"a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		pcs,
+	)
+}
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@ -11,6 +11,7 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )

 // AlertManager represents integration provider with Prometheus alert manager
@ -22,6 +23,8 @@ type AlertManager struct {
 	timeout time.Duration

 	authCfg *promauth.Config
+	// stores already parsed RelabelConfigs object
+	relabelConfigs *promrelabel.ParsedConfigs

 	metrics *metrics
 }
@ -59,7 +62,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {

 func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
-	writeamRequest(b, alerts, am.argFunc)
+	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)

 	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
@ -103,7 +106,8 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"

 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
+	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
@ -131,11 +135,12 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}

 	return &AlertManager{
-		addr:    alertManagerURL,
-		argFunc: fn,
-		authCfg: aCfg,
-		client:  &http.Client{Transport: tr},
-		timeout: timeout,
-		metrics: newMetrics(alertManagerURL),
+		addr:           alertManagerURL,
+		argFunc:        fn,
+		authCfg:        aCfg,
+		relabelConfigs: relabelCfg,
+		client:         &http.Client{Transport: tr},
+		timeout:        timeout,
+		metrics:        newMetrics(alertManagerURL),
 	}, nil
 }
--- a/app/vmalert/notifier/alertmanager_request.qtpl
+++ b/app/vmalert/notifier/alertmanager_request.qtpl
@ -1,9 +1,11 @@
 {% import (
    "time"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 ) %}
 {% stripspace %}

-{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
 {
@ -14,8 +16,9 @@
    {% endif %}
    "labels": {
        "alertname":{%q= alert.Name %}
-        {% for k,v := range alert.Labels %}
-            ,{%q= k %}:{%q= v %}
+        {% code lbls := alert.toPromLabels(relabelCfg) %}
+        {% for _, l := range lbls %}
+            ,{%q= l.Name %}:{%q= l.Value %}
        {% endfor %}
    },
    "annotations": {
--- a/app/vmalert/notifier/alertmanager_request.qtpl.go
+++ b/app/vmalert/notifier/alertmanager_request.qtpl.go
@ -7,124 +7,129 @@ package notifier
 //line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )

-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 import (
 	qtio422016 "io"

 	qt422016 "github.com/valyala/quicktemplate"
 )

-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )

-//line app/vmalert/notifier/alertmanager_request.qtpl:6
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		qw422016.N().S(`"labels": {"alertname":`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-		for k, v := range alert.Labels {
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().Q(alert.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:19
-		qw422016.N().S(`},"annotations": {`)
+		lbls := alert.toPromLabels(relabelCfg)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		for _, l := range lbls {
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Value)
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 		c := len(alert.Annotations)

-//line app/vmalert/notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		for k, v := range alert.Annotations {
-//line app/vmalert/notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
 			c = c - 1

-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	streamamRequest(qw422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	writeamRequest(qb422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
--- a/app/vmalert/notifier/alertmanager_test.go
+++ b/app/vmalert/notifier/alertmanager_test.go
@ -14,7 +14,7 @@ import (

 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@ -89,7 +89,7 @@ func TestAlertManager_Send(t *testing.T) {
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, aCfg, 0)
+	}, aCfg, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@ -34,9 +34,10 @@ type Config struct {

 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
-	// RelabelConfigs contains list of relabeling rules
+	// RelabelConfigs contains list of relabeling rules for entities discovered via SD
 	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
-
+	// AlertRelabelConfigs contains list of relabeling rules alert labels
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
 	// The timeout used when sending alerts.
 	Timeout promutils.Duration `yaml:"timeout,omitempty"`

@ -52,6 +53,8 @@ type Config struct {

 	// stores already parsed RelabelConfigs object
 	parsedRelabelConfigs *promrelabel.ParsedConfigs
+	// stores already parsed AlertRelabelConfigs object
+	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }

 // StaticConfig contains list of static targets in the following form:
@ -78,6 +81,11 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("failed to parse relabeling config: %w", err)
 	}
 	cfg.parsedRelabelConfigs = rCfg
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
+	}
+	cfg.parsedAlertRelabelConfigs = arCfg

 	b, err := yaml.Marshal(cfg)
 	if err != nil {
--- a/app/vmalert/notifier/config_watcher.go
+++ b/app/vmalert/notifier/config_watcher.go
@ -141,7 +141,7 @@ func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator)
 		}
 		duplicates[u] = struct{}{}

-		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			errors = append(errors, err)
 			continue
@ -165,7 +165,7 @@ func (cw *configWatcher) start() error {
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
--- a/app/vmalert/notifier/init.go
+++ b/app/vmalert/notifier/init.go
@ -138,7 +138,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}

 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
 		if err != nil {
 			return nil, err
 		}
--- a/app/vmalert/notifier/testdata/consul.good.yaml
+++ b/app/vmalert/notifier/testdata/consul.good.yaml
@ -10,4 +10,7 @@ relabel_configs:
  - source_labels: [__meta_consul_tags]
    regex: .*,__scheme__=([^,]+),.*
    replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"
--- a/app/vmalert/notifier/testdata/static.good.yaml
+++ b/app/vmalert/notifier/testdata/static.good.yaml
@ -2,3 +2,6 @@ static_configs:
  - targets:
      - localhost:9093
      - localhost:9095
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"