diff --git a/app/vmalert/README.md b/app/vmalert/README.md index b8fd1f650..8a591c139 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -911,12 +911,17 @@ static_configs: consul_sd_configs: [ - ... ] -# List of relabel configurations. +# List of relabel configurations for entities discovere via service discovery. # Supports the same relabeling features as the rest of VictoriaMetrics components. # See https://docs.victoriametrics.com/vmagent.html#relabeling relabel_configs: [ - ... ] +# List of relabel configurations for alert labels sent via Notifier. +# Supports the same relabeling features as the rest of VictoriaMetrics components. +# See https://docs.victoriametrics.com/vmagent.html#relabeling +alert_relabel_configs: + [ - ... ] ``` The configuration file can be [hot-reloaded](#hot-config-reload). diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index a1bcae7c1..233540e90 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -9,6 +9,8 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) // Alert the triggered alert @@ -147,3 +149,18 @@ func templateAnnotation(dst io.Writer, text string, data tplData, funcs template } return nil } + +func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label { + var labels []prompbmarshal.Label + for k, v := range a.Labels { + labels = append(labels, prompbmarshal.Label{ + Name: k, + Value: v, + }) + } + promrelabel.SortLabels(labels) + if relabelCfg != nil { + return relabelCfg.Apply(labels, 0, false) + } + return labels +} diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 27b83aac6..9f5a93220 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -2,9 +2,12 @@ package notifier import ( "fmt" + "reflect" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) func TestAlert_ExecTemplate(t *testing.T) { @@ -146,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) { }) } } + +func TestAlert_toPromLabels(t *testing.T) { + fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) { + t.Helper() + a := Alert{Labels: labels} + got := a.toPromLabels(relabel) + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected to have: \n%v;\ngot:\n%v", + exp, got) + } + } + + fn(nil, nil, nil) + fn( + map[string]string{"foo": "bar", "a": "baz"}, // unsorted + []prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}}, + nil, + ) + + pcs, err := promrelabel.ParseRelabelConfigsData([]byte(` +- target_label: "foo" + replacement: "aaa" +- action: labeldrop + regex: "env.*" +`), false) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + fn( + map[string]string{"a": "baz"}, + []prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}}, + pcs, + ) + fn( + map[string]string{"foo": "bar", "a": "baz"}, + []prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}}, + pcs, + ) + fn( + map[string]string{"qux": "bar", "env": "prod", "environment": "production"}, + []prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}}, + pcs, + ) +} diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index 1caeea692..c64387877 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -11,6 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) // AlertManager represents integration provider with Prometheus alert manager @@ -22,6 +23,8 @@ type AlertManager struct { timeout time.Duration authCfg *promauth.Config + // stores already parsed RelabelConfigs object + relabelConfigs *promrelabel.ParsedConfigs metrics *metrics } @@ -59,7 +62,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error { func (am *AlertManager) send(ctx context.Context, alerts []Alert) error { b := &bytes.Buffer{} - writeamRequest(b, alerts, am.argFunc) + writeamRequest(b, alerts, am.argFunc, am.relabelConfigs) req, err := http.NewRequest("POST", am.addr, b) if err != nil { @@ -103,7 +106,8 @@ type AlertURLGenerator func(Alert) string const alertManagerPath = "/api/v2/alerts" // NewAlertManager is a constructor for AlertManager -func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) { +func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, + relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) { tls := &promauth.TLSConfig{} if authCfg.TLSConfig != nil { tls = authCfg.TLSConfig @@ -131,11 +135,12 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma } return &AlertManager{ - addr: alertManagerURL, - argFunc: fn, - authCfg: aCfg, - client: &http.Client{Transport: tr}, - timeout: timeout, - metrics: newMetrics(alertManagerURL), + addr: alertManagerURL, + argFunc: fn, + authCfg: aCfg, + relabelConfigs: relabelCfg, + client: &http.Client{Transport: tr}, + timeout: timeout, + metrics: newMetrics(alertManagerURL), }, nil } diff --git a/app/vmalert/notifier/alertmanager_request.qtpl b/app/vmalert/notifier/alertmanager_request.qtpl index 9592b5158..dca573ac7 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl +++ b/app/vmalert/notifier/alertmanager_request.qtpl @@ -1,9 +1,11 @@ {% import ( "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) %} {% stripspace %} -{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %} +{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %} [ {% for i, alert := range alerts %} { @@ -14,8 +16,9 @@ {% endif %} "labels": { "alertname":{%q= alert.Name %} - {% for k,v := range alert.Labels %} - ,{%q= k %}:{%q= v %} + {% code lbls := alert.toPromLabels(relabelCfg) %} + {% for _, l := range lbls %} + ,{%q= l.Name %}:{%q= l.Value %} {% endfor %} }, "annotations": { diff --git a/app/vmalert/notifier/alertmanager_request.qtpl.go b/app/vmalert/notifier/alertmanager_request.qtpl.go index 04e3e55ab..8a6ce45e2 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl.go +++ b/app/vmalert/notifier/alertmanager_request.qtpl.go @@ -7,124 +7,129 @@ package notifier //line app/vmalert/notifier/alertmanager_request.qtpl:1 import ( "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) -//line app/vmalert/notifier/alertmanager_request.qtpl:6 +//line app/vmalert/notifier/alertmanager_request.qtpl:8 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmalert/notifier/alertmanager_request.qtpl:6 +//line app/vmalert/notifier/alertmanager_request.qtpl:8 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmalert/notifier/alertmanager_request.qtpl:6 -func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) { -//line app/vmalert/notifier/alertmanager_request.qtpl:6 +//line app/vmalert/notifier/alertmanager_request.qtpl:8 +func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) { +//line app/vmalert/notifier/alertmanager_request.qtpl:8 qw422016.N().S(`[`) -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line app/vmalert/notifier/alertmanager_request.qtpl:10 for i, alert := range alerts { -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line app/vmalert/notifier/alertmanager_request.qtpl:10 qw422016.N().S(`{"startsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:10 +//line app/vmalert/notifier/alertmanager_request.qtpl:12 qw422016.N().Q(alert.Start.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:10 +//line app/vmalert/notifier/alertmanager_request.qtpl:12 qw422016.N().S(`,"generatorURL":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:11 +//line app/vmalert/notifier/alertmanager_request.qtpl:13 qw422016.N().Q(generatorURL(alert)) -//line app/vmalert/notifier/alertmanager_request.qtpl:11 +//line app/vmalert/notifier/alertmanager_request.qtpl:13 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:12 +//line app/vmalert/notifier/alertmanager_request.qtpl:14 if !alert.End.IsZero() { -//line app/vmalert/notifier/alertmanager_request.qtpl:12 +//line app/vmalert/notifier/alertmanager_request.qtpl:14 qw422016.N().S(`"endsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 +//line app/vmalert/notifier/alertmanager_request.qtpl:15 qw422016.N().Q(alert.End.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 +//line app/vmalert/notifier/alertmanager_request.qtpl:15 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:14 - } -//line app/vmalert/notifier/alertmanager_request.qtpl:14 - qw422016.N().S(`"labels": {"alertname":`) //line app/vmalert/notifier/alertmanager_request.qtpl:16 - qw422016.N().Q(alert.Name) -//line app/vmalert/notifier/alertmanager_request.qtpl:17 - for k, v := range alert.Labels { -//line app/vmalert/notifier/alertmanager_request.qtpl:17 - qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:18 - qw422016.N().Q(k) -//line app/vmalert/notifier/alertmanager_request.qtpl:18 - qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:18 - qw422016.N().Q(v) -//line app/vmalert/notifier/alertmanager_request.qtpl:19 } +//line app/vmalert/notifier/alertmanager_request.qtpl:16 + qw422016.N().S(`"labels": {"alertname":`) +//line app/vmalert/notifier/alertmanager_request.qtpl:18 + qw422016.N().Q(alert.Name) //line app/vmalert/notifier/alertmanager_request.qtpl:19 - qw422016.N().S(`},"annotations": {`) + lbls := alert.toPromLabels(relabelCfg) + +//line app/vmalert/notifier/alertmanager_request.qtpl:20 + for _, l := range lbls { +//line app/vmalert/notifier/alertmanager_request.qtpl:20 + qw422016.N().S(`,`) +//line app/vmalert/notifier/alertmanager_request.qtpl:21 + qw422016.N().Q(l.Name) +//line app/vmalert/notifier/alertmanager_request.qtpl:21 + qw422016.N().S(`:`) +//line app/vmalert/notifier/alertmanager_request.qtpl:21 + qw422016.N().Q(l.Value) //line app/vmalert/notifier/alertmanager_request.qtpl:22 + } +//line app/vmalert/notifier/alertmanager_request.qtpl:22 + qw422016.N().S(`},"annotations": {`) +//line app/vmalert/notifier/alertmanager_request.qtpl:25 c := len(alert.Annotations) -//line app/vmalert/notifier/alertmanager_request.qtpl:23 +//line app/vmalert/notifier/alertmanager_request.qtpl:26 for k, v := range alert.Annotations { -//line app/vmalert/notifier/alertmanager_request.qtpl:24 +//line app/vmalert/notifier/alertmanager_request.qtpl:27 c = c - 1 -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 qw422016.N().Q(k) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 qw422016.N().Q(v) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 if c > 0 { -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:28 } -//line app/vmalert/notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 } -//line app/vmalert/notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().S(`}}`) -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 if i != len(alerts)-1 { -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 } -//line app/vmalert/notifier/alertmanager_request.qtpl:30 +//line app/vmalert/notifier/alertmanager_request.qtpl:33 } -//line app/vmalert/notifier/alertmanager_request.qtpl:30 +//line app/vmalert/notifier/alertmanager_request.qtpl:33 qw422016.N().S(`]`) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 } -//line app/vmalert/notifier/alertmanager_request.qtpl:32 -func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) { -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 +func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) { +//line app/vmalert/notifier/alertmanager_request.qtpl:35 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 - streamamRequest(qw422016, alerts, generatorURL) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 + streamamRequest(qw422016, alerts, generatorURL, relabelCfg) +//line app/vmalert/notifier/alertmanager_request.qtpl:35 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 } -//line app/vmalert/notifier/alertmanager_request.qtpl:32 -func amRequest(alerts []Alert, generatorURL func(Alert) string) string { -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 +func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string { +//line app/vmalert/notifier/alertmanager_request.qtpl:35 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/notifier/alertmanager_request.qtpl:32 - writeamRequest(qb422016, alerts, generatorURL) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 + writeamRequest(qb422016, alerts, generatorURL, relabelCfg) +//line app/vmalert/notifier/alertmanager_request.qtpl:35 qs422016 := string(qb422016.B) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 return qs422016 -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:35 } diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go index 646d9e418..1a6fe8b05 100644 --- a/app/vmalert/notifier/alertmanager_test.go +++ b/app/vmalert/notifier/alertmanager_test.go @@ -14,7 +14,7 @@ import ( func TestAlertManager_Addr(t *testing.T) { const addr = "http://localhost" - am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0) + am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0) if err != nil { t.Errorf("unexpected error: %s", err) } @@ -89,7 +89,7 @@ func TestAlertManager_Send(t *testing.T) { } am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string { return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10) - }, aCfg, 0) + }, aCfg, nil, 0) if err != nil { t.Errorf("unexpected error: %s", err) } diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index 07258c40e..c6e75aef8 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -34,9 +34,10 @@ type Config struct { // HTTPClientConfig contains HTTP configuration for Notifier clients HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` - // RelabelConfigs contains list of relabeling rules + // RelabelConfigs contains list of relabeling rules for entities discovered via SD RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"` - + // AlertRelabelConfigs contains list of relabeling rules alert labels + AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"` // The timeout used when sending alerts. Timeout promutils.Duration `yaml:"timeout,omitempty"` @@ -52,6 +53,8 @@ type Config struct { // stores already parsed RelabelConfigs object parsedRelabelConfigs *promrelabel.ParsedConfigs + // stores already parsed AlertRelabelConfigs object + parsedAlertRelabelConfigs *promrelabel.ParsedConfigs } // StaticConfig contains list of static targets in the following form: @@ -78,6 +81,11 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { return fmt.Errorf("failed to parse relabeling config: %w", err) } cfg.parsedRelabelConfigs = rCfg + arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false) + if err != nil { + return fmt.Errorf("failed to parse alert relabeling config: %w", err) + } + cfg.parsedAlertRelabelConfigs = arCfg b, err := yaml.Marshal(cfg) if err != nil { diff --git a/app/vmalert/notifier/config_watcher.go b/app/vmalert/notifier/config_watcher.go index a49990151..0fe8c53f5 100644 --- a/app/vmalert/notifier/config_watcher.go +++ b/app/vmalert/notifier/config_watcher.go @@ -141,7 +141,7 @@ func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) } duplicates[u] = struct{}{} - am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration()) + am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration()) if err != nil { errors = append(errors, err) continue @@ -165,7 +165,7 @@ func (cw *configWatcher) start() error { if err != nil { return fmt.Errorf("failed to parse labels for target %q: %s", target, err) } - notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration()) + notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration()) if err != nil { return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err) } diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index 8efa9ecb1..b00c1bcd6 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -138,7 +138,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) { } addr = strings.TrimSuffix(addr, "/") - am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute) + am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute) if err != nil { return nil, err } diff --git a/app/vmalert/notifier/testdata/consul.good.yaml b/app/vmalert/notifier/testdata/consul.good.yaml index 7aae6bc9a..c79e1668f 100644 --- a/app/vmalert/notifier/testdata/consul.good.yaml +++ b/app/vmalert/notifier/testdata/consul.good.yaml @@ -10,4 +10,7 @@ relabel_configs: - source_labels: [__meta_consul_tags] regex: .*,__scheme__=([^,]+),.* replacement: '${1}' - target_label: __scheme__ \ No newline at end of file + target_label: __scheme__ +alert_relabel_configs: + - target_label: "foo" + replacement: "aaa" \ No newline at end of file diff --git a/app/vmalert/notifier/testdata/static.good.yaml b/app/vmalert/notifier/testdata/static.good.yaml index a9027a714..99cb98eb2 100644 --- a/app/vmalert/notifier/testdata/static.good.yaml +++ b/app/vmalert/notifier/testdata/static.good.yaml @@ -2,3 +2,6 @@ static_configs: - targets: - localhost:9093 - localhost:9095 +alert_relabel_configs: + - target_label: "foo" + replacement: "aaa" \ No newline at end of file