diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index 2dd13ec92..b1dfc697c 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -187,7 +187,7 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl. return nil } -func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label { +func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label { var labels []prompbmarshal.Label for k, v := range a.Labels { labels = append(labels, prompbmarshal.Label{ diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 30c2db57b..9338b3fd2 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -187,7 +187,7 @@ func TestAlert_toPromLabels(t *testing.T) { fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) { t.Helper() a := Alert{Labels: labels} - got := a.toPromLabels(relabel) + got := a.applyRelabelingIfNeeded(relabel) if !reflect.DeepEqual(got, exp) { t.Fatalf("expected to have: \n%v;\ngot:\n%v", exp, got) diff --git a/app/vmalert/notifier/alertmanager_request.qtpl b/app/vmalert/notifier/alertmanager_request.qtpl index 74e974b1e..4972eee24 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl +++ b/app/vmalert/notifier/alertmanager_request.qtpl @@ -8,6 +8,8 @@ {% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %} [ {% for i, alert := range alerts %} +{% code lbls := alert.applyRelabelingIfNeeded(relabelCfg) %} +{% if len(lbls) == 0 %} {% continue %} {% endif %} { "startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %}, "generatorURL": {%q= generatorURL(alert) %}, @@ -15,7 +17,6 @@ "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %}, {% endif %} "labels": { - {% code lbls := alert.toPromLabels(relabelCfg) %} {% code ll := len(lbls) %} {% for idx, l := range lbls %} {%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %} diff --git a/app/vmalert/notifier/alertmanager_request.qtpl.go b/app/vmalert/notifier/alertmanager_request.qtpl.go index a3ce41531..ae29f26f2 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl.go +++ b/app/vmalert/notifier/alertmanager_request.qtpl.go @@ -30,111 +30,117 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun qw422016.N().S(`[`) //line app/vmalert/notifier/alertmanager_request.qtpl:10 for i, alert := range alerts { -//line app/vmalert/notifier/alertmanager_request.qtpl:10 - qw422016.N().S(`{"startsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:12 - qw422016.N().Q(alert.Start.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:12 - qw422016.N().S(`,"generatorURL":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 - qw422016.N().Q(generatorURL(alert)) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 - qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:14 - if !alert.End.IsZero() { -//line app/vmalert/notifier/alertmanager_request.qtpl:14 - qw422016.N().S(`"endsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:15 - qw422016.N().Q(alert.End.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:15 - qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:16 - } -//line app/vmalert/notifier/alertmanager_request.qtpl:16 - qw422016.N().S(`"labels": {`) -//line app/vmalert/notifier/alertmanager_request.qtpl:18 - lbls := alert.toPromLabels(relabelCfg) +//line app/vmalert/notifier/alertmanager_request.qtpl:11 + lbls := alert.applyRelabelingIfNeeded(relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:19 +//line app/vmalert/notifier/alertmanager_request.qtpl:12 + if len(lbls) == 0 { +//line app/vmalert/notifier/alertmanager_request.qtpl:12 + continue +//line app/vmalert/notifier/alertmanager_request.qtpl:12 + } +//line app/vmalert/notifier/alertmanager_request.qtpl:12 + qw422016.N().S(`{"startsAt":`) +//line app/vmalert/notifier/alertmanager_request.qtpl:14 + qw422016.N().Q(alert.Start.Format(time.RFC3339Nano)) +//line app/vmalert/notifier/alertmanager_request.qtpl:14 + qw422016.N().S(`,"generatorURL":`) +//line app/vmalert/notifier/alertmanager_request.qtpl:15 + qw422016.N().Q(generatorURL(alert)) +//line app/vmalert/notifier/alertmanager_request.qtpl:15 + qw422016.N().S(`,`) +//line app/vmalert/notifier/alertmanager_request.qtpl:16 + if !alert.End.IsZero() { +//line app/vmalert/notifier/alertmanager_request.qtpl:16 + qw422016.N().S(`"endsAt":`) +//line app/vmalert/notifier/alertmanager_request.qtpl:17 + qw422016.N().Q(alert.End.Format(time.RFC3339Nano)) +//line app/vmalert/notifier/alertmanager_request.qtpl:17 + qw422016.N().S(`,`) +//line app/vmalert/notifier/alertmanager_request.qtpl:18 + } +//line app/vmalert/notifier/alertmanager_request.qtpl:18 + qw422016.N().S(`"labels": {`) +//line app/vmalert/notifier/alertmanager_request.qtpl:20 ll := len(lbls) -//line app/vmalert/notifier/alertmanager_request.qtpl:20 +//line app/vmalert/notifier/alertmanager_request.qtpl:21 for idx, l := range lbls { -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 qw422016.N().Q(l.Name) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 qw422016.N().Q(l.Value) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 if idx != ll-1 { -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 } -//line app/vmalert/notifier/alertmanager_request.qtpl:22 +//line app/vmalert/notifier/alertmanager_request.qtpl:23 } -//line app/vmalert/notifier/alertmanager_request.qtpl:22 +//line app/vmalert/notifier/alertmanager_request.qtpl:23 qw422016.N().S(`},"annotations": {`) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:26 c := len(alert.Annotations) -//line app/vmalert/notifier/alertmanager_request.qtpl:26 - for k, v := range alert.Annotations { //line app/vmalert/notifier/alertmanager_request.qtpl:27 + for k, v := range alert.Annotations { +//line app/vmalert/notifier/alertmanager_request.qtpl:28 c = c - 1 -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().Q(k) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().Q(v) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 if c > 0 { -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 } -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line app/vmalert/notifier/alertmanager_request.qtpl:30 } -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line app/vmalert/notifier/alertmanager_request.qtpl:30 qw422016.N().S(`}}`) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:33 if i != len(alerts)-1 { -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:33 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line app/vmalert/notifier/alertmanager_request.qtpl:33 } -//line app/vmalert/notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:34 } -//line app/vmalert/notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:34 qw422016.N().S(`]`) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 } -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) { -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 streamamRequest(qw422016, alerts, generatorURL, relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 } -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string { -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 writeamRequest(qb422016, alerts, generatorURL, relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 qs422016 := string(qb422016.B) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 return qs422016 -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line app/vmalert/notifier/alertmanager_request.qtpl:36 } diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go index e792090c4..937ee04cd 100644 --- a/app/vmalert/notifier/alertmanager_test.go +++ b/app/vmalert/notifier/alertmanager_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) func TestAlertManager_Addr(t *testing.T) { @@ -76,12 +77,33 @@ func TestAlertManager_Send(t *testing.T) { if a[0].EndAt.IsZero() { t.Fatalf("expected non-zero end time") } + if len(a[0].Labels) != 1 { + t.Fatalf("expected 1 labels got %d", len(a[0].Labels)) + } + if len(a[0].Annotations) != 2 { + t.Fatalf("expected 2 annotations got %d", len(a[0].Annotations)) + } if r.Header.Get(headerKey) != "bar" { t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey)) } case 3: - if r.Header.Get(headerKey) != headerValue { - t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey)) + var a []struct { + Labels map[string]string `json:"labels"` + } + if err := json.NewDecoder(r.Body).Decode(&a); err != nil { + t.Fatalf("can not unmarshal data into alert %s", err) + } + if len(a) != 1 { + t.Fatalf("expected 1 alert in array got %d", len(a)) + } + if len(a[0].Labels) != 3 { + t.Fatalf("expected 3 labels got %d", len(a[0].Labels)) + } + if a[0].Labels["env"] != "prod" { + t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"]) + } + if r.Header.Get(headerKey) != "bar" { + t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, "bar", r.Header.Get(headerKey)) } } }) @@ -95,31 +117,58 @@ func TestAlertManager_Send(t *testing.T) { }, Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)}, } + parsedConfigs, err := promrelabel.ParseRelabelConfigsData([]byte(` +- action: drop + if: '{tenant="0"}' + regex: ".*" +- target_label: "env" + replacement: "prod" + if: '{tenant="1"}' +`)) + if err != nil { + t.Fatalf("unexpected error when parse relabeling config: %s", err) + } am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string { return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10) - }, aCfg, nil, 0) + }, aCfg, parsedConfigs, 0) if err != nil { t.Fatalf("unexpected error: %s", err) } - if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil { + + if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil { t.Fatalf("expected connection error got nil") } - if err := am.Send(context.Background(), []Alert{}, nil); err == nil { + + if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil { t.Fatalf("expected wrong http code error got nil") } + if err := am.Send(context.Background(), []Alert{{ GroupID: 0, Name: "alert0", Start: time.Now().UTC(), End: time.Now().UTC(), - Annotations: map[string]string{"a": "b", "c": "d", "e": "f"}, + Labels: map[string]string{"alertname": "alert0"}, + Annotations: map[string]string{"a": "b", "c": "d"}, }}, map[string]string{headerKey: "bar"}); err != nil { t.Fatalf("unexpected error %s", err) } - if c != 2 { - t.Fatalf("expected 2 calls(count from zero) to server got %d", c) - } - if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil { + + if err := am.Send(context.Background(), []Alert{ + // drop tenant0 alert message during relabeling + { + Name: "alert1", + Labels: map[string]string{"rule": "test", "tenant": "0"}, + }, + { + Name: "alert2", + Labels: map[string]string{"rule": "test", "tenant": "1"}, + }, + }, map[string]string{headerKey: "bar"}); err != nil { t.Fatalf("unexpected error %s", err) } + + if c != 3 { + t.Fatalf("expected 3 calls(count from zero) to server got %d", c) + } } diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index bc918f324..7c6768ac9 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -35,6 +35,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric `vm_object_references{type="indexdb"}`. Previously, it was overcounted. * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest stale NaN samples. Previously it could be dropped if series didn't exist at storage node. See this issue [https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069] for details. +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): do not send notifications without labels to Alertmanager. Such notifications are rejected by Alertmanager anyway. Before, vmalert could send alert notifications even if no label-value pairs left after applying `alert_relabel_configs` from [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file). + ## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0) Released at 2024-08-28 diff --git a/docs/vmalert.md b/docs/vmalert.md index fa8012c08..2c00e8db4 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -198,6 +198,7 @@ headers: # Optional list of HTTP headers in form `header-name: value` # applied for all alert notifications sent to notifiers # generated by rules of this group. +# It has higher priority over headers defined in notifier config. # For example: # notifier_headers: # - "TenantID: foo"