mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-03-11 15:34:56 +00:00
vmalert: do not send message to alertmanager when alert has no label … (#6823)
…pair
`alert_relabel_configs` in [notifier
config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file)
can drop alert labels when used to filter different tenant alert message
to different notifier.
alertmanager would report error like `msg="Failed to validate alerts"
err="at least one label pair required"` in this case, but the rest of
the alerts inside one request would still be valid in alertmanager, so
it's not severe.
(cherry picked from commit ae4d376e41)
This commit is contained in:
Hui Wang
hagen1778
parent
d5cfce693a
commit
c7fc0d0d2f
7 changed files with 135 additions and 76 deletions
|
|
@ -187,7 +187,7 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
|
|||
return nil
|
||||
}
|
||||
|
||||
func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
var labels []prompbmarshal.Label
|
||||
for k, v := range a.Labels {
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ func TestAlert_toPromLabels(t *testing.T) {
|
|||
fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
|
||||
t.Helper()
|
||||
a := Alert{Labels: labels}
|
||||
got := a.toPromLabels(relabel)
|
||||
got := a.applyRelabelingIfNeeded(relabel)
|
||||
if !reflect.DeepEqual(got, exp) {
|
||||
t.Fatalf("expected to have: \n%v;\ngot:\n%v",
|
||||
exp, got)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
|
||||
[
|
||||
{% for i, alert := range alerts %}
|
||||
{% code lbls := alert.applyRelabelingIfNeeded(relabelCfg) %}
|
||||
{% if len(lbls) == 0 %} {% continue %} {% endif %}
|
||||
{
|
||||
"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
|
||||
"generatorURL": {%q= generatorURL(alert) %},
|
||||
|
|
@ -15,7 +17,6 @@
|
|||
"endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
|
||||
{% endif %}
|
||||
"labels": {
|
||||
{% code lbls := alert.toPromLabels(relabelCfg) %}
|
||||
{% code ll := len(lbls) %}
|
||||
{% for idx, l := range lbls %}
|
||||
{%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %}
|
||||
|
|
|
|||
|
|
@ -30,111 +30,117 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
|
|||
qw422016.N().S(`[`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
for i, alert := range alerts {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
qw422016.N().S(`{"startsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().S(`,"generatorURL":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().Q(generatorURL(alert))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
if !alert.End.IsZero() {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().S(`"endsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
qw422016.N().S(`"labels": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
lbls := alert.toPromLabels(relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:11
|
||||
lbls := alert.applyRelabelingIfNeeded(relabelCfg)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:19
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
if len(lbls) == 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
continue
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().S(`{"startsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().S(`,"generatorURL":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().Q(generatorURL(alert))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
if !alert.End.IsZero() {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
qw422016.N().S(`"endsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
qw422016.N().S(`"labels": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
ll := len(lbls)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
for idx, l := range lbls {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().Q(l.Name)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().Q(l.Value)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
if idx != ll-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
qw422016.N().S(`},"annotations": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:25
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
c := len(alert.Annotations)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
for k, v := range alert.Annotations {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:27
|
||||
for k, v := range alert.Annotations {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
c = c - 1
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().Q(k)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().Q(v)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
if c > 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
qw422016.N().S(`}}`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
if i != len(alerts)-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
return qs422016
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
)
|
||||
|
||||
func TestAlertManager_Addr(t *testing.T) {
|
||||
|
|
@ -76,12 +77,33 @@ func TestAlertManager_Send(t *testing.T) {
|
|||
if a[0].EndAt.IsZero() {
|
||||
t.Fatalf("expected non-zero end time")
|
||||
}
|
||||
if len(a[0].Labels) != 1 {
|
||||
t.Fatalf("expected 1 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if len(a[0].Annotations) != 2 {
|
||||
t.Fatalf("expected 2 annotations got %d", len(a[0].Annotations))
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
case 3:
|
||||
if r.Header.Get(headerKey) != headerValue {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
var a []struct {
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
|
||||
t.Fatalf("can not unmarshal data into alert %s", err)
|
||||
}
|
||||
if len(a) != 1 {
|
||||
t.Fatalf("expected 1 alert in array got %d", len(a))
|
||||
}
|
||||
if len(a[0].Labels) != 3 {
|
||||
t.Fatalf("expected 3 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if a[0].Labels["env"] != "prod" {
|
||||
t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, "bar", r.Header.Get(headerKey))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
@ -95,31 +117,58 @@ func TestAlertManager_Send(t *testing.T) {
|
|||
},
|
||||
Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
|
||||
}
|
||||
parsedConfigs, err := promrelabel.ParseRelabelConfigsData([]byte(`
|
||||
- action: drop
|
||||
if: '{tenant="0"}'
|
||||
regex: ".*"
|
||||
- target_label: "env"
|
||||
replacement: "prod"
|
||||
if: '{tenant="1"}'
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when parse relabeling config: %s", err)
|
||||
}
|
||||
am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
|
||||
return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
|
||||
}, aCfg, nil, 0)
|
||||
}, aCfg, parsedConfigs, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
t.Fatalf("expected connection error got nil")
|
||||
}
|
||||
if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
t.Fatalf("expected wrong http code error got nil")
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{
|
||||
GroupID: 0,
|
||||
Name: "alert0",
|
||||
Start: time.Now().UTC(),
|
||||
End: time.Now().UTC(),
|
||||
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
|
||||
Labels: map[string]string{"alertname": "alert0"},
|
||||
Annotations: map[string]string{"a": "b", "c": "d"},
|
||||
}}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
if c != 2 {
|
||||
t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{
|
||||
// drop tenant0 alert message during relabeling
|
||||
{
|
||||
Name: "alert1",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "0"},
|
||||
},
|
||||
{
|
||||
Name: "alert2",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "1"},
|
||||
},
|
||||
}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
if c != 3 {
|
||||
t.Fatalf("expected 3 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
|
|||
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric `vm_object_references{type="indexdb"}`. Previously, it was overcounted.
|
||||
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest stale NaN samples. Previously it could be dropped if series didn't exist at storage node. See this issue [https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069] for details.
|
||||
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): do not send notifications without labels to Alertmanager. Such notifications are rejected by Alertmanager anyway. Before, vmalert could send alert notifications even if no label-value pairs left after applying `alert_relabel_configs` from [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file).
|
||||
|
||||
## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0)
|
||||
|
||||
Released at 2024-08-28
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ headers:
|
|||
# Optional list of HTTP headers in form `header-name: value`
|
||||
# applied for all alert notifications sent to notifiers
|
||||
# generated by rules of this group.
|
||||
# It has higher priority over headers defined in notifier config.
|
||||
# For example:
|
||||
# notifier_headers:
|
||||
# - "TenantID: foo"
|
||||
|
|
|
|||
Loading…
Reference in a new issue