diff --git a/app/vmalert/config/testdata/rules2-good.rules b/app/vmalert/config/testdata/rules2-good.rules index c307375ac..1f431b32f 100644 --- a/app/vmalert/config/testdata/rules2-good.rules +++ b/app/vmalert/config/testdata/rules2-good.rules @@ -25,6 +25,7 @@ groups: dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}' annotations: description: Job {{ $labels.job }} is up! + external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }} summary: All instances up {{ range query "up" }} {{ . | label "instance" }} {{ end }} diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 16cd5e39b..0d23d56b6 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -167,7 +167,20 @@ func newManager(ctx context.Context) (*manager, error) { if err != nil { return nil, fmt.Errorf("failed to init datasource: %w", err) } - nts, err := notifier.Init(alertURLGeneratorFn) + + labels := make(map[string]string, 0) + for _, s := range *externalLabels { + if len(s) == 0 { + continue + } + n := strings.IndexByte(s, '=') + if n < 0 { + return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s) + } + labels[s[:n]] = s[n+1:] + } + + nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL) if err != nil { return nil, fmt.Errorf("failed to init notifier: %w", err) } @@ -175,7 +188,7 @@ func newManager(ctx context.Context) (*manager, error) { groups: make(map[uint64]*Group), querierBuilder: q, notifiers: nts, - labels: map[string]string{}, + labels: labels, } rw, err := remotewrite.Init(ctx) if err != nil { @@ -189,16 +202,6 @@ func newManager(ctx context.Context) (*manager, error) { } manager.rr = rr - for _, s := range *externalLabels { - if len(s) == 0 { - continue - } - n := strings.IndexByte(s, '=') - if n < 0 { - return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s) - } - manager.labels[s[:n]] = s[n+1:] - } return manager, nil } diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index c2889b94f..4aa661d24 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -70,7 +70,13 @@ type AlertTplData struct { Expr string } -const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}` +var tplHeaders = []string{ + "{{ $value := .Value }}", + "{{ $labels := .Labels }}", + "{{ $expr := .Expr }}", + "{{ $externalLabels := .ExternalLabels }}", + "{{ $externalURL := .ExternalURL }}", +} // ExecTemplate executes the Alert template for given // map of annotations. @@ -100,13 +106,15 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs var buf bytes.Buffer eg := new(utils.ErrGroup) r := make(map[string]string, len(annotations)) + tData := tplData{data, externalLabels, externalURL} + header := strings.Join(tplHeaders, "") for key, text := range annotations { buf.Reset() builder.Reset() - builder.Grow(len(tplHeader) + len(text)) - builder.WriteString(tplHeader) + builder.Grow(len(header) + len(text)) + builder.WriteString(header) builder.WriteString(text) - if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil { + if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil { r[key] = text eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err)) continue @@ -116,7 +124,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs return r, eg.Err() } -func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error { +type tplData struct { + AlertTplData + ExternalLabels map[string]string + ExternalURL string +} + +func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error { t := template.New("").Funcs(funcs).Option("missingkey=zero") tpl, err := t.Parse(text) if err != nil { diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 769116979..f8e0c77ee 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -1,12 +1,24 @@ package notifier import ( + "fmt" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" ) func TestAlert_ExecTemplate(t *testing.T) { + extLabels := make(map[string]string, 0) + const ( + extCluster = "prod" + extDC = "east" + extURL = "https://foo.bar" + ) + extLabels["cluster"] = extCluster + extLabels["dc"] = extDC + _, err := Init(nil, extLabels, extURL) + checkErr(t, err) + testCases := []struct { name string alert *Alert @@ -74,6 +86,26 @@ func TestAlert_ExecTemplate(t *testing.T) { "desc": "bar 1;garply 2;", }, }, + { + name: "external", + alert: &Alert{ + Value: 1e4, + Labels: map[string]string{ + "job": "staging", + "instance": "localhost", + }, + }, + annotations: map[string]string{ + "url": "{{ $externalURL }}", + "summary": "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}", + "description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})", + }, + expTpl: map[string]string{ + "url": extURL, + "summary": fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC), + "description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster), + }, + }, } qFn := func(q string) ([]datasource.Metric, error) { diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index d550de2fc..784751d02 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -46,13 +46,29 @@ func Reload() error { var staticNotifiersFn func() []Notifier +var ( + // externalLabels is a global variable for holding external labels configured via flags + // It is supposed to be inited via Init function only. + externalLabels map[string]string + // externalURL is a global variable for holding external URL value configured via flag + // It is supposed to be inited via Init function only. + externalURL string +) + // Init returns a function for retrieving actual list of Notifier objects. // Init works in two mods: // * configuration via flags (for backward compatibility). Is always static // and don't support live reloads. // * configuration via file. Supports live reloads and service discovery. // Init returns an error if both mods are used. -func Init(gen AlertURLGenerator) (func() []Notifier, error) { +func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) { + if externalLabels != nil || externalURL != "" { + return nil, fmt.Errorf("BUG: notifier.Init was called multiple times") + } + + externalURL = extURL + externalLabels = extLabels + if *configPath == "" && len(*addrs) == 0 { return nil, nil }