diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 9cfb90283..8b62812b7 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "time" @@ -57,9 +58,11 @@ absolute path to all .yaml files in root.`) remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+ " For example, if lookback=1h then range from now() to now()-1h will be scanned.") - evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules") - notifierURL = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093") - externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier") + evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules") + notifierURL = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093") + externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier") + externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. +eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`) ) func main() { @@ -76,13 +79,15 @@ func main() { logger.Fatalf("can not get external url: %s ", err) } notifier.InitTemplateFunc(eu) + aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates) + if err != nil { + logger.Fatalf("URL generator error: %s", err) + } manager := &manager{ - groups: make(map[uint64]*Group), - storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}), - notifier: notifier.NewAlertManager(*notifierURL, func(group, alert string) string { - return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, alert) - }, &http.Client{}), + groups: make(map[uint64]*Group), + storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}), + notifier: notifier.NewAlertManager(*notifierURL, aug, &http.Client{}), } if *remoteWriteURL != "" { c, err := remotewrite.NewClient(ctx, remotewrite.Config{ @@ -166,6 +171,31 @@ func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port)) } +func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) { + if externalAlertSource == "" { + return func(alert notifier.Alert) string { + return fmt.Sprintf("%s/api/v1/%s/%s/status", externalURL, strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)) + }, nil + } + if validateTemplate { + if err := notifier.ValidateTemplates(map[string]string{ + "tpl": externalAlertSource, + }); err != nil { + return nil, fmt.Errorf("error validating source template %s:%w", externalAlertSource, err) + } + } + m := map[string]string{ + "tpl": externalAlertSource, + } + return func(alert notifier.Alert) string { + templated, err := alert.ExecTemplate(m) + if err != nil { + logger.Errorf("can not exec source template %s", err) + } + return fmt.Sprintf("%s/%s", externalURL, templated["tpl"]) + }, nil +} + func checkFlags() { if *notifierURL == "" { flag.PrintDefaults() diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go new file mode 100644 index 000000000..853cdcf3f --- /dev/null +++ b/app/vmalert/main_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "net/url" + "os" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" +) + +func TestGetExternalURL(t *testing.T) { + expURL := "https://vicotriametrics.com/path" + u, err := getExternalURL(expURL, "", false) + if err != nil { + t.Errorf("unexpected error %s", err) + } + if u.String() != expURL { + t.Errorf("unexpected url want %s, got %s", expURL, u.String()) + } + h, _ := os.Hostname() + expURL = fmt.Sprintf("https://%s:4242", h) + u, err = getExternalURL("", "0.0.0.0:4242", true) + if err != nil { + t.Errorf("unexpected error %s", err) + } + if u.String() != expURL { + t.Errorf("unexpected url want %s, got %s", expURL, u.String()) + } +} + +func TestGetAlertURLGenerator(t *testing.T) { + testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4} + u, _ := url.Parse("https://victoriametrics.com/path") + fn, err := getAlertURLGenerator(u, "", false) + if err != nil { + t.Errorf("unexpected error %s", err) + } + if exp := "https://victoriametrics.com/path/api/v1/42/2/status"; exp != fn(testAlert) { + t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert)) + } + _, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true) + if err == nil { + t.Errorf("exptected tempalte validation error got nil") + } + fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true) + if err != nil { + t.Errorf("unexpected error %s", err) + } + if exp := "https://victoriametrics.com/path/foo?query=4"; exp != fn(testAlert) { + t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert)) + } +} diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 3cd032d6a..f663be1c0 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -1,13 +1,10 @@ package notifier import ( - "net/url" "testing" ) func TestAlert_ExecTemplate(t *testing.T) { - u, _ := url.Parse("https://victoriametrics.com/path") - InitTemplateFunc(u) testCases := []struct { name string alert *Alert diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index 9a970454c..6e06949ce 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -46,7 +46,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error { } // AlertURLGenerator returns URL to single alert by given name -type AlertURLGenerator func(group, alert string) string +type AlertURLGenerator func(Alert) string const alertManagerPath = "/api/v2/alerts" diff --git a/app/vmalert/notifier/alertmanager_request.qtpl b/app/vmalert/notifier/alertmanager_request.qtpl index 249523948..9592b5158 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl +++ b/app/vmalert/notifier/alertmanager_request.qtpl @@ -1,15 +1,14 @@ {% import ( - "strconv" "time" ) %} {% stripspace %} -{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %} +{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %} [ {% for i, alert := range alerts %} { - "startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %}, - "generatorURL": {%q= generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)) %}, + "startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %}, + "generatorURL": {%q= generatorURL(alert) %}, {% if !alert.End.IsZero() %} "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %}, {% endif %} diff --git a/app/vmalert/notifier/alertmanager_request.qtpl.go b/app/vmalert/notifier/alertmanager_request.qtpl.go index 188fc3f37..04e3e55ab 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl.go +++ b/app/vmalert/notifier/alertmanager_request.qtpl.go @@ -1,131 +1,130 @@ // Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT. // See https://github.com/valyala/quicktemplate for details. -//line notifier/alertmanager_request.qtpl:1 +//line app/vmalert/notifier/alertmanager_request.qtpl:1 package notifier -//line notifier/alertmanager_request.qtpl:1 +//line app/vmalert/notifier/alertmanager_request.qtpl:1 import ( - "strconv" "time" ) -//line notifier/alertmanager_request.qtpl:7 +//line app/vmalert/notifier/alertmanager_request.qtpl:6 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line notifier/alertmanager_request.qtpl:7 +//line app/vmalert/notifier/alertmanager_request.qtpl:6 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line notifier/alertmanager_request.qtpl:7 -func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) { -//line notifier/alertmanager_request.qtpl:7 +//line app/vmalert/notifier/alertmanager_request.qtpl:6 +func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) { +//line app/vmalert/notifier/alertmanager_request.qtpl:6 qw422016.N().S(`[`) -//line notifier/alertmanager_request.qtpl:9 +//line app/vmalert/notifier/alertmanager_request.qtpl:8 for i, alert := range alerts { -//line notifier/alertmanager_request.qtpl:9 +//line app/vmalert/notifier/alertmanager_request.qtpl:8 qw422016.N().S(`{"startsAt":`) -//line notifier/alertmanager_request.qtpl:11 +//line app/vmalert/notifier/alertmanager_request.qtpl:10 qw422016.N().Q(alert.Start.Format(time.RFC3339Nano)) -//line notifier/alertmanager_request.qtpl:11 +//line app/vmalert/notifier/alertmanager_request.qtpl:10 qw422016.N().S(`,"generatorURL":`) -//line notifier/alertmanager_request.qtpl:12 - qw422016.N().Q(generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10))) -//line notifier/alertmanager_request.qtpl:12 +//line app/vmalert/notifier/alertmanager_request.qtpl:11 + qw422016.N().Q(generatorURL(alert)) +//line app/vmalert/notifier/alertmanager_request.qtpl:11 qw422016.N().S(`,`) -//line notifier/alertmanager_request.qtpl:13 +//line app/vmalert/notifier/alertmanager_request.qtpl:12 if !alert.End.IsZero() { -//line notifier/alertmanager_request.qtpl:13 +//line app/vmalert/notifier/alertmanager_request.qtpl:12 qw422016.N().S(`"endsAt":`) -//line notifier/alertmanager_request.qtpl:14 +//line app/vmalert/notifier/alertmanager_request.qtpl:13 qw422016.N().Q(alert.End.Format(time.RFC3339Nano)) -//line notifier/alertmanager_request.qtpl:14 +//line app/vmalert/notifier/alertmanager_request.qtpl:13 qw422016.N().S(`,`) -//line notifier/alertmanager_request.qtpl:15 +//line app/vmalert/notifier/alertmanager_request.qtpl:14 } -//line notifier/alertmanager_request.qtpl:15 +//line app/vmalert/notifier/alertmanager_request.qtpl:14 qw422016.N().S(`"labels": {"alertname":`) -//line notifier/alertmanager_request.qtpl:17 +//line app/vmalert/notifier/alertmanager_request.qtpl:16 qw422016.N().Q(alert.Name) -//line notifier/alertmanager_request.qtpl:18 +//line app/vmalert/notifier/alertmanager_request.qtpl:17 for k, v := range alert.Labels { -//line notifier/alertmanager_request.qtpl:18 +//line app/vmalert/notifier/alertmanager_request.qtpl:17 qw422016.N().S(`,`) -//line notifier/alertmanager_request.qtpl:19 +//line app/vmalert/notifier/alertmanager_request.qtpl:18 qw422016.N().Q(k) -//line notifier/alertmanager_request.qtpl:19 +//line app/vmalert/notifier/alertmanager_request.qtpl:18 qw422016.N().S(`:`) -//line notifier/alertmanager_request.qtpl:19 +//line app/vmalert/notifier/alertmanager_request.qtpl:18 qw422016.N().Q(v) -//line notifier/alertmanager_request.qtpl:20 +//line app/vmalert/notifier/alertmanager_request.qtpl:19 } -//line notifier/alertmanager_request.qtpl:20 +//line app/vmalert/notifier/alertmanager_request.qtpl:19 qw422016.N().S(`},"annotations": {`) -//line notifier/alertmanager_request.qtpl:23 +//line app/vmalert/notifier/alertmanager_request.qtpl:22 c := len(alert.Annotations) -//line notifier/alertmanager_request.qtpl:24 +//line app/vmalert/notifier/alertmanager_request.qtpl:23 for k, v := range alert.Annotations { -//line notifier/alertmanager_request.qtpl:25 +//line app/vmalert/notifier/alertmanager_request.qtpl:24 c = c - 1 -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 qw422016.N().Q(k) -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 qw422016.N().S(`:`) -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 qw422016.N().Q(v) -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 if c > 0 { -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 qw422016.N().S(`,`) -//line notifier/alertmanager_request.qtpl:26 +//line app/vmalert/notifier/alertmanager_request.qtpl:25 } -//line notifier/alertmanager_request.qtpl:27 +//line app/vmalert/notifier/alertmanager_request.qtpl:26 } -//line notifier/alertmanager_request.qtpl:27 +//line app/vmalert/notifier/alertmanager_request.qtpl:26 qw422016.N().S(`}}`) -//line notifier/alertmanager_request.qtpl:30 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 if i != len(alerts)-1 { -//line notifier/alertmanager_request.qtpl:30 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 qw422016.N().S(`,`) -//line notifier/alertmanager_request.qtpl:30 +//line app/vmalert/notifier/alertmanager_request.qtpl:29 } -//line notifier/alertmanager_request.qtpl:31 +//line app/vmalert/notifier/alertmanager_request.qtpl:30 } -//line notifier/alertmanager_request.qtpl:31 +//line app/vmalert/notifier/alertmanager_request.qtpl:30 qw422016.N().S(`]`) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 } -//line notifier/alertmanager_request.qtpl:33 -func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) { -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 +func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) { +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qw422016 := qt422016.AcquireWriter(qq422016) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 streamamRequest(qw422016, alerts, generatorURL) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qt422016.ReleaseWriter(qw422016) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 } -//line notifier/alertmanager_request.qtpl:33 -func amRequest(alerts []Alert, generatorURL func(string, string) string) string { -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 +func amRequest(alerts []Alert, generatorURL func(Alert) string) string { +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qb422016 := qt422016.AcquireByteBuffer() -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 writeamRequest(qb422016, alerts, generatorURL) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qs422016 := string(qb422016.B) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 qt422016.ReleaseByteBuffer(qb422016) -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 return qs422016 -//line notifier/alertmanager_request.qtpl:33 +//line app/vmalert/notifier/alertmanager_request.qtpl:32 } diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go index 0aa8469ef..cfdcc334c 100644 --- a/app/vmalert/notifier/alertmanager_test.go +++ b/app/vmalert/notifier/alertmanager_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "testing" "time" ) @@ -57,8 +58,8 @@ func TestAlertManager_Send(t *testing.T) { }) srv := httptest.NewServer(mux) defer srv.Close() - am := NewAlertManager(srv.URL, func(group, name string) string { - return group + "/" + name + am := NewAlertManager(srv.URL, func(alert Alert) string { + return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10) }, srv.Client()) if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil { t.Error("expected connection error got nil") diff --git a/app/vmalert/notifier/package_test.go b/app/vmalert/notifier/package_test.go new file mode 100644 index 000000000..11876ee3a --- /dev/null +++ b/app/vmalert/notifier/package_test.go @@ -0,0 +1,13 @@ +package notifier + +import ( + "net/url" + "os" + "testing" +) + +func TestMain(m *testing.M) { + u, _ := url.Parse("https://victoriametrics.com/path") + InitTemplateFunc(u) + os.Exit(m.Run()) +}