From 90de3086b39c5643f71e20b1aa1bff77b8d32b03 Mon Sep 17 00:00:00 2001
From: kreedom <60944649+kreedom@users.noreply.github.com>
Date: Sat, 11 Apr 2020 12:40:24 +0300
Subject: [PATCH] [vmalert] add webserver (#410)
* [vmalert] add webserver
---
app/vmalert/main.go | 6 +-
app/vmalert/notifier/alert.go | 12 ++
app/vmalert/notifier/alertmanager.go | 2 +-
.../notifier/alertmanager_request.qtpl | 3 +-
.../notifier/alertmanager_request.qtpl.go | 113 ++++++++--------
app/vmalert/notifier/alertmanager_test.go | 4 +-
app/vmalert/rule.go | 33 ++++-
app/vmalert/testdata/rules0-good.rules | 2 +-
app/vmalert/web.go | 122 ++++++++++++++++++
9 files changed, 230 insertions(+), 67 deletions(-)
create mode 100644 app/vmalert/web.go
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index ef6490da28..afc03af09d 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -61,7 +61,7 @@ func main() {
w := &watchdog{
storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
alertProvider: notifier.NewAlertManager(*providerURL, func(group, name string) string {
- return strings.Replace(fmt.Sprintf("%s/%s/%s/status", eu, group, name), "//", "/", -1)
+ return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, name)
}, &http.Client{}),
}
wg := sync.WaitGroup{}
@@ -73,9 +73,7 @@ func main() {
}(groups[i])
}
- go httpserver.Serve(*httpListenAddr, func(w http.ResponseWriter, r *http.Request) bool {
- panic("not implemented")
- })
+ go httpserver.Serve(*httpListenAddr, (&requestHandler{groups: groups}).handler)
sig := procutil.WaitForSigterm()
logger.Infof("service received signal %s", sig)
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index d1f027afee..51887532ba 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -21,6 +21,7 @@ type Alert struct {
Start time.Time
End time.Time
Value float64
+ ID uint64
}
// AlertState type indicates the Alert state
@@ -37,6 +38,17 @@ const (
StateFiring
)
+// String stringer for AlertState
+func (as AlertState) String() string {
+ switch as {
+ case StateFiring:
+ return "firing"
+ case StatePending:
+ return "pending"
+ }
+ return "inactive"
+}
+
type alertTplData struct {
Labels map[string]string
Value float64
diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go
index d7e02b7573..ab1ba6cc63 100644
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@@ -37,7 +37,7 @@ func (am *AlertManager) Send(alerts []Alert) error {
}
// AlertURLGenerator returns URL to single alert by given name
-type AlertURLGenerator func(group, name string) string
+type AlertURLGenerator func(group, id string) string
const alertManagerPath = "/api/v2/alerts"
diff --git a/app/vmalert/notifier/alertmanager_request.qtpl b/app/vmalert/notifier/alertmanager_request.qtpl
index b831bcf7c5..1c64f1f373 100644
--- a/app/vmalert/notifier/alertmanager_request.qtpl
+++ b/app/vmalert/notifier/alertmanager_request.qtpl
@@ -1,4 +1,5 @@
{% import (
+ "strconv"
"time"
) %}
{% stripspace %}
@@ -8,7 +9,7 @@
{% for i, alert := range alerts %}
{
"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
- "generatorURL": {%q= generatorURL(alert.Group, alert.Name) %},
+ "generatorURL": {%q= generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)) %},
{% if !alert.End.IsZero() %}
"endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
{% endif %}
diff --git a/app/vmalert/notifier/alertmanager_request.qtpl.go b/app/vmalert/notifier/alertmanager_request.qtpl.go
index 16880aab25..26aebf5848 100644
--- a/app/vmalert/notifier/alertmanager_request.qtpl.go
+++ b/app/vmalert/notifier/alertmanager_request.qtpl.go
@@ -1,130 +1,131 @@
// Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
// See https://github.com/valyala/quicktemplate for details.
-//line notifier/alertmanager_request.qtpl:1
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
package notifier
-//line notifier/alertmanager_request.qtpl:1
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
import (
+ "strconv"
"time"
)
-//line notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
import (
qtio422016 "io"
qt422016 "github.com/valyala/quicktemplate"
)
-//line notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
var (
_ = qtio422016.Copy
_ = qt422016.AcquireByteBuffer
)
-//line notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
qw422016.N().S(`[`)
-//line notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
for i, alert := range alerts {
-//line notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
qw422016.N().S(`{"startsAt":`)
-//line notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
qw422016.N().S(`,"generatorURL":`)
-//line notifier/alertmanager_request.qtpl:11
- qw422016.N().Q(generatorURL(alert.Group, alert.Name))
-//line notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+ qw422016.N().Q(generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)))
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
if !alert.End.IsZero() {
-//line notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
qw422016.N().S(`"endsAt":`)
-//line notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:14
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
}
-//line notifier/alertmanager_request.qtpl:14
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
qw422016.N().S(`"labels": {"alertname":`)
-//line notifier/alertmanager_request.qtpl:16
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
qw422016.N().Q(alert.Name)
-//line notifier/alertmanager_request.qtpl:17
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
for k, v := range alert.Labels {
-//line notifier/alertmanager_request.qtpl:17
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:18
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().Q(k)
-//line notifier/alertmanager_request.qtpl:18
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().S(`:`)
-//line notifier/alertmanager_request.qtpl:18
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().Q(v)
-//line notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
}
-//line notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
qw422016.N().S(`},"annotations": {`)
-//line notifier/alertmanager_request.qtpl:22
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
c := len(alert.Annotations)
-//line notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:24
for k, v := range alert.Annotations {
-//line notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
c = c - 1
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().Q(k)
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().S(`:`)
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().Q(v)
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
if c > 0 {
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
}
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
}
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
qw422016.N().S(`}}`)
-//line notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
if i != len(alerts)-1 {
-//line notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
}
-//line notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
}
-//line notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
qw422016.N().S(`]`)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
}
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
qw422016 := qt422016.AcquireWriter(qq422016)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
streamamRequest(qw422016, alerts, generatorURL)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
qt422016.ReleaseWriter(qw422016)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
}
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
qb422016 := qt422016.AcquireByteBuffer()
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
writeamRequest(qb422016, alerts, generatorURL)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
qs422016 := string(qb422016.B)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
qt422016.ReleaseByteBuffer(qb422016)
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
return qs422016
-//line notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
}
diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go
index b701271835..710d147bc9 100644
--- a/app/vmalert/notifier/alertmanager_test.go
+++ b/app/vmalert/notifier/alertmanager_test.go
@@ -40,7 +40,7 @@ func TestAlertManager_Send(t *testing.T) {
if len(a) != 1 {
t.Errorf("expected 1 alert in array got %d", len(a))
}
- if a[0].GeneratorURL != "group0alert0" {
+ if a[0].GeneratorURL != "group0" {
t.Errorf("exptected alert0 as generatorURL got %s", a[0].GeneratorURL)
}
if a[0].Labels["alertname"] != "alert0" {
@@ -66,7 +66,7 @@ func TestAlertManager_Send(t *testing.T) {
t.Error("expected wrong http code error got nil")
}
if err := am.Send([]Alert{{
- Group: "group0",
+ Group: "group",
Name: "alert0",
Start: time.Now().UTC(),
End: time.Now().UTC(),
diff --git a/app/vmalert/rule.go b/app/vmalert/rule.go
index 16f8546d0c..5cf2298140 100644
--- a/app/vmalert/rule.go
+++ b/app/vmalert/rule.go
@@ -21,6 +21,15 @@ type Group struct {
Rules []*Rule
}
+// ActiveAlerts returns list of active alert for all rules
+func (g *Group) ActiveAlerts() []*notifier.Alert {
+ var list []*notifier.Alert
+ for i := range g.Rules {
+ list = append(list, g.Rules[i].listActiveAlerts()...)
+ }
+ return list
+}
+
// Rule is basic alert entity
type Rule struct {
Name string `yaml:"alert"`
@@ -61,7 +70,6 @@ func (r *Rule) Validate() error {
// Based on the Querier results Rule maintains notifier.Alerts
func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
metrics, err := q.Query(ctx, r.Expr)
-
r.mu.Lock()
defer r.mu.Unlock()
@@ -91,6 +99,7 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
r.lastExecError = err
return fmt.Errorf("failed to create alert: %s", err)
}
+ a.ID = h
a.State = notifier.StatePending
r.alerts[h] = a
}
@@ -119,7 +128,7 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
// notifier.Notifier.
// See for reference https://prometheus.io/docs/alerting/clients/
// TODO: add tests for endAt value
-func (r *Rule) Send(ctx context.Context, ap notifier.Notifier) error {
+func (r *Rule) Send(_ context.Context, ap notifier.Notifier) error {
// copy alerts to new list to avoid locks
var alertsCopy []notifier.Alert
r.mu.Lock()
@@ -178,3 +187,23 @@ func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
a.Annotations, err = a.ExecTemplate(r.Annotations)
return a, err
}
+
+func (r *Rule) listActiveAlerts() []*notifier.Alert {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ var list []*notifier.Alert
+ for _, a := range r.alerts {
+ a := a
+ if a.State == notifier.StateFiring {
+ list = append(list, a)
+ }
+ }
+ return list
+}
+
+// Alert returns single alert by its id(hash)
+func (r *Rule) Alert(id uint64) *notifier.Alert {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.alerts[id]
+}
diff --git a/app/vmalert/testdata/rules0-good.rules b/app/vmalert/testdata/rules0-good.rules
index d8ae2f8f5c..5b4a01cb81 100644
--- a/app/vmalert/testdata/rules0-good.rules
+++ b/app/vmalert/testdata/rules0-good.rules
@@ -2,7 +2,7 @@ groups:
- name: groupGorSingleAlert
rules:
- alert: VMRows
- for: 5m
+ for: 10s
expr: vm_rows > 0
labels:
label: bar
diff --git a/app/vmalert/web.go b/app/vmalert/web.go
new file mode 100644
index 0000000000..ad383f7373
--- /dev/null
+++ b/app/vmalert/web.go
@@ -0,0 +1,122 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+// apiAlert has info for an alert.
+type apiAlert struct {
+ Labels map[string]string `json:"labels"`
+ Annotations map[string]string `json:"annotations"`
+ State string `json:"state"`
+ ActiveAt time.Time `json:"activeAt"`
+ Value string `json:"value"`
+}
+
+type requestHandler struct {
+ groups []Group
+}
+
+func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+ resph := responseHandler{w}
+ switch r.URL.Path {
+ default:
+ if strings.HasSuffix(r.URL.Path, "/status") {
+ resph.handle(rh.alert(r.URL.Path))
+ return true
+ }
+ return false
+ case "/api/v1/alerts":
+ resph.handle(rh.listActiveAlerts())
+ return true
+ }
+}
+
+func (rh *requestHandler) listActiveAlerts() ([]byte, error) {
+ type listAlertsResponse struct {
+ Data struct {
+ Alerts []apiAlert `json:"alerts"`
+ } `json:"data"`
+ Status string `json:"status"`
+ }
+ lr := listAlertsResponse{Status: "success"}
+ for _, g := range rh.groups {
+ alerts := g.ActiveAlerts()
+ for i := range alerts {
+ alert := alerts[i]
+ lr.Data.Alerts = append(lr.Data.Alerts, apiAlert{
+ Labels: alert.Labels,
+ Annotations: alert.Annotations,
+ State: alert.State.String(),
+ ActiveAt: alert.Start,
+ Value: strconv.FormatFloat(alert.Value, 'e', -1, 64),
+ })
+ }
+ }
+
+ b, err := json.Marshal(lr)
+ if err != nil {
+ return nil, &httpserver.ErrorWithStatusCode{
+ Err: fmt.Errorf(`error encoding list of active alerts: %s`, err),
+ StatusCode: http.StatusInternalServerError,
+ }
+ }
+ return b, nil
+}
+
+func (rh *requestHandler) alert(path string) ([]byte, error) {
+ parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+ if len(parts) != 3 {
+ return nil, &httpserver.ErrorWithStatusCode{
+ Err: fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+ StatusCode: http.StatusBadRequest,
+ }
+ }
+ group := strings.TrimRight(parts[0], "/")
+ idStr := strings.TrimRight(parts[1], "/")
+ id, err := strconv.ParseUint(idStr, 10, 0)
+ if err != nil {
+ return nil, &httpserver.ErrorWithStatusCode{
+ Err: fmt.Errorf(`cannot parse int from %s"`, idStr),
+ StatusCode: http.StatusBadRequest,
+ }
+ }
+ for _, g := range rh.groups {
+ if g.Name == group {
+ for i := range g.Rules {
+ if alert := g.Rules[i].Alert(id); alert != nil {
+ return json.Marshal(apiAlert{
+ Labels: alert.Labels,
+ Annotations: alert.Annotations,
+ State: alert.State.String(),
+ ActiveAt: alert.Start,
+ Value: strconv.FormatFloat(alert.Value, 'e', -1, 64),
+ })
+ }
+ }
+ }
+ }
+ return nil, &httpserver.ErrorWithStatusCode{
+ Err: fmt.Errorf(`cannot find alert %s in %s"`, idStr, group),
+ StatusCode: http.StatusNotFound,
+ }
+}
+
+// responseHandler wrapper on http.ResponseWriter with sugar
+type responseHandler struct{ http.ResponseWriter }
+
+func (w responseHandler) handle(b []byte, err error) {
+ if err != nil {
+ httpserver.Errorf(w, "%s", err)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write(b)
+}