[vmalert] - parse template annotaions (#387)

* [vmalert] - parse template annotations

app/vmalert/main.go

@@ -81,6 +81,7 @@ func (w *watchdog) run(ctx context.Context, a config.Group, evaluationInterval t
 	for {
 		select {
 		case <-t.C:
+			start := time.Now()
 			for _, r := range a.Rules {
 				if metrics, err = w.storage.Query(ctx, r.Expr); err != nil {
 					logger.Errorf("error reading metrics %s", err)
@@ -90,7 +91,8 @@ func (w *watchdog) run(ctx context.Context, a config.Group, evaluationInterval t
 				if len(metrics) < 1 {
 					continue
 				}
-				alerts = provider.AlertsFromMetrics(metrics, a.Name, r)
+				// todo define alert end time
+				alerts = provider.AlertsFromMetrics(metrics, a.Name, r, start, time.Time{})
 				// todo save to storage
 				if err := w.alertProvider.Send(alerts); err != nil {
 					logger.Errorf("error sending alerts %s", err)

app/vmalert/provider/common.go

@@ -1,11 +1,14 @@
 package provider
 
 import (
-	"sort"
+	"bytes"
+	"strings"
+	"text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 // AlertProvider is common interface for alert manager provider
@@ -25,41 +28,74 @@ type Alert struct {
 	Value float64
 }
 
+type alertTplData struct {
+	Labels         map[string]string
+	ExternalLabels map[string]string
+	Value          float64
+}
+
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $externalLabels := .ExternalLabels }}`
+
 // AlertsFromMetrics converts metrics to alerts by alert Rule
-func AlertsFromMetrics(metrics []datasource.Metric, group string, rule config.Rule) []Alert {
+func AlertsFromMetrics(metrics []datasource.Metric, group string, rule config.Rule, start, end time.Time) []Alert {
 	alerts := make([]Alert, 0, len(metrics))
 	for i, m := range metrics {
 		a := Alert{
-			Group:  group,
-			Name:   rule.Name,
-			Labels: metrics[i].Labels,
-			// todo eval template in annotations
-			Annotations: rule.Annotations,
-			Start:       time.Unix(m.Timestamp, 0),
+			Group: group,
+			Name:  rule.Name,
+			Start: start,
+			End:   end,
+			Value: m.Value,
 		}
-		for k, v := range rule.Labels {
-			a.Labels = append(a.Labels, datasource.Label{
-				Name:  k,
-				Value: v,
-			})
-		}
-		a.Labels = removeDuplicated(a.Labels)
+		tplData := alertTplData{Value: m.Value, ExternalLabels: make(map[string]string)}
+		tplData.Labels, a.Labels = mergeLabels(metrics[i].Labels, rule.Labels)
+		a.Annotations = templateAnnotations(rule.Annotations, tplHeader, tplData)
 		alerts = append(alerts, a)
 	}
 	return alerts
 }
 
-func removeDuplicated(l []datasource.Label) []datasource.Label {
-	sort.Slice(l, func(i, j int) bool {
-		return l[i].Name < l[j].Name
-	})
-	j := 0
-	for i := 1; i < len(l); i++ {
-		if l[j] == l[i] {
+func mergeLabels(ml []datasource.Label, rl map[string]string) (map[string]string, []datasource.Label) {
+	set := make(map[string]string, len(ml)+len(rl))
+	sl := append([]datasource.Label(nil), ml...)
+	for _, i := range ml {
+		set[i.Name] = i.Value
+	}
+	for name, value := range rl {
+		if _, ok := set[name]; ok {
 			continue
 		}
-		j++
-		l[j] = l[i]
+		set[name] = value
+		sl = append(sl, datasource.Label{
+			Name:  name,
+			Value: value,
+		})
 	}
-	return l[:j+1]
+	return set, sl
+}
+
+func templateAnnotations(annotations map[string]string, header string, data alertTplData) map[string]string {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	r := make(map[string]string, len(annotations))
+	for key, text := range annotations {
+		r[key] = text
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		// todo add template helper func from Prometheus
+		tpl, err := template.New("").Option("missingkey=zero").Parse(builder.String())
+		if err != nil {
+			logger.Errorf("error parsing annotation template %q for %q:%s", text, key, err)
+			continue
+		}
+		if err = tpl.Execute(&buf, data); err != nil {
+			logger.Errorf("error evaluating annotation template %s for %s:%s", text, key, err)
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r
 }

app/vmalert/provider/common_test.go

@@ -0,0 +1,100 @@
+package provider
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+)
+
+func TestAlertsFromMetrics(t *testing.T) {
+	now := time.Now()
+	metrics := []datasource.Metric{
+		{
+			Labels: []datasource.Label{
+				{Name: "__name__", Value: "foo"},
+				{Name: "label", Value: "value"},
+			},
+			Timestamp: 10,
+			Value:     20,
+		},
+		{
+			Labels: []datasource.Label{
+				{Name: "__name__", Value: "bar"},
+				{Name: "label", Value: "value"},
+			},
+			Timestamp: 10,
+			Value:     30,
+		},
+	}
+	rule := config.Rule{
+		Name: "alertname",
+		Expr: "up==0",
+		Labels: map[string]string{
+			"label2": "value",
+		},
+		Annotations: map[string]string{
+			"tpl": "{{$value}} {{ $labels.label}}",
+		},
+	}
+	alerts := AlertsFromMetrics(metrics, "group", rule, now, now)
+	if len(alerts) != 2 {
+		t.Fatalf("expecting 2 alerts got %d", len(alerts))
+	}
+
+	f := func(got, exp Alert) {
+		t.Helper()
+		if got.Group != exp.Group ||
+			got.Value != exp.Value ||
+			got.End != exp.End ||
+			got.Name != exp.Name ||
+			got.Start != exp.Start {
+			t.Errorf("alerts are not equal: \nwant %#v \ngot  %#v", exp, got)
+		}
+		sort.Slice(got.Labels, func(i, j int) bool {
+			return got.Labels[i].Name < got.Labels[j].Name
+		})
+		sort.Slice(exp.Labels, func(i, j int) bool {
+			return got.Labels[i].Name < got.Labels[j].Name
+		})
+		if !reflect.DeepEqual(got.Labels, exp.Labels) {
+			t.Errorf("alerts labels are not equal: want  %+v got %+v", exp.Labels, got.Labels)
+		}
+		if !reflect.DeepEqual(got.Annotations, exp.Annotations) {
+			t.Errorf("alerts annotations are not equal: want %+v got %+v", exp.Annotations, got.Annotations)
+		}
+	}
+	f(alerts[0], Alert{
+		Group: "group",
+		Name:  "alertname",
+		Labels: []datasource.Label{
+			{Name: "__name__", Value: "foo"},
+			{Name: "label", Value: "value"},
+			{Name: "label2", Value: "value"},
+		},
+		Annotations: map[string]string{
+			"tpl": "20 value",
+		},
+		Start: now,
+		End:   now,
+		Value: 20,
+	})
+	f(alerts[1], Alert{
+		Group: "group",
+		Name:  "alertname",
+		Labels: []datasource.Label{
+			{Name: "__name__", Value: "bar"},
+			{Name: "label", Value: "value"},
+			{Name: "label2", Value: "value"},
+		},
+		Annotations: map[string]string{
+			"tpl": "30 value",
+		},
+		Start: now,
+		End:   now,
+		Value: 30,
+	})
+}