2020-03-28 23:48:30 +00:00
|
|
|
package common
|
2020-02-16 18:59:02 +00:00
|
|
|
|
2020-03-13 10:19:31 +00:00
|
|
|
import (
|
2020-03-27 16:31:16 +00:00
|
|
|
"bytes"
|
2020-03-28 23:48:30 +00:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2020-03-27 16:31:16 +00:00
|
|
|
"strings"
|
|
|
|
"text/template"
|
2020-03-13 10:19:31 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
2020-03-27 16:31:16 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
2020-03-13 10:19:31 +00:00
|
|
|
)
|
2020-02-16 18:59:02 +00:00
|
|
|
|
2020-03-13 10:19:31 +00:00
|
|
|
// Alert the triggered alert
|
|
|
|
type Alert struct {
|
|
|
|
Group string
|
|
|
|
Name string
|
|
|
|
Labels []datasource.Label
|
|
|
|
Annotations map[string]string
|
|
|
|
|
|
|
|
Start time.Time
|
|
|
|
End time.Time
|
|
|
|
Value float64
|
|
|
|
}
|
|
|
|
|
2020-03-27 16:31:16 +00:00
|
|
|
type alertTplData struct {
|
|
|
|
Labels map[string]string
|
|
|
|
ExternalLabels map[string]string
|
|
|
|
Value float64
|
|
|
|
}
|
|
|
|
|
|
|
|
const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $externalLabels := .ExternalLabels }}`
|
|
|
|
|
2020-03-13 10:19:31 +00:00
|
|
|
// AlertsFromMetrics converts metrics to alerts by alert Rule
|
2020-03-28 23:48:30 +00:00
|
|
|
func AlertsFromMetrics(metrics []datasource.Metric, group string, rule Rule, start, end time.Time) []Alert {
|
2020-03-13 10:19:31 +00:00
|
|
|
alerts := make([]Alert, 0, len(metrics))
|
2020-03-28 23:48:30 +00:00
|
|
|
var err error
|
2020-03-13 10:19:31 +00:00
|
|
|
for i, m := range metrics {
|
|
|
|
a := Alert{
|
2020-03-27 16:31:16 +00:00
|
|
|
Group: group,
|
|
|
|
Name: rule.Name,
|
|
|
|
Start: start,
|
|
|
|
End: end,
|
|
|
|
Value: m.Value,
|
2020-03-13 10:19:31 +00:00
|
|
|
}
|
2020-03-27 16:31:16 +00:00
|
|
|
tplData := alertTplData{Value: m.Value, ExternalLabels: make(map[string]string)}
|
|
|
|
tplData.Labels, a.Labels = mergeLabels(metrics[i].Labels, rule.Labels)
|
2020-03-28 23:48:30 +00:00
|
|
|
a.Annotations, err = templateAnnotations(rule.Annotations, tplHeader, tplData)
|
|
|
|
if err != nil {
|
|
|
|
logger.Errorf("%s", err)
|
|
|
|
}
|
2020-03-13 10:19:31 +00:00
|
|
|
alerts = append(alerts, a)
|
|
|
|
}
|
|
|
|
return alerts
|
|
|
|
}
|
|
|
|
|
2020-03-27 16:31:16 +00:00
|
|
|
func mergeLabels(ml []datasource.Label, rl map[string]string) (map[string]string, []datasource.Label) {
|
|
|
|
set := make(map[string]string, len(ml)+len(rl))
|
|
|
|
sl := append([]datasource.Label(nil), ml...)
|
|
|
|
for _, i := range ml {
|
|
|
|
set[i.Name] = i.Value
|
|
|
|
}
|
|
|
|
for name, value := range rl {
|
|
|
|
if _, ok := set[name]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
set[name] = value
|
|
|
|
sl = append(sl, datasource.Label{
|
|
|
|
Name: name,
|
|
|
|
Value: value,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return set, sl
|
|
|
|
}
|
|
|
|
|
2020-03-28 23:48:30 +00:00
|
|
|
func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
|
2020-03-27 16:31:16 +00:00
|
|
|
var builder strings.Builder
|
|
|
|
var buf bytes.Buffer
|
2020-03-28 23:48:30 +00:00
|
|
|
eg := errGroup{}
|
2020-03-27 16:31:16 +00:00
|
|
|
r := make(map[string]string, len(annotations))
|
|
|
|
for key, text := range annotations {
|
|
|
|
r[key] = text
|
|
|
|
buf.Reset()
|
|
|
|
builder.Reset()
|
|
|
|
builder.Grow(len(header) + len(text))
|
|
|
|
builder.WriteString(header)
|
|
|
|
builder.WriteString(text)
|
2020-03-28 23:48:30 +00:00
|
|
|
if err := templateAnnotation(&buf, builder.String(), data); err != nil {
|
|
|
|
eg.errs = append(eg.errs, fmt.Sprintf("key %s, template %s:%s", key, text, err))
|
2020-03-13 10:19:31 +00:00
|
|
|
continue
|
|
|
|
}
|
2020-03-27 16:31:16 +00:00
|
|
|
r[key] = buf.String()
|
2020-03-13 10:19:31 +00:00
|
|
|
}
|
2020-03-28 23:48:30 +00:00
|
|
|
return r, eg.err()
|
|
|
|
}
|
|
|
|
|
|
|
|
// ValidateAnnotations validate annotations for possible template error, uses empty data for template population
|
|
|
|
func ValidateAnnotations(annotations map[string]string) error {
|
|
|
|
_, err := templateAnnotations(annotations, tplHeader, alertTplData{
|
|
|
|
Labels: map[string]string{},
|
|
|
|
ExternalLabels: map[string]string{},
|
|
|
|
Value: 0,
|
|
|
|
})
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
|
|
|
|
// todo add template helper func from Prometheus
|
|
|
|
tpl, err := template.New("").Option("missingkey=zero").Parse(text)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("error parsing annotation:%w", err)
|
|
|
|
}
|
|
|
|
if err = tpl.Execute(dst, data); err != nil {
|
|
|
|
return fmt.Errorf("error evaluating annotation template:%w", err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type errGroup struct {
|
|
|
|
errs []string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (eg *errGroup) err() error {
|
|
|
|
if eg == nil || len(eg.errs) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return eg
|
|
|
|
}
|
|
|
|
|
|
|
|
func (eg *errGroup) Error() string {
|
|
|
|
return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
|
2020-02-16 18:59:02 +00:00
|
|
|
}
|