2020-04-06 11:44:03 +00:00
|
|
|
package notifier
|
2020-02-16 18:59:02 +00:00
|
|
|
|
2020-03-13 10:19:31 +00:00
|
|
|
import (
|
2020-03-27 16:31:16 +00:00
|
|
|
"bytes"
|
2020-03-28 23:48:30 +00:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2020-03-27 16:31:16 +00:00
|
|
|
"strings"
|
|
|
|
"text/template"
|
2020-03-13 10:19:31 +00:00
|
|
|
"time"
|
2020-06-29 19:21:03 +00:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
2022-04-09 06:21:16 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
2020-03-13 10:19:31 +00:00
|
|
|
)
|
2020-02-16 18:59:02 +00:00
|
|
|
|
2020-03-13 10:19:31 +00:00
|
|
|
// Alert the triggered alert
|
2020-04-06 11:44:03 +00:00
|
|
|
// TODO: Looks like alert name isn't unique
|
2020-03-13 10:19:31 +00:00
|
|
|
type Alert struct {
|
2021-09-13 12:48:18 +00:00
|
|
|
// GroupID contains the ID of the parent rules group
|
|
|
|
GroupID uint64
|
|
|
|
// Name represents Alert name
|
|
|
|
Name string
|
|
|
|
// Labels is the list of label-value pairs attached to the Alert
|
|
|
|
Labels map[string]string
|
|
|
|
// Annotations is the list of annotations generated on Alert evaluation
|
2020-03-13 10:19:31 +00:00
|
|
|
Annotations map[string]string
|
2021-09-13 12:48:18 +00:00
|
|
|
// State represents the current state of the Alert
|
|
|
|
State AlertState
|
|
|
|
// Expr contains expression that was executed to generate the Alert
|
|
|
|
Expr string
|
2022-03-29 13:09:07 +00:00
|
|
|
// ActiveAt defines the moment of time when Alert has become active
|
|
|
|
ActiveAt time.Time
|
|
|
|
// Start defines the moment of time when Alert has become firing
|
2020-03-13 10:19:31 +00:00
|
|
|
Start time.Time
|
2021-09-13 12:48:18 +00:00
|
|
|
// End defines the moment of time when Alert supposed to expire
|
|
|
|
End time.Time
|
2022-03-29 13:09:07 +00:00
|
|
|
// ResolvedAt defines the moment when Alert was switched from Firing to Inactive
|
|
|
|
ResolvedAt time.Time
|
2022-03-16 15:26:33 +00:00
|
|
|
// LastSent defines the moment when Alert was sent last time
|
|
|
|
LastSent time.Time
|
2021-09-13 12:48:18 +00:00
|
|
|
// Value stores the value returned from evaluating expression from Expr field
|
2020-03-13 10:19:31 +00:00
|
|
|
Value float64
|
2021-09-13 12:48:18 +00:00
|
|
|
// ID is the unique identifer for the Alert
|
|
|
|
ID uint64
|
2021-10-22 09:30:38 +00:00
|
|
|
// Restored is true if Alert was restored after restart
|
|
|
|
Restored bool
|
2020-03-13 10:19:31 +00:00
|
|
|
}
|
|
|
|
|
2020-04-06 11:44:03 +00:00
|
|
|
// AlertState type indicates the Alert state
|
|
|
|
type AlertState int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// StateInactive is the state of an alert that is neither firing nor pending.
|
|
|
|
StateInactive AlertState = iota
|
|
|
|
// StatePending is the state of an alert that has been active for less than
|
|
|
|
// the configured threshold duration.
|
|
|
|
StatePending
|
|
|
|
// StateFiring is the state of an alert that has been active for longer than
|
|
|
|
// the configured threshold duration.
|
|
|
|
StateFiring
|
|
|
|
)
|
|
|
|
|
2020-04-11 09:40:24 +00:00
|
|
|
// String stringer for AlertState
|
|
|
|
func (as AlertState) String() string {
|
|
|
|
switch as {
|
|
|
|
case StateFiring:
|
|
|
|
return "firing"
|
|
|
|
case StatePending:
|
|
|
|
return "pending"
|
|
|
|
}
|
|
|
|
return "inactive"
|
|
|
|
}
|
|
|
|
|
2020-12-19 12:10:59 +00:00
|
|
|
// AlertTplData is used to execute templating
|
|
|
|
type AlertTplData struct {
|
2020-04-06 11:44:03 +00:00
|
|
|
Labels map[string]string
|
|
|
|
Value float64
|
2020-05-18 08:55:16 +00:00
|
|
|
Expr string
|
2020-03-27 16:31:16 +00:00
|
|
|
}
|
|
|
|
|
2022-02-15 13:59:45 +00:00
|
|
|
var tplHeaders = []string{
|
|
|
|
"{{ $value := .Value }}",
|
|
|
|
"{{ $labels := .Labels }}",
|
|
|
|
"{{ $expr := .Expr }}",
|
|
|
|
"{{ $externalLabels := .ExternalLabels }}",
|
|
|
|
"{{ $externalURL := .ExternalURL }}",
|
|
|
|
}
|
2020-03-27 16:31:16 +00:00
|
|
|
|
2020-12-19 12:10:59 +00:00
|
|
|
// ExecTemplate executes the Alert template for given
|
2020-04-06 11:44:03 +00:00
|
|
|
// map of annotations.
|
2020-12-14 18:11:45 +00:00
|
|
|
// Every alert could have a different datasource, so function
|
|
|
|
// requires a queryFunction as an argument.
|
vmalert: fix labels and annotations processing for alerts (#2403)
To improve compatibility with Prometheus alerting the order of
templates processing has changed.
Before, vmalert did all labels processing beforehand. It meant
all extra labels (such as `alertname`, `alertgroup` or rule labels)
were available in templating. All collisions were resolved in favour
of extra labels.
In Prometheus, only labels from the received metric are available in
templating, so no collisions are possible.
This change makes vmalert's behaviour similar to Prometheus.
For example, consider alerting rule which is triggered by time series
with `alertname` label. In vmalert, this label would be overriden
by alerting rule's name everywhere: for alert labels, for annotations, etc.
In Prometheus, it would be overriden for alert's labels only, but in annotations
the original label value would be available.
See more details here https://github.com/prometheus/compliance/issues/80
Signed-off-by: hagen1778 <roman@victoriametrics.com>
2022-04-06 18:24:45 +00:00
|
|
|
func (a *Alert) ExecTemplate(q QueryFn, labels, annotations map[string]string) (map[string]string, error) {
|
|
|
|
tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
|
2022-05-02 08:16:16 +00:00
|
|
|
return templateAnnotations(annotations, tplData, funcsWithQuery(q), true)
|
2020-03-13 10:19:31 +00:00
|
|
|
}
|
|
|
|
|
2020-12-19 12:10:59 +00:00
|
|
|
// ExecTemplate executes the given template for given annotations map.
|
|
|
|
func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) {
|
2022-05-02 08:16:16 +00:00
|
|
|
return templateAnnotations(annotations, tpl, funcsWithQuery(q), true)
|
2020-12-19 12:10:59 +00:00
|
|
|
}
|
|
|
|
|
2020-04-26 11:15:04 +00:00
|
|
|
// ValidateTemplates validate annotations for possible template error, uses empty data for template population
|
|
|
|
func ValidateTemplates(annotations map[string]string) error {
|
2020-12-19 12:10:59 +00:00
|
|
|
_, err := templateAnnotations(annotations, AlertTplData{
|
2020-04-06 11:44:03 +00:00
|
|
|
Labels: map[string]string{},
|
|
|
|
Value: 0,
|
2022-05-02 08:16:16 +00:00
|
|
|
}, tmplFunc, false)
|
2020-04-06 11:44:03 +00:00
|
|
|
return err
|
2020-03-27 16:31:16 +00:00
|
|
|
}
|
|
|
|
|
2022-05-02 08:16:16 +00:00
|
|
|
func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap, execute bool) (map[string]string, error) {
|
2020-03-27 16:31:16 +00:00
|
|
|
var builder strings.Builder
|
|
|
|
var buf bytes.Buffer
|
2020-06-29 19:21:03 +00:00
|
|
|
eg := new(utils.ErrGroup)
|
2020-03-27 16:31:16 +00:00
|
|
|
r := make(map[string]string, len(annotations))
|
2022-02-15 13:59:45 +00:00
|
|
|
tData := tplData{data, externalLabels, externalURL}
|
|
|
|
header := strings.Join(tplHeaders, "")
|
2020-03-27 16:31:16 +00:00
|
|
|
for key, text := range annotations {
|
|
|
|
buf.Reset()
|
|
|
|
builder.Reset()
|
2022-02-15 13:59:45 +00:00
|
|
|
builder.Grow(len(header) + len(text))
|
|
|
|
builder.WriteString(header)
|
2020-03-27 16:31:16 +00:00
|
|
|
builder.WriteString(text)
|
2022-05-02 08:16:16 +00:00
|
|
|
if err := templateAnnotation(&buf, builder.String(), tData, funcs, execute); err != nil {
|
2021-07-28 16:26:20 +00:00
|
|
|
r[key] = text
|
2020-06-30 19:58:18 +00:00
|
|
|
eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
|
2020-03-13 10:19:31 +00:00
|
|
|
continue
|
|
|
|
}
|
2020-03-27 16:31:16 +00:00
|
|
|
r[key] = buf.String()
|
2020-03-13 10:19:31 +00:00
|
|
|
}
|
2020-06-29 19:21:03 +00:00
|
|
|
return r, eg.Err()
|
2020-03-28 23:48:30 +00:00
|
|
|
}
|
|
|
|
|
2022-02-15 13:59:45 +00:00
|
|
|
type tplData struct {
|
|
|
|
AlertTplData
|
|
|
|
ExternalLabels map[string]string
|
|
|
|
ExternalURL string
|
|
|
|
}
|
|
|
|
|
2022-05-02 08:16:16 +00:00
|
|
|
func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap, execute bool) error {
|
2020-12-14 18:11:45 +00:00
|
|
|
t := template.New("").Funcs(funcs).Option("missingkey=zero")
|
|
|
|
tpl, err := t.Parse(text)
|
2020-03-28 23:48:30 +00:00
|
|
|
if err != nil {
|
2020-06-01 10:46:37 +00:00
|
|
|
return fmt.Errorf("error parsing annotation: %w", err)
|
2020-03-28 23:48:30 +00:00
|
|
|
}
|
2022-05-02 08:16:16 +00:00
|
|
|
if !execute {
|
|
|
|
return nil
|
|
|
|
}
|
2020-03-28 23:48:30 +00:00
|
|
|
if err = tpl.Execute(dst, data); err != nil {
|
2020-06-01 10:46:37 +00:00
|
|
|
return fmt.Errorf("error evaluating annotation template: %w", err)
|
2020-03-28 23:48:30 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2022-04-09 06:21:16 +00:00
|
|
|
|
|
|
|
func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
|
|
|
var labels []prompbmarshal.Label
|
|
|
|
for k, v := range a.Labels {
|
|
|
|
labels = append(labels, prompbmarshal.Label{
|
|
|
|
Name: k,
|
|
|
|
Value: v,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
promrelabel.SortLabels(labels)
|
|
|
|
if relabelCfg != nil {
|
|
|
|
return relabelCfg.Apply(labels, 0, false)
|
|
|
|
}
|
|
|
|
return labels
|
|
|
|
}
|