2020-04-27 21:19:27 +00:00
|
|
|
package notifier
|
|
|
|
|
|
|
|
import (
|
2022-02-15 13:59:45 +00:00
|
|
|
"fmt"
|
2022-04-09 06:21:16 +00:00
|
|
|
"reflect"
|
2020-04-27 21:19:27 +00:00
|
|
|
"testing"
|
2022-08-22 11:32:36 +00:00
|
|
|
"time"
|
2020-12-14 18:11:45 +00:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
2022-04-09 06:21:16 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
2020-04-27 21:19:27 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func TestAlert_ExecTemplate(t *testing.T) {
|
2022-05-09 08:11:56 +00:00
|
|
|
extLabels := make(map[string]string)
|
2022-02-15 13:59:45 +00:00
|
|
|
const (
|
|
|
|
extCluster = "prod"
|
|
|
|
extDC = "east"
|
|
|
|
extURL = "https://foo.bar"
|
|
|
|
)
|
|
|
|
extLabels["cluster"] = extCluster
|
|
|
|
extLabels["dc"] = extDC
|
|
|
|
_, err := Init(nil, extLabels, extURL)
|
|
|
|
checkErr(t, err)
|
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
testCases := []struct {
|
2020-05-19 09:59:46 +00:00
|
|
|
name string
|
2020-04-27 21:19:27 +00:00
|
|
|
alert *Alert
|
|
|
|
annotations map[string]string
|
|
|
|
expTpl map[string]string
|
|
|
|
}{
|
|
|
|
{
|
2020-05-19 09:59:46 +00:00
|
|
|
name: "empty-alert",
|
2020-04-27 21:19:27 +00:00
|
|
|
alert: &Alert{},
|
|
|
|
annotations: map[string]string{},
|
|
|
|
expTpl: map[string]string{},
|
|
|
|
},
|
|
|
|
{
|
2020-05-18 08:55:16 +00:00
|
|
|
name: "no-template",
|
2020-04-27 21:19:27 +00:00
|
|
|
alert: &Alert{
|
|
|
|
Value: 1e4,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"instance": "localhost",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
annotations: map[string]string{},
|
|
|
|
expTpl: map[string]string{},
|
|
|
|
},
|
|
|
|
{
|
2020-05-18 08:55:16 +00:00
|
|
|
name: "label-template",
|
2020-04-27 21:19:27 +00:00
|
|
|
alert: &Alert{
|
|
|
|
Value: 1e4,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"job": "staging",
|
|
|
|
"instance": "localhost",
|
|
|
|
},
|
2022-12-12 19:16:10 +00:00
|
|
|
For: 5 * time.Minute,
|
2020-04-27 21:19:27 +00:00
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"summary": "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
|
2022-12-12 19:16:10 +00:00
|
|
|
"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
|
2020-04-27 21:19:27 +00:00
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"summary": "Too high connection number for localhost for job staging",
|
2022-12-12 19:16:10 +00:00
|
|
|
"description": "It is 10000 connections for localhost for more than 5m0s",
|
2020-04-27 21:19:27 +00:00
|
|
|
},
|
|
|
|
},
|
2020-05-18 08:55:16 +00:00
|
|
|
{
|
|
|
|
name: "expression-template",
|
|
|
|
alert: &Alert{
|
app/vmalert/templates: properly escape all the special chars in `quotesEscape` function
Previously the `quotesEscape` function was escaping only double quotes.
This wasn't enough, since the input string could contain other special chars,
which must be escaped when put inside JSON string. For example, carriage return and line feed chars (\n\r),
backslash char, etc. This led to the following issues, which were improperly fixed:
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890 - this issue
was "fixed" by introducing the `crlfEscape` function, which led to unnecessary
complications in user templates, while not fixing various corner cases
such as backslash chars in the input string.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/1de15ad490dbde84ad2a657f3b65a6311991f372
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3139 - this issue
was "fixed" by urlencoding the whole string passed to -external.alert.source
command-line flag. This led to invalid urls, which couldn't be parsed by Grafana.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/00c838353d1246495fd7c7546f3d71095e855eab
and https://github.com/VictoriaMetrics/VictoriaMetrics/commit/4bd024459931a0671dee4abae4bc3556795ee398
This commit properly encodes the input string passed to `quotesEscape`, so it can be safely embedded inside JSON strings.
This commit deprecates crlfEscape template function and adds the following new template functions:
- strvalue and stripDomain - these functions are supported by Prometheus, so they were added
for compatibility purposes.
- jsonEscape and htmlEscape for converting the input string to valid quoted JSON string
and for html-escaping the input string, so it could be safely embedded as a plaintext
into html.
This commit also documents all supported template functions at https://docs.victoriametrics.com/vmalert.html#template-functions
The deprecated crlfEscape function isn't documented on purpose, since its usefulness is negative in general case.
2022-10-27 20:38:19 +00:00
|
|
|
Expr: `vm_rows{"label"="bar"}<0`,
|
2020-05-18 08:55:16 +00:00
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
app/vmalert/templates: properly escape all the special chars in `quotesEscape` function
Previously the `quotesEscape` function was escaping only double quotes.
This wasn't enough, since the input string could contain other special chars,
which must be escaped when put inside JSON string. For example, carriage return and line feed chars (\n\r),
backslash char, etc. This led to the following issues, which were improperly fixed:
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890 - this issue
was "fixed" by introducing the `crlfEscape` function, which led to unnecessary
complications in user templates, while not fixing various corner cases
such as backslash chars in the input string.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/1de15ad490dbde84ad2a657f3b65a6311991f372
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3139 - this issue
was "fixed" by urlencoding the whole string passed to -external.alert.source
command-line flag. This led to invalid urls, which couldn't be parsed by Grafana.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/00c838353d1246495fd7c7546f3d71095e855eab
and https://github.com/VictoriaMetrics/VictoriaMetrics/commit/4bd024459931a0671dee4abae4bc3556795ee398
This commit properly encodes the input string passed to `quotesEscape`, so it can be safely embedded inside JSON strings.
This commit deprecates crlfEscape template function and adds the following new template functions:
- strvalue and stripDomain - these functions are supported by Prometheus, so they were added
for compatibility purposes.
- jsonEscape and htmlEscape for converting the input string to valid quoted JSON string
and for html-escaping the input string, so it could be safely embedded as a plaintext
into html.
This commit also documents all supported template functions at https://docs.victoriametrics.com/vmalert.html#template-functions
The deprecated crlfEscape function isn't documented on purpose, since its usefulness is negative in general case.
2022-10-27 20:38:19 +00:00
|
|
|
"exprEscapedQuery": "{{ $expr|queryEscape }}",
|
|
|
|
"exprEscapedPath": "{{ $expr|pathEscape }}",
|
|
|
|
"exprEscapedJSON": "{{ $expr|jsonEscape }}",
|
|
|
|
"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
|
|
|
|
"exprEscapedHTML": "{{ $expr|htmlEscape }}",
|
2020-05-18 08:55:16 +00:00
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
app/vmalert/templates: properly escape all the special chars in `quotesEscape` function
Previously the `quotesEscape` function was escaping only double quotes.
This wasn't enough, since the input string could contain other special chars,
which must be escaped when put inside JSON string. For example, carriage return and line feed chars (\n\r),
backslash char, etc. This led to the following issues, which were improperly fixed:
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890 - this issue
was "fixed" by introducing the `crlfEscape` function, which led to unnecessary
complications in user templates, while not fixing various corner cases
such as backslash chars in the input string.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/1de15ad490dbde84ad2a657f3b65a6311991f372
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3139 - this issue
was "fixed" by urlencoding the whole string passed to -external.alert.source
command-line flag. This led to invalid urls, which couldn't be parsed by Grafana.
See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/00c838353d1246495fd7c7546f3d71095e855eab
and https://github.com/VictoriaMetrics/VictoriaMetrics/commit/4bd024459931a0671dee4abae4bc3556795ee398
This commit properly encodes the input string passed to `quotesEscape`, so it can be safely embedded inside JSON strings.
This commit deprecates crlfEscape template function and adds the following new template functions:
- strvalue and stripDomain - these functions are supported by Prometheus, so they were added
for compatibility purposes.
- jsonEscape and htmlEscape for converting the input string to valid quoted JSON string
and for html-escaping the input string, so it could be safely embedded as a plaintext
into html.
This commit also documents all supported template functions at https://docs.victoriametrics.com/vmalert.html#template-functions
The deprecated crlfEscape function isn't documented on purpose, since its usefulness is negative in general case.
2022-10-27 20:38:19 +00:00
|
|
|
"exprEscapedQuery": "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
|
|
|
|
"exprEscapedPath": "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
|
|
|
|
"exprEscapedJSON": `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
|
|
|
|
"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
|
|
|
|
"exprEscapedHTML": "vm_rows{"label"="bar"}<0",
|
2020-05-18 08:55:16 +00:00
|
|
|
},
|
|
|
|
},
|
2020-12-14 18:11:45 +00:00
|
|
|
{
|
|
|
|
name: "query",
|
|
|
|
alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"summary": `{{ query "foo" | first | value }}`,
|
|
|
|
"desc": `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"summary": "1",
|
|
|
|
"desc": "bar 1;garply 2;",
|
|
|
|
},
|
|
|
|
},
|
2022-02-15 13:59:45 +00:00
|
|
|
{
|
|
|
|
name: "external",
|
|
|
|
alert: &Alert{
|
|
|
|
Value: 1e4,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"job": "staging",
|
|
|
|
"instance": "localhost",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"url": "{{ $externalURL }}",
|
|
|
|
"summary": "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
|
|
|
|
"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"url": extURL,
|
|
|
|
"summary": fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
|
|
|
|
"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
|
|
|
|
},
|
|
|
|
},
|
2022-08-16 06:08:27 +00:00
|
|
|
{
|
|
|
|
name: "alert and group IDs",
|
|
|
|
alert: &Alert{
|
|
|
|
ID: 42,
|
|
|
|
GroupID: 24,
|
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"url": "/api/v1/alert?alertID=42&groupID=24",
|
|
|
|
},
|
|
|
|
},
|
2022-08-22 11:32:36 +00:00
|
|
|
{
|
|
|
|
name: "ActiveAt time",
|
|
|
|
alert: &Alert{
|
|
|
|
ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
|
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"diagram": "![](http://example.com?render=1660941298",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "ActiveAt time is nil",
|
|
|
|
alert: &Alert{},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"default_time": "{{$activeAt}}",
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"default_time": "0001-01-01 00:00:00 +0000 UTC",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
2022-12-12 19:16:10 +00:00
|
|
|
name: "ActiveAt custom format",
|
2022-08-22 11:32:36 +00:00
|
|
|
alert: &Alert{
|
|
|
|
ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
|
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"fire_time": "2022/08/19 20:34:58",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "ActiveAt query range",
|
|
|
|
alert: &Alert{
|
|
|
|
ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
|
|
|
|
},
|
|
|
|
annotations: map[string]string{
|
|
|
|
"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
|
|
|
|
},
|
|
|
|
expTpl: map[string]string{
|
|
|
|
"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
|
|
|
|
},
|
|
|
|
},
|
2020-04-27 21:19:27 +00:00
|
|
|
}
|
|
|
|
|
2020-12-14 18:11:45 +00:00
|
|
|
qFn := func(q string) ([]datasource.Metric, error) {
|
|
|
|
return []datasource.Metric{
|
|
|
|
{
|
|
|
|
Labels: []datasource.Label{
|
|
|
|
{Name: "foo", Value: "bar"},
|
|
|
|
{Name: "baz", Value: "qux"},
|
|
|
|
},
|
2021-06-09 09:20:38 +00:00
|
|
|
Values: []float64{1},
|
|
|
|
Timestamps: []int64{1},
|
2020-12-14 18:11:45 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
Labels: []datasource.Label{
|
|
|
|
{Name: "foo", Value: "garply"},
|
|
|
|
{Name: "baz", Value: "fred"},
|
|
|
|
},
|
2021-06-09 09:20:38 +00:00
|
|
|
Values: []float64{2},
|
|
|
|
Timestamps: []int64{1},
|
2020-12-14 18:11:45 +00:00
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
2020-05-18 08:55:16 +00:00
|
|
|
for _, tc := range testCases {
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
2023-03-17 14:57:24 +00:00
|
|
|
if err := ValidateTemplates(tc.annotations); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
vmalert: fix labels and annotations processing for alerts (#2403)
To improve compatibility with Prometheus alerting the order of
templates processing has changed.
Before, vmalert did all labels processing beforehand. It meant
all extra labels (such as `alertname`, `alertgroup` or rule labels)
were available in templating. All collisions were resolved in favour
of extra labels.
In Prometheus, only labels from the received metric are available in
templating, so no collisions are possible.
This change makes vmalert's behaviour similar to Prometheus.
For example, consider alerting rule which is triggered by time series
with `alertname` label. In vmalert, this label would be overriden
by alerting rule's name everywhere: for alert labels, for annotations, etc.
In Prometheus, it would be overriden for alert's labels only, but in annotations
the original label value would be available.
See more details here https://github.com/prometheus/compliance/issues/80
Signed-off-by: hagen1778 <roman@victoriametrics.com>
2022-04-06 18:24:45 +00:00
|
|
|
tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
|
2020-04-27 21:19:27 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if len(tpl) != len(tc.expTpl) {
|
|
|
|
t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
|
|
|
|
}
|
|
|
|
for k := range tc.expTpl {
|
|
|
|
got, exp := tpl[k], tc.expTpl[k]
|
|
|
|
if got != exp {
|
|
|
|
t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
2022-04-09 06:21:16 +00:00
|
|
|
|
|
|
|
func TestAlert_toPromLabels(t *testing.T) {
|
|
|
|
fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
|
|
|
|
t.Helper()
|
|
|
|
a := Alert{Labels: labels}
|
|
|
|
got := a.toPromLabels(relabel)
|
|
|
|
if !reflect.DeepEqual(got, exp) {
|
|
|
|
t.Fatalf("expected to have: \n%v;\ngot:\n%v",
|
|
|
|
exp, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn(nil, nil, nil)
|
|
|
|
fn(
|
|
|
|
map[string]string{"foo": "bar", "a": "baz"}, // unsorted
|
|
|
|
[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
|
|
|
|
nil,
|
|
|
|
)
|
|
|
|
|
|
|
|
pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
|
|
|
|
- target_label: "foo"
|
|
|
|
replacement: "aaa"
|
|
|
|
- action: labeldrop
|
|
|
|
regex: "env.*"
|
2022-12-10 10:09:21 +00:00
|
|
|
`))
|
2022-04-09 06:21:16 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn(
|
|
|
|
map[string]string{"a": "baz"},
|
|
|
|
[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
|
|
|
|
pcs,
|
|
|
|
)
|
|
|
|
fn(
|
|
|
|
map[string]string{"foo": "bar", "a": "baz"},
|
|
|
|
[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
|
|
|
|
pcs,
|
|
|
|
)
|
|
|
|
fn(
|
|
|
|
map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
|
|
|
|
[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
|
|
|
|
pcs,
|
|
|
|
)
|
|
|
|
}
|