diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 46ebbd063f..a466c1e1d8 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -675,8 +675,8 @@ The shortlist of configuration flags is the following: -evaluationInterval duration How often to evaluate the rules (default 1m0s) -external.alert.source string - External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. - eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/vmalert/api/v1/alert?group_id=&alert_id=' is used + External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used + If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used. -external.label array Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. Supports an array of values separated by comma or specified via multiple flags. diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 2b7fdcba77..803338a29d 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -58,8 +58,11 @@ absolute path to all .tpl files in root.`) resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier") externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier") - externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. -eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/vmalert/api/v1/alert?group_id=&alert_id=' is used`) + externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+ + `for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+ + `Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+ + `For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . `+ + `If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`) externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+ "Pass multiple -label flags in order to add multiple label sets.") diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 1a55b99f7a..c026a9eda0 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -66,15 +66,21 @@ func TestAlert_ExecTemplate(t *testing.T) { { name: "expression-template", alert: &Alert{ - Expr: `vm_rows{"label"="bar"}>0`, + Expr: `vm_rows{"label"="bar"}<0`, }, annotations: map[string]string{ - "exprEscapedQuery": "{{ $expr|quotesEscape|queryEscape }}", - "exprEscapedPath": "{{ $expr|quotesEscape|pathEscape }}", + "exprEscapedQuery": "{{ $expr|queryEscape }}", + "exprEscapedPath": "{{ $expr|pathEscape }}", + "exprEscapedJSON": "{{ $expr|jsonEscape }}", + "exprEscapedQuotes": "{{ $expr|quotesEscape }}", + "exprEscapedHTML": "{{ $expr|htmlEscape }}", }, expTpl: map[string]string{ - "exprEscapedQuery": "vm_rows%7B%5C%22label%5C%22%3D%5C%22bar%5C%22%7D%3E0", - "exprEscapedPath": "vm_rows%7B%5C%22label%5C%22=%5C%22bar%5C%22%7D%3E0", + "exprEscapedQuery": "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0", + "exprEscapedPath": "vm_rows%7B%22label%22=%22bar%22%7D%3C0", + "exprEscapedJSON": `"vm_rows{\"label\"=\"bar\"}\u003c0"`, + "exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`, + "exprEscapedHTML": "vm_rows{"label"="bar"}<0", }, }, { diff --git a/app/vmalert/templates/funcs.qtpl b/app/vmalert/templates/funcs.qtpl new file mode 100644 index 0000000000..00f4982399 --- /dev/null +++ b/app/vmalert/templates/funcs.qtpl @@ -0,0 +1,15 @@ +{% stripspace %} + +{% func quotesEscape(s string) %} + {%j= s %} +{% endfunc %} + +{% func jsonEscape(s string) %} + {%q= s %} +{% endfunc %} + +{% func htmlEscape(s string) %} + {%s s %} +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmalert/templates/funcs.qtpl.go b/app/vmalert/templates/funcs.qtpl.go new file mode 100644 index 0000000000..55132e2d4e --- /dev/null +++ b/app/vmalert/templates/funcs.qtpl.go @@ -0,0 +1,117 @@ +// Code generated by qtc from "funcs.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +//line app/vmalert/templates/funcs.qtpl:3 +package templates + +//line app/vmalert/templates/funcs.qtpl:3 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmalert/templates/funcs.qtpl:3 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmalert/templates/funcs.qtpl:3 +func streamquotesEscape(qw422016 *qt422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:4 + qw422016.N().J(s) +//line app/vmalert/templates/funcs.qtpl:5 +} + +//line app/vmalert/templates/funcs.qtpl:5 +func writequotesEscape(qq422016 qtio422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:5 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/templates/funcs.qtpl:5 + streamquotesEscape(qw422016, s) +//line app/vmalert/templates/funcs.qtpl:5 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/templates/funcs.qtpl:5 +} + +//line app/vmalert/templates/funcs.qtpl:5 +func quotesEscape(s string) string { +//line app/vmalert/templates/funcs.qtpl:5 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/templates/funcs.qtpl:5 + writequotesEscape(qb422016, s) +//line app/vmalert/templates/funcs.qtpl:5 + qs422016 := string(qb422016.B) +//line app/vmalert/templates/funcs.qtpl:5 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/templates/funcs.qtpl:5 + return qs422016 +//line app/vmalert/templates/funcs.qtpl:5 +} + +//line app/vmalert/templates/funcs.qtpl:7 +func streamjsonEscape(qw422016 *qt422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:8 + qw422016.N().Q(s) +//line app/vmalert/templates/funcs.qtpl:9 +} + +//line app/vmalert/templates/funcs.qtpl:9 +func writejsonEscape(qq422016 qtio422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:9 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/templates/funcs.qtpl:9 + streamjsonEscape(qw422016, s) +//line app/vmalert/templates/funcs.qtpl:9 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/templates/funcs.qtpl:9 +} + +//line app/vmalert/templates/funcs.qtpl:9 +func jsonEscape(s string) string { +//line app/vmalert/templates/funcs.qtpl:9 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/templates/funcs.qtpl:9 + writejsonEscape(qb422016, s) +//line app/vmalert/templates/funcs.qtpl:9 + qs422016 := string(qb422016.B) +//line app/vmalert/templates/funcs.qtpl:9 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/templates/funcs.qtpl:9 + return qs422016 +//line app/vmalert/templates/funcs.qtpl:9 +} + +//line app/vmalert/templates/funcs.qtpl:11 +func streamhtmlEscape(qw422016 *qt422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:12 + qw422016.E().S(s) +//line app/vmalert/templates/funcs.qtpl:13 +} + +//line app/vmalert/templates/funcs.qtpl:13 +func writehtmlEscape(qq422016 qtio422016.Writer, s string) { +//line app/vmalert/templates/funcs.qtpl:13 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/templates/funcs.qtpl:13 + streamhtmlEscape(qw422016, s) +//line app/vmalert/templates/funcs.qtpl:13 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/templates/funcs.qtpl:13 +} + +//line app/vmalert/templates/funcs.qtpl:13 +func htmlEscape(s string) string { +//line app/vmalert/templates/funcs.qtpl:13 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/templates/funcs.qtpl:13 + writehtmlEscape(qb422016, s) +//line app/vmalert/templates/funcs.qtpl:13 + qs422016 := string(qb422016.B) +//line app/vmalert/templates/funcs.qtpl:13 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/templates/funcs.qtpl:13 + return qs422016 +//line app/vmalert/templates/funcs.qtpl:13 +} diff --git a/app/vmalert/templates/template.go b/app/vmalert/templates/template.go index 3907ba97e2..14f5967fa7 100644 --- a/app/vmalert/templates/template.go +++ b/app/vmalert/templates/template.go @@ -207,23 +207,10 @@ func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap { // templateFuncs initiates template helper functions func templateFuncs() textTpl.FuncMap { // See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/ + // and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150 return textTpl.FuncMap{ /* Strings */ - // reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with - // the replacement string repl. Inside repl, $ signs are interpreted as in Expand, - // so for instance $1 represents the text of the first submatch. - // alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString - "reReplaceAll": func(pattern, repl, text string) string { - re := regexp.MustCompile(pattern) - return re.ReplaceAllString(text, repl) - }, - - // match reports whether the string s - // contains any match of the regular expression pattern. - // alias for https://golang.org/pkg/regexp/#MatchString - "match": regexp.MatchString, - // title returns a copy of the string s with all Unicode letters // that begin words mapped to their Unicode title case. // alias for https://golang.org/pkg/strings/#Title @@ -237,6 +224,31 @@ func templateFuncs() textTpl.FuncMap { // alias for https://golang.org/pkg/strings/#ToLower "toLower": strings.ToLower, + // crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`. + // This funcion is deprectated. + // + // It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead - + // these functions properly escape `\n` and `\r` chars according to their purpose. + "crlfEscape": func(q string) string { + q = strings.Replace(q, "\n", `\n`, -1) + return strings.Replace(q, "\r", `\r`, -1) + }, + + // quotesEscape escapes the string, so it can be safely put inside JSON string. + // + // See also jsonEscape. + "quotesEscape": quotesEscape, + + // jsonEscape converts the string to properly encoded JSON string. + // + // See also quotesEscape. + "jsonEscape": jsonEscape, + + // htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html. + // + // See also safeHtml. + "htmlEscape": htmlEscape, + // stripPort splits string into host and port, then returns only host. "stripPort": func(hostPort string) string { host, _, err := net.SplitHostPort(hostPort) @@ -246,6 +258,37 @@ func templateFuncs() textTpl.FuncMap { return host }, + // stripDomain removes the domain part of a FQDN. Leaves port untouched. + "stripDomain": func(hostPort string) string { + host, port, err := net.SplitHostPort(hostPort) + if err != nil { + host = hostPort + } + ip := net.ParseIP(host) + if ip != nil { + return hostPort + } + host = strings.Split(host, ".")[0] + if port != "" { + return net.JoinHostPort(host, port) + } + return host + }, + + // match reports whether the string s + // contains any match of the regular expression pattern. + // alias for https://golang.org/pkg/regexp/#MatchString + "match": regexp.MatchString, + + // reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with + // the replacement string repl. Inside repl, $ signs are interpreted as in Expand, + // so for instance $1 represents the text of the first submatch. + // alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString + "reReplaceAll": func(pattern, repl, text string) string { + re := regexp.MustCompile(pattern) + return re.ReplaceAllString(text, repl) + }, + // parseDuration parses a duration string such as "1h" into the number of seconds it represents "parseDuration": func(s string) (float64, error) { d, err := promutils.ParseDuration(s) @@ -399,31 +442,15 @@ func templateFuncs() textTpl.FuncMap { return "" }, - // pathEscape escapes the string so it can be safely placed inside a URL path segment, - // replacing special characters (including /) with %XX sequences as needed. - // alias for https://golang.org/pkg/net/url/#PathEscape - "pathEscape": func(u string) string { - return url.PathEscape(u) - }, + // pathEscape escapes the string so it can be safely placed inside a URL path segment. + // + // See also queryEscape. + "pathEscape": url.PathEscape, - // queryEscape escapes the string so it can be safely placed - // inside a URL query. - // alias for https://golang.org/pkg/net/url/#QueryEscape - "queryEscape": func(q string) string { - return url.QueryEscape(q) - }, - - // crlfEscape replaces new line chars to skip URL encoding. - // see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890 - "crlfEscape": func(q string) string { - q = strings.Replace(q, "\n", `\n`, -1) - return strings.Replace(q, "\r", `\r`, -1) - }, - - // quotesEscape escapes quote char - "quotesEscape": func(q string) string { - return strings.Replace(q, `"`, `\"`, -1) - }, + // queryEscape escapes the string so it can be safely placed inside a query arg in URL. + // + // See also queryEscape. + "queryEscape": url.QueryEscape, // query executes the MetricsQL/PromQL query against // configured `datasource.url` address. @@ -455,6 +482,17 @@ func templateFuncs() textTpl.FuncMap { return m.Labels[label] }, + // value returns the value of the given metric. + // usually used alongside with `query` template function. + "value": func(m metric) float64 { + return m.Value + }, + + // strvalue returns metric name. + "strvalue": func(m metric) string { + return m.Labels["__name__"] + }, + // sortByLabel sorts the given metrics by provided label key "sortByLabel": func(label string, metrics []metric) []metric { sort.SliceStable(metrics, func(i, j int) bool { @@ -463,12 +501,6 @@ func templateFuncs() textTpl.FuncMap { return metrics }, - // value returns the value of the given metric. - // usually used alongside with `query` template function. - "value": func(m metric) float64 { - return m.Value - }, - /* Helpers */ // Converts a list of objects to a map with keys arg0, arg1 etc. @@ -482,6 +514,8 @@ func templateFuncs() textTpl.FuncMap { }, // safeHtml marks string as HTML not requiring auto-escaping. + // + // See also htmlEscape. "safeHtml": func(text string) htmlTpl.HTML { return htmlTpl.HTML(text) }, diff --git a/app/vmalert/templates/template_test.go b/app/vmalert/templates/template_test.go index 16e4b5c252..04daca3722 100644 --- a/app/vmalert/templates/template_test.go +++ b/app/vmalert/templates/template_test.go @@ -6,6 +6,52 @@ import ( textTpl "text/template" ) +func TestTemplateFuncs(t *testing.T) { + funcs := templateFuncs() + f := func(funcName, s, resultExpected string) { + t.Helper() + v := funcs[funcName] + fLocal := v.(func(s string) string) + result := fLocal(s) + if result != resultExpected { + t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected) + } + } + f("title", "foo bar", "Foo Bar") + f("toUpper", "foo", "FOO") + f("toLower", "FOO", "foo") + f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz") + f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz") + f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`) + f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`) + f("htmlEscape", "foo < 10\nabc", "foo < 10\nabc") + f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`) + f("stripPort", "foo", "foo") + f("stripPort", "foo:1234", "foo") + f("stripDomain", "foo.bar.baz", "foo") + f("stripDomain", "foo.bar:123", "foo:123") + + // check "match" func + matchFunc := funcs["match"].(func(pattern, s string) (bool, error)) + if _, err := matchFunc("invalid[regexp", "abc"); err == nil { + t.Fatalf("expecting non-nil error on invalid regexp") + } + ok, err := matchFunc("abc", "def") + if err != nil { + t.Fatalf("unexpected error") + } + if ok { + t.Fatalf("unexpected match") + } + ok, err = matchFunc("a.+b", "acsdb") + if err != nil { + t.Fatalf("unexpected error") + } + if !ok { + t.Fatalf("unexpected mismatch") + } +} + func mkTemplate(current, replacement interface{}) textTemplate { tmpl := textTemplate{} if current != nil { diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index e3a0c4bd51..fe82788221 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -72,7 +72,8 @@ services: - "--rule=/etc/alerts/*.yml" # display source of alerts in grafana - "--external.url=http://127.0.0.1:3000" #grafana outside container - - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":"{{$$expr|quotesEscape|crlfEscape|queryEscape}}"},{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' ## when copypaste the line be aware of '$$' for escaping in '$expr' + # when copypaste the line be aware of '$$' for escaping in '$expr' + - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' networks: - vm_net restart: always diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ff278f7d89..2be56bec06 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,9 +15,12 @@ The following tip changes can be tested by building VictoriaMetrics components f ## v1.79.x long-time support release (LTS) +**Update note 1:** [vmalert](https://docs.victoriametrics.com/vmalert.html): the `crlfEscape` [template function](https://docs.victoriametrics.com/vmalert.html#template-functions) becames obsolete starting from this release. It can be safely removed from alerting templates, since `\n` chars are properly escaped with other `*Escape` functions now. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3139) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890) issue for details. + * SECURITY: update Go builder to v1.19.3. This fixes [CVE-2022 security issue](https://github.com/golang/go/issues/56328). See [the changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.19.3+label%3ACherryPickApproved). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change severity level for log messages about failed attempts for sending data to remote storage from `error` to `warn`. The message for about all failed send attempts remains at `error` severity level. +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly escape string passed to `quotesEscape` [template function](https://docs.victoriametrics.com/vmalert.html#template-functions), so it can be safely embedded into JSON string. This makes obsolete the `crlfEscape` function. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3139) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890) issue. * BUGFIX: `vmselect`: expose missing metric `vm_cache_size_max_bytes{type="promql/rollupResult"}` . This metric is used for monitoring rollup cache usage with the query `vm_cache_size_bytes{type="promql/rollupResult"} / vm_cache_size_max_bytes{type="promql/rollupResult"}` in the same way as this is done for other cache types. diff --git a/docs/vmalert.md b/docs/vmalert.md index dac3b750b6..cf6a7bc5b7 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -679,8 +679,8 @@ The shortlist of configuration flags is the following: -evaluationInterval duration How often to evaluate the rules (default 1m0s) -external.alert.source string - External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. - eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/vmalert/api/v1/alert?group_id=&alert_id=' is used + External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used + If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used. -external.label array Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. Supports an array of values separated by comma or specified via multiple flags.