From 0ac1cdfff59f6285ed4750c2cb030587cd8b0f65 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk <andrew.chubatiuk@gmail.com> Date: Fri, 13 May 2022 23:43:07 +0300 Subject: [PATCH] added reusable templates support Signed-off-by: Andrii Chubatiuk <andrew.chubatiuk@gmail.com> --- app/vmalert/Makefile | 1 + app/vmalert/alerting.go | 5 +- app/vmalert/config/config_test.go | 13 +- .../testdata/{ => rules}/kube-good.rules | 0 .../{ => rules}/rules-query-good.rules | 0 .../{ => rules}/rules-replay-good.rules | 0 .../testdata/{ => rules}/rules0-bad.rules | 0 .../testdata/{ => rules}/rules0-good.rules | 0 .../testdata/{ => rules}/rules1-bad.rules | 0 .../testdata/{ => rules}/rules1-good.rules | 0 .../testdata/{ => rules}/rules2-good.rules | 0 .../testdata/{ => rules}/rules3-good.rules | 0 .../testdata/{ => rules}/rules4-good.rules | 0 .../{ => rules}/rules_interval_good.rules | 0 .../testdata/templates/templates0-good.tmpl | 3 + .../testdata/templates/templates1-good.tmpl | 3 + .../testdata/templates/templates2-good.tmpl | 3 + .../testdata/templates/templates3-good.tmpl | 3 + .../testdata/templates/templates4-good-tmpl | 3 + app/vmalert/group_test.go | 2 +- app/vmalert/main.go | 33 ++- app/vmalert/manager_test.go | 35 +-- app/vmalert/notifier/alert.go | 45 ++- app/vmalert/notifier/init.go | 9 +- app/vmalert/notifier/package_test.go | 7 +- .../template.go} | 184 ++++++++++-- app/vmalert/templates/template_test.go | 275 ++++++++++++++++++ .../templates/other/nested/bad0-test.tpl | 3 + .../templates/other/nested/good0-test.tpl | 9 + .../templates/templates/test/good0-test.tpl | 9 + docs/vmalert.md | 55 +++- 31 files changed, 624 insertions(+), 76 deletions(-) rename app/vmalert/config/testdata/{ => rules}/kube-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules-query-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules-replay-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules0-bad.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules0-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules1-bad.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules1-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules2-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules3-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules4-good.rules (100%) rename app/vmalert/config/testdata/{ => rules}/rules_interval_good.rules (100%) create mode 100644 app/vmalert/config/testdata/templates/templates0-good.tmpl create mode 100644 app/vmalert/config/testdata/templates/templates1-good.tmpl create mode 100644 app/vmalert/config/testdata/templates/templates2-good.tmpl create mode 100644 app/vmalert/config/testdata/templates/templates3-good.tmpl create mode 100644 app/vmalert/config/testdata/templates/templates4-good-tmpl rename app/vmalert/{notifier/template_func.go => templates/template.go} (70%) create mode 100644 app/vmalert/templates/template_test.go create mode 100644 app/vmalert/templates/templates/other/nested/bad0-test.tpl create mode 100644 app/vmalert/templates/templates/other/nested/good0-test.tpl create mode 100644 app/vmalert/templates/templates/test/good0-test.tpl diff --git a/app/vmalert/Makefile b/app/vmalert/Makefile index eebac7f01f..c96708456c 100644 --- a/app/vmalert/Makefile +++ b/app/vmalert/Makefile @@ -62,6 +62,7 @@ publish-vmalert: test-vmalert: go test -v -race -cover ./app/vmalert -loggerLevel=ERROR + go test -v -race -cover ./app/vmalert/templates go test -v -race -cover ./app/vmalert/datasource go test -v -race -cover ./app/vmalert/notifier go test -v -race -cover ./app/vmalert/config diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index 0f819cbd64..f294626158 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -12,6 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -152,7 +153,7 @@ type labelSet struct { // toLabels converts labels from given Metric // to labelSet which contains original and processed labels. -func (ar *AlertingRule) toLabels(m datasource.Metric, qFn notifier.QueryFn) (*labelSet, error) { +func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) { ls := &labelSet{ origin: make(map[string]string, len(m.Labels)), processed: make(map[string]string), @@ -382,7 +383,7 @@ func hash(labels map[string]string) uint64 { return hash.Sum64() } -func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) { +func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) { var err error if ls == nil { ls, err = ar.toLabels(m, qFn) diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 2b84c3effc..c81717ed01 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -10,18 +10,19 @@ import ( "gopkg.in/yaml.v2" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) func TestMain(m *testing.M) { - u, _ := url.Parse("https://victoriametrics.com/path") - notifier.InitTemplateFunc(u) + if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil { + os.Exit(1) + } os.Exit(m.Run()) } func TestParseGood(t *testing.T) { - if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil { + if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil { t.Errorf("error parsing files %s", err) } } @@ -32,7 +33,7 @@ func TestParseBad(t *testing.T) { expErr string }{ { - []string{"testdata/rules0-bad.rules"}, + []string{"testdata/rules/rules0-bad.rules"}, "unexpected token", }, { @@ -56,7 +57,7 @@ func TestParseBad(t *testing.T) { "either `record` or `alert` must be set", }, { - []string{"testdata/rules1-bad.rules"}, + []string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr", }, } diff --git a/app/vmalert/config/testdata/kube-good.rules b/app/vmalert/config/testdata/rules/kube-good.rules similarity index 100% rename from app/vmalert/config/testdata/kube-good.rules rename to app/vmalert/config/testdata/rules/kube-good.rules diff --git a/app/vmalert/config/testdata/rules-query-good.rules b/app/vmalert/config/testdata/rules/rules-query-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules-query-good.rules rename to app/vmalert/config/testdata/rules/rules-query-good.rules diff --git a/app/vmalert/config/testdata/rules-replay-good.rules b/app/vmalert/config/testdata/rules/rules-replay-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules-replay-good.rules rename to app/vmalert/config/testdata/rules/rules-replay-good.rules diff --git a/app/vmalert/config/testdata/rules0-bad.rules b/app/vmalert/config/testdata/rules/rules0-bad.rules similarity index 100% rename from app/vmalert/config/testdata/rules0-bad.rules rename to app/vmalert/config/testdata/rules/rules0-bad.rules diff --git a/app/vmalert/config/testdata/rules0-good.rules b/app/vmalert/config/testdata/rules/rules0-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules0-good.rules rename to app/vmalert/config/testdata/rules/rules0-good.rules diff --git a/app/vmalert/config/testdata/rules1-bad.rules b/app/vmalert/config/testdata/rules/rules1-bad.rules similarity index 100% rename from app/vmalert/config/testdata/rules1-bad.rules rename to app/vmalert/config/testdata/rules/rules1-bad.rules diff --git a/app/vmalert/config/testdata/rules1-good.rules b/app/vmalert/config/testdata/rules/rules1-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules1-good.rules rename to app/vmalert/config/testdata/rules/rules1-good.rules diff --git a/app/vmalert/config/testdata/rules2-good.rules b/app/vmalert/config/testdata/rules/rules2-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules2-good.rules rename to app/vmalert/config/testdata/rules/rules2-good.rules diff --git a/app/vmalert/config/testdata/rules3-good.rules b/app/vmalert/config/testdata/rules/rules3-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules3-good.rules rename to app/vmalert/config/testdata/rules/rules3-good.rules diff --git a/app/vmalert/config/testdata/rules4-good.rules b/app/vmalert/config/testdata/rules/rules4-good.rules similarity index 100% rename from app/vmalert/config/testdata/rules4-good.rules rename to app/vmalert/config/testdata/rules/rules4-good.rules diff --git a/app/vmalert/config/testdata/rules_interval_good.rules b/app/vmalert/config/testdata/rules/rules_interval_good.rules similarity index 100% rename from app/vmalert/config/testdata/rules_interval_good.rules rename to app/vmalert/config/testdata/rules/rules_interval_good.rules diff --git a/app/vmalert/config/testdata/templates/templates0-good.tmpl b/app/vmalert/config/testdata/templates/templates0-good.tmpl new file mode 100644 index 0000000000..617b712d8c --- /dev/null +++ b/app/vmalert/config/testdata/templates/templates0-good.tmpl @@ -0,0 +1,3 @@ +{{ define "template0" }} +Visit {{ externalURL }} +{{ end }} \ No newline at end of file diff --git a/app/vmalert/config/testdata/templates/templates1-good.tmpl b/app/vmalert/config/testdata/templates/templates1-good.tmpl new file mode 100644 index 0000000000..69448a65f5 --- /dev/null +++ b/app/vmalert/config/testdata/templates/templates1-good.tmpl @@ -0,0 +1,3 @@ +{{ define "template1" }} +{{ 1048576 | humanize1024 }} +{{ end }} \ No newline at end of file diff --git a/app/vmalert/config/testdata/templates/templates2-good.tmpl b/app/vmalert/config/testdata/templates/templates2-good.tmpl new file mode 100644 index 0000000000..d0cf31764c --- /dev/null +++ b/app/vmalert/config/testdata/templates/templates2-good.tmpl @@ -0,0 +1,3 @@ +{{ define "template2" }} +{{ 1048576 | humanize1024 }} +{{ end }} \ No newline at end of file diff --git a/app/vmalert/config/testdata/templates/templates3-good.tmpl b/app/vmalert/config/testdata/templates/templates3-good.tmpl new file mode 100644 index 0000000000..05e26ac484 --- /dev/null +++ b/app/vmalert/config/testdata/templates/templates3-good.tmpl @@ -0,0 +1,3 @@ +{{ define "template3" }} +{{ printf "%s to %s!" "welcome" "hell" | toUpper }} +{{ end }} \ No newline at end of file diff --git a/app/vmalert/config/testdata/templates/templates4-good-tmpl b/app/vmalert/config/testdata/templates/templates4-good-tmpl new file mode 100644 index 0000000000..312eed6db3 --- /dev/null +++ b/app/vmalert/config/testdata/templates/templates4-good-tmpl @@ -0,0 +1,3 @@ +{{ define "template3" }} +{{ 1230912039102391023.0 | humanizeDuration }} +{{ end }} \ No newline at end of file diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go index 8322e65d02..247ed05d67 100644 --- a/app/vmalert/group_test.go +++ b/app/vmalert/group_test.go @@ -157,7 +157,7 @@ func TestUpdateWith(t *testing.T) { func TestGroupStart(t *testing.T) { // TODO: make parsing from string instead of file - groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true) + groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, true, true) if err != nil { t.Fatalf("failed to parse rules: %s", err) } diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 9366ad3500..34609d1ee4 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -15,6 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo" "github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" @@ -34,6 +35,13 @@ Examples: absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`) + ruleTemplatesPath = flagutil.NewArray("rule.templates", `Path or glob pattern to location with go template definitions + for rules annotations templating. Flag can be specified multiple times. +Examples: + -rule.templates="/path/to/file". Path to a single file with go templates + -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, +absolute path to all .tpl files in root.`) + rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+ "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead") @@ -73,10 +81,12 @@ func main() { envflag.Parse() buildinfo.Init() logger.Init() + err := templates.Load(*ruleTemplatesPath, true) + if err != nil { + logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err) + } if *dryRun { - u, _ := url.Parse("https://victoriametrics.com/") - notifier.InitTemplateFunc(u) groups, err := config.Parse(*rulePath, true, true) if err != nil { logger.Fatalf("failed to parse %q: %s", *rulePath, err) @@ -91,7 +101,7 @@ func main() { if err != nil { logger.Fatalf("failed to init `external.url`: %s", err) } - notifier.InitTemplateFunc(eu) + alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates) if err != nil { logger.Fatalf("failed to init `external.alert.source`: %s", err) @@ -105,7 +115,6 @@ func main() { if rw == nil { logger.Fatalf("remoteWrite.url can't be empty in replay mode") } - notifier.InitTemplateFunc(eu) groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions) if err != nil { logger.Fatalf("cannot parse configuration file: %s", err) @@ -127,7 +136,6 @@ func main() { if err != nil { logger.Fatalf("failed to init: %s", err) } - logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";")) groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions) if err != nil { @@ -281,7 +289,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig case <-ctx.Done(): return case <-sighupCh: - logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath) + tmplMsg := "" + if len(*ruleTemplatesPath) > 0 { + tmplMsg = fmt.Sprintf("and templates %q ", *ruleTemplatesPath) + } + logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg) configReloads.Inc() case <-configCheckCh: } @@ -291,6 +303,13 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig logger.Errorf("failed to reload notifier config: %s", err) continue } + err := templates.Load(*ruleTemplatesPath, false) + if err != nil { + configReloadErrors.Inc() + configSuccess.Set(0) + logger.Errorf("failed to load new templates: %s", err) + continue + } newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions) if err != nil { configReloadErrors.Inc() @@ -299,6 +318,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig continue } if configsEqual(newGroupsCfg, groupsCfg) { + templates.Reload() // set success to 1 since previous reload // could have been unsuccessful configSuccess.Set(1) @@ -311,6 +331,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig logger.Errorf("error while reloading rules: %s", err) continue } + templates.Reload() groupsCfg = newGroupsCfg configSuccess.Set(1) configTimestamp.Set(fasttime.UnixTimestamp()) diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go index 9b7b1faf15..2ac3574caa 100644 --- a/app/vmalert/manager_test.go +++ b/app/vmalert/manager_test.go @@ -3,7 +3,6 @@ package main import ( "context" "math/rand" - "net/url" "os" "strings" "sync" @@ -14,11 +13,13 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" ) func TestMain(m *testing.M) { - u, _ := url.Parse("https://victoriametrics.com/path") - notifier.InitTemplateFunc(u) + if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil { + os.Exit(1) + } os.Exit(m.Run()) } @@ -47,9 +48,9 @@ func TestManagerUpdateConcurrent(t *testing.T) { "config/testdata/dir/rules0-bad.rules", "config/testdata/dir/rules1-good.rules", "config/testdata/dir/rules1-bad.rules", - "config/testdata/rules0-good.rules", - "config/testdata/rules1-good.rules", - "config/testdata/rules2-good.rules", + "config/testdata/rules/rules0-good.rules", + "config/testdata/rules/rules1-good.rules", + "config/testdata/rules/rules2-good.rules", } evalInterval := *evaluationInterval defer func() { *evaluationInterval = evalInterval }() @@ -125,7 +126,7 @@ func TestManagerUpdate(t *testing.T) { }{ { name: "update good rules", - initPath: "config/testdata/rules0-good.rules", + initPath: "config/testdata/rules/rules0-good.rules", updatePath: "config/testdata/dir/rules1-good.rules", want: []*Group{ { @@ -150,18 +151,18 @@ func TestManagerUpdate(t *testing.T) { }, { name: "update good rules from 1 to 2 groups", - initPath: "config/testdata/dir/rules1-good.rules", - updatePath: "config/testdata/rules0-good.rules", + initPath: "config/testdata/dir/rules/rules1-good.rules", + updatePath: "config/testdata/rules/rules0-good.rules", want: []*Group{ { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Name: "groupGorSingleAlert", Type: datasource.NewPrometheusType(), Rules: []Rule{VMRows}, Interval: defaultEvalInterval, }, { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Interval: defaultEvalInterval, Type: datasource.NewPrometheusType(), Name: "TestGroup", Rules: []Rule{ @@ -172,18 +173,18 @@ func TestManagerUpdate(t *testing.T) { }, { name: "update with one bad rule file", - initPath: "config/testdata/rules0-good.rules", + initPath: "config/testdata/rules/rules0-good.rules", updatePath: "config/testdata/dir/rules2-bad.rules", want: []*Group{ { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Name: "groupGorSingleAlert", Type: datasource.NewPrometheusType(), Interval: defaultEvalInterval, Rules: []Rule{VMRows}, }, { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Interval: defaultEvalInterval, Name: "TestGroup", Type: datasource.NewPrometheusType(), @@ -196,17 +197,17 @@ func TestManagerUpdate(t *testing.T) { { name: "update empty dir rules from 0 to 2 groups", initPath: "config/testdata/empty/*", - updatePath: "config/testdata/rules0-good.rules", + updatePath: "config/testdata/rules/rules0-good.rules", want: []*Group{ { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Name: "groupGorSingleAlert", Type: datasource.NewPrometheusType(), Interval: defaultEvalInterval, Rules: []Rule{VMRows}, }, { - File: "config/testdata/rules0-good.rules", + File: "config/testdata/rules/rules0-good.rules", Interval: defaultEvalInterval, Type: datasource.NewPrometheusType(), Name: "TestGroup", Rules: []Rule{ diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index 96f5e5d1a1..8c3e7f2f5a 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -5,9 +5,10 @@ import ( "fmt" "io" "strings" - "text/template" + textTpl "text/template" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" @@ -90,26 +91,38 @@ var tplHeaders = []string{ // map of annotations. // Every alert could have a different datasource, so function // requires a queryFunction as an argument. -func (a *Alert) ExecTemplate(q QueryFn, labels, annotations map[string]string) (map[string]string, error) { +func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) { tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr} - return templateAnnotations(annotations, tplData, funcsWithQuery(q), true) + tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q)) + if err != nil { + return nil, fmt.Errorf("error getting a template: %w", err) + } + return templateAnnotations(annotations, tplData, tmpl, true) } // ExecTemplate executes the given template for given annotations map. -func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) { - return templateAnnotations(annotations, tpl, funcsWithQuery(q), true) +func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) { + tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q)) + if err != nil { + return nil, fmt.Errorf("error cloning template: %w", err) + } + return templateAnnotations(annotations, tplData, tmpl, true) } // ValidateTemplates validate annotations for possible template error, uses empty data for template population func ValidateTemplates(annotations map[string]string) error { - _, err := templateAnnotations(annotations, AlertTplData{ + tmpl, err := templates.Get() + if err != nil { + return err + } + _, err = templateAnnotations(annotations, AlertTplData{ Labels: map[string]string{}, Value: 0, - }, tmplFunc, false) + }, tmpl, false) return err } -func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap, execute bool) (map[string]string, error) { +func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) { var builder strings.Builder var buf bytes.Buffer eg := new(utils.ErrGroup) @@ -122,7 +135,7 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs builder.Grow(len(header) + len(text)) builder.WriteString(header) builder.WriteString(text) - if err := templateAnnotation(&buf, builder.String(), tData, funcs, execute); err != nil { + if err := templateAnnotation(&buf, builder.String(), tData, tmpl, execute); err != nil { r[key] = text eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err)) continue @@ -138,11 +151,17 @@ type tplData struct { ExternalURL string } -func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap, execute bool) error { - t := template.New("").Funcs(funcs).Option("missingkey=zero") - tpl, err := t.Parse(text) +func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error { + tpl, err := tmpl.Clone() if err != nil { - return fmt.Errorf("error parsing annotation: %w", err) + return fmt.Errorf("error cloning template before parse annotation: %w", err) + } + tpl, err = tpl.Parse(text) + if err != nil { + return fmt.Errorf("error parsing annotation template: %w", err) + } + if !execute { + return nil } if !execute { return nil diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index 99892d2b48..fdc49af1a7 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -3,9 +3,11 @@ package notifier import ( "flag" "fmt" + "net/url" "strings" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -83,6 +85,12 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu externalURL = extURL externalLabels = extLabels + eu, err := url.Parse(externalURL) + if err != nil { + return nil, fmt.Errorf("failed to parse external URL: %s", err) + } + + templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu)) if *configPath == "" && len(*addrs) == 0 { return nil, nil @@ -102,7 +110,6 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu return staticNotifiersFn, nil } - var err error cw, err = newWatcher(*configPath, gen) if err != nil { return nil, fmt.Errorf("failed to init config watcher: %s", err) diff --git a/app/vmalert/notifier/package_test.go b/app/vmalert/notifier/package_test.go index 11876ee3ac..3da0a93066 100644 --- a/app/vmalert/notifier/package_test.go +++ b/app/vmalert/notifier/package_test.go @@ -1,13 +1,14 @@ package notifier import ( - "net/url" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" "os" "testing" ) func TestMain(m *testing.M) { - u, _ := url.Parse("https://victoriametrics.com/path") - InitTemplateFunc(u) + if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil { + os.Exit(1) + } os.Exit(m.Run()) } diff --git a/app/vmalert/notifier/template_func.go b/app/vmalert/templates/template.go similarity index 70% rename from app/vmalert/notifier/template_func.go rename to app/vmalert/templates/template.go index c217bc6ff5..12de9b3c17 100644 --- a/app/vmalert/notifier/template_func.go +++ b/app/vmalert/templates/template.go @@ -11,26 +11,117 @@ // See the License for the specific language governing permissions and // limitations under the License. -package notifier +package templates import ( "errors" "fmt" + htmlTpl "html/template" + "io/ioutil" "math" "net" "net/url" + "path/filepath" "regexp" "sort" "strings" + "sync" "time" - htmlTpl "html/template" textTpl "text/template" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) +// go template execution fails when it's tree is empty +const defaultTemplate = `{{- define "default.template" -}}{{- end -}}` + +var tplMu sync.RWMutex + +type textTemplate struct { + current *textTpl.Template + replacement *textTpl.Template +} + +var masterTmpl textTemplate + +func newTemplate() *textTpl.Template { + tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs()) + return textTpl.Must(tmpl.Parse(defaultTemplate)) +} + +// Load func loads templates from multiple globs specified in pathPatterns and either +// sets them directly to current template if it's undefined or with overwrite=true +// or sets replacement templates and adds templates with new names to a current +func Load(pathPatterns []string, overwrite bool) error { + var err error + tmpl := newTemplate() + for _, tp := range pathPatterns { + p, err := filepath.Glob(tp) + if err != nil { + return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err) + } + if len(p) > 0 { + tmpl, err = tmpl.ParseGlob(tp) + if err != nil { + return fmt.Errorf("failed to parse template glob %q: %w", tp, err) + } + } + } + if len(tmpl.Templates()) > 0 { + err := tmpl.Execute(ioutil.Discard, nil) + if err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + } + tplMu.Lock() + defer tplMu.Unlock() + if masterTmpl.current == nil || overwrite { + masterTmpl.replacement = nil + masterTmpl.current = newTemplate() + } else { + masterTmpl.replacement = newTemplate() + if err = copyTemplates(tmpl, masterTmpl.replacement, overwrite); err != nil { + return err + } + } + return copyTemplates(tmpl, masterTmpl.current, overwrite) +} + +func copyTemplates(from *textTpl.Template, to *textTpl.Template, overwrite bool) error { + if from == nil { + return nil + } + if to == nil { + to = newTemplate() + } + tmpl, err := from.Clone() + if err != nil { + return err + } + for _, t := range tmpl.Templates() { + if to.Lookup(t.Name()) == nil || overwrite { + to, err = to.AddParseTree(t.Name(), t.Tree) + if err != nil { + return fmt.Errorf("failed to add template %q: %w", t.Name(), err) + } + } + } + return nil +} + +// Reload func replaces current template with a replacement template +// which was set by Load with override=false +func Reload() { + tplMu.Lock() + defer tplMu.Unlock() + if masterTmpl.replacement != nil { + masterTmpl.current = masterTmpl.replacement + masterTmpl.replacement = nil + } +} + // metric is private copy of datasource.Metric, // it is used for templating annotations, // Labels as map simplifies templates evaluation. @@ -60,12 +151,62 @@ func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric { // for templating functions. type QueryFn func(query string) ([]datasource.Metric, error) -var tmplFunc textTpl.FuncMap +// UpdateWithFuncs updates existing or sets a new function map for a template +func UpdateWithFuncs(funcs textTpl.FuncMap) { + tplMu.Lock() + defer tplMu.Unlock() + masterTmpl.current = masterTmpl.current.Funcs(funcs) +} -// InitTemplateFunc initiates template helper functions -func InitTemplateFunc(externalURL *url.URL) { +// GetWithFuncs returns a copy of current template with additional FuncMap +// provided with funcs argument +func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) { + tplMu.RLock() + defer tplMu.RUnlock() + tmpl, err := masterTmpl.current.Clone() + if err != nil { + return nil, err + } + return tmpl.Funcs(funcs), nil +} + +// Get returns a copy of a template +func Get() (*textTpl.Template, error) { + tplMu.RLock() + defer tplMu.RUnlock() + return masterTmpl.current.Clone() +} + +// FuncsWithQuery returns a function map that depends on metric data +func FuncsWithQuery(query QueryFn) textTpl.FuncMap { + return textTpl.FuncMap{ + "query": func(q string) ([]metric, error) { + result, err := query(q) + if err != nil { + return nil, err + } + return datasourceMetricsToTemplateMetrics(result), nil + }, + } +} + +// FuncsWithExternalURL returns a function map that depends on externalURL value +func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap { + return textTpl.FuncMap{ + "externalURL": func() string { + return externalURL.String() + }, + + "pathPrefix": func() string { + return externalURL.Path + }, + } +} + +// templateFuncs initiates template helper functions +func templateFuncs() textTpl.FuncMap { // See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/ - tmplFunc = textTpl.FuncMap{ + return textTpl.FuncMap{ /* Strings */ // reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with @@ -219,12 +360,22 @@ func InitTemplateFunc(externalURL *url.URL) { // externalURL returns value of `external.url` flag "externalURL": func() string { - return externalURL.String() + // externalURL function supposed to be substituted at FuncsWithExteralURL(). + // it is present here only for validation purposes, when there is no + // provided datasource. + // + // return non-empty slice to pass validation with chained functions in template + return "" }, // pathPrefix returns a Path segment from the URL value in `external.url` flag "pathPrefix": func() string { - return externalURL.Path + // pathPrefix function supposed to be substituted at FuncsWithExteralURL(). + // it is present here only for validation purposes, when there is no + // provided datasource. + // + // return non-empty slice to pass validation with chained functions in template + return "" }, // pathEscape escapes the string so it can be safely placed inside a URL path segment, @@ -259,7 +410,7 @@ func InitTemplateFunc(externalURL *url.URL) { // execute "/api/v1/query?query=foo" request and will return // the first value in response. "query": func(q string) ([]metric, error) { - // query function supposed to be substituted at funcsWithQuery(). + // query function supposed to be substituted at FuncsWithQuery(). // it is present here only for validation purposes, when there is no // provided datasource. // @@ -316,21 +467,6 @@ func InitTemplateFunc(externalURL *url.URL) { } } -func funcsWithQuery(query QueryFn) textTpl.FuncMap { - fm := make(textTpl.FuncMap) - for k, fn := range tmplFunc { - fm[k] = fn - } - fm["query"] = func(q string) ([]metric, error) { - result, err := query(q) - if err != nil { - return nil, err - } - return datasourceMetricsToTemplateMetrics(result), nil - } - return fm -} - // Time is the number of milliseconds since the epoch // (1970-01-01 00:00 UTC) excluding leap seconds. type Time int64 diff --git a/app/vmalert/templates/template_test.go b/app/vmalert/templates/template_test.go new file mode 100644 index 0000000000..dc2cb09ce3 --- /dev/null +++ b/app/vmalert/templates/template_test.go @@ -0,0 +1,275 @@ +package templates + +import ( + "strings" + "testing" + textTpl "text/template" +) + +func mkTemplate(current, replacement interface{}) textTemplate { + tmpl := textTemplate{} + if current != nil { + switch val := current.(type) { + case string: + tmpl.current = textTpl.Must(newTemplate().Parse(val)) + } + } + if replacement != nil { + switch val := replacement.(type) { + case string: + tmpl.replacement = textTpl.Must(newTemplate().Parse(val)) + } + } + return tmpl +} + +func equalTemplates(t *testing.T, tmpls ...*textTpl.Template) bool { + var cmp *textTpl.Template + for i, tmpl := range tmpls { + if i == 0 { + cmp = tmpl + } else { + if cmp == nil || tmpl == nil { + if cmp != tmpl { + return false + } + continue + } + if len(tmpl.Templates()) != len(cmp.Templates()) { + return false + } + for _, t := range tmpl.Templates() { + tp := cmp.Lookup(t.Name()) + if tp == nil { + return false + } + if tp.Root.String() != t.Root.String() { + return false + } + } + } + } + return true +} + +func TestTemplates_Load(t *testing.T) { + testCases := []struct { + name string + initialTemplate textTemplate + pathPatterns []string + overwrite bool + expectedTemplate textTemplate + expErr string + }{ + { + "non existing path undefined template override", + mkTemplate(nil, nil), + []string{ + "templates/non-existing/good-*.tpl", + "templates/absent/good-*.tpl", + }, + true, + mkTemplate(``, nil), + "", + }, + { + "non existing path defined template override", + mkTemplate(` + {{- define "test.1" -}} + {{- printf "value" -}} + {{- end -}} + `, nil), + []string{ + "templates/non-existing/good-*.tpl", + "templates/absent/good-*.tpl", + }, + true, + mkTemplate(``, nil), + "", + }, + { + "existing path undefined template override", + mkTemplate(nil, nil), + []string{ + "templates/other/nested/good0-*.tpl", + "templates/test/good0-*.tpl", + }, + false, + mkTemplate(` + {{- define "good0-test.tpl" -}}{{- end -}} + {{- define "test.0" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.1" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.2" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.3" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + `, nil), + "", + }, + { + "existing path defined template override", + mkTemplate(` + {{- define "test.1" -}} + {{ printf "Hello %s!" "world" }} + {{- end -}} + `, nil), + []string{ + "templates/other/nested/good0-*.tpl", + "templates/test/good0-*.tpl", + }, + false, + mkTemplate(` + {{- define "good0-test.tpl" -}}{{- end -}} + {{- define "test.0" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.1" -}} + {{ printf "Hello %s!" "world" }} + {{- end -}} + {{- define "test.2" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.3" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + `, ` + {{- define "good0-test.tpl" -}}{{- end -}} + {{- define "test.0" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.1" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.2" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + {{- define "test.3" -}} + {{ printf "Hello %s!" externalURL }} + {{- end -}} + `), + "", + }, + { + "load template with syntax error", + mkTemplate(` + {{- define "test.1" -}} + {{ printf "Hello %s!" "world" }} + {{- end -}} + `, nil), + []string{ + "templates/other/nested/bad0-*.tpl", + "templates/test/good0-*.tpl", + }, + false, + mkTemplate(` + {{- define "test.1" -}} + {{ printf "Hello %s!" "world" }} + {{- end -}} + `, nil), + "failed to parse template glob", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + masterTmpl = tc.initialTemplate + err := Load(tc.pathPatterns, tc.overwrite) + if tc.expErr == "" && err != nil { + t.Error("happened error that wasn't expected: %w", err) + } + if tc.expErr != "" && err == nil { + t.Error("%+w", err) + t.Error("expected error that didn't happend") + } + if err != nil && !strings.Contains(err.Error(), tc.expErr) { + t.Error("%+w", err) + t.Error("expected string doesn't exist in error message") + } + if !equalTemplates(t, masterTmpl.replacement, tc.expectedTemplate.replacement) { + t.Fatalf("replacement template is not as expected") + } + if !equalTemplates(t, masterTmpl.current, tc.expectedTemplate.current) { + t.Fatalf("current template is not as expected") + } + }) + } +} + +func TestTemplates_Reload(t *testing.T) { + testCases := []struct { + name string + initialTemplate textTemplate + expectedTemplate textTemplate + }{ + { + "empty current and replacement templates", + mkTemplate(nil, nil), + mkTemplate(nil, nil), + }, + { + "empty current template only", + mkTemplate(` + {{- define "test.1" -}} + {{- printf "value" -}} + {{- end -}} + `, nil), + mkTemplate(` + {{- define "test.1" -}} + {{- printf "value" -}} + {{- end -}} + `, nil), + }, + { + "empty replacement template only", + mkTemplate(nil, ` + {{- define "test.1" -}} + {{- printf "value" -}} + {{- end -}} + `), + mkTemplate(` + {{- define "test.1" -}} + {{- printf "value" -}} + {{- end -}} + `, nil), + }, + { + "defined both templates", + mkTemplate(` + {{- define "test.0" -}} + {{- printf "value" -}} + {{- end -}} + {{- define "test.1" -}} + {{- printf "before" -}} + {{- end -}} + `, ` + {{- define "test.1" -}} + {{- printf "after" -}} + {{- end -}} + `), + mkTemplate(` + {{- define "test.1" -}} + {{- printf "after" -}} + {{- end -}} + `, nil), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + masterTmpl = tc.initialTemplate + Reload() + if !equalTemplates(t, masterTmpl.replacement, tc.expectedTemplate.replacement) { + t.Fatalf("replacement template is not as expected") + } + if !equalTemplates(t, masterTmpl.current, tc.expectedTemplate.current) { + t.Fatalf("current template is not as expected") + } + }) + } +} diff --git a/app/vmalert/templates/templates/other/nested/bad0-test.tpl b/app/vmalert/templates/templates/other/nested/bad0-test.tpl new file mode 100644 index 0000000000..4ed6658825 --- /dev/null +++ b/app/vmalert/templates/templates/other/nested/bad0-test.tpl @@ -0,0 +1,3 @@ +{{- define "test.1" -}} + {{ printf "Hello %s!" externalURL" }} +{{- end -}} \ No newline at end of file diff --git a/app/vmalert/templates/templates/other/nested/good0-test.tpl b/app/vmalert/templates/templates/other/nested/good0-test.tpl new file mode 100644 index 0000000000..7c8b2924d7 --- /dev/null +++ b/app/vmalert/templates/templates/other/nested/good0-test.tpl @@ -0,0 +1,9 @@ +{{- define "test.1" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} +{{- define "test.0" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} +{{- define "test.3" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} \ No newline at end of file diff --git a/app/vmalert/templates/templates/test/good0-test.tpl b/app/vmalert/templates/templates/test/good0-test.tpl new file mode 100644 index 0000000000..2b33463eac --- /dev/null +++ b/app/vmalert/templates/templates/test/good0-test.tpl @@ -0,0 +1,9 @@ +{{- define "test.2" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} +{{- define "test.0" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} +{{- define "test.3" -}} + {{ printf "Hello %s!" externalURL }} +{{- end -}} \ No newline at end of file diff --git a/docs/vmalert.md b/docs/vmalert.md index 1433856eb8..990ab73687 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -25,6 +25,7 @@ implementation and aims to be compatible with its syntax. * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite); * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling); * Lightweight without extra dependencies. +* Supports [reusable templates](#reusable-templates) for annotations. ## Limitations @@ -188,10 +189,53 @@ annotations: [ <labelname>: <tmpl_string> ] ``` -It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations -to format data, iterate over it or execute expressions. +It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations to format data, iterate over it or execute expressions. Additionally, `vmalert` provides some extra templating functions -listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go). +listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go) and [reusable templates](#reusable-templates). + +#### Reusable templates + +Like in Alertmanager you can use reusable templates to share same templates across anotations. Path to files with templates is provided with `-rule.templates` cli argument. E.g: + +`/etc/vmalert/templates/global/common.tpl` + +``` +{{ define "grafana.filter" -}} + {{- $labels := .arg0 -}} + {{- range $name, $label := . -}} + {{- if (ne $name "arg0") -}} + {{- ( or (index $labels $label) "All" ) | printf "&var-%s=%s" $label -}} + {{- end -}} + {{- end -}} +{{- end -}} +``` + +`/etc/vmalert/rules/project/rule.yaml` + +```yaml +groups: + - name: AlertGroupName + rules: + - alert: AlertName + expr: any_metric > 100 + for: 30s + labels: + alertname: 'Any metric is too high' + severity: 'warning' + annotations: + dashboard: '{{ $externalURL }}/d/dashboard?orgId=1{{ template "grafana.filter" (args .CommonLabels "account_id" "any_label") }}' +``` + +`vmalert` configuration flags: + +``` +./bin/vmalert -rule=/etc/vmalert/rules/**/*.yaml \ # Path to the fules with rules configuration + -rule.templates=/etc/vmalert/templates/**/*.tpl \ # Path to the files with rule templates + -datasource.url=http://victoriametrics:8428 \ # VM-single addr for executing rules expressions + -remoteWrite.url=http://victoriametrics:8428 \ # VM-single addr to persist alerts state and recording rules results + -remoteRead.url=http://victoriametrics:8428 \ # VM-single addr for restoring alerts state after restart + -notifier.url=http://alertmanager:9093 # AlertManager addr to send alerts when they trigger +``` #### Recording rules @@ -793,6 +837,11 @@ The shortlist of configuration flags is the following: absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Supports an array of values separated by comma or specified via multiple flags. + -rule.templates + Path or glob pattern to location with go template definitions for rules annotations templating. Flag can be specified multiple times. + Examples: + -rule.templates="/path/to/file". Path to a single file with go templates + -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, absolute path to all .tpl files in root. -rule.configCheckInterval duration Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead -rule.maxResolveDuration duration