diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 0762cc6344..912daefa46 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -8,6 +8,7 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/VictoriaMetrics/metrics" @@ -326,8 +327,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig } // init reload metrics with positive values to improve alerting conditions - configSuccess.Set(1) - configTimestamp.Set(fasttime.UnixTimestamp()) + setConfigSuccess(fasttime.UnixTimestamp()) parseFn := config.Parse for { select { @@ -347,22 +347,19 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig parseFn = config.ParseSilent } if err := notifier.Reload(); err != nil { - configReloadErrors.Inc() - configSuccess.Set(0) + setConfigError(err) logger.Errorf("failed to reload notifier config: %s", err) continue } err := templates.Load(*ruleTemplatesPath, false) if err != nil { - configReloadErrors.Inc() - configSuccess.Set(0) + setConfigError(err) logger.Errorf("failed to load new templates: %s", err) continue } newGroupsCfg, err := parseFn(*rulePath, validateTplFn, *validateExpressions) if err != nil { - configReloadErrors.Inc() - configSuccess.Set(0) + setConfigError(err) logger.Errorf("cannot parse configuration file: %s", err) continue } @@ -371,19 +368,18 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig // set success to 1 since previous reload // could have been unsuccessful configSuccess.Set(1) + setConfigError(nil) // config didn't change - skip it continue } if err := m.update(ctx, newGroupsCfg, false); err != nil { - configReloadErrors.Inc() - configSuccess.Set(0) + setConfigError(err) logger.Errorf("error while reloading rules: %s", err) continue } templates.Reload() groupsCfg = newGroupsCfg - configSuccess.Set(1) - configTimestamp.Set(fasttime.UnixTimestamp()) + setConfigSuccess(fasttime.UnixTimestamp()) logger.Infof("Rules reloaded successfully from %q", *rulePath) } } @@ -399,3 +395,40 @@ func configsEqual(a, b []config.Group) bool { } return true } + +// setConfigSuccess sets config reload status to 1. +func setConfigSuccess(at uint64) { + configSuccess.Set(1) + configTimestamp.Set(fasttime.UnixTimestamp()) + // reset the error if any + setConfigErr(nil) +} + +// setConfigError sets config reload status to 0. +func setConfigError(err error) { + configReloadErrors.Inc() + configSuccess.Set(0) + setConfigErr(err) +} + +var ( + configErrMu sync.RWMutex + // configErr represent the error message from the last + // config reload. + configErr error +) + +func setConfigErr(err error) { + configErrMu.Lock() + configErr = err + configErrMu.Unlock() +} + +func configError() error { + configErrMu.RLock() + defer configErrMu.RUnlock() + if configErr != nil { + return configErr + } + return nil +} diff --git a/app/vmalert/static/js/custom.js b/app/vmalert/static/js/custom.js index 8e06ca2a7f..5a1f7f2e7c 100644 --- a/app/vmalert/static/js/custom.js +++ b/app/vmalert/static/js/custom.js @@ -7,9 +7,11 @@ function collapseAll() { } function toggleByID(id) { - let el = $("#" + id); - if (el.length > 0) { - el.click(); + if (id) { + let el = $("#" + id); + if (el.length > 0) { + el.click(); + } } } @@ -36,4 +38,4 @@ $(document).ready(function () { $(document).ready(function () { $('[data-bs-toggle="tooltip"]').tooltip(); -}); \ No newline at end of file +}); diff --git a/app/vmalert/tpl/header.qtpl b/app/vmalert/tpl/header.qtpl index d08ae138c8..e2c6b0fd5c 100644 --- a/app/vmalert/tpl/header.qtpl +++ b/app/vmalert/tpl/header.qtpl @@ -6,8 +6,8 @@ "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" ) %} -{% func Header(r *http.Request, navItems []NavItem, title string) %} - {%code prefix := utils.Prefix(r.URL.Path) %} +{% func Header(r *http.Request, navItems []NavItem, title string, userErr error) %} +{%code prefix := utils.Prefix(r.URL.Path) %} @@ -70,8 +70,9 @@ - {%= printNavItems(r, title, navItems) %} + {%= printNavItems(r, title, navItems, userErr) %}
+ {%= errorBody(userErr) %} {% endfunc %} @@ -82,7 +83,7 @@ type NavItem struct { } %} -{% func printNavItems(r *http.Request, current string, items []NavItem) %} +{% func printNavItems(r *http.Request, current string, items []NavItem, userErr error) %} {%code prefix := utils.Prefix(r.URL.Path) %} @@ -103,5 +104,30 @@ type NavItem struct { {% endfor %} + {%= errorIcon(userErr) %} {% endfunc %} + +{% func errorIcon(err error) %} +{% if err != nil %} +
+ + + + + + + +
+{% endif %} +{% endfunc %} + +{% func errorBody(err error) %} +{% if err != nil %} +
+
+ {%s err.Error() %} +
+
+{% endif %} +{% endfunc %} diff --git a/app/vmalert/tpl/header.qtpl.go b/app/vmalert/tpl/header.qtpl.go index 0d0b46a498..424e17aceb 100644 --- a/app/vmalert/tpl/header.qtpl.go +++ b/app/vmalert/tpl/header.qtpl.go @@ -27,10 +27,10 @@ var ( ) //line app/vmalert/tpl/header.qtpl:9 -func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string) { +func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string, userErr error) { //line app/vmalert/tpl/header.qtpl:9 qw422016.N().S(` - `) +`) //line app/vmalert/tpl/header.qtpl:10 prefix := utils.Prefix(r.URL.Path) @@ -114,135 +114,251 @@ func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem `) //line app/vmalert/tpl/header.qtpl:73 - streamprintNavItems(qw422016, r, title, navItems) + streamprintNavItems(qw422016, r, title, navItems, userErr) //line app/vmalert/tpl/header.qtpl:73 qw422016.N().S(`
+ `) +//line app/vmalert/tpl/header.qtpl:75 + streamerrorBody(qw422016, userErr) +//line app/vmalert/tpl/header.qtpl:75 + qw422016.N().S(` `) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 } -//line app/vmalert/tpl/header.qtpl:75 -func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string) { -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 +func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string, userErr error) { +//line app/vmalert/tpl/header.qtpl:76 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/tpl/header.qtpl:75 - StreamHeader(qw422016, r, navItems, title) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 + StreamHeader(qw422016, r, navItems, title, userErr) +//line app/vmalert/tpl/header.qtpl:76 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 } -//line app/vmalert/tpl/header.qtpl:75 -func Header(r *http.Request, navItems []NavItem, title string) string { -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 +func Header(r *http.Request, navItems []NavItem, title string, userErr error) string { +//line app/vmalert/tpl/header.qtpl:76 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/tpl/header.qtpl:75 - WriteHeader(qb422016, r, navItems, title) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 + WriteHeader(qb422016, r, navItems, title, userErr) +//line app/vmalert/tpl/header.qtpl:76 qs422016 := string(qb422016.B) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 return qs422016 -//line app/vmalert/tpl/header.qtpl:75 +//line app/vmalert/tpl/header.qtpl:76 } -//line app/vmalert/tpl/header.qtpl:79 +//line app/vmalert/tpl/header.qtpl:80 type NavItem struct { Name string Url string } -//line app/vmalert/tpl/header.qtpl:85 -func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem) { -//line app/vmalert/tpl/header.qtpl:85 +//line app/vmalert/tpl/header.qtpl:86 +func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem, userErr error) { +//line app/vmalert/tpl/header.qtpl:86 qw422016.N().S(` `) -//line app/vmalert/tpl/header.qtpl:87 +//line app/vmalert/tpl/header.qtpl:88 prefix := utils.Prefix(r.URL.Path) -//line app/vmalert/tpl/header.qtpl:88 +//line app/vmalert/tpl/header.qtpl:89 qw422016.N().S(` `) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 } -//line app/vmalert/tpl/header.qtpl:107 -func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem) { -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 +func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem, userErr error) { +//line app/vmalert/tpl/header.qtpl:109 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/tpl/header.qtpl:107 - streamprintNavItems(qw422016, r, current, items) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 + streamprintNavItems(qw422016, r, current, items, userErr) +//line app/vmalert/tpl/header.qtpl:109 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 } -//line app/vmalert/tpl/header.qtpl:107 -func printNavItems(r *http.Request, current string, items []NavItem) string { -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 +func printNavItems(r *http.Request, current string, items []NavItem, userErr error) string { +//line app/vmalert/tpl/header.qtpl:109 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/tpl/header.qtpl:107 - writeprintNavItems(qb422016, r, current, items) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 + writeprintNavItems(qb422016, r, current, items, userErr) +//line app/vmalert/tpl/header.qtpl:109 qs422016 := string(qb422016.B) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 return qs422016 -//line app/vmalert/tpl/header.qtpl:107 +//line app/vmalert/tpl/header.qtpl:109 +} + +//line app/vmalert/tpl/header.qtpl:111 +func streamerrorIcon(qw422016 *qt422016.Writer, err error) { +//line app/vmalert/tpl/header.qtpl:111 + qw422016.N().S(` +`) +//line app/vmalert/tpl/header.qtpl:112 + if err != nil { +//line app/vmalert/tpl/header.qtpl:112 + qw422016.N().S(` +
+ + + + + + + +
+`) +//line app/vmalert/tpl/header.qtpl:122 + } +//line app/vmalert/tpl/header.qtpl:122 + qw422016.N().S(` +`) +//line app/vmalert/tpl/header.qtpl:123 +} + +//line app/vmalert/tpl/header.qtpl:123 +func writeerrorIcon(qq422016 qtio422016.Writer, err error) { +//line app/vmalert/tpl/header.qtpl:123 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/tpl/header.qtpl:123 + streamerrorIcon(qw422016, err) +//line app/vmalert/tpl/header.qtpl:123 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/tpl/header.qtpl:123 +} + +//line app/vmalert/tpl/header.qtpl:123 +func errorIcon(err error) string { +//line app/vmalert/tpl/header.qtpl:123 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/tpl/header.qtpl:123 + writeerrorIcon(qb422016, err) +//line app/vmalert/tpl/header.qtpl:123 + qs422016 := string(qb422016.B) +//line app/vmalert/tpl/header.qtpl:123 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/tpl/header.qtpl:123 + return qs422016 +//line app/vmalert/tpl/header.qtpl:123 +} + +//line app/vmalert/tpl/header.qtpl:125 +func streamerrorBody(qw422016 *qt422016.Writer, err error) { +//line app/vmalert/tpl/header.qtpl:125 + qw422016.N().S(` +`) +//line app/vmalert/tpl/header.qtpl:126 + if err != nil { +//line app/vmalert/tpl/header.qtpl:126 + qw422016.N().S(` +
+
+ `) +//line app/vmalert/tpl/header.qtpl:129 + qw422016.E().S(err.Error()) +//line app/vmalert/tpl/header.qtpl:129 + qw422016.N().S(` +
+
+`) +//line app/vmalert/tpl/header.qtpl:132 + } +//line app/vmalert/tpl/header.qtpl:132 + qw422016.N().S(` +`) +//line app/vmalert/tpl/header.qtpl:133 +} + +//line app/vmalert/tpl/header.qtpl:133 +func writeerrorBody(qq422016 qtio422016.Writer, err error) { +//line app/vmalert/tpl/header.qtpl:133 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/tpl/header.qtpl:133 + streamerrorBody(qw422016, err) +//line app/vmalert/tpl/header.qtpl:133 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/tpl/header.qtpl:133 +} + +//line app/vmalert/tpl/header.qtpl:133 +func errorBody(err error) string { +//line app/vmalert/tpl/header.qtpl:133 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/tpl/header.qtpl:133 + writeerrorBody(qb422016, err) +//line app/vmalert/tpl/header.qtpl:133 + qs422016 := string(qb422016.B) +//line app/vmalert/tpl/header.qtpl:133 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/tpl/header.qtpl:133 + return qs422016 +//line app/vmalert/tpl/header.qtpl:133 } diff --git a/app/vmalert/web.qtpl b/app/vmalert/web.qtpl index 55bc0e3e58..1b412c7c30 100644 --- a/app/vmalert/web.qtpl +++ b/app/vmalert/web.qtpl @@ -12,7 +12,7 @@ {% func Welcome(r *http.Request) %} - {%= tpl.Header(r, navItems, "vmalert") %} + {%= tpl.Header(r, navItems, "vmalert", configError()) %}

API:
{% for _, p := range apiLinks %} @@ -40,7 +40,7 @@ btn-primary {% func ListGroups(r *http.Request, originGroups []APIGroup) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "Groups") %} + {%= tpl.Header(r, navItems, "Groups", configError()) %} {%code filter := r.URL.Query().Get("filter") rOk := make(map[string]int) @@ -164,7 +164,7 @@ btn-primary {% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "Alerts") %} + {%= tpl.Header(r, navItems, "Alerts", configError()) %} {% if len(groupAlerts) > 0 %} Collapse All Expand All @@ -250,7 +250,7 @@ btn-primary {% endfunc %} {% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %} - {%= tpl.Header(r, navItems, "Notifiers") %} + {%= tpl.Header(r, navItems, "Notifiers", configError()) %} {% if len(targets) > 0 %} Collapse All Expand All @@ -307,7 +307,7 @@ btn-primary {% func Alert(r *http.Request, alert *APIAlert) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "") %} + {%= tpl.Header(r, navItems, "", configError()) %} {%code var labelKeys []string for k := range alert.Labels { @@ -394,7 +394,7 @@ btn-primary {% func RuleDetails(r *http.Request, rule APIRule) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "") %} + {%= tpl.Header(r, navItems, "", configError()) %} {%code var labelKeys []string for k := range rule.Labels { @@ -578,4 +578,4 @@ btn-primary func isNoMatch (r APIRule) bool { return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0 } -%} \ No newline at end of file +%} diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go index 029bc80642..97999a74f8 100644 --- a/app/vmalert/web.qtpl.go +++ b/app/vmalert/web.qtpl.go @@ -34,7 +34,7 @@ func StreamWelcome(qw422016 *qt422016.Writer, r *http.Request) { qw422016.N().S(` `) //line app/vmalert/web.qtpl:15 - tpl.StreamHeader(qw422016, r, navItems, "vmalert") + tpl.StreamHeader(qw422016, r, navItems, "vmalert", configError()) //line app/vmalert/web.qtpl:15 qw422016.N().S(`

@@ -207,7 +207,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, originGroups [ qw422016.N().S(` `) //line app/vmalert/web.qtpl:43 - tpl.StreamHeader(qw422016, r, navItems, "Groups") + tpl.StreamHeader(qw422016, r, navItems, "Groups", configError()) //line app/vmalert/web.qtpl:43 qw422016.N().S(` `) @@ -619,7 +619,7 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts [] qw422016.N().S(` `) //line app/vmalert/web.qtpl:167 - tpl.StreamHeader(qw422016, r, navItems, "Alerts") + tpl.StreamHeader(qw422016, r, navItems, "Alerts", configError()) //line app/vmalert/web.qtpl:167 qw422016.N().S(` `) @@ -885,7 +885,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n qw422016.N().S(` `) //line app/vmalert/web.qtpl:253 - tpl.StreamHeader(qw422016, r, navItems, "Notifiers") + tpl.StreamHeader(qw422016, r, navItems, "Notifiers", configError()) //line app/vmalert/web.qtpl:253 qw422016.N().S(` `) @@ -1065,7 +1065,7 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { qw422016.N().S(` `) //line app/vmalert/web.qtpl:310 - tpl.StreamHeader(qw422016, r, navItems, "") + tpl.StreamHeader(qw422016, r, navItems, "", configError()) //line app/vmalert/web.qtpl:310 qw422016.N().S(` `) @@ -1274,7 +1274,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) qw422016.N().S(` `) //line app/vmalert/web.qtpl:397 - tpl.StreamHeader(qw422016, r, navItems, "") + tpl.StreamHeader(qw422016, r, navItems, "", configError()) //line app/vmalert/web.qtpl:397 qw422016.N().S(` `) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5cad4bcd39..e3f7ccc09a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: accept timestamps in milliseconds at `start`, `end` and `time` query args in [Prometheus querying API](https://docs.victoriametrics.com/#prometheus-querying-api-usage). See [these docs](https://docs.victoriametrics.com/#timestamp-formats) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4459). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): update retry policy for pushing data to `-remoteWrite.url`. By default, vmalert will make multiple retry attempts with exponential delay. The total time spent during retry attempts shouldn't exceed `-remoteWrite.retryMaxTime` (default is 30s). When retry time is exceeded vmalert drops the data dedicated for `-remoteWrite.url`. Before, vmalert dropped data after 5 retry attempts with 1s delay between attempts (not configurable). See `-remoteWrite.retryMinInterval` and `-remoteWrite.retryMaxTime` cmd-line flags. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): expose `vmalert_remotewrite_send_duration_seconds_total` counter, which can be used for determining high saturation of every connection to remote storage with an alerting query `sum(rate(vmalert_remotewrite_send_duration_seconds_total[5m])) by(job, instance) > 0.9 * max(vmalert_remotewrite_concurrency) by(job, instance)`. This query triggers when a connection is saturated by more than 90%. This usually means that `-remoteWrite.concurrency` command-line flag must be increased in order to increase the number of concurrent writings into remote endpoint. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4516). +* FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): display the error message received during unsuccessful config reload in vmalert's UI. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4076) for details. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_duration_seconds` and `vmauth_unauthorized_user_request_duration_seconds` summary metrics for measuring requests latency per user. * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): show backup progress percentage in log during backup uploading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460). * FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): show restoring progress percentage in log during backup downloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460).