diff --git a/app/vmalert/Makefile b/app/vmalert/Makefile index 8a3dbf24a..c56d28f61 100644 --- a/app/vmalert/Makefile +++ b/app/vmalert/Makefile @@ -70,6 +70,13 @@ run-vmalert: vmalert -evaluationInterval=3s \ -rule.configCheckInterval=10s +run-vmalert-sd: vmalert + ./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \ + -datasource.url=http://localhost:8428 \ + -remoteWrite.url=http://localhost:8428 \ + -notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \ + -configCheckInterval=10s + replay-vmalert: vmalert ./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \ -datasource.url=http://localhost:8428 \ diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 26a703019..4be872e14 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -43,7 +43,8 @@ To start using `vmalert` you will need the following things: * list of rules - PromQL/MetricsQL expressions to execute; * datasource address - reachable MetricsQL endpoint to run queries against; * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing, -aggregating alerts, and sending notifications. +aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via +[config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go). * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations) compatible storage to persist rules and alerts state info; * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from. @@ -689,8 +690,8 @@ The shortlist of configuration flags is the following: absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Supports an array of values separated by comma or specified via multiple flags. - -rule.configCheckInterval duration - Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -configCheckInterval duration + Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -rule.maxResolveDuration duration Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group. -rule.validateExpressions @@ -703,6 +704,14 @@ The shortlist of configuration flags is the following: Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower -tlsKeyFile string Path to file with TLS key. Used only if -tls is set + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set + -promscrape.consulSDCheckInterval duration + Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) + -promscrape.discovery.concurrency int + The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) + -promscrape.discovery.concurrentWaitTime duration + The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -version Show VictoriaMetrics version ``` @@ -711,7 +720,7 @@ The shortlist of configuration flags is the following: `vmalert` supports "hot" config reload via the following methods: * send SIGHUP signal to `vmalert` process; * send GET request to `/-/reload` endpoint; -* configure `-rule.configCheckInterval` flag for periodic reload +* configure `-configCheckInterval` flag for periodic reload on config change. ### URL params @@ -732,6 +741,88 @@ Please note, `params` are used only for executing rules expressions (requests to If there would be a conflict between URL params set in `datasource.url` flag and params in group definition the latter will have higher priority. +### Notifier configuration file + +Notifier also supports configuration vai file specified with flag `notifier.config`: +``` +./bin/vmalert -rule=app/vmalert/config/testdata/rules.good.rules \ + -datasource.url=http://localhost:8428 \ + -notifier.config=app/vmalert/notifier/testdata/consul.good.yaml +``` + +The configuration file allows to configure static notifiers or discover notifiers via +[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config). +For example: +``` +static_configs: + - targets: + - localhost:9093 + - localhost:9095 + +consul_sd_configs: + - server: localhost:8500 + services: + - alertmanager +``` + +The list of configured or discovered Notifiers can be explored via [UI](#Web). + +The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go) +is the following: +``` +# Per-target Notifier timeout when pushing alerts. +[ timeout: | default = 10s ] + +# Prefix for the HTTP path alerts are pushed to. +[ path_prefix: | default = / ] + +# Configures the protocol scheme used for requests. +[ scheme: | default = http ] + +# Sets the `Authorization` header on every request with the +# configured username and password. +# password and password_file are mutually exclusive. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional `Authorization` header configuration. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials to the credentials read from the configured file. + # It is mutually exclusive with `credentials`. + [ credentials_file: ] + +# Configures the scrape request's TLS settings. +# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config +tls_config: + [ ] + +# List of labeled statically configured Notifiers. +static_configs: + targets: + [ - '' ] + +# List of Consul service discovery configurations. +# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config +consul_sd_configs: + [ - ... ] + +# List of relabel configurations. +# Supports the same relabeling features as the rest of VictoriaMetrics components. +# See https://docs.victoriametrics.com/vmagent.html#relabeling +relabel_configs: + [ - ... ] + +``` + +The configuration file can be [hot-reloaded](#hot-config-reload). + ## Contributing diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index cafa5a0d3..9cd86fd21 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -12,9 +12,9 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" - "github.com/VictoriaMetrics/metrics" ) // AlertingRule is basic alert entity @@ -50,10 +50,10 @@ type AlertingRule struct { } type alertingRuleMetrics struct { - errors *gauge - pending *gauge - active *gauge - samples *gauge + errors *utils.Gauge + pending *utils.Gauge + active *utils.Gauge + samples *utils.Gauge } func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule { @@ -78,7 +78,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule } labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID()) - ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels), + ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels), func() float64 { ar.mu.RLock() defer ar.mu.RUnlock() @@ -90,7 +90,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule } return float64(num) }) - ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels), + ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels), func() float64 { ar.mu.RLock() defer ar.mu.RUnlock() @@ -102,7 +102,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule } return float64(num) }) - ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels), + ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels), func() float64 { ar.mu.RLock() defer ar.mu.RUnlock() @@ -111,7 +111,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule } return 1 }) - ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels), + ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels), func() float64 { ar.mu.RLock() defer ar.mu.RUnlock() @@ -122,10 +122,10 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule // Close unregisters rule metrics func (ar *AlertingRule) Close() { - metrics.UnregisterMetric(ar.metrics.active.name) - metrics.UnregisterMetric(ar.metrics.pending.name) - metrics.UnregisterMetric(ar.metrics.errors.name) - metrics.UnregisterMetric(ar.metrics.samples.name) + ar.metrics.active.Unregister() + ar.metrics.pending.Unregister() + ar.metrics.errors.Unregister() + ar.metrics.samples.Unregister() } // String implements Stringer interface @@ -153,7 +153,7 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([] return nil, fmt.Errorf("`query` template isn't supported in replay mode") } for _, s := range series { - // set additional labels to identify group and rule name + // set additional labels to identify group and rule Name if ar.Name != "" { s.SetLabel(alertNameLabel, ar.Name) } diff --git a/app/vmalert/group.go b/app/vmalert/group.go index dc4979895..addcfa3ff 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -41,15 +41,15 @@ type Group struct { } type groupMetrics struct { - iterationTotal *counter - iterationDuration *summary + iterationTotal *utils.Counter + iterationDuration *utils.Summary } func newGroupMetrics(name, file string) *groupMetrics { m := &groupMetrics{} labels := fmt.Sprintf(`group=%q, file=%q`, name, file) - m.iterationTotal = getOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels)) - m.iterationDuration = getOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels)) + m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels)) + m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels)) return m } @@ -122,7 +122,7 @@ func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule { } // ID return unique group ID that consists of -// rules file and group name +// rules file and group Name func (g *Group) ID() uint64 { g.mu.RLock() defer g.mu.RUnlock() @@ -213,8 +213,8 @@ func (g *Group) close() { close(g.doneCh) <-g.finishedCh - metrics.UnregisterMetric(g.metrics.iterationDuration.name) - metrics.UnregisterMetric(g.metrics.iterationTotal.name) + g.metrics.iterationDuration.Unregister() + g.metrics.iterationTotal.Unregister() for _, rule := range g.Rules { rule.Close() } @@ -222,7 +222,7 @@ func (g *Group) close() { var skipRandSleepOnGroupStart bool -func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewrite.Client) { +func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) { defer func() { close(g.finishedCh) }() // Spread group rules evaluation over time in order to reduce load on VictoriaMetrics. @@ -246,16 +246,7 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr } logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency) - e := &executor{rw: rw} - for _, nt := range nts { - ent := eNotifier{ - Notifier: nt, - alertsSent: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", nt.Addr())), - alertsSendErrors: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", nt.Addr())), - } - e.notifiers = append(e.notifiers, ent) - } - + e := &executor{rw: rw, notifiers: nts} t := time.NewTicker(g.Interval) defer t.Stop() for { @@ -310,16 +301,10 @@ func getResolveDuration(groupInterval time.Duration) time.Duration { } type executor struct { - notifiers []eNotifier + notifiers func() []notifier.Notifier rw *remotewrite.Client } -type eNotifier struct { - notifier.Notifier - alertsSent *counter - alertsSendErrors *counter -} - func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error { res := make(chan error, len(rules)) if concurrency == 1 { @@ -400,11 +385,9 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur } errGr := new(utils.ErrGroup) - for _, nt := range e.notifiers { - nt.alertsSent.Add(len(alerts)) + for _, nt := range e.notifiers() { if err := nt.Send(ctx, alerts); err != nil { - nt.alertsSendErrors.Inc() - errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err)) + errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err)) } } return errGr.Err() diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go index b06252256..5cbbab0ad 100644 --- a/app/vmalert/group_test.go +++ b/app/vmalert/group_test.go @@ -212,7 +212,7 @@ func TestGroupStart(t *testing.T) { fs.add(m1) fs.add(m2) go func() { - g.start(context.Background(), []notifier.Notifier{fn}, nil) + g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil) close(finished) }() diff --git a/app/vmalert/helpers_test.go b/app/vmalert/helpers_test.go index bc0ce54cb..5fac5b1e0 100644 --- a/app/vmalert/helpers_test.go +++ b/app/vmalert/helpers_test.go @@ -63,6 +63,7 @@ type fakeNotifier struct { alerts []notifier.Alert } +func (*fakeNotifier) Close() {} func (*fakeNotifier) Addr() string { return "" } func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error { fn.Lock() diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 5fdeb811d..16cd5e39b 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -35,7 +35,10 @@ absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`) rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+ - "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes") + "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead") + + configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+ + "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.") httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections") evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules") @@ -47,14 +50,14 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier") externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`) - externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+ + externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+ "Pass multiple -label flags in order to add multiple label sets.") remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+ " For example, if lookback=1h then range from now() to now()-1h will be scanned.") remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.") - disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's name as label to generated alerts and time series.") + disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.") dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.") ) @@ -192,7 +195,7 @@ func newManager(ctx context.Context) (*manager, error) { } n := strings.IndexByte(s, '=') if n < 0 { - return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s) + return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s) } manager.labels[s[:n]] = s[n+1:] } @@ -254,8 +257,13 @@ See the docs at https://docs.victoriametrics.com/vmalert.html . func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) { var configCheckCh <-chan time.Time - if *rulesCheckInterval > 0 { - ticker := time.NewTicker(*rulesCheckInterval) + checkInterval := *configCheckInterval + if checkInterval == 0 && *rulesCheckInterval > 0 { + logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead") + checkInterval = *rulesCheckInterval + } + if checkInterval > 0 { + ticker := time.NewTicker(checkInterval) configCheckCh = ticker.C defer ticker.Stop() } @@ -272,6 +280,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig configReloads.Inc() case <-configCheckCh: } + if err := notifier.Reload(); err != nil { + configReloadErrors.Inc() + configSuccess.Set(0) + logger.Errorf("failed to reload notifier config: %s", err) + continue + } newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions) if err != nil { configReloadErrors.Inc() diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go index a52576421..4999723ea 100644 --- a/app/vmalert/main_test.go +++ b/app/vmalert/main_test.go @@ -100,7 +100,7 @@ groups: querierBuilder: &fakeQuerier{}, groups: make(map[uint64]*Group), labels: map[string]string{}, - notifiers: []notifier.Notifier{&fakeNotifier{}}, + notifiers: func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }, rw: &remotewrite.Client{}, } diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go index 9c2823cb7..24f872faf 100644 --- a/app/vmalert/manager.go +++ b/app/vmalert/manager.go @@ -17,7 +17,7 @@ import ( // manager controls group states type manager struct { querierBuilder datasource.QuerierBuilder - notifiers []notifier.Notifier + notifiers func() []notifier.Notifier rw *remotewrite.Client // remote read builder. @@ -109,7 +109,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set") } if arPresent && m.notifiers == nil { - return fmt.Errorf("config contains alerting rules but `-notifier.url` isn't set") + return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set") } type updateItem struct { diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go index cb21906dd..9b7b1faf1 100644 --- a/app/vmalert/manager_test.go +++ b/app/vmalert/manager_test.go @@ -40,7 +40,7 @@ func TestManagerUpdateConcurrent(t *testing.T) { m := &manager{ groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}, - notifiers: []notifier.Notifier{&fakeNotifier{}}, + notifiers: func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }, } paths := []string{ "config/testdata/dir/rules0-good.rules", @@ -223,7 +223,7 @@ func TestManagerUpdate(t *testing.T) { m := &manager{ groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}, - notifiers: []notifier.Notifier{&fakeNotifier{}}, + notifiers: func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }, } cfgInit := loadCfg(t, []string{tc.initPath}, true, true) @@ -311,9 +311,11 @@ func TestManagerUpdateNegative(t *testing.T) { m := &manager{ groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}, - notifiers: tc.notifiers, rw: tc.rw, } + if tc.notifiers != nil { + m.notifiers = func() []notifier.Notifier { return tc.notifiers } + } err := m.update(context.Background(), []config.Group{tc.cfg}, false) if err == nil { t.Fatalf("expected to get error; got nil") diff --git a/app/vmalert/metrics.go b/app/vmalert/metrics.go deleted file mode 100644 index 011e394f7..000000000 --- a/app/vmalert/metrics.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import "github.com/VictoriaMetrics/metrics" - -type gauge struct { - name string - *metrics.Gauge -} - -func getOrCreateGauge(name string, f func() float64) *gauge { - return &gauge{ - name: name, - Gauge: metrics.GetOrCreateGauge(name, f), - } -} - -type counter struct { - name string - *metrics.Counter -} - -func getOrCreateCounter(name string) *counter { - return &counter{ - name: name, - Counter: metrics.GetOrCreateCounter(name), - } -} - -type summary struct { - name string - *metrics.Summary -} - -func getOrCreateSummary(name string) *summary { - return &summary{ - name: name, - Summary: metrics.GetOrCreateSummary(name), - } -} diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index cf132f828..4ecc5a55e 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -6,18 +6,41 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) // AlertManager represents integration provider with Prometheus alert manager // https://github.com/prometheus/alertmanager type AlertManager struct { - addr string - alertURL string - basicAuthUser string - basicAuthPass string - argFunc AlertURLGenerator - client *http.Client + addr string + argFunc AlertURLGenerator + client *http.Client + timeout time.Duration + + authCfg *promauth.Config + + metrics *metrics +} + +type metrics struct { + alertsSent *utils.Counter + alertsSendErrors *utils.Counter +} + +func newMetrics(addr string) *metrics { + return &metrics{ + alertsSent: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)), + alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)), + } +} + +// Close is a destructor method for AlertManager +func (am *AlertManager) Close() { + am.metrics.alertsSent.Unregister() + am.metrics.alertsSendErrors.Unregister() } // Addr returns address where alerts are sent. @@ -25,17 +48,36 @@ func (am AlertManager) Addr() string { return am.addr } // Send an alert or resolve message func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error { + am.metrics.alertsSent.Add(len(alerts)) + err := am.send(ctx, alerts) + if err != nil { + am.metrics.alertsSendErrors.Add(len(alerts)) + } + return err +} + +func (am *AlertManager) send(ctx context.Context, alerts []Alert) error { b := &bytes.Buffer{} writeamRequest(b, alerts, am.argFunc) - req, err := http.NewRequest("POST", am.alertURL, b) + req, err := http.NewRequest("POST", am.addr, b) if err != nil { return err } req.Header.Set("Content-Type", "application/json") + + if am.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, am.timeout) + defer cancel() + } + req = req.WithContext(ctx) - if am.basicAuthPass != "" { - req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass) + + if am.authCfg != nil { + if auth := am.authCfg.GetAuthHeader(); auth != "" { + req.Header.Set("Authorization", auth) + } } resp, err := am.client.Do(req) if err != nil { @@ -47,9 +89,9 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error { if resp.StatusCode != http.StatusOK { body, err := ioutil.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("failed to read response from %q: %w", am.alertURL, err) + return fmt.Errorf("failed to read response from %q: %w", am.addr, err) } - return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body)) + return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body)) } return nil } @@ -60,14 +102,31 @@ type AlertURLGenerator func(Alert) string const alertManagerPath = "/api/v2/alerts" // NewAlertManager is a constructor for AlertManager -func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager { - url := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath - return &AlertManager{ - addr: alertManagerURL, - alertURL: url, - argFunc: fn, - client: c, - basicAuthUser: user, - basicAuthPass: pass, +func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) { + tls := &promauth.TLSConfig{} + if authCfg.TLSConfig != nil { + tls = authCfg.TLSConfig } + tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify) + if err != nil { + return nil, fmt.Errorf("failed to create transport: %w", err) + } + + ba := &promauth.BasicAuthConfig{} + if authCfg.BasicAuth != nil { + ba = authCfg.BasicAuth + } + aCfg, err := utils.AuthConfig(ba.Username, ba.Password.String(), ba.PasswordFile, authCfg.BearerToken.String(), authCfg.BearerTokenFile) + if err != nil { + return nil, fmt.Errorf("failed to configure auth: %w", err) + } + + return &AlertManager{ + addr: alertManagerURL, + argFunc: fn, + authCfg: aCfg, + client: &http.Client{Transport: tr}, + timeout: timeout, + metrics: newMetrics(alertManagerURL), + }, nil } diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go index 0cf0fa148..646d9e418 100644 --- a/app/vmalert/notifier/alertmanager_test.go +++ b/app/vmalert/notifier/alertmanager_test.go @@ -8,11 +8,16 @@ import ( "strconv" "testing" "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) func TestAlertManager_Addr(t *testing.T) { const addr = "http://localhost" - am := NewAlertManager(addr, "", "", nil, nil) + am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0) + if err != nil { + t.Errorf("unexpected error: %s", err) + } if am.Addr() != addr { t.Errorf("expected to have %q; got %q", addr, am.Addr()) } @@ -75,9 +80,19 @@ func TestAlertManager_Send(t *testing.T) { }) srv := httptest.NewServer(mux) defer srv.Close() - am := NewAlertManager(srv.URL, baUser, baPass, func(alert Alert) string { + + aCfg := promauth.HTTPClientConfig{ + BasicAuth: &promauth.BasicAuthConfig{ + Username: baUser, + Password: promauth.NewSecret(baPass), + }, + } + am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string { return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10) - }, srv.Client()) + }, aCfg, 0) + if err != nil { + t.Errorf("unexpected error: %s", err) + } if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil { t.Error("expected connection error got nil") } diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go new file mode 100644 index 000000000..72e0663b2 --- /dev/null +++ b/app/vmalert/notifier/config.go @@ -0,0 +1,182 @@ +package notifier + +import ( + "crypto/md5" + "fmt" + "gopkg.in/yaml.v2" + "io/ioutil" + "net/url" + "path" + "path/filepath" + "strings" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" +) + +// Config contains list of supported configuration settings +// for Notifier +type Config struct { + // Scheme defines the HTTP scheme for Notifier address + Scheme string `yaml:"scheme,omitempty"` + // PathPrefix is added to URL path before adding alertManagerPath value + PathPrefix string `yaml:"path_prefix,omitempty"` + + // ConsulSDConfigs contains list of settings for service discovery via Consul + // see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config + ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + // StaticConfigs contains list of static targets + StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"` + + // HTTPClientConfig contains HTTP configuration for Notifier clients + HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` + // RelabelConfigs contains list of relabeling rules + RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"` + + // The timeout used when sending alerts. + Timeout utils.PromDuration `yaml:"timeout,omitempty"` + + // Checksum stores the hash of yaml definition for the config. + // May be used to detect any changes to the config file. + Checksum string + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` + + // This is set to the directory from where the config has been loaded. + baseDir string + + // stores already parsed RelabelConfigs object + parsedRelabelConfigs *promrelabel.ParsedConfigs +} + +// StaticConfig contains list of static targets in the following form: +// targets: +// [ - '' ] +type StaticConfig struct { + Targets []string `yaml:"targets"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + type config Config + if err := unmarshal((*config)(cfg)); err != nil { + return err + } + if cfg.Scheme == "" { + cfg.Scheme = "http" + } + if cfg.Timeout.Duration() == 0 { + cfg.Timeout = utils.NewPromDuration(time.Second * 10) + } + rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false) + if err != nil { + return fmt.Errorf("failed to parse relabeling config: %w", err) + } + cfg.parsedRelabelConfigs = rCfg + + b, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("failed to marshal configuration for checksum: %w", err) + } + h := md5.New() + h.Write(b) + cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil)) + return nil +} + +func parseConfig(path string) (*Config, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error reading config file: %w", err) + } + var cfg *Config + err = yaml.Unmarshal(data, &cfg) + if err != nil { + return nil, err + } + if len(cfg.XXX) > 0 { + var keys []string + for k := range cfg.XXX { + keys = append(keys, k) + } + return nil, fmt.Errorf("unknown fields in %s", strings.Join(keys, ", ")) + } + absPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err) + } + cfg.baseDir = filepath.Dir(absPath) + return cfg, nil +} + +func parseLabels(target string, metaLabels map[string]string, cfg *Config) (string, []prompbmarshal.Label, error) { + labels := mergeLabels(target, metaLabels, cfg) + labels = cfg.parsedRelabelConfigs.Apply(labels, 0, false) + labels = promrelabel.RemoveMetaLabels(labels[:0], labels) + // Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels). + // This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details. + labels = append([]prompbmarshal.Label{}, labels...) + + if len(labels) == 0 { + return "", nil, nil + } + schemeRelabeled := promrelabel.GetLabelValueByName(labels, "__scheme__") + if len(schemeRelabeled) == 0 { + schemeRelabeled = "http" + } + addressRelabeled := promrelabel.GetLabelValueByName(labels, "__address__") + if len(addressRelabeled) == 0 { + return "", nil, nil + } + if strings.Contains(addressRelabeled, "/") { + return "", nil, nil + } + addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled) + alertsPathRelabeled := promrelabel.GetLabelValueByName(labels, "__alerts_path__") + if !strings.HasPrefix(alertsPathRelabeled, "/") { + alertsPathRelabeled = "/" + alertsPathRelabeled + } + u := fmt.Sprintf("%s://%s%s", schemeRelabeled, addressRelabeled, alertsPathRelabeled) + if _, err := url.Parse(u); err != nil { + return "", nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q, metrics_path=%q (%q): %w", + u, cfg.Scheme, schemeRelabeled, target, addressRelabeled, alertsPathRelabeled, err) + } + return u, labels, nil +} + +func addMissingPort(scheme, target string) string { + if strings.Contains(target, ":") { + return target + } + if scheme == "https" { + target += ":443" + } else { + target += ":80" + } + return target +} + +func mergeLabels(target string, metaLabels map[string]string, cfg *Config) []prompbmarshal.Label { + // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + m := make(map[string]string) + m["__address__"] = target + m["__scheme__"] = cfg.Scheme + m["__alerts_path__"] = path.Join("/", cfg.PathPrefix, alertManagerPath) + for k, v := range metaLabels { + m[k] = v + } + result := make([]prompbmarshal.Label, 0, len(m)) + for k, v := range m { + result = append(result, prompbmarshal.Label{ + Name: k, + Value: v, + }) + } + return result +} diff --git a/app/vmalert/notifier/config_test.go b/app/vmalert/notifier/config_test.go new file mode 100644 index 000000000..e3dfe6b50 --- /dev/null +++ b/app/vmalert/notifier/config_test.go @@ -0,0 +1,31 @@ +package notifier + +import ( + "strings" + "testing" +) + +func TestConfigParseGood(t *testing.T) { + f := func(path string) { + _, err := parseConfig(path) + checkErr(t, err) + } + f("testdata/mixed.good.yaml") + f("testdata/consul.good.yaml") + f("testdata/static.good.yaml") +} + +func TestConfigParseBad(t *testing.T) { + f := func(path, expErr string) { + _, err := parseConfig(path) + if err == nil { + t.Fatalf("expected to get non-nil err for config %q", path) + } + if !strings.Contains(err.Error(), expErr) { + t.Errorf("expected err to contain %q; got %q instead", expErr, err) + } + } + + f("testdata/unknownFields.bad.yaml", "unknown field") + f("non-existing-file", "error reading") +} diff --git a/app/vmalert/notifier/config_watcher.go b/app/vmalert/notifier/config_watcher.go new file mode 100644 index 000000000..5cba98bd5 --- /dev/null +++ b/app/vmalert/notifier/config_watcher.go @@ -0,0 +1,244 @@ +package notifier + +import ( + "fmt" + "sync" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" +) + +// configWatcher supports dynamic reload of Notifier objects +// from static configuration and service discovery. +// Use newWatcher to create a new object. +type configWatcher struct { + cfg *Config + genFn AlertURLGenerator + wg sync.WaitGroup + + reloadCh chan struct{} + syncCh chan struct{} + + targetsMu sync.RWMutex + targets map[TargetType][]Target +} + +func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) { + cfg, err := parseConfig(path) + if err != nil { + return nil, err + } + cw := &configWatcher{ + cfg: cfg, + wg: sync.WaitGroup{}, + reloadCh: make(chan struct{}, 1), + syncCh: make(chan struct{}), + genFn: gen, + targetsMu: sync.RWMutex{}, + targets: make(map[TargetType][]Target), + } + return cw, cw.start() +} + +func (cw *configWatcher) notifiers() []Notifier { + cw.targetsMu.RLock() + defer cw.targetsMu.RUnlock() + + var notifiers []Notifier + for _, ns := range cw.targets { + for _, n := range ns { + notifiers = append(notifiers, n.Notifier) + } + + } + return notifiers +} + +func (cw *configWatcher) reload(path string) error { + select { + case cw.reloadCh <- struct{}{}: + default: + return nil + } + + defer func() { <-cw.reloadCh }() + + cfg, err := parseConfig(path) + if err != nil { + return err + } + if cfg.Checksum == cw.cfg.Checksum { + return nil + } + + // stop existing discovery + close(cw.syncCh) + cw.wg.Wait() + + // re-start cw with new config + cw.syncCh = make(chan struct{}) + cw.cfg = cfg + + cw.resetTargets() + return cw.start() +} + +const ( + addRetryBackoff = time.Millisecond * 100 + addRetryCount = 2 +) + +func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error { + var targets []Target + var errors []error + var count int + for { // retry addRetryCount times if first discovery attempts gave no results + targets, errors = targetsFromLabels(labelsFn, cw.cfg, cw.genFn) + for _, err := range errors { + return fmt.Errorf("failed to init notifier for %q: %s", typeK, err) + } + if len(targets) > 0 || count >= addRetryCount { + break + } + time.Sleep(addRetryBackoff) + } + + cw.setTargets(typeK, targets) + + cw.wg.Add(1) + go func() { + defer cw.wg.Done() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-cw.syncCh: + return + case <-ticker.C: + } + updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn) + for _, err := range errors { + logger.Errorf("failed to init notifier for %q: %s", typeK, err) + } + cw.setTargets(typeK, updateTargets) + } + }() + return nil +} + +func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) { + metaLabels, err := labelsFn() + if err != nil { + return nil, []error{fmt.Errorf("failed to get labels: %s", err)} + } + var targets []Target + var errors []error + duplicates := make(map[string]struct{}) + for _, labels := range metaLabels { + target := labels["__address__"] + u, processedLabels, err := parseLabels(target, labels, cfg) + if err != nil { + errors = append(errors, err) + continue + } + if len(u) == 0 { + continue + } + if _, ok := duplicates[u]; ok { // check for duplicates + if !*suppressDuplicateTargetErrors { + logger.Errorf("skipping duplicate target with identical address %q; "+ + "make sure service discovery and relabeling is set up properly; "+ + "original labels: %s; resulting labels: %s", + u, labels, processedLabels) + } + continue + } + duplicates[u] = struct{}{} + + am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration()) + if err != nil { + errors = append(errors, err) + continue + } + targets = append(targets, Target{ + Notifier: am, + Labels: processedLabels, + }) + } + return targets, errors +} + +type getLabels func() ([]map[string]string, error) + +func (cw *configWatcher) start() error { + if len(cw.cfg.StaticConfigs) > 0 { + var targets []Target + for _, cfg := range cw.cfg.StaticConfigs { + for _, target := range cfg.Targets { + address, labels, err := parseLabels(target, nil, cw.cfg) + if err != nil { + return fmt.Errorf("failed to parse labels for target %q: %s", target, err) + } + notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration()) + if err != nil { + return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err) + } + targets = append(targets, Target{ + Notifier: notifier, + Labels: labels, + }) + } + } + cw.setTargets(TargetStatic, targets) + } + + if len(cw.cfg.ConsulSDConfigs) > 0 { + err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]map[string]string, error) { + var labels []map[string]string + for i := range cw.cfg.ConsulSDConfigs { + sdc := &cw.cfg.ConsulSDConfigs[i] + targetLabels, err := sdc.GetLabels(cw.cfg.baseDir) + if err != nil { + return nil, fmt.Errorf("got labels err: %s", err) + } + labels = append(labels, targetLabels...) + } + return labels, nil + }) + if err != nil { + return fmt.Errorf("failed to start consulSD discovery: %s", err) + } + } + return nil +} + +func (cw *configWatcher) resetTargets() { + cw.targetsMu.Lock() + for _, targets := range cw.targets { + for _, t := range targets { + t.Close() + } + } + cw.targets = make(map[TargetType][]Target) + cw.targetsMu.Unlock() +} + +func (cw *configWatcher) setTargets(key TargetType, targets []Target) { + cw.targetsMu.Lock() + newT := make(map[string]Target) + for _, t := range targets { + newT[t.Addr()] = t + } + oldT := cw.targets[key] + + for _, ot := range oldT { + if _, ok := newT[ot.Addr()]; !ok { + ot.Notifier.Close() + } + } + cw.targets[key] = targets + cw.targetsMu.Unlock() +} diff --git a/app/vmalert/notifier/config_watcher_test.go b/app/vmalert/notifier/config_watcher_test.go new file mode 100644 index 000000000..9107157b4 --- /dev/null +++ b/app/vmalert/notifier/config_watcher_test.go @@ -0,0 +1,307 @@ +package notifier + +import ( + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" +) + +func TestConfigWatcherReload(t *testing.T) { + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(f.Name()) }() + + writeToFile(t, f.Name(), ` +static_configs: + - targets: + - localhost:9093 + - localhost:9094 +`) + cw, err := newWatcher(f.Name(), nil) + if err != nil { + t.Fatalf("failed to start config watcher: %s", err) + } + ns := cw.notifiers() + if len(ns) != 2 { + t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns) + } + + f2, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(f2.Name()) }() + + writeToFile(t, f2.Name(), ` +static_configs: + - targets: + - 127.0.0.1:9093 +`) + checkErr(t, cw.reload(f2.Name())) + + ns = cw.notifiers() + if len(ns) != 1 { + t.Fatalf("expected to have 1 notifier; got %d", len(ns)) + } + expAddr := "http://127.0.0.1:9093/api/v2/alerts" + if ns[0].Addr() != expAddr { + t.Fatalf("expected to get %q; got %q instead", expAddr, ns[0].Addr()) + } +} + +func TestConfigWatcherStart(t *testing.T) { + consulSDServer := newFakeConsulServer() + defer consulSDServer.Close() + + consulSDFile, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(consulSDFile.Name()) }() + + writeToFile(t, consulSDFile.Name(), fmt.Sprintf(` +scheme: https +path_prefix: proxy +consul_sd_configs: + - server: %s + services: + - alertmanager +`, consulSDServer.URL)) + + prevCheckInterval := *consul.SDCheckInterval + defer func() { *consul.SDCheckInterval = prevCheckInterval }() + + *consul.SDCheckInterval = time.Millisecond * 100 + + cw, err := newWatcher(consulSDFile.Name(), nil) + if err != nil { + t.Fatalf("failed to start config watcher: %s", err) + } + time.Sleep(*consul.SDCheckInterval * 2) + + if len(cw.notifiers()) != 2 { + t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers())) + } + + expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1) + expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2) + + n1, n2 := cw.notifiers()[0], cw.notifiers()[1] + if n1.Addr() != expAddr1 { + t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr()) + } + if n2.Addr() != expAddr2 { + t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr()) + } +} + +// TestConfigWatcherReloadConcurrent supposed to test concurrent +// execution of configuration update. +// Should be executed with -race flag +func TestConfigWatcherReloadConcurrent(t *testing.T) { + consulSDServer1 := newFakeConsulServer() + defer consulSDServer1.Close() + consulSDServer2 := newFakeConsulServer() + defer consulSDServer2.Close() + + consulSDFile, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(consulSDFile.Name()) }() + + writeToFile(t, consulSDFile.Name(), fmt.Sprintf(` +consul_sd_configs: + - server: %s + services: + - alertmanager + - server: %s + services: + - consul +`, consulSDServer1.URL, consulSDServer2.URL)) + + staticAndConsulSDFile, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(staticAndConsulSDFile.Name()) }() + + writeToFile(t, staticAndConsulSDFile.Name(), fmt.Sprintf(` +static_configs: + - targets: + - localhost:9093 + - localhost:9095 +consul_sd_configs: + - server: %s + services: + - alertmanager + - server: %s + services: + - consul +`, consulSDServer1.URL, consulSDServer2.URL)) + + paths := []string{ + staticAndConsulSDFile.Name(), + consulSDFile.Name(), + "testdata/static.good.yaml", + "unknownFields.bad.yaml", + } + + cw, err := newWatcher(paths[0], nil) + if err != nil { + t.Fatalf("failed to start config watcher: %s", err) + } + + const workers = 500 + const iterations = 10 + wg := sync.WaitGroup{} + wg.Add(workers) + for i := 0; i < workers; i++ { + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + rnd := rand.Intn(len(paths)) + _ = cw.reload(paths[rnd]) // update can fail and this is expected + _ = cw.notifiers() + } + }() + } + wg.Wait() +} + +func writeToFile(t *testing.T, file, b string) { + t.Helper() + checkErr(t, ioutil.WriteFile(file, []byte(b), 0644)) +} + +func checkErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("unexpected err: %s", err) + } +} + +const ( + fakeConsulService1 = "127.0.0.1:9093" + fakeConsulService2 = "127.0.0.1:9095" +) + +func newFakeConsulServer() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) { + rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`)) + }) + mux.HandleFunc("/v1/catalog/services", func(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("X-Consul-Index", "1") + rw.Write([]byte(`{ + "alertmanager": [ + "alertmanager", + "__scheme__=http" + ] +}`)) + }) + mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("X-Consul-Index", "1") + rw.Write([]byte(` +[ + { + "Node": { + "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72", + "Node": "machine", + "Address": "127.0.0.1", + "Datacenter": "dc1", + "TaggedAddresses": { + "lan": "127.0.0.1", + "lan_ipv4": "127.0.0.1", + "wan": "127.0.0.1", + "wan_ipv4": "127.0.0.1" + }, + "Meta": { + "consul-network-segment": "" + }, + "CreateIndex": 13, + "ModifyIndex": 14 + }, + "Service": { + "ID": "am1", + "Service": "alertmanager", + "Tags": [ + "alertmanager", + "__scheme__=http" + ], + "Address": "", + "Meta": null, + "Port": 9093, + "Weights": { + "Passing": 1, + "Warning": 1 + }, + "EnableTagOverride": false, + "Proxy": { + "Mode": "", + "MeshGateway": {}, + "Expose": {} + }, + "Connect": {}, + "CreateIndex": 16, + "ModifyIndex": 16 + } + }, + { + "Node": { + "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72", + "Node": "machine", + "Address": "127.0.0.1", + "Datacenter": "dc1", + "TaggedAddresses": { + "lan": "127.0.0.1", + "lan_ipv4": "127.0.0.1", + "wan": "127.0.0.1", + "wan_ipv4": "127.0.0.1" + }, + "Meta": { + "consul-network-segment": "" + }, + "CreateIndex": 13, + "ModifyIndex": 14 + }, + "Service": { + "ID": "am2", + "Service": "alertmanager", + "Tags": [ + "alertmanager", + "bad-node" + ], + "Address": "", + "Meta": null, + "Port": 9095, + "Weights": { + "Passing": 1, + "Warning": 1 + }, + "EnableTagOverride": false, + "Proxy": { + "Mode": "", + "MeshGateway": {}, + "Expose": {} + }, + "Connect": {}, + "CreateIndex": 15, + "ModifyIndex": 15 + } + } +]`)) + }) + + return httptest.NewServer(mux) +} diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index 0263e7e10..d6dc778ef 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -1,14 +1,19 @@ package notifier import ( + "flag" "fmt" - "net/http" + "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) var ( + configPath = flag.String("notifier.config", "", "Path to configuration file for notifiers") + suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery") + addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093") basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url") basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url") @@ -22,20 +27,117 @@ var ( "By default the server name from -notifier.url is used") ) -// Init creates a Notifier object based on provided flags. -func Init(gen AlertURLGenerator) ([]Notifier, error) { - var notifiers []Notifier - for i, addr := range *addrs { - cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i) - ca, serverName := tlsCAFile.GetOptionalArg(i), tlsServerName.GetOptionalArg(i) - tr, err := utils.Transport(addr, cert, key, ca, serverName, tlsInsecureSkipVerify.GetOptionalArg(i)) - if err != nil { - return nil, fmt.Errorf("failed to create transport: %w", err) - } - user, pass := basicAuthUsername.GetOptionalArg(i), basicAuthPassword.GetOptionalArg(i) - am := NewAlertManager(addr, user, pass, gen, &http.Client{Transport: tr}) - notifiers = append(notifiers, am) +// cw holds a configWatcher for configPath configuration file +// configWatcher provides a list of Notifier objects discovered +// from static config or via service discovery. +// cw is not nil only if configPath is provided. +var cw *configWatcher + +// Reload checks the changes in configPath configuration file +// and applies changes if any. +func Reload() error { + if cw == nil { + return nil + } + return cw.reload(*configPath) +} + +var staticNotifiersFn func() []Notifier + +// Init returns a function for retrieving actual list of Notifier objects. +// Init works in two mods: +// * configuration via flags (for backward compatibility). Is always static +// and don't support live reloads. +// * configuration via file. Supports live reloads and service discovery. +// Init returns an error if both mods are used. +func Init(gen AlertURLGenerator) (func() []Notifier, error) { + if *configPath == "" && len(*addrs) == 0 { + return nil, nil + } + if *configPath != "" && len(*addrs) > 0 { + return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified") } + if len(*addrs) > 0 { + notifiers, err := notifiersFromFlags(gen) + if err != nil { + return nil, fmt.Errorf("failed to create notifier from flag values: %s", err) + } + staticNotifiersFn = func() []Notifier { + return notifiers + } + return staticNotifiersFn, nil + } + + var err error + cw, err = newWatcher(*configPath, gen) + if err != nil { + return nil, fmt.Errorf("failed to init config watcher: %s", err) + } + return cw.notifiers, nil +} + +func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) { + var notifiers []Notifier + for i, addr := range *addrs { + authCfg := promauth.HTTPClientConfig{ + TLSConfig: &promauth.TLSConfig{ + CAFile: tlsCAFile.GetOptionalArg(i), + CertFile: tlsCertFile.GetOptionalArg(i), + KeyFile: tlsKeyFile.GetOptionalArg(i), + ServerName: tlsServerName.GetOptionalArg(i), + InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i), + }, + BasicAuth: &promauth.BasicAuthConfig{ + Username: basicAuthUsername.GetOptionalArg(i), + Password: promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)), + }, + } + am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute) + if err != nil { + return nil, err + } + notifiers = append(notifiers, am) + } return notifiers, nil } + +// Target represents a Notifier and optional +// list of labels added during discovery. +type Target struct { + Notifier + Labels []prompbmarshal.Label +} + +// TargetType defines how the Target was discovered +type TargetType string + +const ( + // TargetStatic is for targets configured statically + TargetStatic TargetType = "static" + // TargetConsul is for targets discovered via Consul + TargetConsul TargetType = "consulSD" +) + +// GetTargets returns list of static or discovered targets +// via notifier configuration. +func GetTargets() map[TargetType][]Target { + var targets = make(map[TargetType][]Target) + + if staticNotifiersFn != nil { + for _, ns := range staticNotifiersFn() { + targets[TargetStatic] = append(targets[TargetStatic], Target{ + Notifier: ns, + }) + } + } + + if cw != nil { + cw.targetsMu.RLock() + for key, ns := range cw.targets { + targets[key] = append(targets[key], ns...) + } + cw.targetsMu.RUnlock() + } + return targets +} diff --git a/app/vmalert/notifier/notifier.go b/app/vmalert/notifier/notifier.go index 8135a19ea..996805a71 100644 --- a/app/vmalert/notifier/notifier.go +++ b/app/vmalert/notifier/notifier.go @@ -10,4 +10,6 @@ type Notifier interface { Send(ctx context.Context, alerts []Alert) error // Addr returns address where alerts are sent. Addr() string + // Close is a destructor for the Notifier + Close() } diff --git a/app/vmalert/notifier/testdata/consul.good.yaml b/app/vmalert/notifier/testdata/consul.good.yaml new file mode 100644 index 000000000..7aae6bc9a --- /dev/null +++ b/app/vmalert/notifier/testdata/consul.good.yaml @@ -0,0 +1,13 @@ +consul_sd_configs: + - server: localhost:8500 + scheme: http + services: + - alertmanager + - server: localhost:8500 + services: + - consul +relabel_configs: + - source_labels: [__meta_consul_tags] + regex: .*,__scheme__=([^,]+),.* + replacement: '${1}' + target_label: __scheme__ \ No newline at end of file diff --git a/app/vmalert/notifier/testdata/mixed.good.yaml b/app/vmalert/notifier/testdata/mixed.good.yaml new file mode 100644 index 000000000..f6a5c5a62 --- /dev/null +++ b/app/vmalert/notifier/testdata/mixed.good.yaml @@ -0,0 +1,18 @@ +static_configs: + - targets: + - localhost:9093 + - localhost:9095 + +consul_sd_configs: + - server: localhost:8500 + scheme: http + services: + - alertmanager + - server: localhost:8500 + services: + - consul +relabel_configs: + - source_labels: [__meta_consul_tags] + regex: .*,__scheme__=([^,]+),.* + replacement: '${1}' + target_label: __scheme__ \ No newline at end of file diff --git a/app/vmalert/notifier/testdata/static.good.yaml b/app/vmalert/notifier/testdata/static.good.yaml new file mode 100644 index 000000000..a9027a714 --- /dev/null +++ b/app/vmalert/notifier/testdata/static.good.yaml @@ -0,0 +1,4 @@ +static_configs: + - targets: + - localhost:9093 + - localhost:9095 diff --git a/app/vmalert/notifier/testdata/unknownFields.bad.yaml b/app/vmalert/notifier/testdata/unknownFields.bad.yaml new file mode 100644 index 000000000..4689247cf --- /dev/null +++ b/app/vmalert/notifier/testdata/unknownFields.bad.yaml @@ -0,0 +1,5 @@ +scheme: https +unknown: field +static_configs: + - targets: + - localhost:9093 \ No newline at end of file diff --git a/app/vmalert/recording.go b/app/vmalert/recording.go index 94f7f241f..d05ffc17d 100644 --- a/app/vmalert/recording.go +++ b/app/vmalert/recording.go @@ -10,8 +10,8 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" - "github.com/VictoriaMetrics/metrics" ) // RecordingRule is a Rule that supposed @@ -43,8 +43,8 @@ type RecordingRule struct { } type recordingRuleMetrics struct { - errors *gauge - samples *gauge + errors *utils.Gauge + samples *utils.Gauge } // String implements Stringer interface @@ -75,7 +75,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul } labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID()) - rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels), + rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels), func() float64 { rr.mu.RLock() defer rr.mu.RUnlock() @@ -84,7 +84,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul } return 1 }) - rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels), + rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels), func() float64 { rr.mu.RLock() defer rr.mu.RUnlock() @@ -95,8 +95,8 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul // Close unregisters rule metrics func (rr *RecordingRule) Close() { - metrics.UnregisterMetric(rr.metrics.errors.name) - metrics.UnregisterMetric(rr.metrics.samples.name) + rr.metrics.errors.Unregister() + rr.metrics.samples.Unregister() } // ExecRange executes recording rule on the given time range similarly to Exec. diff --git a/app/vmalert/utils/metrics.go b/app/vmalert/utils/metrics.go new file mode 100644 index 000000000..cd179ba7b --- /dev/null +++ b/app/vmalert/utils/metrics.go @@ -0,0 +1,54 @@ +package utils + +import "github.com/VictoriaMetrics/metrics" + +type namedMetric struct { + Name string +} + +// Unregister removes the metric by name from default registry +func (nm namedMetric) Unregister() { + metrics.UnregisterMetric(nm.Name) +} + +// Gauge is a metrics.Gauge with Name +type Gauge struct { + namedMetric + *metrics.Gauge +} + +// GetOrCreateGauge creates a new Gauge with the given name +func GetOrCreateGauge(name string, f func() float64) *Gauge { + return &Gauge{ + namedMetric: namedMetric{Name: name}, + Gauge: metrics.GetOrCreateGauge(name, f), + } +} + +// Counter is a metrics.Counter with Name +type Counter struct { + namedMetric + *metrics.Counter +} + +// GetOrCreateCounter creates a new Counter with the given name +func GetOrCreateCounter(name string) *Counter { + return &Counter{ + namedMetric: namedMetric{Name: name}, + Counter: metrics.GetOrCreateCounter(name), + } +} + +// Summary is a metrics.Summary with Name +type Summary struct { + namedMetric + *metrics.Summary +} + +// GetOrCreateSummary creates a new Summary with the given name +func GetOrCreateSummary(name string) *Summary { + return &Summary{ + namedMetric: namedMetric{Name: name}, + Summary: metrics.GetOrCreateSummary(name), + } +} diff --git a/app/vmalert/web.go b/app/vmalert/web.go index 5100d0e36..51a22d2be 100644 --- a/app/vmalert/web.go +++ b/app/vmalert/web.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -33,9 +34,10 @@ func initLinks() { {path.Join(pathPrefix, "-/reload"), "reload configuration"}, } navItems = []tpl.NavItem{ - {Name: "vmalert", Url: pathPrefix}, + {Name: "vmalert", Url: path.Join(pathPrefix, "/")}, {Name: "Groups", Url: path.Join(pathPrefix, "groups")}, {Name: "Alerts", Url: path.Join(pathPrefix, "alerts")}, + {Name: "Notifiers", Url: path.Join(pathPrefix, "notifiers")}, {Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"}, } } @@ -62,6 +64,9 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool { case "/groups": WriteListGroups(w, rh.groups()) return true + case "/notifiers": + WriteListTargets(w, notifier.GetTargets()) + return true case "/api/v1/groups": data, err := rh.listGroups() if err != nil { diff --git a/app/vmalert/web.qtpl b/app/vmalert/web.qtpl index 98e33c05f..e17b1b2ce 100644 --- a/app/vmalert/web.qtpl +++ b/app/vmalert/web.qtpl @@ -5,6 +5,7 @@ "sort" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" ) %} @@ -205,6 +206,62 @@ {% endfunc %} +{% func ListTargets(targets map[notifier.TargetType][]notifier.Target) %} + {%= tpl.Header("Notifiers", navItems) %} + {% if len(targets) > 0 %} + Collapse All + Expand All + + {%code + var keys []string + for key := range targets { + keys = append(keys, string(key)) + } + sort.Strings(keys) + %} + + {% for i := range keys %} + {%code typeK, ns := keys[i], targets[notifier.TargetType(keys[i])] + count := len(ns) + %} + +
+ + + + + + + + + {% for _, n := range ns %} + + + + + {% endfor %} + +
LabelsAddress
+ {% for _, l := range n.Labels %} + {%s l.Name %}={%s l.Value %} + {% endfor %} + {%s n.Notifier.Addr() %}
+
+ {% endfor %} + + {% else %} +
+

No items...

+
+ {% endif %} + + {%= tpl.Footer() %} + +{% endfunc %} + {% func Alert(alert *APIAlert) %} {%= tpl.Header("", navItems) %} {%code diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go index 27c829378..67f5199ee 100644 --- a/app/vmalert/web.qtpl.go +++ b/app/vmalert/web.qtpl.go @@ -9,114 +9,115 @@ import ( "sort" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl" ) -//line app/vmalert/web.qtpl:11 +//line app/vmalert/web.qtpl:12 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmalert/web.qtpl:11 +//line app/vmalert/web.qtpl:12 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmalert/web.qtpl:11 +//line app/vmalert/web.qtpl:12 func StreamWelcome(qw422016 *qt422016.Writer) { -//line app/vmalert/web.qtpl:11 +//line app/vmalert/web.qtpl:12 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:12 +//line app/vmalert/web.qtpl:13 tpl.StreamHeader(qw422016, "vmalert", navItems) -//line app/vmalert/web.qtpl:12 +//line app/vmalert/web.qtpl:13 qw422016.N().S(`

API:
`) -//line app/vmalert/web.qtpl:15 +//line app/vmalert/web.qtpl:16 for _, p := range apiLinks { -//line app/vmalert/web.qtpl:15 +//line app/vmalert/web.qtpl:16 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:17 +//line app/vmalert/web.qtpl:18 p, doc := p[0], p[1] -//line app/vmalert/web.qtpl:18 +//line app/vmalert/web.qtpl:19 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:19 +//line app/vmalert/web.qtpl:20 qw422016.E().S(p) -//line app/vmalert/web.qtpl:19 +//line app/vmalert/web.qtpl:20 qw422016.N().S(` - `) -//line app/vmalert/web.qtpl:19 +//line app/vmalert/web.qtpl:20 qw422016.E().S(doc) -//line app/vmalert/web.qtpl:19 +//line app/vmalert/web.qtpl:20 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:20 +//line app/vmalert/web.qtpl:21 } -//line app/vmalert/web.qtpl:20 +//line app/vmalert/web.qtpl:21 qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:22 +//line app/vmalert/web.qtpl:23 tpl.StreamFooter(qw422016) -//line app/vmalert/web.qtpl:22 +//line app/vmalert/web.qtpl:23 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 } -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 func WriteWelcome(qq422016 qtio422016.Writer) { -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 StreamWelcome(qw422016) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 } -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 func Welcome() string { -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 WriteWelcome(qb422016) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 return qs422016 -//line app/vmalert/web.qtpl:23 +//line app/vmalert/web.qtpl:24 } -//line app/vmalert/web.qtpl:25 +//line app/vmalert/web.qtpl:26 func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) { -//line app/vmalert/web.qtpl:25 +//line app/vmalert/web.qtpl:26 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:26 +//line app/vmalert/web.qtpl:27 tpl.StreamHeader(qw422016, "Groups", navItems) -//line app/vmalert/web.qtpl:26 +//line app/vmalert/web.qtpl:27 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:27 +//line app/vmalert/web.qtpl:28 if len(groups) > 0 { -//line app/vmalert/web.qtpl:27 +//line app/vmalert/web.qtpl:28 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:29 +//line app/vmalert/web.qtpl:30 rOk := make(map[string]int) rNotOk := make(map[string]int) for _, g := range groups { @@ -136,111 +137,111 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) { } } -//line app/vmalert/web.qtpl:47 +//line app/vmalert/web.qtpl:48 qw422016.N().S(` Collapse All Expand All `) -//line app/vmalert/web.qtpl:50 +//line app/vmalert/web.qtpl:51 for _, g := range groups { -//line app/vmalert/web.qtpl:50 +//line app/vmalert/web.qtpl:51 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.E().S(g.Name) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 if g.Type != "prometheus" { -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.N().S(` (`) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.E().S(g.Type) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.N().S(`)`) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 } -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.N().S(` (every `) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.E().S(g.Interval) -//line app/vmalert/web.qtpl:53 +//line app/vmalert/web.qtpl:54 qw422016.N().S(`) `) -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 if rNotOk[g.Name] > 0 { -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 qw422016.N().S(``) -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 qw422016.N().D(rNotOk[g.Name]) -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 } -//line app/vmalert/web.qtpl:54 +//line app/vmalert/web.qtpl:55 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:55 +//line app/vmalert/web.qtpl:56 qw422016.N().D(rOk[g.Name]) -//line app/vmalert/web.qtpl:55 +//line app/vmalert/web.qtpl:56 qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:56 +//line app/vmalert/web.qtpl:57 qw422016.E().S(g.File) -//line app/vmalert/web.qtpl:56 +//line app/vmalert/web.qtpl:57 qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:57 +//line app/vmalert/web.qtpl:58 if len(g.Params) > 0 { -//line app/vmalert/web.qtpl:57 +//line app/vmalert/web.qtpl:58 qw422016.N().S(`
Extra params `) -//line app/vmalert/web.qtpl:59 +//line app/vmalert/web.qtpl:60 for _, param := range g.Params { -//line app/vmalert/web.qtpl:59 +//line app/vmalert/web.qtpl:60 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:60 +//line app/vmalert/web.qtpl:61 qw422016.E().S(param) -//line app/vmalert/web.qtpl:60 +//line app/vmalert/web.qtpl:61 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:61 +//line app/vmalert/web.qtpl:62 } -//line app/vmalert/web.qtpl:61 +//line app/vmalert/web.qtpl:62 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:63 +//line app/vmalert/web.qtpl:64 } -//line app/vmalert/web.qtpl:63 +//line app/vmalert/web.qtpl:64 qw422016.N().S(`
@@ -253,280 +254,280 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) { `) -//line app/vmalert/web.qtpl:76 +//line app/vmalert/web.qtpl:77 for _, ar := range g.AlertingRules { -//line app/vmalert/web.qtpl:76 +//line app/vmalert/web.qtpl:77 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:90 +//line app/vmalert/web.qtpl:91 } -//line app/vmalert/web.qtpl:90 +//line app/vmalert/web.qtpl:91 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:91 +//line app/vmalert/web.qtpl:92 for _, rr := range g.RecordingRules { -//line app/vmalert/web.qtpl:91 +//line app/vmalert/web.qtpl:92 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:105 +//line app/vmalert/web.qtpl:106 } -//line app/vmalert/web.qtpl:105 +//line app/vmalert/web.qtpl:106 qw422016.N().S(`
alert: `) -//line app/vmalert/web.qtpl:79 +//line app/vmalert/web.qtpl:80 qw422016.E().S(ar.Name) -//line app/vmalert/web.qtpl:79 +//line app/vmalert/web.qtpl:80 qw422016.N().S(` (for: `) -//line app/vmalert/web.qtpl:79 +//line app/vmalert/web.qtpl:80 qw422016.E().V(ar.For) -//line app/vmalert/web.qtpl:79 +//line app/vmalert/web.qtpl:80 qw422016.N().S(`)
`)
-//line app/vmalert/web.qtpl:80
+//line app/vmalert/web.qtpl:81
 				qw422016.E().S(ar.Expression)
-//line app/vmalert/web.qtpl:80
+//line app/vmalert/web.qtpl:81
 				qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:81 +//line app/vmalert/web.qtpl:82 if len(ar.Labels) > 0 { -//line app/vmalert/web.qtpl:81 +//line app/vmalert/web.qtpl:82 qw422016.N().S(` Labels:`) -//line app/vmalert/web.qtpl:81 +//line app/vmalert/web.qtpl:82 } -//line app/vmalert/web.qtpl:81 +//line app/vmalert/web.qtpl:82 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:82 +//line app/vmalert/web.qtpl:83 for k, v := range ar.Labels { -//line app/vmalert/web.qtpl:82 +//line app/vmalert/web.qtpl:83 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:83 +//line app/vmalert/web.qtpl:84 qw422016.E().S(k) -//line app/vmalert/web.qtpl:83 +//line app/vmalert/web.qtpl:84 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:83 +//line app/vmalert/web.qtpl:84 qw422016.E().S(v) -//line app/vmalert/web.qtpl:83 +//line app/vmalert/web.qtpl:84 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:84 +//line app/vmalert/web.qtpl:85 } -//line app/vmalert/web.qtpl:84 +//line app/vmalert/web.qtpl:85 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:86 +//line app/vmalert/web.qtpl:87 qw422016.E().S(ar.LastError) -//line app/vmalert/web.qtpl:86 +//line app/vmalert/web.qtpl:87 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:87 +//line app/vmalert/web.qtpl:88 qw422016.N().D(ar.LastSamples) -//line app/vmalert/web.qtpl:87 +//line app/vmalert/web.qtpl:88 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:88 +//line app/vmalert/web.qtpl:89 qw422016.N().FPrec(time.Since(ar.LastExec).Seconds(), 3) -//line app/vmalert/web.qtpl:88 +//line app/vmalert/web.qtpl:89 qw422016.N().S(`s ago
record: `) -//line app/vmalert/web.qtpl:94 +//line app/vmalert/web.qtpl:95 qw422016.E().S(rr.Name) -//line app/vmalert/web.qtpl:94 +//line app/vmalert/web.qtpl:95 qw422016.N().S(`
`)
-//line app/vmalert/web.qtpl:95
+//line app/vmalert/web.qtpl:96
 				qw422016.E().S(rr.Expression)
-//line app/vmalert/web.qtpl:95
+//line app/vmalert/web.qtpl:96
 				qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:96 +//line app/vmalert/web.qtpl:97 if len(rr.Labels) > 0 { -//line app/vmalert/web.qtpl:96 +//line app/vmalert/web.qtpl:97 qw422016.N().S(` Labels:`) -//line app/vmalert/web.qtpl:96 +//line app/vmalert/web.qtpl:97 } -//line app/vmalert/web.qtpl:96 +//line app/vmalert/web.qtpl:97 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:97 +//line app/vmalert/web.qtpl:98 for k, v := range rr.Labels { -//line app/vmalert/web.qtpl:97 +//line app/vmalert/web.qtpl:98 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:98 +//line app/vmalert/web.qtpl:99 qw422016.E().S(k) -//line app/vmalert/web.qtpl:98 +//line app/vmalert/web.qtpl:99 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:98 +//line app/vmalert/web.qtpl:99 qw422016.E().S(v) -//line app/vmalert/web.qtpl:98 +//line app/vmalert/web.qtpl:99 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:99 +//line app/vmalert/web.qtpl:100 } -//line app/vmalert/web.qtpl:99 +//line app/vmalert/web.qtpl:100 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:101 +//line app/vmalert/web.qtpl:102 qw422016.E().S(rr.LastError) -//line app/vmalert/web.qtpl:101 +//line app/vmalert/web.qtpl:102 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:102 +//line app/vmalert/web.qtpl:103 qw422016.N().D(rr.LastSamples) -//line app/vmalert/web.qtpl:102 +//line app/vmalert/web.qtpl:103 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:103 +//line app/vmalert/web.qtpl:104 qw422016.N().FPrec(time.Since(rr.LastExec).Seconds(), 3) -//line app/vmalert/web.qtpl:103 +//line app/vmalert/web.qtpl:104 qw422016.N().S(`s ago
`) -//line app/vmalert/web.qtpl:109 +//line app/vmalert/web.qtpl:110 } -//line app/vmalert/web.qtpl:109 +//line app/vmalert/web.qtpl:110 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:111 +//line app/vmalert/web.qtpl:112 } else { -//line app/vmalert/web.qtpl:111 +//line app/vmalert/web.qtpl:112 qw422016.N().S(`

No items...

`) -//line app/vmalert/web.qtpl:115 +//line app/vmalert/web.qtpl:116 } -//line app/vmalert/web.qtpl:115 +//line app/vmalert/web.qtpl:116 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:117 +//line app/vmalert/web.qtpl:118 tpl.StreamFooter(qw422016) -//line app/vmalert/web.qtpl:117 +//line app/vmalert/web.qtpl:118 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 } -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 func WriteListGroups(qq422016 qtio422016.Writer, groups []APIGroup) { -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 StreamListGroups(qw422016, groups) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 } -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 func ListGroups(groups []APIGroup) string { -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 WriteListGroups(qb422016, groups) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 return qs422016 -//line app/vmalert/web.qtpl:119 +//line app/vmalert/web.qtpl:120 } -//line app/vmalert/web.qtpl:122 +//line app/vmalert/web.qtpl:123 func StreamListAlerts(qw422016 *qt422016.Writer, groupAlerts []GroupAlerts) { -//line app/vmalert/web.qtpl:122 +//line app/vmalert/web.qtpl:123 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:123 +//line app/vmalert/web.qtpl:124 tpl.StreamHeader(qw422016, "Alerts", navItems) -//line app/vmalert/web.qtpl:123 +//line app/vmalert/web.qtpl:124 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:124 +//line app/vmalert/web.qtpl:125 if len(groupAlerts) > 0 { -//line app/vmalert/web.qtpl:124 +//line app/vmalert/web.qtpl:125 qw422016.N().S(` Collapse All Expand All `) -//line app/vmalert/web.qtpl:127 +//line app/vmalert/web.qtpl:128 for _, ga := range groupAlerts { -//line app/vmalert/web.qtpl:127 +//line app/vmalert/web.qtpl:128 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:128 +//line app/vmalert/web.qtpl:129 g := ga.Group -//line app/vmalert/web.qtpl:128 +//line app/vmalert/web.qtpl:129 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:137 +//line app/vmalert/web.qtpl:138 var keys []string alertsByRule := make(map[string][]*APIAlert) for _, alert := range ga.Alerts { @@ -537,20 +538,20 @@ func StreamListAlerts(qw422016 *qt422016.Writer, groupAlerts []GroupAlerts) { } sort.Strings(keys) -//line app/vmalert/web.qtpl:146 +//line app/vmalert/web.qtpl:147 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:148 +//line app/vmalert/web.qtpl:149 for _, ruleID := range keys { -//line app/vmalert/web.qtpl:148 +//line app/vmalert/web.qtpl:149 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:150 +//line app/vmalert/web.qtpl:151 defaultAR := alertsByRule[ruleID][0] var labelKeys []string for k := range defaultAR.Labels { @@ -558,28 +559,28 @@ func StreamListAlerts(qw422016 *qt422016.Writer, groupAlerts []GroupAlerts) { } sort.Strings(labelKeys) -//line app/vmalert/web.qtpl:156 +//line app/vmalert/web.qtpl:157 qw422016.N().S(`
alert: `) -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:159 qw422016.E().S(defaultAR.Name) -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:159 qw422016.N().S(` (`) -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:159 qw422016.N().D(len(alertsByRule[ruleID])) -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:159 qw422016.N().S(`) | Source
expr:
`)
-//line app/vmalert/web.qtpl:161
+//line app/vmalert/web.qtpl:162
 				qw422016.E().S(defaultAR.Expression)
-//line app/vmalert/web.qtpl:161
+//line app/vmalert/web.qtpl:162
 				qw422016.N().S(`
@@ -593,151 +594,325 @@ func StreamListAlerts(qw422016 *qt422016.Writer, groupAlerts []GroupAlerts) { `) -//line app/vmalert/web.qtpl:173 +//line app/vmalert/web.qtpl:174 for _, ar := range alertsByRule[ruleID] { -//line app/vmalert/web.qtpl:173 +//line app/vmalert/web.qtpl:174 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:190 +//line app/vmalert/web.qtpl:191 } -//line app/vmalert/web.qtpl:190 +//line app/vmalert/web.qtpl:191 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:176 +//line app/vmalert/web.qtpl:177 for _, k := range labelKeys { -//line app/vmalert/web.qtpl:176 +//line app/vmalert/web.qtpl:177 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:177 +//line app/vmalert/web.qtpl:178 qw422016.E().S(k) -//line app/vmalert/web.qtpl:177 +//line app/vmalert/web.qtpl:178 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:177 +//line app/vmalert/web.qtpl:178 qw422016.E().S(ar.Labels[k]) -//line app/vmalert/web.qtpl:177 +//line app/vmalert/web.qtpl:178 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:178 +//line app/vmalert/web.qtpl:179 } -//line app/vmalert/web.qtpl:178 +//line app/vmalert/web.qtpl:179 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:180 +//line app/vmalert/web.qtpl:181 streambadgeState(qw422016, ar.State) -//line app/vmalert/web.qtpl:180 +//line app/vmalert/web.qtpl:181 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:182 +//line app/vmalert/web.qtpl:183 qw422016.E().S(ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00")) -//line app/vmalert/web.qtpl:182 +//line app/vmalert/web.qtpl:183 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:183 +//line app/vmalert/web.qtpl:184 if ar.Restored { -//line app/vmalert/web.qtpl:183 +//line app/vmalert/web.qtpl:184 streambadgeRestored(qw422016) -//line app/vmalert/web.qtpl:183 +//line app/vmalert/web.qtpl:184 } -//line app/vmalert/web.qtpl:183 +//line app/vmalert/web.qtpl:184 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:185 +//line app/vmalert/web.qtpl:186 qw422016.E().S(ar.Value) -//line app/vmalert/web.qtpl:185 +//line app/vmalert/web.qtpl:186 qw422016.N().S(` Details
`) -//line app/vmalert/web.qtpl:193 +//line app/vmalert/web.qtpl:194 } -//line app/vmalert/web.qtpl:193 +//line app/vmalert/web.qtpl:194 qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:196 +//line app/vmalert/web.qtpl:197 } -//line app/vmalert/web.qtpl:196 +//line app/vmalert/web.qtpl:197 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:198 +//line app/vmalert/web.qtpl:199 } else { -//line app/vmalert/web.qtpl:198 +//line app/vmalert/web.qtpl:199 qw422016.N().S(`

No items...

`) -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:203 } -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:203 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:204 +//line app/vmalert/web.qtpl:205 tpl.StreamFooter(qw422016) -//line app/vmalert/web.qtpl:204 +//line app/vmalert/web.qtpl:205 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 } -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 func WriteListAlerts(qq422016 qtio422016.Writer, groupAlerts []GroupAlerts) { -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 StreamListAlerts(qw422016, groupAlerts) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 } -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 func ListAlerts(groupAlerts []GroupAlerts) string { -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 WriteListAlerts(qb422016, groupAlerts) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 return qs422016 -//line app/vmalert/web.qtpl:206 +//line app/vmalert/web.qtpl:207 } -//line app/vmalert/web.qtpl:208 -func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) { -//line app/vmalert/web.qtpl:208 +//line app/vmalert/web.qtpl:209 +func StreamListTargets(qw422016 *qt422016.Writer, targets map[notifier.TargetType][]notifier.Target) { +//line app/vmalert/web.qtpl:209 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:209 - tpl.StreamHeader(qw422016, "", navItems) -//line app/vmalert/web.qtpl:209 +//line app/vmalert/web.qtpl:210 + tpl.StreamHeader(qw422016, "Notifiers", navItems) +//line app/vmalert/web.qtpl:210 qw422016.N().S(` `) //line app/vmalert/web.qtpl:211 + if len(targets) > 0 { +//line app/vmalert/web.qtpl:211 + qw422016.N().S(` + Collapse All + Expand All + + `) +//line app/vmalert/web.qtpl:216 + var keys []string + for key := range targets { + keys = append(keys, string(key)) + } + sort.Strings(keys) + +//line app/vmalert/web.qtpl:221 + qw422016.N().S(` + + `) +//line app/vmalert/web.qtpl:223 + for i := range keys { +//line app/vmalert/web.qtpl:223 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:224 + typeK, ns := keys[i], targets[notifier.TargetType(keys[i])] + count := len(ns) + +//line app/vmalert/web.qtpl:226 + qw422016.N().S(` + +
+ + + + + + + + + `) +//line app/vmalert/web.qtpl:240 + for _, n := range ns { +//line app/vmalert/web.qtpl:240 + qw422016.N().S(` + + + + + `) +//line app/vmalert/web.qtpl:249 + } +//line app/vmalert/web.qtpl:249 + qw422016.N().S(` + +
LabelsAddress
+ `) +//line app/vmalert/web.qtpl:243 + for _, l := range n.Labels { +//line app/vmalert/web.qtpl:243 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:244 + qw422016.E().S(l.Name) +//line app/vmalert/web.qtpl:244 + qw422016.N().S(`=`) +//line app/vmalert/web.qtpl:244 + qw422016.E().S(l.Value) +//line app/vmalert/web.qtpl:244 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:245 + } +//line app/vmalert/web.qtpl:245 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:247 + qw422016.E().S(n.Notifier.Addr()) +//line app/vmalert/web.qtpl:247 + qw422016.N().S(`
+
+ `) +//line app/vmalert/web.qtpl:253 + } +//line app/vmalert/web.qtpl:253 + qw422016.N().S(` + + `) +//line app/vmalert/web.qtpl:255 + } else { +//line app/vmalert/web.qtpl:255 + qw422016.N().S(` +
+

No items...

+
+ `) +//line app/vmalert/web.qtpl:259 + } +//line app/vmalert/web.qtpl:259 + qw422016.N().S(` + + `) +//line app/vmalert/web.qtpl:261 + tpl.StreamFooter(qw422016) +//line app/vmalert/web.qtpl:261 + qw422016.N().S(` + +`) +//line app/vmalert/web.qtpl:263 +} + +//line app/vmalert/web.qtpl:263 +func WriteListTargets(qq422016 qtio422016.Writer, targets map[notifier.TargetType][]notifier.Target) { +//line app/vmalert/web.qtpl:263 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/web.qtpl:263 + StreamListTargets(qw422016, targets) +//line app/vmalert/web.qtpl:263 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/web.qtpl:263 +} + +//line app/vmalert/web.qtpl:263 +func ListTargets(targets map[notifier.TargetType][]notifier.Target) string { +//line app/vmalert/web.qtpl:263 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/web.qtpl:263 + WriteListTargets(qb422016, targets) +//line app/vmalert/web.qtpl:263 + qs422016 := string(qb422016.B) +//line app/vmalert/web.qtpl:263 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/web.qtpl:263 + return qs422016 +//line app/vmalert/web.qtpl:263 +} + +//line app/vmalert/web.qtpl:265 +func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) { +//line app/vmalert/web.qtpl:265 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:266 + tpl.StreamHeader(qw422016, "", navItems) +//line app/vmalert/web.qtpl:266 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:268 var labelKeys []string for k := range alert.Labels { labelKeys = append(labelKeys, k) @@ -750,28 +925,28 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) { } sort.Strings(annotationKeys) -//line app/vmalert/web.qtpl:222 +//line app/vmalert/web.qtpl:279 qw422016.N().S(`
`) -//line app/vmalert/web.qtpl:223 +//line app/vmalert/web.qtpl:280 qw422016.E().S(alert.Name) -//line app/vmalert/web.qtpl:223 +//line app/vmalert/web.qtpl:280 qw422016.N().S(``) -//line app/vmalert/web.qtpl:223 +//line app/vmalert/web.qtpl:280 qw422016.E().S(alert.State) -//line app/vmalert/web.qtpl:223 +//line app/vmalert/web.qtpl:280 qw422016.N().S(`
@@ -780,9 +955,9 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) {
`) -//line app/vmalert/web.qtpl:230 +//line app/vmalert/web.qtpl:287 qw422016.E().S(alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00")) -//line app/vmalert/web.qtpl:230 +//line app/vmalert/web.qtpl:287 qw422016.N().S(`
@@ -794,9 +969,9 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) {
`)
-//line app/vmalert/web.qtpl:240
+//line app/vmalert/web.qtpl:297
 	qw422016.E().S(alert.Expression)
-//line app/vmalert/web.qtpl:240
+//line app/vmalert/web.qtpl:297
 	qw422016.N().S(`
@@ -808,23 +983,23 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) {
`) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:307 for _, k := range labelKeys { -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:307 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:251 +//line app/vmalert/web.qtpl:308 qw422016.E().S(k) -//line app/vmalert/web.qtpl:251 +//line app/vmalert/web.qtpl:308 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:251 +//line app/vmalert/web.qtpl:308 qw422016.E().S(alert.Labels[k]) -//line app/vmalert/web.qtpl:251 +//line app/vmalert/web.qtpl:308 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:252 +//line app/vmalert/web.qtpl:309 } -//line app/vmalert/web.qtpl:252 +//line app/vmalert/web.qtpl:309 qw422016.N().S(`
@@ -836,24 +1011,24 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) {
`) -//line app/vmalert/web.qtpl:262 +//line app/vmalert/web.qtpl:319 for _, k := range annotationKeys { -//line app/vmalert/web.qtpl:262 +//line app/vmalert/web.qtpl:319 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:263 +//line app/vmalert/web.qtpl:320 qw422016.E().S(k) -//line app/vmalert/web.qtpl:263 +//line app/vmalert/web.qtpl:320 qw422016.N().S(`:

`) -//line app/vmalert/web.qtpl:264 +//line app/vmalert/web.qtpl:321 qw422016.E().S(alert.Annotations[k]) -//line app/vmalert/web.qtpl:264 +//line app/vmalert/web.qtpl:321 qw422016.N().S(`

`) -//line app/vmalert/web.qtpl:265 +//line app/vmalert/web.qtpl:322 } -//line app/vmalert/web.qtpl:265 +//line app/vmalert/web.qtpl:322 qw422016.N().S(`
@@ -865,13 +1040,13 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) { @@ -883,132 +1058,132 @@ func StreamAlert(qw422016 *qt422016.Writer, alert *APIAlert) { `) -//line app/vmalert/web.qtpl:289 +//line app/vmalert/web.qtpl:346 tpl.StreamFooter(qw422016) -//line app/vmalert/web.qtpl:289 +//line app/vmalert/web.qtpl:346 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 } -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 func WriteAlert(qq422016 qtio422016.Writer, alert *APIAlert) { -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 StreamAlert(qw422016, alert) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 } -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 func Alert(alert *APIAlert) string { -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 WriteAlert(qb422016, alert) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 return qs422016 -//line app/vmalert/web.qtpl:291 +//line app/vmalert/web.qtpl:348 } -//line app/vmalert/web.qtpl:293 +//line app/vmalert/web.qtpl:350 func streambadgeState(qw422016 *qt422016.Writer, state string) { -//line app/vmalert/web.qtpl:293 +//line app/vmalert/web.qtpl:350 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:295 +//line app/vmalert/web.qtpl:352 badgeClass := "bg-warning text-dark" if state == "firing" { badgeClass = "bg-danger" } -//line app/vmalert/web.qtpl:299 +//line app/vmalert/web.qtpl:356 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:300 +//line app/vmalert/web.qtpl:357 qw422016.E().S(state) -//line app/vmalert/web.qtpl:300 +//line app/vmalert/web.qtpl:357 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 } -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 func writebadgeState(qq422016 qtio422016.Writer, state string) { -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 streambadgeState(qw422016, state) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 } -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 func badgeState(state string) string { -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 writebadgeState(qb422016, state) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 return qs422016 -//line app/vmalert/web.qtpl:301 +//line app/vmalert/web.qtpl:358 } -//line app/vmalert/web.qtpl:303 +//line app/vmalert/web.qtpl:360 func streambadgeRestored(qw422016 *qt422016.Writer) { -//line app/vmalert/web.qtpl:303 +//line app/vmalert/web.qtpl:360 qw422016.N().S(` restored `) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 } -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 func writebadgeRestored(qq422016 qtio422016.Writer) { -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 streambadgeRestored(qw422016) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 } -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 func badgeRestored() string { -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 writebadgeRestored(qb422016) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 return qs422016 -//line app/vmalert/web.qtpl:305 +//line app/vmalert/web.qtpl:362 }