diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 6e06c822ba..68c375cf6b 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -203,6 +203,10 @@ expr: <string> # as firing once they return. [ for: <duration> | default = 0s ] +# Alert will continue firing for this long even when the alerting expression no longer has results. +# This allows you to delay alert resolution. +[ keep_firing_for: <duration> | default = 0s ] + # Whether to print debug information into logs. # Information includes alerts state changes and requests sent to the datasource. # Please note, that if rule's query params contain sensitive @@ -736,6 +740,7 @@ See full description for these flags in `./vmalert -help`. * Graphite engine isn't supported yet; * `query` template function is disabled for performance reasons (might be changed in future); * `limit` group's param has no effect during replay (might be changed in future); +* `keep_firing_for` alerting rule param has no effect during replay (might be changed in future). ## Unit Testing for Rules diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index e8ab1816b5..779423b6f7 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -21,17 +21,18 @@ import ( // AlertingRule is basic alert entity type AlertingRule struct { - Type config.Type - RuleID uint64 - Name string - Expr string - For time.Duration - Labels map[string]string - Annotations map[string]string - GroupID uint64 - GroupName string - EvalInterval time.Duration - Debug bool + Type config.Type + RuleID uint64 + Name string + Expr string + For time.Duration + KeepFiringFor time.Duration + Labels map[string]string + Annotations map[string]string + GroupID uint64 + GroupName string + EvalInterval time.Duration + Debug bool q datasource.Querier @@ -56,17 +57,18 @@ type alertingRuleMetrics struct { func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule { ar := &AlertingRule{ - Type: group.Type, - RuleID: cfg.ID, - Name: cfg.Alert, - Expr: cfg.Expr, - For: cfg.For.Duration(), - Labels: cfg.Labels, - Annotations: cfg.Annotations, - GroupID: group.ID(), - GroupName: group.Name, - EvalInterval: group.Interval, - Debug: cfg.Debug, + Type: group.Type, + RuleID: cfg.ID, + Name: cfg.Alert, + Expr: cfg.Expr, + For: cfg.For.Duration(), + KeepFiringFor: cfg.KeepFiringFor.Duration(), + Labels: cfg.Labels, + Annotations: cfg.Annotations, + GroupID: group.ID(), + GroupName: group.Name, + EvalInterval: group.Interval, + Debug: cfg.Debug, q: qb.BuildWithParams(datasource.QuerierParams{ DataSourceType: group.Type.String(), EvaluationInterval: group.Interval, @@ -366,6 +368,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]pr if err != nil { return nil, err } + a.KeepFiringSince = time.Time{} continue } a, err := ar.newAlert(m, ls, start, qFn) @@ -391,12 +394,24 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]pr ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round") continue } + // check if alert should keep StateFiring if rule has + // `keep_firing_for` field if a.State == notifier.StateFiring { - a.State = notifier.StateInactive - a.ResolvedAt = ts - ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round") + if ar.KeepFiringFor > 0 { + if a.KeepFiringSince.IsZero() { + a.KeepFiringSince = ts + } + } + // alerts with ar.KeepFiringFor>0 may remain FIRING + // even if their expression isn't true anymore + if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor { + a.State = notifier.StateInactive + a.ResolvedAt = ts + ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round") + continue + } + ar.logDebugf(ts, a, "KEEP_FIRING: will keep firing for %fs since %v", ar.KeepFiringFor.Seconds(), a.KeepFiringSince) } - continue } numActivePending++ if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For { @@ -436,6 +451,7 @@ func (ar *AlertingRule) UpdateWith(r Rule) error { } ar.Expr = nr.Expr ar.For = nr.For + ar.KeepFiringFor = nr.KeepFiringFor ar.Labels = nr.Labels ar.Annotations = nr.Annotations ar.EvalInterval = nr.EvalInterval @@ -508,6 +524,7 @@ func (ar *AlertingRule) ToAPI() APIRule { Name: ar.Name, Query: ar.Expr, Duration: ar.For.Seconds(), + KeepFiringFor: ar.KeepFiringFor.Seconds(), Labels: ar.Labels, Annotations: ar.Annotations, LastEvaluation: lastState.time, @@ -576,6 +593,9 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert { if alertURLGeneratorFn != nil { aa.SourceLink = alertURLGeneratorFn(a) } + if a.State == notifier.StateFiring && !a.KeepFiringSince.IsZero() { + aa.Stabilizing = true + } return aa } diff --git a/app/vmalert/alerting_test.go b/app/vmalert/alerting_test.go index d05305c90d..9c779d7386 100644 --- a/app/vmalert/alerting_test.go +++ b/app/vmalert/alerting_test.go @@ -113,7 +113,7 @@ func TestAlertingRule_Exec(t *testing.T) { testCases := []struct { rule *AlertingRule steps [][]datasource.Metric - expAlerts []testAlert + expAlerts map[int][]testAlert }{ { newTestAlertingRule("empty", 0), @@ -125,50 +125,8 @@ func TestAlertingRule_Exec(t *testing.T) { [][]datasource.Metric{ {datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}}, }, - []testAlert{ - {alert: ¬ifier.Alert{State: notifier.StateFiring}}, - }, - }, - { - newTestAlertingRule("single-firing", 0), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, - }, - }, - { - newTestAlertingRule("single-firing=>inactive", 0), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - {}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, - }, - }, - { - newTestAlertingRule("single-firing=>inactive=>firing", 0), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - {}, - {metricWithLabels(t, "name", "foo")}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, - }, - }, - { - newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - {}, - {metricWithLabels(t, "name", "foo")}, - {}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, + map[int][]testAlert{ + 0: {{alert: ¬ifier.Alert{State: notifier.StateFiring}}}, }, }, { @@ -180,12 +138,16 @@ func TestAlertingRule_Exec(t *testing.T) { {}, {}, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 2: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 4: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, }, }, { - newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0), + newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive=>firing", 0), [][]datasource.Metric{ {metricWithLabels(t, "name", "foo")}, {}, @@ -194,8 +156,13 @@ func TestAlertingRule_Exec(t *testing.T) { {}, {metricWithLabels(t, "name", "foo")}, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 2: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 4: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 5: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, }, }, { @@ -207,10 +174,12 @@ func TestAlertingRule_Exec(t *testing.T) { metricWithLabels(t, "name", "foo2"), }, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, - {labels: []string{"name", "foo1"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, - {labels: []string{"name", "foo2"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + map[int][]testAlert{ + 0: { + {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + {labels: []string{"name", "foo1"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + {labels: []string{"name", "foo2"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + }, }, }, { @@ -223,10 +192,19 @@ func TestAlertingRule_Exec(t *testing.T) { // 1: fire first alert // 2: fire second alert, set first inactive // 3: fire third alert, set second inactive - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, - {labels: []string{"name", "foo1"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, - {labels: []string{"name", "foo2"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + map[int][]testAlert{ + 0: { + {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + }, + 1: { + {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, + {labels: []string{"name", "foo1"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + }, + 2: { + {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, + {labels: []string{"name", "foo1"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, + {labels: []string{"name", "foo2"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + }, }, }, { @@ -234,8 +212,8 @@ func TestAlertingRule_Exec(t *testing.T) { [][]datasource.Metric{ {metricWithLabels(t, "name", "foo")}, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, }, }, { @@ -244,8 +222,9 @@ func TestAlertingRule_Exec(t *testing.T) { {metricWithLabels(t, "name", "foo")}, {metricWithLabels(t, "name", "foo")}, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, }, }, { @@ -253,34 +232,13 @@ func TestAlertingRule_Exec(t *testing.T) { [][]datasource.Metric{ {metricWithLabels(t, "name", "foo")}, {metricWithLabels(t, "name", "foo")}, - // empty step to reset and delete pending alerts + // empty step to delete pending alerts {}, }, - nil, - }, - { - newTestAlertingRule("for-pending=>firing=>inactive", defaultStep), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - {metricWithLabels(t, "name", "foo")}, - // empty step to reset pending alerts - {}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}, - }, - }, - { - newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep), - [][]datasource.Metric{ - {metricWithLabels(t, "name", "foo")}, - {metricWithLabels(t, "name", "foo")}, - // empty step to reset pending alerts - {}, - {metricWithLabels(t, "name", "foo")}, - }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 2: {}, }, }, { @@ -288,13 +246,57 @@ func TestAlertingRule_Exec(t *testing.T) { [][]datasource.Metric{ {metricWithLabels(t, "name", "foo")}, {metricWithLabels(t, "name", "foo")}, - // empty step to reset pending alerts + // empty step to set alert inactive {}, {metricWithLabels(t, "name", "foo")}, {metricWithLabels(t, "name", "foo")}, }, - []testAlert{ - {labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 2: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 4: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + }, + }, + { + newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>firing", defaultStep, defaultStep), + [][]datasource.Metric{ + {metricWithLabels(t, "name", "foo")}, + {metricWithLabels(t, "name", "foo")}, + // empty step to keep firing + {}, + {metricWithLabels(t, "name", "foo")}, + }, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 2: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + }, + }, + { + newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 2*defaultStep), + [][]datasource.Metric{ + {metricWithLabels(t, "name", "foo")}, + {metricWithLabels(t, "name", "foo")}, + // empty step to keep firing + {}, + // another empty step to keep firing + {}, + // empty step to set alert inactive + {}, + {metricWithLabels(t, "name", "foo")}, + {metricWithLabels(t, "name", "foo")}, + }, + map[int][]testAlert{ + 0: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 1: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 2: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, + 4: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateInactive}}}, + 5: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StatePending}}}, + 6: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}}, }, }, } @@ -304,7 +306,7 @@ func TestAlertingRule_Exec(t *testing.T) { fq := &fakeQuerier{} tc.rule.q = fq tc.rule.GroupID = fakeGroup.ID() - for _, step := range tc.steps { + for i, step := range tc.steps { fq.reset() fq.add(step...) if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil { @@ -312,28 +314,31 @@ func TestAlertingRule_Exec(t *testing.T) { } // artificial delay between applying steps time.Sleep(defaultStep) - } - if len(tc.rule.alerts) != len(tc.expAlerts) { - t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts)) - } - expAlerts := make(map[uint64]*notifier.Alert) - for _, ta := range tc.expAlerts { - labels := make(map[string]string) - for i := 0; i < len(ta.labels); i += 2 { - k, v := ta.labels[i], ta.labels[i+1] - labels[k] = v + if _, ok := tc.expAlerts[i]; !ok { + continue } - labels[alertNameLabel] = tc.rule.Name - h := hash(labels) - expAlerts[h] = ta.alert - } - for key, exp := range expAlerts { - got, ok := tc.rule.alerts[key] - if !ok { - t.Fatalf("expected to have key %d", key) + if len(tc.rule.alerts) != len(tc.expAlerts[i]) { + t.Fatalf("evalIndex %d: expected %d alerts; got %d", i, len(tc.expAlerts[i]), len(tc.rule.alerts)) } - if got.State != exp.State { - t.Fatalf("expected state %d; got %d", exp.State, got.State) + expAlerts := make(map[uint64]*notifier.Alert) + for _, ta := range tc.expAlerts[i] { + labels := make(map[string]string) + for i := 0; i < len(ta.labels); i += 2 { + k, v := ta.labels[i], ta.labels[i+1] + labels[k] = v + } + labels[alertNameLabel] = tc.rule.Name + h := hash(labels) + expAlerts[h] = ta.alert + } + for key, exp := range expAlerts { + got, ok := tc.rule.alerts[key] + if !ok { + t.Fatalf("evalIndex %d: expected to have key %d", i, key) + } + if got.State != exp.State { + t.Fatalf("evalIndex %d: expected state %d; got %d", i, exp.State, got.State) + } } } }) @@ -867,7 +872,6 @@ func TestAlertingRule_Template(t *testing.T) { gotAlert := tc.rule.alerts[hash] if gotAlert == nil { t.Fatalf("alert %d is missing; labels: %v; annotations: %v", hash, expAlert.Labels, expAlert.Annotations) - break } if !reflect.DeepEqual(expAlert.Annotations, gotAlert.Annotations) { t.Fatalf("expected to have annotations %#v; got %#v", expAlert.Annotations, gotAlert.Annotations) @@ -970,11 +974,18 @@ func newTestRuleWithLabels(name string, labels ...string) *AlertingRule { } func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule { - return &AlertingRule{ + rule := AlertingRule{ Name: name, For: waitFor, EvalInterval: waitFor, alerts: make(map[uint64]*notifier.Alert), state: newRuleState(10), } + return &rule +} + +func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule { + rule := newTestAlertingRule(name, waitFor) + rule.KeepFiringFor = keepFiringFor + return rule } diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 0fc201ef90..7719974685 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -105,14 +105,16 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) // Rule describes entity that represent either // recording rule or alerting rule. type Rule struct { - ID uint64 - Record string `yaml:"record,omitempty"` - Alert string `yaml:"alert,omitempty"` - Expr string `yaml:"expr"` - For *promutils.Duration `yaml:"for,omitempty"` - Labels map[string]string `yaml:"labels,omitempty"` - Annotations map[string]string `yaml:"annotations,omitempty"` - Debug bool `yaml:"debug,omitempty"` + ID uint64 + Record string `yaml:"record,omitempty"` + Alert string `yaml:"alert,omitempty"` + Expr string `yaml:"expr"` + For *promutils.Duration `yaml:"for,omitempty"` + // Alert will continue firing for this long even when the alerting expression no longer has results. + KeepFiringFor *promutils.Duration `yaml:"keep_firing_for,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + Annotations map[string]string `yaml:"annotations,omitempty"` + Debug bool `yaml:"debug,omitempty"` // UpdateEntriesLimit defines max number of rule's state updates stored in memory. // Overrides `-rule.updateEntriesLimit`. UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"` diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 90df89d8b4..b548f55f13 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -404,7 +404,7 @@ func TestHashRule(t *testing.T) { true, }, { - Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)}, + Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)}, Rule{Alert: "alert", Expr: "up == 1"}, true, }, diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go index 29199a4bd8..c166b61b49 100644 --- a/app/vmalert/group_test.go +++ b/app/vmalert/group_test.go @@ -46,18 +46,36 @@ func TestUpdateWith(t *testing.T) { "summary": "{{ $value|humanize }}", "description": "{{$labels}}", }, - }}, - []config.Rule{{ - Alert: "foo", - Expr: "up > 10", - For: promutils.NewDuration(time.Second), - Labels: map[string]string{ - "baz": "bar", + }, + { + Alert: "bar", + Expr: "up > 0", + For: promutils.NewDuration(time.Second), + Labels: map[string]string{ + "bar": "baz", + }, + }}, + []config.Rule{ + { + Alert: "foo", + Expr: "up > 10", + For: promutils.NewDuration(time.Second), + Labels: map[string]string{ + "baz": "bar", + }, + Annotations: map[string]string{ + "summary": "none", + }, }, - Annotations: map[string]string{ - "summary": "none", - }, - }}, + { + Alert: "bar", + Expr: "up > 0", + For: promutils.NewDuration(2 * time.Second), + KeepFiringFor: promutils.NewDuration(time.Minute), + Labels: map[string]string{ + "bar": "baz", + }, + }}, }, { "update recording rule", diff --git a/app/vmalert/helpers_test.go b/app/vmalert/helpers_test.go index de0db34620..95e7c9f444 100644 --- a/app/vmalert/helpers_test.go +++ b/app/vmalert/helpers_test.go @@ -272,6 +272,9 @@ func compareAlertingRules(t *testing.T, a, b *AlertingRule) error { if a.For != b.For { return fmt.Errorf("expected to have for %q; got %q", a.For, b.For) } + if a.KeepFiringFor != b.KeepFiringFor { + return fmt.Errorf("expected to have KeepFiringFor %q; got %q", a.KeepFiringFor, b.KeepFiringFor) + } if !reflect.DeepEqual(a.Annotations, b.Annotations) { return fmt.Errorf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations) } diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index 9be76aeedd..3bdd3a8f85 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -39,6 +39,8 @@ type Alert struct { ResolvedAt time.Time // LastSent defines the moment when Alert was sent last time LastSent time.Time + // KeepFiringSince defines the moment when StateFiring was kept because of `keep_firing_for` instead of real alert + KeepFiringSince time.Time // Value stores the value returned from evaluating expression from Expr field Value float64 // ID is the unique identifier for the Alert diff --git a/app/vmalert/web.qtpl b/app/vmalert/web.qtpl index 1b412c7c30..2cbf0a2ce7 100644 --- a/app/vmalert/web.qtpl +++ b/app/vmalert/web.qtpl @@ -116,7 +116,11 @@ btn-primary <div class="row"> <div class="col-12 mb-2"> {% if r.Type == "alerting" %} + {% if r.KeepFiringFor > 0 %} + <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds, keep_firing_for: {%v r.KeepFiringFor %} seconds) + {% else %} <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds) + {% endif %} {% else %} <b>record:</b> {%s r.Name %} {% endif %} @@ -225,6 +229,7 @@ btn-primary <td> {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %} {% if ar.Restored %}{%= badgeRestored() %}{% endif %} + {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %} </td> <td>{%s ar.Value %}</td> <td> @@ -442,6 +447,18 @@ btn-primary </div> </div> </div> + {% if rule.KeepFiringFor > 0 %} + <div class="container border-bottom p-2"> + <div class="row"> + <div class="col-2"> + Keep firing for + </div> + <div class="col"> + {%v rule.KeepFiringFor %} seconds + </div> + </div> + </div> + {% endif %} {% endif %} <div class="container border-bottom p-2"> <div class="row"> @@ -561,6 +578,10 @@ btn-primary <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span> {% endfunc %} +{% func badgeStabilizing() %} +<span class="badge bg-warning text-dark" title="This firing state is kept because of `keep_firing_for`">stabilizing</span> +{% endfunc %} + {% func seriesFetchedWarn(r APIRule) %} {% if isNoMatch(r) %} <svg xmlns="http://www.w3.org/2000/svg" diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go index 97999a74f8..0fa83c873c 100644 --- a/app/vmalert/web.qtpl.go +++ b/app/vmalert/web.qtpl.go @@ -438,251 +438,279 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, originGroups [ if r.Type == "alerting" { //line app/vmalert/web.qtpl:118 qw422016.N().S(` - <b>alert:</b> `) -//line app/vmalert/web.qtpl:119 - qw422016.E().S(r.Name) -//line app/vmalert/web.qtpl:119 - qw422016.N().S(` (for: `) -//line app/vmalert/web.qtpl:119 - qw422016.E().V(r.Duration) -//line app/vmalert/web.qtpl:119 - qw422016.N().S(` seconds) `) +//line app/vmalert/web.qtpl:119 + if r.KeepFiringFor > 0 { +//line app/vmalert/web.qtpl:119 + qw422016.N().S(` + <b>alert:</b> `) //line app/vmalert/web.qtpl:120 + qw422016.E().S(r.Name) +//line app/vmalert/web.qtpl:120 + qw422016.N().S(` (for: `) +//line app/vmalert/web.qtpl:120 + qw422016.E().V(r.Duration) +//line app/vmalert/web.qtpl:120 + qw422016.N().S(` seconds, keep_firing_for: `) +//line app/vmalert/web.qtpl:120 + qw422016.E().V(r.KeepFiringFor) +//line app/vmalert/web.qtpl:120 + qw422016.N().S(` seconds) + `) +//line app/vmalert/web.qtpl:121 + } else { +//line app/vmalert/web.qtpl:121 + qw422016.N().S(` + <b>alert:</b> `) +//line app/vmalert/web.qtpl:122 + qw422016.E().S(r.Name) +//line app/vmalert/web.qtpl:122 + qw422016.N().S(` (for: `) +//line app/vmalert/web.qtpl:122 + qw422016.E().V(r.Duration) +//line app/vmalert/web.qtpl:122 + qw422016.N().S(` seconds) + `) +//line app/vmalert/web.qtpl:123 + } +//line app/vmalert/web.qtpl:123 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:124 } else { -//line app/vmalert/web.qtpl:120 +//line app/vmalert/web.qtpl:124 qw422016.N().S(` <b>record:</b> `) -//line app/vmalert/web.qtpl:121 +//line app/vmalert/web.qtpl:125 qw422016.E().S(r.Name) -//line app/vmalert/web.qtpl:121 +//line app/vmalert/web.qtpl:125 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:122 +//line app/vmalert/web.qtpl:126 } -//line app/vmalert/web.qtpl:122 +//line app/vmalert/web.qtpl:126 qw422016.N().S(` | `) -//line app/vmalert/web.qtpl:124 +//line app/vmalert/web.qtpl:128 streamseriesFetchedWarn(qw422016, r) -//line app/vmalert/web.qtpl:124 +//line app/vmalert/web.qtpl:128 qw422016.N().S(` <span><a target="_blank" href="`) -//line app/vmalert/web.qtpl:125 +//line app/vmalert/web.qtpl:129 qw422016.E().S(prefix + r.WebLink()) -//line app/vmalert/web.qtpl:125 +//line app/vmalert/web.qtpl:129 qw422016.N().S(`">Details</a></span> </div> <div class="col-12"> <code><pre>`) -//line app/vmalert/web.qtpl:128 +//line app/vmalert/web.qtpl:132 qw422016.E().S(r.Query) -//line app/vmalert/web.qtpl:128 +//line app/vmalert/web.qtpl:132 qw422016.N().S(`</pre></code> </div> <div class="col-12 mb-2"> `) -//line app/vmalert/web.qtpl:131 +//line app/vmalert/web.qtpl:135 if len(r.Labels) > 0 { -//line app/vmalert/web.qtpl:131 +//line app/vmalert/web.qtpl:135 qw422016.N().S(` <b>Labels:</b>`) -//line app/vmalert/web.qtpl:131 +//line app/vmalert/web.qtpl:135 } -//line app/vmalert/web.qtpl:131 +//line app/vmalert/web.qtpl:135 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:132 +//line app/vmalert/web.qtpl:136 for k, v := range r.Labels { -//line app/vmalert/web.qtpl:132 +//line app/vmalert/web.qtpl:136 qw422016.N().S(` <span class="ms-1 badge bg-primary">`) -//line app/vmalert/web.qtpl:133 +//line app/vmalert/web.qtpl:137 qw422016.E().S(k) -//line app/vmalert/web.qtpl:133 +//line app/vmalert/web.qtpl:137 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:133 +//line app/vmalert/web.qtpl:137 qw422016.E().S(v) -//line app/vmalert/web.qtpl:133 +//line app/vmalert/web.qtpl:137 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:134 +//line app/vmalert/web.qtpl:138 } -//line app/vmalert/web.qtpl:134 +//line app/vmalert/web.qtpl:138 qw422016.N().S(` </div> `) -//line app/vmalert/web.qtpl:136 +//line app/vmalert/web.qtpl:140 if r.LastError != "" { -//line app/vmalert/web.qtpl:136 +//line app/vmalert/web.qtpl:140 qw422016.N().S(` <div class="col-12"> <b>Error:</b> <div class="error-cell"> `) -//line app/vmalert/web.qtpl:140 +//line app/vmalert/web.qtpl:144 qw422016.E().S(r.LastError) -//line app/vmalert/web.qtpl:140 +//line app/vmalert/web.qtpl:144 qw422016.N().S(` </div> </div> `) -//line app/vmalert/web.qtpl:143 +//line app/vmalert/web.qtpl:147 } -//line app/vmalert/web.qtpl:143 +//line app/vmalert/web.qtpl:147 qw422016.N().S(` </div> </td> <td class="text-center">`) -//line app/vmalert/web.qtpl:146 +//line app/vmalert/web.qtpl:150 qw422016.N().D(r.LastSamples) -//line app/vmalert/web.qtpl:146 +//line app/vmalert/web.qtpl:150 qw422016.N().S(`</td> <td class="text-center">`) -//line app/vmalert/web.qtpl:147 +//line app/vmalert/web.qtpl:151 qw422016.N().FPrec(time.Since(r.LastEvaluation).Seconds(), 3) -//line app/vmalert/web.qtpl:147 +//line app/vmalert/web.qtpl:151 qw422016.N().S(`s ago</td> </tr> `) -//line app/vmalert/web.qtpl:149 +//line app/vmalert/web.qtpl:153 } -//line app/vmalert/web.qtpl:149 +//line app/vmalert/web.qtpl:153 qw422016.N().S(` </tbody> </table> </div> `) -//line app/vmalert/web.qtpl:153 +//line app/vmalert/web.qtpl:157 } -//line app/vmalert/web.qtpl:153 +//line app/vmalert/web.qtpl:157 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:154 +//line app/vmalert/web.qtpl:158 } else { -//line app/vmalert/web.qtpl:154 +//line app/vmalert/web.qtpl:158 qw422016.N().S(` <div> <p>No groups...</p> </div> `) -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:162 } -//line app/vmalert/web.qtpl:158 +//line app/vmalert/web.qtpl:162 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:160 +//line app/vmalert/web.qtpl:164 tpl.StreamFooter(qw422016, r) -//line app/vmalert/web.qtpl:160 +//line app/vmalert/web.qtpl:164 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 } -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, originGroups []APIGroup) { -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 StreamListGroups(qw422016, r, originGroups) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 } -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 func ListGroups(r *http.Request, originGroups []APIGroup) string { -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 WriteListGroups(qb422016, r, originGroups) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 return qs422016 -//line app/vmalert/web.qtpl:162 +//line app/vmalert/web.qtpl:166 } -//line app/vmalert/web.qtpl:165 +//line app/vmalert/web.qtpl:169 func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts []GroupAlerts) { -//line app/vmalert/web.qtpl:165 +//line app/vmalert/web.qtpl:169 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:166 +//line app/vmalert/web.qtpl:170 prefix := utils.Prefix(r.URL.Path) -//line app/vmalert/web.qtpl:166 +//line app/vmalert/web.qtpl:170 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:167 +//line app/vmalert/web.qtpl:171 tpl.StreamHeader(qw422016, r, navItems, "Alerts", configError()) -//line app/vmalert/web.qtpl:167 +//line app/vmalert/web.qtpl:171 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:168 +//line app/vmalert/web.qtpl:172 if len(groupAlerts) > 0 { -//line app/vmalert/web.qtpl:168 +//line app/vmalert/web.qtpl:172 qw422016.N().S(` <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a> <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a> `) -//line app/vmalert/web.qtpl:171 +//line app/vmalert/web.qtpl:175 for _, ga := range groupAlerts { -//line app/vmalert/web.qtpl:171 +//line app/vmalert/web.qtpl:175 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:172 +//line app/vmalert/web.qtpl:176 g := ga.Group -//line app/vmalert/web.qtpl:172 +//line app/vmalert/web.qtpl:176 qw422016.N().S(` <div class="group-heading alert-danger" data-bs-target="rules-`) -//line app/vmalert/web.qtpl:173 +//line app/vmalert/web.qtpl:177 qw422016.E().S(g.ID) -//line app/vmalert/web.qtpl:173 +//line app/vmalert/web.qtpl:177 qw422016.N().S(`"> <span class="anchor" id="group-`) -//line app/vmalert/web.qtpl:174 +//line app/vmalert/web.qtpl:178 qw422016.E().S(g.ID) -//line app/vmalert/web.qtpl:174 +//line app/vmalert/web.qtpl:178 qw422016.N().S(`"></span> <a href="#group-`) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.E().S(g.ID) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.E().S(g.Name) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 if g.Type != "prometheus" { -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.N().S(` (`) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.E().S(g.Type) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.N().S(`)`) -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 } -//line app/vmalert/web.qtpl:175 +//line app/vmalert/web.qtpl:179 qw422016.N().S(`</a> <span class="badge bg-danger" title="Number of active alerts">`) -//line app/vmalert/web.qtpl:176 +//line app/vmalert/web.qtpl:180 qw422016.N().D(len(ga.Alerts)) -//line app/vmalert/web.qtpl:176 +//line app/vmalert/web.qtpl:180 qw422016.N().S(`</span> <br> <p class="fs-6 fw-lighter">`) -//line app/vmalert/web.qtpl:178 +//line app/vmalert/web.qtpl:182 qw422016.E().S(g.File) -//line app/vmalert/web.qtpl:178 +//line app/vmalert/web.qtpl:182 qw422016.N().S(`</p> </div> `) -//line app/vmalert/web.qtpl:181 +//line app/vmalert/web.qtpl:185 var keys []string alertsByRule := make(map[string][]*APIAlert) for _, alert := range ga.Alerts { @@ -693,20 +721,20 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts [] } sort.Strings(keys) -//line app/vmalert/web.qtpl:190 +//line app/vmalert/web.qtpl:194 qw422016.N().S(` <div class="collapse" id="rules-`) -//line app/vmalert/web.qtpl:191 +//line app/vmalert/web.qtpl:195 qw422016.E().S(g.ID) -//line app/vmalert/web.qtpl:191 +//line app/vmalert/web.qtpl:195 qw422016.N().S(`"> `) -//line app/vmalert/web.qtpl:192 +//line app/vmalert/web.qtpl:196 for _, ruleID := range keys { -//line app/vmalert/web.qtpl:192 +//line app/vmalert/web.qtpl:196 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:194 +//line app/vmalert/web.qtpl:198 defaultAR := alertsByRule[ruleID][0] var labelKeys []string for k := range defaultAR.Labels { @@ -714,28 +742,28 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts [] } sort.Strings(labelKeys) -//line app/vmalert/web.qtpl:200 +//line app/vmalert/web.qtpl:204 qw422016.N().S(` <br> <b>alert:</b> `) -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:206 qw422016.E().S(defaultAR.Name) -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:206 qw422016.N().S(` (`) -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:206 qw422016.N().D(len(alertsByRule[ruleID])) -//line app/vmalert/web.qtpl:202 +//line app/vmalert/web.qtpl:206 qw422016.N().S(`) | <span><a target="_blank" href="`) -//line app/vmalert/web.qtpl:203 +//line app/vmalert/web.qtpl:207 qw422016.E().S(defaultAR.SourceLink) -//line app/vmalert/web.qtpl:203 +//line app/vmalert/web.qtpl:207 qw422016.N().S(`">Source</a></span> <br> <b>expr:</b><code><pre>`) -//line app/vmalert/web.qtpl:205 +//line app/vmalert/web.qtpl:209 qw422016.E().S(defaultAR.Expression) -//line app/vmalert/web.qtpl:205 +//line app/vmalert/web.qtpl:209 qw422016.N().S(`</pre></code> <table class="table table-striped table-hover table-sm"> <thead> @@ -749,204 +777,213 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts [] </thead> <tbody> `) -//line app/vmalert/web.qtpl:217 +//line app/vmalert/web.qtpl:221 for _, ar := range alertsByRule[ruleID] { -//line app/vmalert/web.qtpl:217 +//line app/vmalert/web.qtpl:221 qw422016.N().S(` <tr> <td> `) -//line app/vmalert/web.qtpl:220 +//line app/vmalert/web.qtpl:224 for _, k := range labelKeys { -//line app/vmalert/web.qtpl:220 +//line app/vmalert/web.qtpl:224 qw422016.N().S(` <span class="ms-1 badge bg-primary">`) -//line app/vmalert/web.qtpl:221 +//line app/vmalert/web.qtpl:225 qw422016.E().S(k) -//line app/vmalert/web.qtpl:221 +//line app/vmalert/web.qtpl:225 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:221 +//line app/vmalert/web.qtpl:225 qw422016.E().S(ar.Labels[k]) -//line app/vmalert/web.qtpl:221 +//line app/vmalert/web.qtpl:225 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:222 +//line app/vmalert/web.qtpl:226 } -//line app/vmalert/web.qtpl:222 +//line app/vmalert/web.qtpl:226 qw422016.N().S(` </td> <td>`) -//line app/vmalert/web.qtpl:224 +//line app/vmalert/web.qtpl:228 streambadgeState(qw422016, ar.State) -//line app/vmalert/web.qtpl:224 +//line app/vmalert/web.qtpl:228 qw422016.N().S(`</td> <td> `) -//line app/vmalert/web.qtpl:226 +//line app/vmalert/web.qtpl:230 qw422016.E().S(ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00")) -//line app/vmalert/web.qtpl:226 +//line app/vmalert/web.qtpl:230 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:227 +//line app/vmalert/web.qtpl:231 if ar.Restored { -//line app/vmalert/web.qtpl:227 +//line app/vmalert/web.qtpl:231 streambadgeRestored(qw422016) -//line app/vmalert/web.qtpl:227 +//line app/vmalert/web.qtpl:231 } -//line app/vmalert/web.qtpl:227 +//line app/vmalert/web.qtpl:231 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:232 + if ar.Stabilizing { +//line app/vmalert/web.qtpl:232 + streambadgeStabilizing(qw422016) +//line app/vmalert/web.qtpl:232 + } +//line app/vmalert/web.qtpl:232 qw422016.N().S(` </td> <td>`) -//line app/vmalert/web.qtpl:229 +//line app/vmalert/web.qtpl:234 qw422016.E().S(ar.Value) -//line app/vmalert/web.qtpl:229 +//line app/vmalert/web.qtpl:234 qw422016.N().S(`</td> <td> <a href="`) -//line app/vmalert/web.qtpl:231 +//line app/vmalert/web.qtpl:236 qw422016.E().S(prefix + ar.WebLink()) -//line app/vmalert/web.qtpl:231 +//line app/vmalert/web.qtpl:236 qw422016.N().S(`">Details</a> </td> </tr> `) -//line app/vmalert/web.qtpl:234 +//line app/vmalert/web.qtpl:239 } -//line app/vmalert/web.qtpl:234 +//line app/vmalert/web.qtpl:239 qw422016.N().S(` </tbody> </table> `) -//line app/vmalert/web.qtpl:237 +//line app/vmalert/web.qtpl:242 } -//line app/vmalert/web.qtpl:237 +//line app/vmalert/web.qtpl:242 qw422016.N().S(` </div> <br> `) -//line app/vmalert/web.qtpl:240 +//line app/vmalert/web.qtpl:245 } -//line app/vmalert/web.qtpl:240 +//line app/vmalert/web.qtpl:245 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:242 +//line app/vmalert/web.qtpl:247 } else { -//line app/vmalert/web.qtpl:242 +//line app/vmalert/web.qtpl:247 qw422016.N().S(` <div> <p>No active alerts...</p> </div> `) -//line app/vmalert/web.qtpl:246 +//line app/vmalert/web.qtpl:251 } -//line app/vmalert/web.qtpl:246 +//line app/vmalert/web.qtpl:251 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:248 +//line app/vmalert/web.qtpl:253 tpl.StreamFooter(qw422016, r) -//line app/vmalert/web.qtpl:248 +//line app/vmalert/web.qtpl:253 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 } -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 func WriteListAlerts(qq422016 qtio422016.Writer, r *http.Request, groupAlerts []GroupAlerts) { -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 StreamListAlerts(qw422016, r, groupAlerts) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 } -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) string { -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 WriteListAlerts(qb422016, r, groupAlerts) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 return qs422016 -//line app/vmalert/web.qtpl:250 +//line app/vmalert/web.qtpl:255 } -//line app/vmalert/web.qtpl:252 +//line app/vmalert/web.qtpl:257 func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[notifier.TargetType][]notifier.Target) { -//line app/vmalert/web.qtpl:252 +//line app/vmalert/web.qtpl:257 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:253 +//line app/vmalert/web.qtpl:258 tpl.StreamHeader(qw422016, r, navItems, "Notifiers", configError()) -//line app/vmalert/web.qtpl:253 +//line app/vmalert/web.qtpl:258 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:254 +//line app/vmalert/web.qtpl:259 if len(targets) > 0 { -//line app/vmalert/web.qtpl:254 +//line app/vmalert/web.qtpl:259 qw422016.N().S(` <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a> <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a> `) -//line app/vmalert/web.qtpl:259 +//line app/vmalert/web.qtpl:264 var keys []string for key := range targets { keys = append(keys, string(key)) } sort.Strings(keys) -//line app/vmalert/web.qtpl:264 +//line app/vmalert/web.qtpl:269 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:266 +//line app/vmalert/web.qtpl:271 for i := range keys { -//line app/vmalert/web.qtpl:266 +//line app/vmalert/web.qtpl:271 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:267 +//line app/vmalert/web.qtpl:272 typeK, ns := keys[i], targets[notifier.TargetType(keys[i])] count := len(ns) -//line app/vmalert/web.qtpl:269 +//line app/vmalert/web.qtpl:274 qw422016.N().S(` <div class="group-heading" data-bs-target="notifiers-`) -//line app/vmalert/web.qtpl:270 +//line app/vmalert/web.qtpl:275 qw422016.E().S(typeK) -//line app/vmalert/web.qtpl:270 +//line app/vmalert/web.qtpl:275 qw422016.N().S(`"> <span class="anchor" id="group-`) -//line app/vmalert/web.qtpl:271 +//line app/vmalert/web.qtpl:276 qw422016.E().S(typeK) -//line app/vmalert/web.qtpl:271 +//line app/vmalert/web.qtpl:276 qw422016.N().S(`"></span> <a href="#group-`) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.E().S(typeK) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.E().S(typeK) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.N().S(` (`) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.N().D(count) -//line app/vmalert/web.qtpl:272 +//line app/vmalert/web.qtpl:277 qw422016.N().S(`)</a> </div> <div class="collapse show" id="notifiers-`) -//line app/vmalert/web.qtpl:274 +//line app/vmalert/web.qtpl:279 qw422016.E().S(typeK) -//line app/vmalert/web.qtpl:274 +//line app/vmalert/web.qtpl:279 qw422016.N().S(`"> <table class="table table-striped table-hover table-sm"> <thead> @@ -957,119 +994,119 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n </thead> <tbody> `) -//line app/vmalert/web.qtpl:283 +//line app/vmalert/web.qtpl:288 for _, n := range ns { -//line app/vmalert/web.qtpl:283 +//line app/vmalert/web.qtpl:288 qw422016.N().S(` <tr> <td> `) -//line app/vmalert/web.qtpl:286 +//line app/vmalert/web.qtpl:291 for _, l := range n.Labels.GetLabels() { -//line app/vmalert/web.qtpl:286 +//line app/vmalert/web.qtpl:291 qw422016.N().S(` <span class="ms-1 badge bg-primary">`) -//line app/vmalert/web.qtpl:287 +//line app/vmalert/web.qtpl:292 qw422016.E().S(l.Name) -//line app/vmalert/web.qtpl:287 +//line app/vmalert/web.qtpl:292 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:287 +//line app/vmalert/web.qtpl:292 qw422016.E().S(l.Value) -//line app/vmalert/web.qtpl:287 +//line app/vmalert/web.qtpl:292 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:288 +//line app/vmalert/web.qtpl:293 } -//line app/vmalert/web.qtpl:288 +//line app/vmalert/web.qtpl:293 qw422016.N().S(` </td> <td>`) -//line app/vmalert/web.qtpl:290 +//line app/vmalert/web.qtpl:295 qw422016.E().S(n.Notifier.Addr()) -//line app/vmalert/web.qtpl:290 +//line app/vmalert/web.qtpl:295 qw422016.N().S(`</td> </tr> `) -//line app/vmalert/web.qtpl:292 +//line app/vmalert/web.qtpl:297 } -//line app/vmalert/web.qtpl:292 +//line app/vmalert/web.qtpl:297 qw422016.N().S(` </tbody> </table> </div> `) -//line app/vmalert/web.qtpl:296 +//line app/vmalert/web.qtpl:301 } -//line app/vmalert/web.qtpl:296 +//line app/vmalert/web.qtpl:301 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:298 +//line app/vmalert/web.qtpl:303 } else { -//line app/vmalert/web.qtpl:298 +//line app/vmalert/web.qtpl:303 qw422016.N().S(` <div> <p>No targets...</p> </div> `) -//line app/vmalert/web.qtpl:302 +//line app/vmalert/web.qtpl:307 } -//line app/vmalert/web.qtpl:302 +//line app/vmalert/web.qtpl:307 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:304 +//line app/vmalert/web.qtpl:309 tpl.StreamFooter(qw422016, r) -//line app/vmalert/web.qtpl:304 +//line app/vmalert/web.qtpl:309 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 } -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 func WriteListTargets(qq422016 qtio422016.Writer, r *http.Request, targets map[notifier.TargetType][]notifier.Target) { -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 StreamListTargets(qw422016, r, targets) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 } -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) string { -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 WriteListTargets(qb422016, r, targets) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 return qs422016 -//line app/vmalert/web.qtpl:306 +//line app/vmalert/web.qtpl:311 } -//line app/vmalert/web.qtpl:308 +//line app/vmalert/web.qtpl:313 func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { -//line app/vmalert/web.qtpl:308 +//line app/vmalert/web.qtpl:313 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:309 +//line app/vmalert/web.qtpl:314 prefix := utils.Prefix(r.URL.Path) -//line app/vmalert/web.qtpl:309 +//line app/vmalert/web.qtpl:314 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:310 +//line app/vmalert/web.qtpl:315 tpl.StreamHeader(qw422016, r, navItems, "", configError()) -//line app/vmalert/web.qtpl:310 +//line app/vmalert/web.qtpl:315 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:312 +//line app/vmalert/web.qtpl:317 var labelKeys []string for k := range alert.Labels { labelKeys = append(labelKeys, k) @@ -1082,28 +1119,28 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { } sort.Strings(annotationKeys) -//line app/vmalert/web.qtpl:323 +//line app/vmalert/web.qtpl:328 qw422016.N().S(` <div class="display-6 pb-3 mb-3">Alert: `) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.E().S(alert.Name) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.N().S(`<span class="ms-2 badge `) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 if alert.State == "firing" { -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.N().S(`bg-danger`) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 } else { -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.N().S(` bg-warning text-dark`) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 } -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.E().S(alert.State) -//line app/vmalert/web.qtpl:324 +//line app/vmalert/web.qtpl:329 qw422016.N().S(`</span></div> <div class="container border-bottom p-2"> <div class="row"> @@ -1112,9 +1149,9 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> `) -//line app/vmalert/web.qtpl:331 +//line app/vmalert/web.qtpl:336 qw422016.E().S(alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00")) -//line app/vmalert/web.qtpl:331 +//line app/vmalert/web.qtpl:336 qw422016.N().S(` </div> </div> @@ -1126,9 +1163,9 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> <code><pre>`) -//line app/vmalert/web.qtpl:341 +//line app/vmalert/web.qtpl:346 qw422016.E().S(alert.Expression) -//line app/vmalert/web.qtpl:341 +//line app/vmalert/web.qtpl:346 qw422016.N().S(`</pre></code> </div> </div> @@ -1140,23 +1177,23 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> `) -//line app/vmalert/web.qtpl:351 +//line app/vmalert/web.qtpl:356 for _, k := range labelKeys { -//line app/vmalert/web.qtpl:351 +//line app/vmalert/web.qtpl:356 qw422016.N().S(` <span class="m-1 badge bg-primary">`) -//line app/vmalert/web.qtpl:352 +//line app/vmalert/web.qtpl:357 qw422016.E().S(k) -//line app/vmalert/web.qtpl:352 +//line app/vmalert/web.qtpl:357 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:352 +//line app/vmalert/web.qtpl:357 qw422016.E().S(alert.Labels[k]) -//line app/vmalert/web.qtpl:352 +//line app/vmalert/web.qtpl:357 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:353 +//line app/vmalert/web.qtpl:358 } -//line app/vmalert/web.qtpl:353 +//line app/vmalert/web.qtpl:358 qw422016.N().S(` </div> </div> @@ -1168,24 +1205,24 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> `) -//line app/vmalert/web.qtpl:363 +//line app/vmalert/web.qtpl:368 for _, k := range annotationKeys { -//line app/vmalert/web.qtpl:363 +//line app/vmalert/web.qtpl:368 qw422016.N().S(` <b>`) -//line app/vmalert/web.qtpl:364 +//line app/vmalert/web.qtpl:369 qw422016.E().S(k) -//line app/vmalert/web.qtpl:364 +//line app/vmalert/web.qtpl:369 qw422016.N().S(`:</b><br> <p>`) -//line app/vmalert/web.qtpl:365 +//line app/vmalert/web.qtpl:370 qw422016.E().S(alert.Annotations[k]) -//line app/vmalert/web.qtpl:365 +//line app/vmalert/web.qtpl:370 qw422016.N().S(`</p> `) -//line app/vmalert/web.qtpl:366 +//line app/vmalert/web.qtpl:371 } -//line app/vmalert/web.qtpl:366 +//line app/vmalert/web.qtpl:371 qw422016.N().S(` </div> </div> @@ -1197,17 +1234,17 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> <a target="_blank" href="`) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.E().S(prefix) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.N().S(`groups#group-`) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.E().S(alert.GroupID) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.E().S(alert.GroupID) -//line app/vmalert/web.qtpl:376 +//line app/vmalert/web.qtpl:381 qw422016.N().S(`</a> </div> </div> @@ -1219,66 +1256,66 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { </div> <div class="col"> <a target="_blank" href="`) -//line app/vmalert/web.qtpl:386 +//line app/vmalert/web.qtpl:391 qw422016.E().S(alert.SourceLink) -//line app/vmalert/web.qtpl:386 +//line app/vmalert/web.qtpl:391 qw422016.N().S(`">Link</a> </div> </div> </div> `) -//line app/vmalert/web.qtpl:390 +//line app/vmalert/web.qtpl:395 tpl.StreamFooter(qw422016, r) -//line app/vmalert/web.qtpl:390 +//line app/vmalert/web.qtpl:395 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 } -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 func WriteAlert(qq422016 qtio422016.Writer, r *http.Request, alert *APIAlert) { -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 StreamAlert(qw422016, r, alert) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 } -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 func Alert(r *http.Request, alert *APIAlert) string { -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 WriteAlert(qb422016, r, alert) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 return qs422016 -//line app/vmalert/web.qtpl:392 +//line app/vmalert/web.qtpl:397 } -//line app/vmalert/web.qtpl:395 +//line app/vmalert/web.qtpl:400 func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) { -//line app/vmalert/web.qtpl:395 +//line app/vmalert/web.qtpl:400 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:396 +//line app/vmalert/web.qtpl:401 prefix := utils.Prefix(r.URL.Path) -//line app/vmalert/web.qtpl:396 +//line app/vmalert/web.qtpl:401 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:397 +//line app/vmalert/web.qtpl:402 tpl.StreamHeader(qw422016, r, navItems, "", configError()) -//line app/vmalert/web.qtpl:397 +//line app/vmalert/web.qtpl:402 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:399 +//line app/vmalert/web.qtpl:404 var labelKeys []string for k := range rule.Labels { labelKeys = append(labelKeys, k) @@ -1302,28 +1339,28 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) } } -//line app/vmalert/web.qtpl:422 +//line app/vmalert/web.qtpl:427 qw422016.N().S(` <div class="display-6 pb-3 mb-3">Rule: `) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.E().S(rule.Name) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.N().S(`<span class="ms-2 badge `) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 if rule.Health != "ok" { -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.N().S(`bg-danger`) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 } else { -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.N().S(` bg-success text-dark`) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 } -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.E().S(rule.Health) -//line app/vmalert/web.qtpl:423 +//line app/vmalert/web.qtpl:428 qw422016.N().S(`</span></div> <div class="container border-bottom p-2"> <div class="row"> @@ -1332,17 +1369,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> <code><pre>`) -//line app/vmalert/web.qtpl:430 +//line app/vmalert/web.qtpl:435 qw422016.E().S(rule.Query) -//line app/vmalert/web.qtpl:430 +//line app/vmalert/web.qtpl:435 qw422016.N().S(`</pre></code> </div> </div> </div> `) -//line app/vmalert/web.qtpl:434 +//line app/vmalert/web.qtpl:439 if rule.Type == "alerting" { -//line app/vmalert/web.qtpl:434 +//line app/vmalert/web.qtpl:439 qw422016.N().S(` <div class="container border-bottom p-2"> <div class="row"> @@ -1351,17 +1388,41 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> `) -//line app/vmalert/web.qtpl:441 +//line app/vmalert/web.qtpl:446 qw422016.E().V(rule.Duration) -//line app/vmalert/web.qtpl:441 +//line app/vmalert/web.qtpl:446 qw422016.N().S(` seconds </div> </div> </div> `) -//line app/vmalert/web.qtpl:445 +//line app/vmalert/web.qtpl:450 + if rule.KeepFiringFor > 0 { +//line app/vmalert/web.qtpl:450 + qw422016.N().S(` + <div class="container border-bottom p-2"> + <div class="row"> + <div class="col-2"> + Keep firing for + </div> + <div class="col"> + `) +//line app/vmalert/web.qtpl:457 + qw422016.E().V(rule.KeepFiringFor) +//line app/vmalert/web.qtpl:457 + qw422016.N().S(` seconds + </div> + </div> + </div> + `) +//line app/vmalert/web.qtpl:461 + } +//line app/vmalert/web.qtpl:461 + qw422016.N().S(` + `) +//line app/vmalert/web.qtpl:462 } -//line app/vmalert/web.qtpl:445 +//line app/vmalert/web.qtpl:462 qw422016.N().S(` <div class="container border-bottom p-2"> <div class="row"> @@ -1370,31 +1431,31 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> `) -//line app/vmalert/web.qtpl:452 +//line app/vmalert/web.qtpl:469 for _, k := range labelKeys { -//line app/vmalert/web.qtpl:452 +//line app/vmalert/web.qtpl:469 qw422016.N().S(` <span class="m-1 badge bg-primary">`) -//line app/vmalert/web.qtpl:453 +//line app/vmalert/web.qtpl:470 qw422016.E().S(k) -//line app/vmalert/web.qtpl:453 +//line app/vmalert/web.qtpl:470 qw422016.N().S(`=`) -//line app/vmalert/web.qtpl:453 +//line app/vmalert/web.qtpl:470 qw422016.E().S(rule.Labels[k]) -//line app/vmalert/web.qtpl:453 +//line app/vmalert/web.qtpl:470 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:454 +//line app/vmalert/web.qtpl:471 } -//line app/vmalert/web.qtpl:454 +//line app/vmalert/web.qtpl:471 qw422016.N().S(` </div> </div> </div> `) -//line app/vmalert/web.qtpl:458 +//line app/vmalert/web.qtpl:475 if rule.Type == "alerting" { -//line app/vmalert/web.qtpl:458 +//line app/vmalert/web.qtpl:475 qw422016.N().S(` <div class="container border-bottom p-2"> <div class="row"> @@ -1403,24 +1464,24 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> `) -//line app/vmalert/web.qtpl:465 +//line app/vmalert/web.qtpl:482 for _, k := range annotationKeys { -//line app/vmalert/web.qtpl:465 +//line app/vmalert/web.qtpl:482 qw422016.N().S(` <b>`) -//line app/vmalert/web.qtpl:466 +//line app/vmalert/web.qtpl:483 qw422016.E().S(k) -//line app/vmalert/web.qtpl:466 +//line app/vmalert/web.qtpl:483 qw422016.N().S(`:</b><br> <p>`) -//line app/vmalert/web.qtpl:467 +//line app/vmalert/web.qtpl:484 qw422016.E().S(rule.Annotations[k]) -//line app/vmalert/web.qtpl:467 +//line app/vmalert/web.qtpl:484 qw422016.N().S(`</p> `) -//line app/vmalert/web.qtpl:468 +//line app/vmalert/web.qtpl:485 } -//line app/vmalert/web.qtpl:468 +//line app/vmalert/web.qtpl:485 qw422016.N().S(` </div> </div> @@ -1432,17 +1493,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> `) -//line app/vmalert/web.qtpl:478 +//line app/vmalert/web.qtpl:495 qw422016.E().V(rule.Debug) -//line app/vmalert/web.qtpl:478 +//line app/vmalert/web.qtpl:495 qw422016.N().S(` </div> </div> </div> `) -//line app/vmalert/web.qtpl:482 +//line app/vmalert/web.qtpl:499 } -//line app/vmalert/web.qtpl:482 +//line app/vmalert/web.qtpl:499 qw422016.N().S(` <div class="container border-bottom p-2"> <div class="row"> @@ -1451,17 +1512,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) </div> <div class="col"> <a target="_blank" href="`) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.E().S(prefix) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.N().S(`groups#group-`) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.E().S(rule.GroupID) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.E().S(rule.GroupID) -//line app/vmalert/web.qtpl:489 +//line app/vmalert/web.qtpl:506 qw422016.N().S(`</a> </div> </div> @@ -1469,9 +1530,9 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) <br> `) -//line app/vmalert/web.qtpl:495 +//line app/vmalert/web.qtpl:512 if seriesFetchedWarning { -//line app/vmalert/web.qtpl:495 +//line app/vmalert/web.qtpl:512 qw422016.N().S(` <div class="alert alert-warning" role="alert"> <strong>Warning:</strong> some of updates have "Series fetched" equal to 0.<br> @@ -1485,18 +1546,18 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) See more details about this detection <a target="_blank" href="https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4039">here</a>. </div> `) -//line app/vmalert/web.qtpl:507 +//line app/vmalert/web.qtpl:524 } -//line app/vmalert/web.qtpl:507 +//line app/vmalert/web.qtpl:524 qw422016.N().S(` <div class="display-6 pb-3">Last `) -//line app/vmalert/web.qtpl:508 +//line app/vmalert/web.qtpl:525 qw422016.N().D(len(rule.Updates)) -//line app/vmalert/web.qtpl:508 +//line app/vmalert/web.qtpl:525 qw422016.N().S(`/`) -//line app/vmalert/web.qtpl:508 +//line app/vmalert/web.qtpl:525 qw422016.N().D(rule.MaxUpdates) -//line app/vmalert/web.qtpl:508 +//line app/vmalert/web.qtpl:525 qw422016.N().S(` updates</span>:</div> <table class="table table-striped table-hover table-sm"> <thead> @@ -1504,13 +1565,13 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) <th scope="col" title="The time when event was created">Updated at</th> <th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th> `) -//line app/vmalert/web.qtpl:514 +//line app/vmalert/web.qtpl:531 if seriesFetchedEnabled { -//line app/vmalert/web.qtpl:514 +//line app/vmalert/web.qtpl:531 qw422016.N().S(`<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>`) -//line app/vmalert/web.qtpl:514 +//line app/vmalert/web.qtpl:531 } -//line app/vmalert/web.qtpl:514 +//line app/vmalert/web.qtpl:531 qw422016.N().S(` <th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th> <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th> @@ -1520,242 +1581,285 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) <tbody> `) -//line app/vmalert/web.qtpl:522 +//line app/vmalert/web.qtpl:539 for _, u := range rule.Updates { -//line app/vmalert/web.qtpl:522 +//line app/vmalert/web.qtpl:539 qw422016.N().S(` <tr`) -//line app/vmalert/web.qtpl:523 +//line app/vmalert/web.qtpl:540 if u.err != nil { -//line app/vmalert/web.qtpl:523 +//line app/vmalert/web.qtpl:540 qw422016.N().S(` class="alert-danger"`) -//line app/vmalert/web.qtpl:523 +//line app/vmalert/web.qtpl:540 } -//line app/vmalert/web.qtpl:523 +//line app/vmalert/web.qtpl:540 qw422016.N().S(`> <td> <span class="badge bg-primary rounded-pill me-3" title="Updated at">`) -//line app/vmalert/web.qtpl:525 +//line app/vmalert/web.qtpl:542 qw422016.E().S(u.time.Format(time.RFC3339)) -//line app/vmalert/web.qtpl:525 +//line app/vmalert/web.qtpl:542 qw422016.N().S(`</span> </td> <td class="text-center">`) -//line app/vmalert/web.qtpl:527 +//line app/vmalert/web.qtpl:544 qw422016.N().D(u.samples) -//line app/vmalert/web.qtpl:527 +//line app/vmalert/web.qtpl:544 qw422016.N().S(`</td> `) -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 if seriesFetchedEnabled { -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 qw422016.N().S(`<td class="text-center">`) -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 if u.seriesFetched != nil { -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 qw422016.N().D(*u.seriesFetched) -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 } -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 qw422016.N().S(`</td>`) -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 } -//line app/vmalert/web.qtpl:528 +//line app/vmalert/web.qtpl:545 qw422016.N().S(` <td class="text-center">`) -//line app/vmalert/web.qtpl:529 +//line app/vmalert/web.qtpl:546 qw422016.N().FPrec(u.duration.Seconds(), 3) -//line app/vmalert/web.qtpl:529 +//line app/vmalert/web.qtpl:546 qw422016.N().S(`s</td> <td class="text-center">`) -//line app/vmalert/web.qtpl:530 +//line app/vmalert/web.qtpl:547 qw422016.E().S(u.at.Format(time.RFC3339)) -//line app/vmalert/web.qtpl:530 +//line app/vmalert/web.qtpl:547 qw422016.N().S(`</td> <td> <textarea class="curl-area" rows="1" onclick="this.focus();this.select()">`) -//line app/vmalert/web.qtpl:532 +//line app/vmalert/web.qtpl:549 qw422016.E().S(u.curl) -//line app/vmalert/web.qtpl:532 +//line app/vmalert/web.qtpl:549 qw422016.N().S(`</textarea> </td> </tr> </li> `) -//line app/vmalert/web.qtpl:536 +//line app/vmalert/web.qtpl:553 if u.err != nil { -//line app/vmalert/web.qtpl:536 +//line app/vmalert/web.qtpl:553 qw422016.N().S(` <tr`) -//line app/vmalert/web.qtpl:537 +//line app/vmalert/web.qtpl:554 if u.err != nil { -//line app/vmalert/web.qtpl:537 +//line app/vmalert/web.qtpl:554 qw422016.N().S(` class="alert-danger"`) -//line app/vmalert/web.qtpl:537 +//line app/vmalert/web.qtpl:554 } -//line app/vmalert/web.qtpl:537 +//line app/vmalert/web.qtpl:554 qw422016.N().S(`> <td colspan="`) -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 if seriesFetchedEnabled { -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 qw422016.N().S(`6`) -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 } else { -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 qw422016.N().S(`5`) -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 } -//line app/vmalert/web.qtpl:538 +//line app/vmalert/web.qtpl:555 qw422016.N().S(`"> <span class="alert-danger">`) -//line app/vmalert/web.qtpl:539 +//line app/vmalert/web.qtpl:556 qw422016.E().V(u.err) -//line app/vmalert/web.qtpl:539 +//line app/vmalert/web.qtpl:556 qw422016.N().S(`</span> </td> </tr> `) -//line app/vmalert/web.qtpl:542 +//line app/vmalert/web.qtpl:559 } -//line app/vmalert/web.qtpl:542 +//line app/vmalert/web.qtpl:559 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:543 +//line app/vmalert/web.qtpl:560 } -//line app/vmalert/web.qtpl:543 +//line app/vmalert/web.qtpl:560 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:545 +//line app/vmalert/web.qtpl:562 tpl.StreamFooter(qw422016, r) -//line app/vmalert/web.qtpl:545 +//line app/vmalert/web.qtpl:562 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 } -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 func WriteRuleDetails(qq422016 qtio422016.Writer, r *http.Request, rule APIRule) { -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 StreamRuleDetails(qw422016, r, rule) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 } -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 func RuleDetails(r *http.Request, rule APIRule) string { -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 WriteRuleDetails(qb422016, r, rule) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 return qs422016 -//line app/vmalert/web.qtpl:546 +//line app/vmalert/web.qtpl:563 } -//line app/vmalert/web.qtpl:550 +//line app/vmalert/web.qtpl:567 func streambadgeState(qw422016 *qt422016.Writer, state string) { -//line app/vmalert/web.qtpl:550 +//line app/vmalert/web.qtpl:567 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:552 +//line app/vmalert/web.qtpl:569 badgeClass := "bg-warning text-dark" if state == "firing" { badgeClass = "bg-danger" } -//line app/vmalert/web.qtpl:556 +//line app/vmalert/web.qtpl:573 qw422016.N().S(` <span class="badge `) -//line app/vmalert/web.qtpl:557 +//line app/vmalert/web.qtpl:574 qw422016.E().S(badgeClass) -//line app/vmalert/web.qtpl:557 +//line app/vmalert/web.qtpl:574 qw422016.N().S(`">`) -//line app/vmalert/web.qtpl:557 +//line app/vmalert/web.qtpl:574 qw422016.E().S(state) -//line app/vmalert/web.qtpl:557 +//line app/vmalert/web.qtpl:574 qw422016.N().S(`</span> `) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 } -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 func writebadgeState(qq422016 qtio422016.Writer, state string) { -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 streambadgeState(qw422016, state) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 } -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 func badgeState(state string) string { -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 writebadgeState(qb422016, state) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 return qs422016 -//line app/vmalert/web.qtpl:558 +//line app/vmalert/web.qtpl:575 } -//line app/vmalert/web.qtpl:560 +//line app/vmalert/web.qtpl:577 func streambadgeRestored(qw422016 *qt422016.Writer) { -//line app/vmalert/web.qtpl:560 +//line app/vmalert/web.qtpl:577 qw422016.N().S(` <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span> `) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 } -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 func writebadgeRestored(qq422016 qtio422016.Writer) { -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 streambadgeRestored(qw422016) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 } -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 func badgeRestored() string { -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 writebadgeRestored(qb422016) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 return qs422016 -//line app/vmalert/web.qtpl:562 +//line app/vmalert/web.qtpl:579 } -//line app/vmalert/web.qtpl:564 +//line app/vmalert/web.qtpl:581 +func streambadgeStabilizing(qw422016 *qt422016.Writer) { +//line app/vmalert/web.qtpl:581 + qw422016.N().S(` +<span class="badge bg-warning text-dark" title="This firing state is kept because of `) +//line app/vmalert/web.qtpl:581 + qw422016.N().S("`") +//line app/vmalert/web.qtpl:581 + qw422016.N().S(`keep_firing_for`) +//line app/vmalert/web.qtpl:581 + qw422016.N().S("`") +//line app/vmalert/web.qtpl:581 + qw422016.N().S(`">stabilizing</span> +`) +//line app/vmalert/web.qtpl:583 +} + +//line app/vmalert/web.qtpl:583 +func writebadgeStabilizing(qq422016 qtio422016.Writer) { +//line app/vmalert/web.qtpl:583 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmalert/web.qtpl:583 + streambadgeStabilizing(qw422016) +//line app/vmalert/web.qtpl:583 + qt422016.ReleaseWriter(qw422016) +//line app/vmalert/web.qtpl:583 +} + +//line app/vmalert/web.qtpl:583 +func badgeStabilizing() string { +//line app/vmalert/web.qtpl:583 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmalert/web.qtpl:583 + writebadgeStabilizing(qb422016) +//line app/vmalert/web.qtpl:583 + qs422016 := string(qb422016.B) +//line app/vmalert/web.qtpl:583 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmalert/web.qtpl:583 + return qs422016 +//line app/vmalert/web.qtpl:583 +} + +//line app/vmalert/web.qtpl:585 func streamseriesFetchedWarn(qw422016 *qt422016.Writer, r APIRule) { -//line app/vmalert/web.qtpl:564 +//line app/vmalert/web.qtpl:585 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:565 +//line app/vmalert/web.qtpl:586 if isNoMatch(r) { -//line app/vmalert/web.qtpl:565 +//line app/vmalert/web.qtpl:586 qw422016.N().S(` <svg xmlns="http://www.w3.org/2000/svg" data-bs-toggle="tooltip" @@ -1766,41 +1870,41 @@ func streamseriesFetchedWarn(qw422016 *qt422016.Writer, r APIRule) { <path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/> </svg> `) -//line app/vmalert/web.qtpl:574 +//line app/vmalert/web.qtpl:595 } -//line app/vmalert/web.qtpl:574 +//line app/vmalert/web.qtpl:595 qw422016.N().S(` `) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 } -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 func writeseriesFetchedWarn(qq422016 qtio422016.Writer, r APIRule) { -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 streamseriesFetchedWarn(qw422016, r) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 } -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 func seriesFetchedWarn(r APIRule) string { -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 writeseriesFetchedWarn(qb422016, r) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 qs422016 := string(qb422016.B) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 return qs422016 -//line app/vmalert/web.qtpl:575 +//line app/vmalert/web.qtpl:596 } -//line app/vmalert/web.qtpl:578 +//line app/vmalert/web.qtpl:599 func isNoMatch(r APIRule) bool { return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0 } diff --git a/app/vmalert/web_types.go b/app/vmalert/web_types.go index f9a162769d..4f4a8f911b 100644 --- a/app/vmalert/web_types.go +++ b/app/vmalert/web_types.go @@ -32,6 +32,9 @@ type APIAlert struct { SourceLink string `json:"source"` // Restored shows whether Alert's state was restored on restart Restored bool `json:"restored"` + // Stabilizing shows when firing state is kept because of + // `keep_firing_for` instead of real alert + Stabilizing bool `json:"stabilizing"` } // WebLink returns a link to the alert which can be used in UI. @@ -96,9 +99,11 @@ type APIRule struct { // Query represents Rule's `expression` field Query string `json:"query"` // Duration represents Rule's `for` field - Duration float64 `json:"duration"` - Labels map[string]string `json:"labels,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` + Duration float64 `json:"duration"` + // Alert will continue firing for this long even when the alerting expression no longer has results. + KeepFiringFor float64 `json:"keep_firing_for"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` // LastError contains the error faced while executing the rule. LastError string `json:"lastError"` // EvaluationTime is the time taken to completely evaluate the rule in float seconds. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b59cd20f12..50e8c9a3ad 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -66,7 +66,8 @@ The previous behavior can be restored in the following ways: * FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow disabling of `step` param attached to [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query). This might be useful for using vmalert with datasources that to not support this param, unlike VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4573) for details. * FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support option for "blackholing" alerting notifications if `-notifier.blackhole` cmd-line flag is set. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4122) for details. Thanks to @venkatbvc for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4639). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add unit test for alerting and recording rules, see more [details](https://docs.victoriametrics.com/vmalert.html#unit-testing-for-rules) here. Thanks to @Haleygo for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4596). -* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow overriding default GET params for rules with `graphite` datasource type, in the same way as it happens for `prometheus` type. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4685). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow overriding default GET params for rules with `graphite` datasource type, in the same way as it happens for `prometheus` type. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4685). +* FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support `keep_firing_for` field for alerting rules. See docs updated [here](https://docs.victoriametrics.com/vmalert.html#alerting-rules) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4529). Thanks to @Haleygo for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4669). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_duration_seconds` and `vmauth_unauthorized_user_request_duration_seconds` summary metrics for measuring requests latency per user. * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): show backup progress percentage in log during backup uploading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460). * FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): show restoring progress percentage in log during backup downloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460). diff --git a/docs/vmalert.md b/docs/vmalert.md index 2e2704815f..fc4d6604b6 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -214,6 +214,10 @@ expr: <string> # as firing once they return. [ for: <duration> | default = 0s ] +# Alert will continue firing for this long even when the alerting expression no longer has results. +# This allows you to delay alert resolution. +[ keep_firing_for: <duration> | default = 0s ] + # Whether to print debug information into logs. # Information includes alerts state changes and requests sent to the datasource. # Please note, that if rule's query params contain sensitive @@ -747,6 +751,7 @@ See full description for these flags in `./vmalert -help`. * Graphite engine isn't supported yet; * `query` template function is disabled for performance reasons (might be changed in future); * `limit` group's param has no effect during replay (might be changed in future); +* `keep_firing_for` alerting rule param has no effect during replay (might be changed in future). ## Unit Testing for Rules