From e9fa36348050bb34520cb8a6507da22ff3c53fdd Mon Sep 17 00:00:00 2001 From: Roman Khavronenko <roman@victoriametrics.com> Date: Mon, 9 May 2022 08:11:06 +0000 Subject: [PATCH] Vmalert fix bugs in alerting evaluation (#2557) * vmalert: calculate time for firing alert based on the given timestamp Previously, current time was used for checking the `firing` threshold. This is not correct, since alerts are evaluated at specific timestamps. Hence, this specific timestamp supposed to be used in the calculation. Signed-off-by: hagen1778 <roman@victoriametrics.com> * vmalert: properly calculate evaluation timestamp for rules Timestamp for rules evaluation should be calculated after the artifical delay for groups start. Otherwise, evaluation timestamp can fall back too far in time. Signed-off-by: hagen1778 <roman@victoriametrics.com> --- app/vmalert/alerting.go | 2 +- app/vmalert/group.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index 0f819cbd64..64a86bbea6 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -323,7 +323,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal } continue } - if a.State == notifier.StatePending && time.Since(a.ActiveAt) >= ar.For { + if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For { a.State = notifier.StateFiring a.Start = ts alertsFired.Inc() diff --git a/app/vmalert/group.go b/app/vmalert/group.go index e218e4fc75..d89f5037c3 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -237,8 +237,6 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r notifiers: nts, previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)} - evalTS := time.Now() - // Spread group rules evaluation over time in order to reduce load on VictoriaMetrics. if !skipRandSleepOnGroupStart { randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64))) @@ -259,6 +257,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r } } + evalTS := time.Now() + logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency) eval := func(ts time.Time) {