diff --git a/app/vmalert/group.go b/app/vmalert/group.go index 2d6f1ad95c..ec1eddcd80 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "hash/fnv" + "strconv" "sync" "time" @@ -14,6 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/metrics" + "github.com/cespare/xxhash/v2" ) // Group is an entity for grouping rules @@ -182,7 +184,17 @@ func (g *Group) close() { func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []notifier.Notifier, rw *remotewrite.Client) { defer func() { close(g.finishedCh) }() - logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency) + // This should spread load of rule evaluation by group + h := uint32(xxhash.Sum64([]byte(strconv.FormatUint(g.ID(), 10)))) + randSleep := uint64(float64(g.Interval) * (float64(h) / (1 << 32))) + sleeper := time.NewTimer(time.Duration(randSleep)) + select { + case <-g.finishedCh: + sleeper.Stop() + return + case <-sleeper.C: + } + logger.Infof("group %q started with delay %v; interval=%v; concurrency=%d", time.Duration(randSleep), g.Name, g.Interval, g.Concurrency) e := &executor{querier, nts, rw} t := time.NewTicker(g.Interval) defer t.Stop()