From c1722003a2e99373b69ee36896b5d1cc7df062c2 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 23 Dec 2021 00:20:34 +0200
Subject: [PATCH] lib/promscrape: scrape replicated targets at different
offsets in vmagent replicated clustering mode
This guarantees that the deduplication consistently leaves samples from the same vmagent replica.
See https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets
---
docs/CHANGELOG.md | 2 ++
lib/promscrape/scrapework.go | 7 ++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 95f1789dff..1ff410fb32 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -9,6 +9,8 @@ sort: 15
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): log error message when remote storage returns 400 or 409 http errors. This should simplify detection and debugging of this case. See [this issue](vmagent_remotewrite_packets_dropped_total).
* FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): store `restore-in-progress` file in `-dst` directory while `vmrestore` is running. This file is automatically deleted when `vmrestore` is successfully finished. This helps detecting incompletely restored data on VictoriaMetrics start. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1958).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that `vmagent` replicas scrape the same targets at different time offsets when [replication is enabled in vmagent clustering mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). This guarantees that the [deduplication](https://docs.victoriametrics.com/#deduplication) consistently leaves samples from the same `vmagent` replica.
+
## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0)
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 287d8271de..0454bfa966 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -270,7 +270,12 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}) {
// scrape urls and labels.
// This also makes consistent scrape times across restarts
// for a target with the same ScrapeURL and labels.
- key := fmt.Sprintf("ScrapeURL=%s, Labels=%s", sw.Config.ScrapeURL, sw.Config.LabelsString())
+ //
+ // Include clusterMemberNum to the key in order to guarantee that each member in vmagent cluster
+ // scrapes replicated targets at different time offsets. This guarantees that the deduplication consistently leaves samples
+ // received from the same vmagent replica.
+ // See https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets
+ key := fmt.Sprintf("ClusterMemberNum=%d, ScrapeURL=%s, Labels=%s", *clusterMemberNum, sw.Config.ScrapeURL, sw.Config.LabelsString())
h := xxhash.Sum64(bytesutil.ToUnsafeBytes(key))
randSleep = uint64(float64(scrapeInterval) * (float64(h) / (1 << 64)))
sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval)