From 067c7afebc673d80822c1c6c921d2ee7b91384ed Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 14 Apr 2020 14:51:52 +0300
Subject: [PATCH] lib/promscrape: show information on improperly configured
scrape targets at the bottom of `/targets` page
This is a common error whith improperly configured target autodiscovery and/or relabeling.
This error leads to duplicate scraping of the same targets with the same set of labels, which leads
to duplicate samples in time series.
---
app/vmagent/README.md | 2 +-
docs/vmagent.md | 2 +-
lib/promscrape/targetstatus.go | 13 +++++++++++++
3 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index d167bf9958..2912be4a26 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -183,7 +183,7 @@ Read more about relabeling in the following articles:
`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
-`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format. This page also exports information on improperly configured scrape configs.
### Troubleshooting
diff --git a/docs/vmagent.md b/docs/vmagent.md
index d167bf9958..2912be4a26 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -183,7 +183,7 @@ Read more about relabeling in the following articles:
`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
-`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format. This page also exports information on improperly configured scrape configs.
### Troubleshooting
diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go
index b362d9a84b..081588a2c4 100644
--- a/lib/promscrape/targetstatus.go
+++ b/lib/promscrape/targetstatus.go
@@ -86,6 +86,7 @@ func (tsm *targetStatusMap) WriteHumanReadable(w io.Writer) {
return jss[i].job < jss[j].job
})
+ targetsByEndpoint := make(map[string]int)
for _, js := range jss {
sts := js.statuses
sort.Slice(sts, func(i, j int) bool {
@@ -115,8 +116,20 @@ func (tsm *targetStatusMap) WriteHumanReadable(w io.Writer) {
}
fmt.Fprintf(w, "\tstate=%s, endpoint=%s, labels=%s, last_scrape=%.3fs ago, scrape_duration=%.3fs, error=%q\n",
state, st.sw.ScrapeURL, labelsStr, lastScrape.Seconds(), float64(st.scrapeDuration)/1000, errMsg)
+ key := fmt.Sprintf("endpoint=%s, labels=%s", st.sw.ScrapeURL, labelsStr)
+ targetsByEndpoint[key]++
}
}
+ fmt.Fprintf(w, "\n")
+
+ // Check whether there are targets with duplicate endpoints and labels.
+ for key, n := range targetsByEndpoint {
+ if n <= 1 {
+ continue
+ }
+ fmt.Fprintf(w, "!!! Scrape config error: %d duplicate targets with identical endpoint and labels found:\n", n)
+ fmt.Fprintf(w, "\t%s\n", key)
+ }
}
type jobStatus struct {