diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0abaae99f..212840203 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -7,6 +7,7 @@ sort: 15
## tip
* FEATURE: vmagent: add service discovery for DigitalOcean (aka [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1367).
+* FEATURE: vmagent: show the number of samples the target returned during the last scrape on `/targets` and `/api/v1/targets` pages. This should simplify debugging targets, which may return too big or too low number of samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1377).
## [v1.61.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.1)
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 792234429..281a0e57e 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -324,7 +324,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
// body must be released only after wc is released, since wc refers to body.
sw.prevBodyLen = len(body.B)
leveledbytebufferpool.Put(body)
- tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), err)
+ tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
return err
}
@@ -391,7 +391,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
sw.prevLabelsLen = len(wc.labels)
wc.reset()
writeRequestCtxPool.Put(wc)
- tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), err)
+ tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
return err
}
diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl
index fabd7b103..6120e3678 100644
--- a/lib/promscrape/targets_response.qtpl
+++ b/lib/promscrape/targets_response.qtpl
@@ -19,7 +19,8 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)
{% if showOriginLabels %}, originalLabels={%s= ol %}{% endif %},
last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago,
scrape_duration={%f.3 ts.scrapeDuration.Seconds() %}s,
- error={%q= ts.error %}
+ samples_scraped={%d ts.samplesScraped %},
+ error={%q= ts.errMsg %}
{% newline %}
{% endfor %}
{% endfor %}
@@ -62,6 +63,7 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)
Labels |
Last Scrape |
Scrape Duration |
+ Samples Scraped |
Error |
@@ -76,7 +78,8 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)
{%f.3 ts.lastScrapeTime.Seconds() %}s ago |
{%f.3 ts.scrapeDuration.Seconds() %}s |
- {%s ts.error %} |
+ {%d ts.samplesScraped %} |
+ {%s ts.errMsg %} |
{% endfor %}
diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go
index 0c9d9699a..bce59cdcd 100644
--- a/lib/promscrape/targets_response.qtpl.go
+++ b/lib/promscrape/targets_response.qtpl.go
@@ -94,280 +94,288 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu
//line lib/promscrape/targets_response.qtpl:21
qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3)
//line lib/promscrape/targets_response.qtpl:21
- qw422016.N().S(`s, error=`)
+ qw422016.N().S(`s, samples_scraped=`)
//line lib/promscrape/targets_response.qtpl:22
- qw422016.N().Q(ts.error)
+ qw422016.N().D(ts.samplesScraped)
//line lib/promscrape/targets_response.qtpl:22
- qw422016.N().S(` `)
+ qw422016.N().S(`, error=`)
//line lib/promscrape/targets_response.qtpl:23
+ qw422016.N().Q(ts.errMsg)
+//line lib/promscrape/targets_response.qtpl:23
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:24
qw422016.N().S(`
`)
-//line lib/promscrape/targets_response.qtpl:23
+//line lib/promscrape/targets_response.qtpl:24
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:24
+//line lib/promscrape/targets_response.qtpl:25
}
-//line lib/promscrape/targets_response.qtpl:24
+//line lib/promscrape/targets_response.qtpl:25
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:25
- }
-//line lib/promscrape/targets_response.qtpl:25
- qw422016.N().S(` `)
//line lib/promscrape/targets_response.qtpl:26
+ }
+//line lib/promscrape/targets_response.qtpl:26
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:27
qw422016.N().S(`
`)
-//line lib/promscrape/targets_response.qtpl:26
+//line lib/promscrape/targets_response.qtpl:27
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
}
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
func WriteTargetsResponsePlain(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) {
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
StreamTargetsResponsePlain(qw422016, jts, showOriginLabels)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
}
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) string {
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
WriteTargetsResponsePlain(qb422016, jts, showOriginLabels)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
return qs422016
-//line lib/promscrape/targets_response.qtpl:28
+//line lib/promscrape/targets_response.qtpl:29
}
-//line lib/promscrape/targets_response.qtpl:30
+//line lib/promscrape/targets_response.qtpl:31
func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
-//line lib/promscrape/targets_response.qtpl:30
+//line lib/promscrape/targets_response.qtpl:31
qw422016.N().S(` Scrape targets Scrape targets
`)
-//line lib/promscrape/targets_response.qtpl:51
+//line lib/promscrape/targets_response.qtpl:52
for _, js := range jts {
-//line lib/promscrape/targets_response.qtpl:51
- qw422016.N().S(` `)
//line lib/promscrape/targets_response.qtpl:52
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:53
if onlyUnhealthy && js.upCount == js.targetsTotal {
-//line lib/promscrape/targets_response.qtpl:52
+//line lib/promscrape/targets_response.qtpl:53
continue
-//line lib/promscrape/targets_response.qtpl:52
+//line lib/promscrape/targets_response.qtpl:53
}
-//line lib/promscrape/targets_response.qtpl:52
+//line lib/promscrape/targets_response.qtpl:53
qw422016.N().S(` Endpoint | State | Labels | Last Scrape | Scrape Duration | Error |
`)
-//line lib/promscrape/targets_response.qtpl:69
+//line lib/promscrape/targets_response.qtpl:56
+ qw422016.N().S(` up) Endpoint | State | Labels | Last Scrape | Scrape Duration | Samples Scraped | Error |
`)
+//line lib/promscrape/targets_response.qtpl:71
for _, ts := range js.targetsStatus {
-//line lib/promscrape/targets_response.qtpl:69
+//line lib/promscrape/targets_response.qtpl:71
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:70
+//line lib/promscrape/targets_response.qtpl:72
if onlyUnhealthy && ts.up {
-//line lib/promscrape/targets_response.qtpl:70
+//line lib/promscrape/targets_response.qtpl:72
continue
-//line lib/promscrape/targets_response.qtpl:70
+//line lib/promscrape/targets_response.qtpl:72
}
-//line lib/promscrape/targets_response.qtpl:70
+//line lib/promscrape/targets_response.qtpl:72
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:72
+//line lib/promscrape/targets_response.qtpl:74
qw422016.E().S(ts.endpoint)
-//line lib/promscrape/targets_response.qtpl:72
+//line lib/promscrape/targets_response.qtpl:74
qw422016.N().S(`
| `)
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
if ts.up {
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
qw422016.N().S(`UP`)
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
} else {
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
qw422016.N().S(`DOWN`)
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
}
-//line lib/promscrape/targets_response.qtpl:73
+//line lib/promscrape/targets_response.qtpl:75
qw422016.N().S(` | `)
-//line lib/promscrape/targets_response.qtpl:75
+//line lib/promscrape/targets_response.qtpl:77
streamformatLabel(qw422016, ts.labels)
-//line lib/promscrape/targets_response.qtpl:75
+//line lib/promscrape/targets_response.qtpl:77
qw422016.N().S(` | `)
-//line lib/promscrape/targets_response.qtpl:77
+//line lib/promscrape/targets_response.qtpl:79
qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
-//line lib/promscrape/targets_response.qtpl:77
+//line lib/promscrape/targets_response.qtpl:79
qw422016.N().S(`s ago | `)
-//line lib/promscrape/targets_response.qtpl:78
+//line lib/promscrape/targets_response.qtpl:80
qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3)
-//line lib/promscrape/targets_response.qtpl:78
+//line lib/promscrape/targets_response.qtpl:80
qw422016.N().S(`s | `)
-//line lib/promscrape/targets_response.qtpl:79
- qw422016.E().S(ts.error)
-//line lib/promscrape/targets_response.qtpl:79
+//line lib/promscrape/targets_response.qtpl:81
+ qw422016.N().D(ts.samplesScraped)
+//line lib/promscrape/targets_response.qtpl:81
+ qw422016.N().S(` | `)
+//line lib/promscrape/targets_response.qtpl:82
+ qw422016.E().S(ts.errMsg)
+//line lib/promscrape/targets_response.qtpl:82
qw422016.N().S(` |
`)
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:84
}
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().S(`
`)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:88
}
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:88
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
}
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
}
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string {
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
return qs422016
-//line lib/promscrape/targets_response.qtpl:88
+//line lib/promscrape/targets_response.qtpl:91
}
-//line lib/promscrape/targets_response.qtpl:90
+//line lib/promscrape/targets_response.qtpl:93
func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:90
+//line lib/promscrape/targets_response.qtpl:93
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:91
+//line lib/promscrape/targets_response.qtpl:94
for _, label := range labels {
-//line lib/promscrape/targets_response.qtpl:91
+//line lib/promscrape/targets_response.qtpl:94
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.E().S(label.Name)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.N().S(`=`)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.E().Q(label.Value)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:95
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:93
+//line lib/promscrape/targets_response.qtpl:96
}
-//line lib/promscrape/targets_response.qtpl:93
+//line lib/promscrape/targets_response.qtpl:96
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
}
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
streamformatLabel(qw422016, labels)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
}
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
func formatLabel(labels []prompbmarshal.Label) string {
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
writeformatLabel(qb422016, labels)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
return qs422016
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:97
}
diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go
index 21998f47b..eee6a548a 100644
--- a/lib/promscrape/targetstatus.go
+++ b/lib/promscrape/targetstatus.go
@@ -88,7 +88,7 @@ func (tsm *targetStatusMap) Unregister(sw *ScrapeWork) {
tsm.mu.Unlock()
}
-func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, err error) {
+func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, samplesScraped int, err error) {
tsm.mu.Lock()
ts := tsm.m[sw]
if ts == nil {
@@ -101,6 +101,7 @@ func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrape
ts.scrapeGroup = group
ts.scrapeTime = scrapeTime
ts.scrapeDuration = scrapeDuration
+ ts.samplesScraped = samplesScraped
ts.err = err
tsm.mu.Unlock()
}
@@ -156,6 +157,7 @@ func (tsm *targetStatusMap) WriteActiveTargetsJSON(w io.Writer) {
fmt.Fprintf(w, `,"lastError":%q`, errMsg)
fmt.Fprintf(w, `,"lastScrape":%q`, time.Unix(st.scrapeTime/1000, (st.scrapeTime%1000)*1e6).Format(time.RFC3339Nano))
fmt.Fprintf(w, `,"lastScrapeDuration":%g`, (time.Millisecond * time.Duration(st.scrapeDuration)).Seconds())
+ fmt.Fprintf(w, `,"lastSamplesScraped":%d`, st.samplesScraped)
state := "up"
if !st.up {
state = "down"
@@ -185,6 +187,7 @@ type targetStatus struct {
scrapeGroup string
scrapeTime int64
scrapeDuration int64
+ samplesScraped int
err error
}
@@ -270,7 +273,8 @@ type jobTargetStatus struct {
originalLabels []prompbmarshal.Label
lastScrapeTime time.Duration
scrapeDuration time.Duration
- error string
+ samplesScraped int
+ errMsg string
}
type jobTargetsStatuses struct {
@@ -313,7 +317,8 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses {
originalLabels: st.sw.OriginalLabels,
lastScrapeTime: st.getDurationFromLastScrape(),
scrapeDuration: time.Duration(st.scrapeDuration) * time.Millisecond,
- error: errMsg,
+ samplesScraped: st.samplesScraped,
+ errMsg: errMsg,
})
}
jts = append(jts, jobTargetsStatuses{