From fb72a2133f1d14a4c106bf426ae80e038404f341 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 18 Jun 2021 10:53:10 +0300 Subject: [PATCH] lib/promscrape: show jobs with empty scrape targets on /targets page --- docs/CHANGELOG.md | 1 + lib/promscrape/config.go | 11 + lib/promscrape/targets_response.qtpl | 47 ++- lib/promscrape/targets_response.qtpl.go | 418 ++++++++++++------------ lib/promscrape/targetstatus.go | 41 ++- 5 files changed, 294 insertions(+), 224 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8e4ea7b22..f23b9ad17 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,7 @@ sort: 15 * FEATURE: vmagent: add service discovery for DigitalOcean (aka [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1367). * FEATURE: vmagent: show the number of samples the target returned during the last scrape on `/targets` and `/api/v1/targets` pages. This should simplify debugging targets, which may return too big or too low number of samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1377). * FEATURE: vmagent: change the default value for `-remoteWrite.queues` from 4 to `2 * numCPUs`. This should reduce scrape duration for highly loaded vmagent, which scrapes tens of thousands of targets. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1385). +* FEATURE: vmagent: show jobs with zero discovered targets on `/targets` page. This should help debugging improperly configured scrape configs. * BUGFIX: prevent from adding new samples to deleted time series after the rotation of the inverted index (the rotation is performed once per `-retentionPeriod`). See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1347#issuecomment-861232136) for details. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 6316971d1..02af9abc8 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -67,6 +67,8 @@ func (cfg *Config) mustStart() { for i := range cfg.ScrapeConfigs { cfg.ScrapeConfigs[i].mustStart(cfg.baseDir) } + jobNames := cfg.getJobNames() + tsmGlobal.registerJobNames(jobNames) logger.Infof("started service discovery routines in %.3f seconds", time.Since(startTime).Seconds()) } @@ -79,6 +81,15 @@ func (cfg *Config) mustStop() { logger.Infof("stopped service discovery routines in %.3f seconds", time.Since(startTime).Seconds()) } +// getJobNames returns all the scrape job names from the cfg. +func (cfg *Config) getJobNames() []string { + a := make([]string, 0, len(cfg.ScrapeConfigs)) + for i := range cfg.ScrapeConfigs { + a = append(a, cfg.ScrapeConfigs[i].JobName) + } + return a +} + // GlobalConfig represents essential parts for `global` section of Prometheus config. // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/ diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl index 6120e3678..9583130e9 100644 --- a/lib/promscrape/targets_response.qtpl +++ b/lib/promscrape/targets_response.qtpl @@ -1,9 +1,9 @@ {% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" %} -{% collapsespace %} +{% stripspace %} -{% func TargetsResponsePlain (jts []jobTargetsStatuses, showOriginLabels bool) -%} +{% func TargetsResponsePlain(jts []jobTargetsStatuses, emptyJobs []string, showOriginLabels bool) %} {% for _, js := range jts %} job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) @@ -13,22 +13,26 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) labels := promLabelsString(ts.labels) ol := promLabelsString(ts.originalLabels) %} -{%s= "\t" %}state={% if ts.up %}up{% else %}down{% endif %}, - endpoint={%s= ts.endpoint %}, +{%s= "\t" %}state={% if ts.up %}up{% else %}down{% endif %},{% space %} + endpoint={%s= ts.endpoint %},{ %space %} labels={%s= labels %} - {% if showOriginLabels %}, originalLabels={%s= ol %}{% endif %}, - last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago, - scrape_duration={%f.3 ts.scrapeDuration.Seconds() %}s, - samples_scraped={%d ts.samplesScraped %}, + {% if showOriginLabels %}, originalLabels={%s= ol %}{% endif %},{% space %} + last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago,{% space %} + scrape_duration={%f.3 ts.scrapeDuration.Seconds() %}s,{% space %} + samples_scraped={%d ts.samplesScraped %},{% space %} error={%q= ts.errMsg %} {% newline %} {% endfor %} {% endfor %} + +{% for _, jobName := range emptyJobs %} +job={%q= jobName %} (0/0 up) {% newline %} +{% endfor %} {% endfunc %} -{% func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) %} +{% func TargetsResponseHTML(jts []jobTargetsStatuses, emptyJobs []string, redirectPath string, onlyUnhealthy bool) %} @@ -49,7 +53,7 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) Unhealthy - {% for _,js :=range jts %} + {% for _, js := range jts %} {% if onlyUnhealthy && js.upCount == js.targetsTotal %}{% continue %}{% endif %}

@@ -86,6 +90,27 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)

{% endfor %} + + {% for _, jobName := range emptyJobs %} +
+

+ {%s jobName %} (0/0 up) +

+ + + + + + + + + + + + +
EndpointStateLabelsLast ScrapeScrape DurationSamples ScrapedError
+
+ {% endfor %} {% endfunc %} @@ -96,4 +121,4 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) {% endfor %} {% endfunc %} -{% endcollapsespace %} +{% endstripspace %} diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go index bce59cdcd..1292c9faf 100644 --- a/lib/promscrape/targets_response.qtpl.go +++ b/lib/promscrape/targets_response.qtpl.go @@ -21,15 +21,15 @@ var ( ) //line lib/promscrape/targets_response.qtpl:6 -func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) { +func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, emptyJobs []string, showOriginLabels bool) { //line lib/promscrape/targets_response.qtpl:8 for _, js := range jts { //line lib/promscrape/targets_response.qtpl:8 - qw422016.N().S(` job=`) + qw422016.N().S(`job=`) //line lib/promscrape/targets_response.qtpl:9 qw422016.N().Q(js.job) //line lib/promscrape/targets_response.qtpl:9 - qw422016.N().S(` (`) + qw422016.N().S(`(`) //line lib/promscrape/targets_response.qtpl:9 qw422016.N().D(js.upCount) //line lib/promscrape/targets_response.qtpl:9 @@ -37,22 +37,16 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu //line lib/promscrape/targets_response.qtpl:9 qw422016.N().D(js.targetsTotal) //line lib/promscrape/targets_response.qtpl:9 - qw422016.N().S(` up) `) + qw422016.N().S(`up)`) //line lib/promscrape/targets_response.qtpl:10 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:10 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:11 for _, ts := range js.targetsStatus { -//line lib/promscrape/targets_response.qtpl:11 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:13 labels := promLabelsString(ts.labels) ol := promLabelsString(ts.originalLabels) -//line lib/promscrape/targets_response.qtpl:15 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:16 qw422016.N().S("\t") //line lib/promscrape/targets_response.qtpl:16 @@ -68,15 +62,17 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu //line lib/promscrape/targets_response.qtpl:16 } //line lib/promscrape/targets_response.qtpl:16 - qw422016.N().S(`, endpoint=`) + qw422016.N().S(`,`) +//line lib/promscrape/targets_response.qtpl:16 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:16 + qw422016.N().S(`endpoint=`) //line lib/promscrape/targets_response.qtpl:17 qw422016.N().S(ts.endpoint) //line lib/promscrape/targets_response.qtpl:17 - qw422016.N().S(`, labels=`) + qw422016.N().S(`,{ %space %}labels=`) //line lib/promscrape/targets_response.qtpl:18 qw422016.N().S(labels) -//line lib/promscrape/targets_response.qtpl:18 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:19 if showOriginLabels { //line lib/promscrape/targets_response.qtpl:19 @@ -86,296 +82,308 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu //line lib/promscrape/targets_response.qtpl:19 } //line lib/promscrape/targets_response.qtpl:19 - qw422016.N().S(`, last_scrape=`) + qw422016.N().S(`,`) +//line lib/promscrape/targets_response.qtpl:19 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:19 + qw422016.N().S(`last_scrape=`) //line lib/promscrape/targets_response.qtpl:20 qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3) //line lib/promscrape/targets_response.qtpl:20 - qw422016.N().S(`s ago, scrape_duration=`) + qw422016.N().S(`s ago,`) +//line lib/promscrape/targets_response.qtpl:20 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:20 + qw422016.N().S(`scrape_duration=`) //line lib/promscrape/targets_response.qtpl:21 qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3) //line lib/promscrape/targets_response.qtpl:21 - qw422016.N().S(`s, samples_scraped=`) + qw422016.N().S(`s,`) +//line lib/promscrape/targets_response.qtpl:21 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:21 + qw422016.N().S(`samples_scraped=`) //line lib/promscrape/targets_response.qtpl:22 qw422016.N().D(ts.samplesScraped) //line lib/promscrape/targets_response.qtpl:22 - qw422016.N().S(`, error=`) + qw422016.N().S(`,`) +//line lib/promscrape/targets_response.qtpl:22 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:22 + qw422016.N().S(`error=`) //line lib/promscrape/targets_response.qtpl:23 qw422016.N().Q(ts.errMsg) -//line lib/promscrape/targets_response.qtpl:23 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:24 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:24 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:25 } -//line lib/promscrape/targets_response.qtpl:25 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:26 } -//line lib/promscrape/targets_response.qtpl:26 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:27 - qw422016.N().S(` +//line lib/promscrape/targets_response.qtpl:28 + for _, jobName := range emptyJobs { +//line lib/promscrape/targets_response.qtpl:28 + qw422016.N().S(`job=`) +//line lib/promscrape/targets_response.qtpl:29 + qw422016.N().Q(jobName) +//line lib/promscrape/targets_response.qtpl:29 + qw422016.N().S(`(0/0 up)`) +//line lib/promscrape/targets_response.qtpl:30 + qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:27 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:29 -} - -//line lib/promscrape/targets_response.qtpl:29 -func WriteTargetsResponsePlain(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) { -//line lib/promscrape/targets_response.qtpl:29 - qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:29 - StreamTargetsResponsePlain(qw422016, jts, showOriginLabels) -//line lib/promscrape/targets_response.qtpl:29 - qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:29 -} - -//line lib/promscrape/targets_response.qtpl:29 -func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) string { -//line lib/promscrape/targets_response.qtpl:29 - qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:29 - WriteTargetsResponsePlain(qb422016, jts, showOriginLabels) -//line lib/promscrape/targets_response.qtpl:29 - qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:29 - qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:29 - return qs422016 -//line lib/promscrape/targets_response.qtpl:29 -} - //line lib/promscrape/targets_response.qtpl:31 -func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) { -//line lib/promscrape/targets_response.qtpl:31 - qw422016.N().S(` Scrape targets

Scrape targets

`) //line lib/promscrape/targets_response.qtpl:52 + } +//line lib/promscrape/targets_response.qtpl:52 + qw422016.N().S(`>Unhealthy`) +//line lib/promscrape/targets_response.qtpl:56 for _, js := range jts { -//line lib/promscrape/targets_response.qtpl:52 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:53 +//line lib/promscrape/targets_response.qtpl:57 if onlyUnhealthy && js.upCount == js.targetsTotal { -//line lib/promscrape/targets_response.qtpl:53 +//line lib/promscrape/targets_response.qtpl:57 continue -//line lib/promscrape/targets_response.qtpl:53 +//line lib/promscrape/targets_response.qtpl:57 } -//line lib/promscrape/targets_response.qtpl:53 - qw422016.N().S(`

`) -//line lib/promscrape/targets_response.qtpl:56 +//line lib/promscrape/targets_response.qtpl:57 + qw422016.N().S(`

`) +//line lib/promscrape/targets_response.qtpl:60 qw422016.E().S(js.job) -//line lib/promscrape/targets_response.qtpl:56 - qw422016.N().S(` (`) -//line lib/promscrape/targets_response.qtpl:56 +//line lib/promscrape/targets_response.qtpl:60 + qw422016.N().S(`(`) +//line lib/promscrape/targets_response.qtpl:60 qw422016.N().D(js.upCount) -//line lib/promscrape/targets_response.qtpl:56 +//line lib/promscrape/targets_response.qtpl:60 qw422016.N().S(`/`) -//line lib/promscrape/targets_response.qtpl:56 +//line lib/promscrape/targets_response.qtpl:60 qw422016.N().D(js.targetsTotal) -//line lib/promscrape/targets_response.qtpl:56 - qw422016.N().S(` up)

`) -//line lib/promscrape/targets_response.qtpl:71 +//line lib/promscrape/targets_response.qtpl:60 + qw422016.N().S(`up)
Endpoint State Labels Last Scrape Scrape Duration Samples Scraped Error
`) +//line lib/promscrape/targets_response.qtpl:75 for _, ts := range js.targetsStatus { -//line lib/promscrape/targets_response.qtpl:71 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:72 +//line lib/promscrape/targets_response.qtpl:76 if onlyUnhealthy && ts.up { -//line lib/promscrape/targets_response.qtpl:72 +//line lib/promscrape/targets_response.qtpl:76 continue -//line lib/promscrape/targets_response.qtpl:72 +//line lib/promscrape/targets_response.qtpl:76 } -//line lib/promscrape/targets_response.qtpl:72 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:84 +//line lib/promscrape/targets_response.qtpl:86 + qw422016.N().S(``) +//line lib/promscrape/targets_response.qtpl:88 } -//line lib/promscrape/targets_response.qtpl:84 - qw422016.N().S(`
EndpointStateLabelsLast ScrapeScrape DurationSamples ScrapedError
`) -//line lib/promscrape/targets_response.qtpl:74 +//line lib/promscrape/targets_response.qtpl:78 qw422016.E().S(ts.endpoint) -//line lib/promscrape/targets_response.qtpl:74 - qw422016.N().S(`
 `) -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:78 + qw422016.N().S(`
`) +//line lib/promscrape/targets_response.qtpl:79 if ts.up { -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:79 qw422016.N().S(`UP`) -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:79 } else { -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:79 qw422016.N().S(`DOWN`) -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:79 } -//line lib/promscrape/targets_response.qtpl:75 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:77 +//line lib/promscrape/targets_response.qtpl:80 + qw422016.N().S(`">`) +//line lib/promscrape/targets_response.qtpl:81 streamformatLabel(qw422016, ts.labels) -//line lib/promscrape/targets_response.qtpl:77 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:79 +//line lib/promscrape/targets_response.qtpl:81 + qw422016.N().S(``) +//line lib/promscrape/targets_response.qtpl:83 qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3) -//line lib/promscrape/targets_response.qtpl:79 - qw422016.N().S(`s ago `) -//line lib/promscrape/targets_response.qtpl:80 +//line lib/promscrape/targets_response.qtpl:83 + qw422016.N().S(`s ago`) +//line lib/promscrape/targets_response.qtpl:84 qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3) -//line lib/promscrape/targets_response.qtpl:80 - qw422016.N().S(`s `) -//line lib/promscrape/targets_response.qtpl:81 +//line lib/promscrape/targets_response.qtpl:84 + qw422016.N().S(`s`) +//line lib/promscrape/targets_response.qtpl:85 qw422016.N().D(ts.samplesScraped) -//line lib/promscrape/targets_response.qtpl:81 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:82 +//line lib/promscrape/targets_response.qtpl:85 + qw422016.N().S(``) +//line lib/promscrape/targets_response.qtpl:86 qw422016.E().S(ts.errMsg) -//line lib/promscrape/targets_response.qtpl:82 - qw422016.N().S(`
`) //line lib/promscrape/targets_response.qtpl:88 + qw422016.N().S(`

`) +//line lib/promscrape/targets_response.qtpl:92 } -//line lib/promscrape/targets_response.qtpl:88 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:94 + for _, jobName := range emptyJobs { +//line lib/promscrape/targets_response.qtpl:94 + qw422016.N().S(`

`) +//line lib/promscrape/targets_response.qtpl:97 + qw422016.E().S(jobName) +//line lib/promscrape/targets_response.qtpl:97 + qw422016.N().S(`(0/0 up)

EndpointStateLabelsLast ScrapeScrape DurationSamples ScrapedError
`) +//line lib/promscrape/targets_response.qtpl:113 + } +//line lib/promscrape/targets_response.qtpl:113 + qw422016.N().S(``) +//line lib/promscrape/targets_response.qtpl:116 } -//line lib/promscrape/targets_response.qtpl:91 -func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) { -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 +func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, emptyJobs []string, redirectPath string, onlyUnhealthy bool) { +//line lib/promscrape/targets_response.qtpl:116 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:91 - StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 + StreamTargetsResponseHTML(qw422016, jts, emptyJobs, redirectPath, onlyUnhealthy) +//line lib/promscrape/targets_response.qtpl:116 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 } -//line lib/promscrape/targets_response.qtpl:91 -func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string { -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 +func TargetsResponseHTML(jts []jobTargetsStatuses, emptyJobs []string, redirectPath string, onlyUnhealthy bool) string { +//line lib/promscrape/targets_response.qtpl:116 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:91 - WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 + WriteTargetsResponseHTML(qb422016, jts, emptyJobs, redirectPath, onlyUnhealthy) +//line lib/promscrape/targets_response.qtpl:116 qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 return qs422016 -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:116 } -//line lib/promscrape/targets_response.qtpl:93 +//line lib/promscrape/targets_response.qtpl:118 func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) { -//line lib/promscrape/targets_response.qtpl:93 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:119 for _, label := range labels { -//line lib/promscrape/targets_response.qtpl:94 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:95 +//line lib/promscrape/targets_response.qtpl:120 qw422016.E().S(label.Name) -//line lib/promscrape/targets_response.qtpl:95 +//line lib/promscrape/targets_response.qtpl:120 qw422016.N().S(`=`) -//line lib/promscrape/targets_response.qtpl:95 +//line lib/promscrape/targets_response.qtpl:120 qw422016.E().Q(label.Value) -//line lib/promscrape/targets_response.qtpl:95 +//line lib/promscrape/targets_response.qtpl:120 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:95 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:95 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:96 +//line lib/promscrape/targets_response.qtpl:121 } -//line lib/promscrape/targets_response.qtpl:96 - qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 } -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) { -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 streamformatLabel(qw422016, labels) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 } -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 func formatLabel(labels []prompbmarshal.Label) string { -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 writeformatLabel(qb422016, labels) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 return qs422016 -//line lib/promscrape/targets_response.qtpl:97 +//line lib/promscrape/targets_response.qtpl:122 } diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go index eee6a548a..2bb4c5e33 100644 --- a/lib/promscrape/targetstatus.go +++ b/lib/promscrape/targetstatus.go @@ -58,8 +58,9 @@ func WriteAPIV1Targets(w io.Writer, state string) { } type targetStatusMap struct { - mu sync.Mutex - m map[*ScrapeWork]*targetStatus + mu sync.Mutex + m map[*ScrapeWork]*targetStatus + jobNames []string } func newTargetStatusMap() *targetStatusMap { @@ -74,6 +75,12 @@ func (tsm *targetStatusMap) Reset() { tsm.mu.Unlock() } +func (tsm *targetStatusMap) registerJobNames(jobNames []string) { + tsm.mu.Lock() + tsm.jobNames = append(tsm.jobNames[:0], jobNames...) + tsm.mu.Unlock() +} + func (tsm *targetStatusMap) Register(sw *ScrapeWork) { tsm.mu.Lock() tsm.m[sw] = &targetStatus{ @@ -284,13 +291,14 @@ type jobTargetsStatuses struct { targetsStatus []jobTargetStatus } -func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses { +func (tsm *targetStatusMap) getTargetsStatusByJob() ([]jobTargetsStatuses, []string) { byJob := make(map[string][]targetStatus) tsm.mu.Lock() for _, st := range tsm.m { job := st.sw.Job() byJob[job] = append(byJob[job], *st) } + jobNames := append([]string{}, tsm.jobNames...) tsm.mu.Unlock() var jts []jobTargetsStatuses @@ -331,20 +339,37 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses { sort.Slice(jts, func(i, j int) bool { return jts[i].job < jts[j].job }) - return jts + emptyJobs := getEmptyJobs(jts, jobNames) + return jts, emptyJobs +} + +func getEmptyJobs(jts []jobTargetsStatuses, jobNames []string) []string { + jobNamesMap := make(map[string]struct{}, len(jobNames)) + for _, jobName := range jobNames { + jobNamesMap[jobName] = struct{}{} + } + for i := range jts { + delete(jobNamesMap, jts[i].job) + } + emptyJobs := make([]string, 0, len(jobNamesMap)) + for k := range jobNamesMap { + emptyJobs = append(emptyJobs, k) + } + sort.Strings(emptyJobs) + return emptyJobs } // WriteTargetsHTML writes targets status grouped by job into writer w in html table, // accepts filter to show only unhealthy targets. func (tsm *targetStatusMap) WriteTargetsHTML(w io.Writer, showOnlyUnhealthy bool) { - jss := tsm.getTargetsStatusByJob() + jss, emptyJobs := tsm.getTargetsStatusByJob() targetsPath := path.Join(httpserver.GetPathPrefix(), "/targets") - WriteTargetsResponseHTML(w, jss, targetsPath, showOnlyUnhealthy) + WriteTargetsResponseHTML(w, jss, emptyJobs, targetsPath, showOnlyUnhealthy) } // WriteTargetsPlain writes targets grouped by job into writer w in plain text, // accept filter to show original labels. func (tsm *targetStatusMap) WriteTargetsPlain(w io.Writer, showOriginalLabels bool) { - jss := tsm.getTargetsStatusByJob() - WriteTargetsResponsePlain(w, jss, showOriginalLabels) + jss, emptyJobs := tsm.getTargetsStatusByJob() + WriteTargetsResponsePlain(w, jss, emptyJobs, showOriginalLabels) }