diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index 866dce3292..dcfcf845ec 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -262,6 +262,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
promscrapeTargetsRequests.Inc()
promscrape.WriteHumanReadableTargetsStatus(w, r)
return true
+ case "/target_response":
+ promscrapeTargetResponseRequests.Inc()
+ if err := promscrape.WriteTargetResponse(w, r); err != nil {
+ promscrapeTargetResponseErrors.Inc()
+ httpserver.Errorf(w, r, "%s", err)
+ return true
+ }
+ return true
case "/config":
if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
err := &httpserver.ErrorWithStatusCode{
@@ -443,6 +451,9 @@ var (
promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+ promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
+ promscrapeTargetResponseErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
+
promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index c77d78a4ae..c5b928853b 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -199,6 +199,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
state := r.FormValue("state")
promscrape.WriteAPIV1Targets(w, state)
return true
+ case "/prometheus/target_response", "/target_response":
+ promscrapeTargetResponseRequests.Inc()
+ if err := promscrape.WriteTargetResponse(w, r); err != nil {
+ promscrapeTargetResponseErrors.Inc()
+ httpserver.Errorf(w, r, "%s", err)
+ return true
+ }
+ return true
case "/prometheus/config", "/config":
if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
err := &httpserver.ErrorWithStatusCode{
@@ -266,6 +274,9 @@ var (
promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
+ promscrapeTargetResponseRequests = metrics.NewCounter(`vm_http_requests_total{path="/target_response"}`)
+ promscrapeTargetResponseErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/target_response"}`)
+
promscrapeConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/config"}`)
promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0e22c225fe..61d16bcdf4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -16,6 +16,7 @@ sort: 15
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add ability to configure notifiers (e.g. alertmanager) via a file in the way similar to Prometheus. See [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file), [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2127).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for Consul service discovery for notifiers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1947).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for specifying Basic Auth password for notifiers via a file. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1567).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): provide the ability to fetch target responses on behalf of `vmagent`. Click `fetch response` link for the needed target at `/targets` page. This feature may be useful for debugging responses from targets located in isolated environments.
* BUGFIX: return proper results from `highestMax()` function at [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage). Previously it was incorrectly returning timeseries with min peaks instead of max peaks.
* BUGFIX: properly limit indexdb cache sizes. Previously they could exceed values set via `-memory.allowedPercent` and/or `-memory.allowedBytes` when `indexdb` contained many data parts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007).
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 6ae795575e..d02c09b5ce 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -373,14 +373,14 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) {
sg.activeScrapers.Inc()
sg.scrapersStarted.Inc()
sg.wg.Add(1)
- tsmGlobal.Register(sw)
+ tsmGlobal.Register(&sc.sw)
go func(sw *ScrapeWork) {
defer func() {
sg.wg.Done()
close(sc.stoppedCh)
}()
sc.sw.run(sc.stopCh, sg.globalStopCh)
- tsmGlobal.Unregister(sw)
+ tsmGlobal.Unregister(&sc.sw)
sg.activeScrapers.Dec()
sg.scrapersStopped.Inc()
}(sw)
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 13a08a43a5..aef8f85b18 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -3,6 +3,7 @@ package promscrape
import (
"flag"
"fmt"
+ "io/ioutil"
"math"
"math/bits"
"strconv"
@@ -371,6 +372,22 @@ func (sw *scrapeWork) mustSwitchToStreamParseMode(responseSize int) bool {
return sw.Config.canSwitchToStreamParseMode() && responseSize >= minResponseSizeForStreamParse.N
}
+// getTargetResponse() fetches response from sw target in the same way as when scraping the target.
+func (sw *scrapeWork) getTargetResponse() ([]byte, error) {
+ if *streamParse || sw.Config.StreamParse || sw.mustSwitchToStreamParseMode(sw.prevBodyLen) {
+ // Read the response in stream mode.
+ sr, err := sw.GetStreamReader()
+ if err != nil {
+ return nil, err
+ }
+ data, err := ioutil.ReadAll(sr)
+ sr.MustClose()
+ return data, err
+ }
+ // Read the response in usual mode.
+ return sw.ReadData(nil)
+}
+
func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error {
if *streamParse || sw.Config.StreamParse || sw.mustSwitchToStreamParseMode(sw.prevBodyLen) {
// Read data from scrape targets in streaming manner.
@@ -455,7 +472,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
// This should reduce memory usage when scraping targets which return big responses.
leveledbytebufferpool.Put(body)
}
- tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
+ tsmGlobal.Update(sw, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
return err
}
@@ -558,7 +575,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
sw.storeLastScrape(sbr.body)
}
sw.finalizeLastScrape()
- tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
+ tsmGlobal.Update(sw, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err)
// Do not track active series in streaming mode, since this may need too big amounts of memory
// when the target exports too big number of metrics.
return err
diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl
index b85bc8b8c1..9e50fbff74 100644
--- a/lib/promscrape/targets_response.qtpl
+++ b/lib/promscrape/targets_response.qtpl
@@ -76,7 +76,9 @@ job={%q= jobName %} (0/0 up)
{% for j, ts := range js.targetsStatus %}
{% if onlyUnhealthy && ts.up %}{% continue %}{% endif %}
<tr {% if !ts.up %}{%space%}class="alert alert-danger" role="alert"{% endif %}>
- <td><a href="{%s ts.endpoint %}">{%s ts.endpoint %}</a><br></td>
+ <td><a href="{%s ts.endpoint %}">{%s ts.endpoint %}</a> (
+ <a href="target_response?id={%s ts.targetID %}" target="_blank">fetch response</a>
+ )</td>
<td>{% if ts.up %}UP{% else %}DOWN{% endif %}</td>
<td>
<button type="button" class="btn btn-sm btn-outline-info" onclick="document.getElementById('original_labels_{%d i %}_{%d j %}').style.display='block'">show original labels</button>{% space %}
diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go
index 9055cece39..1c9a1749be 100644
--- a/lib/promscrape/targets_response.qtpl.go
+++ b/lib/promscrape/targets_response.qtpl.go
@@ -265,167 +265,171 @@ func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatus
//line lib/promscrape/targets_response.qtpl:79
qw422016.E().S(ts.endpoint)
//line lib/promscrape/targets_response.qtpl:79
- qw422016.N().S(`</a><br></td><td>`)
+ qw422016.N().S(`</a> (<a href="target_response?id=`)
//line lib/promscrape/targets_response.qtpl:80
+ qw422016.E().S(ts.targetID)
+//line lib/promscrape/targets_response.qtpl:80
+ qw422016.N().S(`" target="_blank">fetch response</a>)</td><td>`)
+//line lib/promscrape/targets_response.qtpl:82
if ts.up {
-//line lib/promscrape/targets_response.qtpl:80
+//line lib/promscrape/targets_response.qtpl:82
qw422016.N().S(`UP`)
-//line lib/promscrape/targets_response.qtpl:80
+//line lib/promscrape/targets_response.qtpl:82
} else {
-//line lib/promscrape/targets_response.qtpl:80
+//line lib/promscrape/targets_response.qtpl:82
qw422016.N().S(`DOWN`)
-//line lib/promscrape/targets_response.qtpl:80
+//line lib/promscrape/targets_response.qtpl:82
}
-//line lib/promscrape/targets_response.qtpl:80
+//line lib/promscrape/targets_response.qtpl:82
qw422016.N().S(`</td><td><button type="button" class="btn btn-sm btn-outline-info" onclick="document.getElementById('original_labels_`)
-//line lib/promscrape/targets_response.qtpl:82
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().D(i)
-//line lib/promscrape/targets_response.qtpl:82
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().S(`_`)
-//line lib/promscrape/targets_response.qtpl:82
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().D(j)
-//line lib/promscrape/targets_response.qtpl:82
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().S(`').style.display='block'">show original labels</button>`)
-//line lib/promscrape/targets_response.qtpl:82
+//line lib/promscrape/targets_response.qtpl:84
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:83
+//line lib/promscrape/targets_response.qtpl:85
streamformatLabel(qw422016, ts.labels)
-//line lib/promscrape/targets_response.qtpl:83
+//line lib/promscrape/targets_response.qtpl:85
qw422016.N().S(`<div style="display:none" id="original_labels_`)
-//line lib/promscrape/targets_response.qtpl:84
+//line lib/promscrape/targets_response.qtpl:86
qw422016.N().D(i)
-//line lib/promscrape/targets_response.qtpl:84
+//line lib/promscrape/targets_response.qtpl:86
qw422016.N().S(`_`)
-//line lib/promscrape/targets_response.qtpl:84
+//line lib/promscrape/targets_response.qtpl:86
qw422016.N().D(j)
-//line lib/promscrape/targets_response.qtpl:84
+//line lib/promscrape/targets_response.qtpl:86
qw422016.N().S(`"><button type="button" class="btn btn-sm btn-outline-info" onclick="document.getElementById('original_labels_`)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:87
qw422016.N().D(i)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:87
qw422016.N().S(`_`)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:87
qw422016.N().D(j)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:87
qw422016.N().S(`').style.display='none'">hide original labels</button>`)
-//line lib/promscrape/targets_response.qtpl:85
+//line lib/promscrape/targets_response.qtpl:87
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:86
+//line lib/promscrape/targets_response.qtpl:88
streamformatLabel(qw422016, ts.originalLabels)
-//line lib/promscrape/targets_response.qtpl:86
+//line lib/promscrape/targets_response.qtpl:88
qw422016.N().S(`</div></td><td>`)
-//line lib/promscrape/targets_response.qtpl:89
+//line lib/promscrape/targets_response.qtpl:91
qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
-//line lib/promscrape/targets_response.qtpl:89
+//line lib/promscrape/targets_response.qtpl:91
qw422016.N().S(`s ago</td><td>`)
-//line lib/promscrape/targets_response.qtpl:90
+//line lib/promscrape/targets_response.qtpl:92
qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3)
-//line lib/promscrape/targets_response.qtpl:90
+//line lib/promscrape/targets_response.qtpl:92
qw422016.N().S(`s</td><td>`)
-//line lib/promscrape/targets_response.qtpl:91
+//line lib/promscrape/targets_response.qtpl:93
qw422016.N().D(ts.samplesScraped)
-//line lib/promscrape/targets_response.qtpl:91
+//line lib/promscrape/targets_response.qtpl:93
qw422016.N().S(`</td><td>`)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:94
qw422016.E().S(ts.errMsg)
-//line lib/promscrape/targets_response.qtpl:92
+//line lib/promscrape/targets_response.qtpl:94
qw422016.N().S(`</td></tr>`)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:96
}
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:96
qw422016.N().S(`</tbody></table></div></div>`)
-//line lib/promscrape/targets_response.qtpl:99
- }
//line lib/promscrape/targets_response.qtpl:101
+ }
+//line lib/promscrape/targets_response.qtpl:103
for _, jobName := range emptyJobs {
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:103
qw422016.N().S(`<div><h4><a>`)
-//line lib/promscrape/targets_response.qtpl:104
+//line lib/promscrape/targets_response.qtpl:106
qw422016.E().S(jobName)
-//line lib/promscrape/targets_response.qtpl:104
+//line lib/promscrape/targets_response.qtpl:106
qw422016.N().S(`(0/0 up)</a></h4><table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col">Endpoint</th><th scope="col">State</th><th scope="col">Labels</th><th scope="col">Last Scrape</th><th scope="col">Scrape Duration</th><th scope="col">Samples Scraped</th><th scope="col">Error</th></tr></thead></table></div>`)
-//line lib/promscrape/targets_response.qtpl:120
+//line lib/promscrape/targets_response.qtpl:122
}
-//line lib/promscrape/targets_response.qtpl:120
+//line lib/promscrape/targets_response.qtpl:122
qw422016.N().S(`</body></html>`)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
}
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, emptyJobs []string, onlyUnhealthy bool) {
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
StreamTargetsResponseHTML(qw422016, jts, emptyJobs, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
}
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
func TargetsResponseHTML(jts []jobTargetsStatuses, emptyJobs []string, onlyUnhealthy bool) string {
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
WriteTargetsResponseHTML(qb422016, jts, emptyJobs, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
return qs422016
-//line lib/promscrape/targets_response.qtpl:123
+//line lib/promscrape/targets_response.qtpl:125
}
-//line lib/promscrape/targets_response.qtpl:125
-func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:125
- qw422016.N().S(`{`)
//line lib/promscrape/targets_response.qtpl:127
+func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
+//line lib/promscrape/targets_response.qtpl:127
+ qw422016.N().S(`{`)
+//line lib/promscrape/targets_response.qtpl:129
for i, label := range labels {
-//line lib/promscrape/targets_response.qtpl:128
+//line lib/promscrape/targets_response.qtpl:130
qw422016.E().S(label.Name)
-//line lib/promscrape/targets_response.qtpl:128
+//line lib/promscrape/targets_response.qtpl:130
qw422016.N().S(`=`)
-//line lib/promscrape/targets_response.qtpl:128
+//line lib/promscrape/targets_response.qtpl:130
qw422016.E().Q(label.Value)
-//line lib/promscrape/targets_response.qtpl:129
+//line lib/promscrape/targets_response.qtpl:131
if i+1 < len(labels) {
-//line lib/promscrape/targets_response.qtpl:129
+//line lib/promscrape/targets_response.qtpl:131
qw422016.N().S(`,`)
-//line lib/promscrape/targets_response.qtpl:129
+//line lib/promscrape/targets_response.qtpl:131
qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:129
+//line lib/promscrape/targets_response.qtpl:131
}
-//line lib/promscrape/targets_response.qtpl:130
+//line lib/promscrape/targets_response.qtpl:132
}
-//line lib/promscrape/targets_response.qtpl:130
+//line lib/promscrape/targets_response.qtpl:132
qw422016.N().S(`}`)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
}
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
streamformatLabel(qw422016, labels)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
}
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
func formatLabel(labels []prompbmarshal.Label) string {
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
writeformatLabel(qb422016, labels)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
return qs422016
-//line lib/promscrape/targets_response.qtpl:132
+//line lib/promscrape/targets_response.qtpl:134
}
diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go
index b585ba1af9..4d5c98677e 100644
--- a/lib/promscrape/targetstatus.go
+++ b/lib/promscrape/targetstatus.go
@@ -10,6 +10,7 @@ import (
"strings"
"sync"
"time"
+ "unsafe"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -22,6 +23,24 @@ var maxDroppedTargets = flag.Int("promscrape.maxDroppedTargets", 1000, "The maxi
var tsmGlobal = newTargetStatusMap()
+// WriteTargetResponse serves requests to /target_response?id=<id>
+//
+// It fetches response for the given target id and returns it.
+func WriteTargetResponse(w http.ResponseWriter, r *http.Request) error {
+ targetID := r.FormValue("id")
+ sw := tsmGlobal.getScrapeWorkByTargetID(targetID)
+ if sw == nil {
+ return fmt.Errorf("cannot find target for id=%s", targetID)
+ }
+ data, err := sw.getTargetResponse()
+ if err != nil {
+ return fmt.Errorf("cannot fetch response from id=%s: %w", targetID, err)
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ _, err = w.Write(data)
+ return err
+}
+
// WriteHumanReadableTargetsStatus writes human-readable status for all the scrape targets to w according to r.
func WriteHumanReadableTargetsStatus(w http.ResponseWriter, r *http.Request) {
showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
@@ -57,19 +76,19 @@ func WriteAPIV1Targets(w io.Writer, state string) {
type targetStatusMap struct {
mu sync.Mutex
- m map[*ScrapeWork]*targetStatus
+ m map[*scrapeWork]*targetStatus
jobNames []string
}
func newTargetStatusMap() *targetStatusMap {
return &targetStatusMap{
- m: make(map[*ScrapeWork]*targetStatus),
+ m: make(map[*scrapeWork]*targetStatus),
}
}
func (tsm *targetStatusMap) Reset() {
tsm.mu.Lock()
- tsm.m = make(map[*ScrapeWork]*targetStatus)
+ tsm.m = make(map[*scrapeWork]*targetStatus)
tsm.mu.Unlock()
}
@@ -79,7 +98,7 @@ func (tsm *targetStatusMap) registerJobNames(jobNames []string) {
tsm.mu.Unlock()
}
-func (tsm *targetStatusMap) Register(sw *ScrapeWork) {
+func (tsm *targetStatusMap) Register(sw *scrapeWork) {
tsm.mu.Lock()
tsm.m[sw] = &targetStatus{
sw: sw,
@@ -87,13 +106,13 @@ func (tsm *targetStatusMap) Register(sw *ScrapeWork) {
tsm.mu.Unlock()
}
-func (tsm *targetStatusMap) Unregister(sw *ScrapeWork) {
+func (tsm *targetStatusMap) Unregister(sw *scrapeWork) {
tsm.mu.Lock()
delete(tsm.m, sw)
tsm.mu.Unlock()
}
-func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, samplesScraped int, err error) {
+func (tsm *targetStatusMap) Update(sw *scrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, samplesScraped int, err error) {
tsm.mu.Lock()
ts := tsm.m[sw]
if ts == nil {
@@ -111,6 +130,21 @@ func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrape
tsm.mu.Unlock()
}
+func (tsm *targetStatusMap) getScrapeWorkByTargetID(targetID string) *scrapeWork {
+ tsm.mu.Lock()
+ defer tsm.mu.Unlock()
+ for sw := range tsm.m {
+ if getTargetID(sw) == targetID {
+ return sw
+ }
+ }
+ return nil
+}
+
+func getTargetID(sw *scrapeWork) string {
+ return fmt.Sprintf("%016x", uintptr(unsafe.Pointer(sw)))
+}
+
// StatusByGroup returns the number of targets with status==up
// for the given group name
func (tsm *targetStatusMap) StatusByGroup(group string, up bool) int {
@@ -134,7 +168,7 @@ func (tsm *targetStatusMap) WriteActiveTargetsJSON(w io.Writer) {
}
kss := make([]keyStatus, 0, len(tsm.m))
for sw, st := range tsm.m {
- key := promLabelsString(sw.OriginalLabels)
+ key := promLabelsString(sw.Config.OriginalLabels)
kss = append(kss, keyStatus{
key: key,
st: *st,
@@ -149,12 +183,12 @@ func (tsm *targetStatusMap) WriteActiveTargetsJSON(w io.Writer) {
for i, ks := range kss {
st := ks.st
fmt.Fprintf(w, `{"discoveredLabels":`)
- writeLabelsJSON(w, st.sw.OriginalLabels)
+ writeLabelsJSON(w, st.sw.Config.OriginalLabels)
fmt.Fprintf(w, `,"labels":`)
- labelsFinalized := promrelabel.FinalizeLabels(nil, st.sw.Labels)
+ labelsFinalized := promrelabel.FinalizeLabels(nil, st.sw.Config.Labels)
writeLabelsJSON(w, labelsFinalized)
- fmt.Fprintf(w, `,"scrapePool":%q`, st.sw.Job())
- fmt.Fprintf(w, `,"scrapeUrl":%q`, st.sw.ScrapeURL)
+ fmt.Fprintf(w, `,"scrapePool":%q`, st.sw.Config.Job())
+ fmt.Fprintf(w, `,"scrapeUrl":%q`, st.sw.Config.ScrapeURL)
errMsg := ""
if st.err != nil {
errMsg = st.err.Error()
@@ -187,7 +221,7 @@ func writeLabelsJSON(w io.Writer, labels []prompbmarshal.Label) {
}
type targetStatus struct {
- sw *ScrapeWork
+ sw *scrapeWork
up bool
scrapeGroup string
scrapeTime int64
@@ -274,6 +308,7 @@ var droppedTargetsMap = &droppedTargets{
type jobTargetStatus struct {
up bool
endpoint string
+ targetID string
labels []prompbmarshal.Label
originalLabels []prompbmarshal.Label
lastScrapeTime time.Duration
@@ -293,7 +328,7 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() ([]jobTargetsStatuses, []str
byJob := make(map[string][]targetStatus)
tsm.mu.Lock()
for _, st := range tsm.m {
- job := st.sw.jobNameOriginal
+ job := st.sw.Config.jobNameOriginal
byJob[job] = append(byJob[job], *st)
}
jobNames := append([]string{}, tsm.jobNames...)
@@ -302,7 +337,7 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() ([]jobTargetsStatuses, []str
var jts []jobTargetsStatuses
for job, statuses := range byJob {
sort.Slice(statuses, func(i, j int) bool {
- return statuses[i].sw.ScrapeURL < statuses[j].sw.ScrapeURL
+ return statuses[i].sw.Config.ScrapeURL < statuses[j].sw.Config.ScrapeURL
})
ups := 0
var targetsStatuses []jobTargetStatus
@@ -318,9 +353,10 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() ([]jobTargetsStatuses, []str
}
targetsStatuses = append(targetsStatuses, jobTargetStatus{
up: st.up,
- endpoint: st.sw.ScrapeURL,
- labels: promrelabel.FinalizeLabels(nil, st.sw.Labels),
- originalLabels: st.sw.OriginalLabels,
+ endpoint: st.sw.Config.ScrapeURL,
+ targetID: getTargetID(st.sw),
+ labels: promrelabel.FinalizeLabels(nil, st.sw.Config.Labels),
+ originalLabels: st.sw.Config.OriginalLabels,
lastScrapeTime: st.getDurationFromLastScrape(),
scrapeDuration: time.Duration(st.scrapeDuration) * time.Millisecond,
samplesScraped: st.samplesScraped,