lib/promrelabel: make target url from labels on target relabel page (#3882)

* lib/promrelabel: make target url from labels on target relabel page

* wip

---------

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

docs/CHANGELOG.md

@@ -21,6 +21,7 @@ created by v1.90.0 or newer versions. The solution is to upgrade to v1.90.0 or n
 
 * FEATURE: publish VictoriaMetrics binaries for Windows. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3236), [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3821) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70) issues.
 * FEATURE: log metrics with truncated labels if the length of label value in the ingested metric exceeds `-maxLabelValueLen`. This should simplify debugging for this case.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): show target URL when debugging [target relabeling](https://docs.victoriametrics.com/vmagent.html#relabel-debug). This should simplify target relabel debugging a bit. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3882).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent.html#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `--kafka.consumer.topic.concurrency` command-line flag. It controls the number of Kafka consumer workers to use by `vmagent`. It should eliminate the need to start multiple `vmagent` instances to improve data transfer rate. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1957).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [Kafka producer and consumer](https://docs.victoriametrics.com/vmagent.html#kafka-integration) on `arm64` machines. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2271).

lib/promrelabel/debug.go

@@ -21,30 +21,32 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 	if metric == "" {
 		metric = "{}"
 	}
+	targetURL := ""
 	if err != nil {
-		WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err)
 		return
 	}
 	labels, err := promutils.NewLabelsFromString(metric)
 	if err != nil {
 		err = fmt.Errorf("cannot parse metric: %s", err)
-		WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err)
 		return
 	}
 	pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
 	if err != nil {
 		err = fmt.Errorf("cannot parse relabel configs: %s", err)
-		WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err)
 		return
 	}
 
-	dss := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
+	dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
-	WriteRelabelDebugSteps(w, isTargetRelabel, targetID, dss, metric, relabelConfigs, nil)
+	WriteRelabelDebugSteps(w, targetURL, targetID, dss, metric, relabelConfigs, nil)
 }
 
-func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTargetRelabel bool) []DebugStep {
+func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTargetRelabel bool) ([]DebugStep, string) {
-	// The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork if isTragetRelabeling=true
+	// The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork if isTargetRelabel=true
 	// and with the code at scrapeWork.addRowToTimeseries when isTargetRelabeling=false
+	targetURL := ""
 
 	// Prevent from modifying the original labels
 	labels = labels.Clone()
@@ -70,6 +72,9 @@ func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTarget
 			}
 		}
 
+		// Generate targetURL
+		targetURL, _ = GetScrapeURL(labels, nil)
+
 		// Remove labels with __ prefix
 		inStr := outStr
 		labels.RemoveLabelsWithDoubleUnderscorePrefix()
@@ -96,7 +101,7 @@ func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTarget
 	}
 
 	// There is no need in labels' sorting, since LabelsToString() automatically sorts labels.
-	return dss
+	return dss, targetURL
 }
 
 func getChangedLabelNames(in, out *promutils.Labels) map[string]struct{} {

lib/promrelabel/debug.qtpl

@@ -5,7 +5,7 @@
 
 {% stripspace %}
 
-{% func RelabelDebugSteps(isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugSteps(targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -27,7 +27,7 @@ function submitRelabelDebugForm(e) {
     <div class="container-fluid">
         <a href="https://docs.victoriametrics.com/relabeling.html" target="_blank">Relabeling docs</a>{% space %}
 
-        {% if isTargetRelabel %}
+        {% if targetURL != "" %}
           <a href="metric-relabel-debug{% if targetID != "" %}?id={%s targetID %}{% endif %}">Metric relabel debug</a>
         {% else %}
           <a href="target-relabel-debug{% if targetID != "" %}?id={%s targetID %}{% endif %}">Target relabel debug</a>
@@ -53,7 +53,7 @@ function submitRelabelDebugForm(e) {
 
         <div class="row">
             <main class="col-12">
-              {%= relabelDebugSteps(dss) %}
+              {%= relabelDebugSteps(dss, targetURL, targetID) %}
             </main>
         </div>
     </div>
@@ -73,10 +73,10 @@ function submitRelabelDebugForm(e) {
     </div>
 {% endfunc %}
 
-{% func relabelDebugSteps(dss []DebugStep) %}
+{% func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) %}
 {% if len(dss) > 0 %}
 <div class="m-3">
-        <b>Original labels:</b> <samp>{%= mustFormatLabels(dss[0].In) %}</samp>
+    <b>Original labels:</b> <samp>{%= mustFormatLabels(dss[0].In) %}</samp>
 </div>
 {% endif %}
 <table class="table table-striped table-hover table-bordered table-sm">
@@ -114,7 +114,16 @@ function submitRelabelDebugForm(e) {
 </table>
 {% if len(dss) > 0 %}
 <div class="m-3">
-        <b>Resulting labels:</b> <samp>{%= mustFormatLabels(dss[len(dss)-1].Out) %}</samp>
+    <b>Resulting labels:</b> <samp>{%= mustFormatLabels(dss[len(dss)-1].Out) %}</samp>
+    {% if targetURL != "" %}
+        <div>
+            <b>Target URL:</b>{% space %}<a href="{%s targetURL %}" target="_blank">{%s targetURL %}</a>
+            {% if targetID != "" %}
+                {% space %}
+                (<a href="target_response?id={%s targetID %}" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)
+            {% endif %}
+        </div>
+    {% endif %}
 </div>
 {% endif %}
 {% endfunc %}

lib/promrelabel/debug.qtpl.go

@@ -24,7 +24,7 @@ var (
 )
 
 //line lib/promrelabel/debug.qtpl:8
-func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
 //line lib/promrelabel/debug.qtpl:8
 	qw422016.N().S(`<!DOCTYPE html><html lang="en"><head>`)
 //line lib/promrelabel/debug.qtpl:12
@@ -38,7 +38,7 @@ func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, isTargetRelabel bool, ta
 //line lib/promrelabel/debug.qtpl:28
 	qw422016.N().S(` `)
 //line lib/promrelabel/debug.qtpl:30
-	if isTargetRelabel {
+	if targetURL != "" {
 //line lib/promrelabel/debug.qtpl:30
 		qw422016.N().S(`<a href="metric-relabel-debug`)
 //line lib/promrelabel/debug.qtpl:31
@@ -104,29 +104,29 @@ func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, isTargetRelabel bool, ta
 //line lib/promrelabel/debug.qtpl:50
 	qw422016.N().S(`</form></div><div class="row"><main class="col-12">`)
 //line lib/promrelabel/debug.qtpl:56
-	streamrelabelDebugSteps(qw422016, dss)
+	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
 //line lib/promrelabel/debug.qtpl:56
 	qw422016.N().S(`</main></div></div></body></html>`)
 //line lib/promrelabel/debug.qtpl:62
 }
 
 //line lib/promrelabel/debug.qtpl:62
-func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
 //line lib/promrelabel/debug.qtpl:62
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line lib/promrelabel/debug.qtpl:62
-	StreamRelabelDebugSteps(qw422016, isTargetRelabel, targetID, dss, metric, relabelConfigs, err)
+	StreamRelabelDebugSteps(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
 //line lib/promrelabel/debug.qtpl:62
 	qt422016.ReleaseWriter(qw422016)
 //line lib/promrelabel/debug.qtpl:62
 }
 
 //line lib/promrelabel/debug.qtpl:62
-func RelabelDebugSteps(isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
+func RelabelDebugSteps(targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
 //line lib/promrelabel/debug.qtpl:62
 	qb422016 := qt422016.AcquireByteBuffer()
 //line lib/promrelabel/debug.qtpl:62
-	WriteRelabelDebugSteps(qb422016, isTargetRelabel, targetID, dss, metric, relabelConfigs, err)
+	WriteRelabelDebugSteps(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
 //line lib/promrelabel/debug.qtpl:62
 	qs422016 := string(qb422016.B)
 //line lib/promrelabel/debug.qtpl:62
@@ -178,7 +178,7 @@ func relabelDebugFormInputs(metric, relabelConfigs string) string {
 }
 
 //line lib/promrelabel/debug.qtpl:76
-func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep) {
+func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) {
 //line lib/promrelabel/debug.qtpl:77
 	if len(dss) > 0 {
 //line lib/promrelabel/debug.qtpl:77
@@ -227,41 +227,75 @@ func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep) {
 //line lib/promrelabel/debug.qtpl:117
 		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
 //line lib/promrelabel/debug.qtpl:117
-		qw422016.N().S(`</samp></div>`)
+		qw422016.N().S(`</samp>`)
-//line lib/promrelabel/debug.qtpl:119
+//line lib/promrelabel/debug.qtpl:118
-	}
+		if targetURL != "" {
+//line lib/promrelabel/debug.qtpl:118
+			qw422016.N().S(`<div><b>Target URL:</b>`)
 //line lib/promrelabel/debug.qtpl:120
-}
+			qw422016.N().S(` `)
-
 //line lib/promrelabel/debug.qtpl:120
-func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep) {
+			qw422016.N().S(`<a href="`)
 //line lib/promrelabel/debug.qtpl:120
-	qw422016 := qt422016.AcquireWriter(qq422016)
+			qw422016.E().S(targetURL)
 //line lib/promrelabel/debug.qtpl:120
-	streamrelabelDebugSteps(qw422016, dss)
+			qw422016.N().S(`" target="_blank">`)
 //line lib/promrelabel/debug.qtpl:120
-	qt422016.ReleaseWriter(qw422016)
+			qw422016.E().S(targetURL)
 //line lib/promrelabel/debug.qtpl:120
-}
+			qw422016.N().S(`</a>`)
-
+//line lib/promrelabel/debug.qtpl:121
-//line lib/promrelabel/debug.qtpl:120
+			if targetID != "" {
-func relabelDebugSteps(dss []DebugStep) string {
-//line lib/promrelabel/debug.qtpl:120
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:120
-	writerelabelDebugSteps(qb422016, dss)
-//line lib/promrelabel/debug.qtpl:120
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:120
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:120
-	return qs422016
-//line lib/promrelabel/debug.qtpl:120
-}
-
 //line lib/promrelabel/debug.qtpl:122
-func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+				qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:122
+				qw422016.N().S(`(<a href="target_response?id=`)
+//line lib/promrelabel/debug.qtpl:123
+				qw422016.E().S(targetID)
+//line lib/promrelabel/debug.qtpl:123
+				qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
 //line lib/promrelabel/debug.qtpl:124
+			}
+//line lib/promrelabel/debug.qtpl:124
+			qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:126
+		}
+//line lib/promrelabel/debug.qtpl:126
+		qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:128
+	}
+//line lib/promrelabel/debug.qtpl:129
+}
+
+//line lib/promrelabel/debug.qtpl:129
+func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) {
+//line lib/promrelabel/debug.qtpl:129
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:129
+	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
+//line lib/promrelabel/debug.qtpl:129
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:129
+}
+
+//line lib/promrelabel/debug.qtpl:129
+func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string {
+//line lib/promrelabel/debug.qtpl:129
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:129
+	writerelabelDebugSteps(qb422016, dss, targetURL, targetID)
+//line lib/promrelabel/debug.qtpl:129
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:129
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:129
+	return qs422016
+//line lib/promrelabel/debug.qtpl:129
+}
+
+//line lib/promrelabel/debug.qtpl:131
+func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:133
 	labelsList := labels.GetLabels()
 	metricName := ""
 	for i, label := range labelsList {
@@ -272,40 +306,10 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labe
 		}
 	}
 
-//line lib/promrelabel/debug.qtpl:134
-	if metricName != "" {
-//line lib/promrelabel/debug.qtpl:135
-		if _, ok := highlight["__name__"]; ok {
-//line lib/promrelabel/debug.qtpl:135
-			qw422016.N().S(`<span style="font-weight:bold;color:`)
-//line lib/promrelabel/debug.qtpl:136
-			qw422016.E().S(color)
-//line lib/promrelabel/debug.qtpl:136
-			qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:136
-			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:136
-			qw422016.N().S(`</span>`)
-//line lib/promrelabel/debug.qtpl:137
-		} else {
-//line lib/promrelabel/debug.qtpl:138
-			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:139
-		}
-//line lib/promrelabel/debug.qtpl:140
-		if len(labelsList) == 0 {
-//line lib/promrelabel/debug.qtpl:140
-			return
-//line lib/promrelabel/debug.qtpl:140
-		}
-//line lib/promrelabel/debug.qtpl:141
-	}
-//line lib/promrelabel/debug.qtpl:141
-	qw422016.N().S(`{`)
 //line lib/promrelabel/debug.qtpl:143
-	for i, label := range labelsList {
+	if metricName != "" {
 //line lib/promrelabel/debug.qtpl:144
-		if _, ok := highlight[label.Name]; ok {
+		if _, ok := highlight["__name__"]; ok {
 //line lib/promrelabel/debug.qtpl:144
 			qw422016.N().S(`<span style="font-weight:bold;color:`)
 //line lib/promrelabel/debug.qtpl:145
@@ -313,96 +317,126 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labe
 //line lib/promrelabel/debug.qtpl:145
 			qw422016.N().S(`">`)
 //line lib/promrelabel/debug.qtpl:145
-			qw422016.E().S(label.Name)
+			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:145
-			qw422016.N().S(`=`)
-//line lib/promrelabel/debug.qtpl:145
-			qw422016.E().Q(label.Value)
 //line lib/promrelabel/debug.qtpl:145
 			qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:146
 		} else {
 //line lib/promrelabel/debug.qtpl:147
-			qw422016.E().S(label.Name)
+			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:147
-			qw422016.N().S(`=`)
-//line lib/promrelabel/debug.qtpl:147
-			qw422016.E().Q(label.Value)
 //line lib/promrelabel/debug.qtpl:148
 		}
 //line lib/promrelabel/debug.qtpl:149
-		if i < len(labelsList)-1 {
+		if len(labelsList) == 0 {
 //line lib/promrelabel/debug.qtpl:149
-			qw422016.N().S(`,`)
+			return
-//line lib/promrelabel/debug.qtpl:149
-			qw422016.N().S(` `)
 //line lib/promrelabel/debug.qtpl:149
 		}
 //line lib/promrelabel/debug.qtpl:150
 	}
 //line lib/promrelabel/debug.qtpl:150
-	qw422016.N().S(`}`)
+	qw422016.N().S(`{`)
 //line lib/promrelabel/debug.qtpl:152
-}
+	for i, label := range labelsList {
-
+//line lib/promrelabel/debug.qtpl:153
-//line lib/promrelabel/debug.qtpl:152
+		if _, ok := highlight[label.Name]; ok {
-func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:153
-//line lib/promrelabel/debug.qtpl:152
+			qw422016.N().S(`<span style="font-weight:bold;color:`)
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:152
-	streamlabelsWithHighlight(qw422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:152
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:152
-}
-
-//line lib/promrelabel/debug.qtpl:152
-func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string {
-//line lib/promrelabel/debug.qtpl:152
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:152
-	writelabelsWithHighlight(qb422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:152
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:152
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:152
-	return qs422016
-//line lib/promrelabel/debug.qtpl:152
-}
-
 //line lib/promrelabel/debug.qtpl:154
-func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
+			qw422016.E().S(color)
+//line lib/promrelabel/debug.qtpl:154
+			qw422016.N().S(`">`)
+//line lib/promrelabel/debug.qtpl:154
+			qw422016.E().S(label.Name)
+//line lib/promrelabel/debug.qtpl:154
+			qw422016.N().S(`=`)
+//line lib/promrelabel/debug.qtpl:154
+			qw422016.E().Q(label.Value)
+//line lib/promrelabel/debug.qtpl:154
+			qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:155
+		} else {
+//line lib/promrelabel/debug.qtpl:156
+			qw422016.E().S(label.Name)
+//line lib/promrelabel/debug.qtpl:156
+			qw422016.N().S(`=`)
+//line lib/promrelabel/debug.qtpl:156
+			qw422016.E().Q(label.Value)
+//line lib/promrelabel/debug.qtpl:157
+		}
+//line lib/promrelabel/debug.qtpl:158
+		if i < len(labelsList)-1 {
+//line lib/promrelabel/debug.qtpl:158
+			qw422016.N().S(`,`)
+//line lib/promrelabel/debug.qtpl:158
+			qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:158
+		}
+//line lib/promrelabel/debug.qtpl:159
+	}
+//line lib/promrelabel/debug.qtpl:159
+	qw422016.N().S(`}`)
+//line lib/promrelabel/debug.qtpl:161
+}
+
+//line lib/promrelabel/debug.qtpl:161
+func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:161
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:161
+	streamlabelsWithHighlight(qw422016, labels, highlight, color)
+//line lib/promrelabel/debug.qtpl:161
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:161
+}
+
+//line lib/promrelabel/debug.qtpl:161
+func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string {
+//line lib/promrelabel/debug.qtpl:161
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:161
+	writelabelsWithHighlight(qb422016, labels, highlight, color)
+//line lib/promrelabel/debug.qtpl:161
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:161
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:161
+	return qs422016
+//line lib/promrelabel/debug.qtpl:161
+}
+
+//line lib/promrelabel/debug.qtpl:163
+func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
+//line lib/promrelabel/debug.qtpl:164
 	labels := promutils.MustNewLabelsFromString(s)
 
-//line lib/promrelabel/debug.qtpl:156
+//line lib/promrelabel/debug.qtpl:165
 	streamlabelsWithHighlight(qw422016, labels, nil, "")
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 }
 
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 func writemustFormatLabels(qq422016 qtio422016.Writer, s string) {
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	streammustFormatLabels(qw422016, s)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 }
 
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 func mustFormatLabels(s string) string {
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	writemustFormatLabels(qb422016, s)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 	return qs422016
-//line lib/promrelabel/debug.qtpl:157
+//line lib/promrelabel/debug.qtpl:166
 }

lib/promrelabel/relabel.go

@@ -49,7 +49,7 @@ type DebugStep struct {
 	// Rule contains string representation of the rule step
 	Rule string
 
-	// In contains the input labels before the exeuction of the rule step
+	// In contains the input labels before the execution of the rule step
 	In string
 
 	// Out contains the output labels after the execution of the rule step

lib/promrelabel/scrape_url.go

@@ -0,0 +1,114 @@
+package promrelabel
+
+import (
+	"net/url"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// GetScrapeURL makes scrape url and __address_ labels for the given labels and extraParams.
+func GetScrapeURL(labels *promutils.Labels, extraParams map[string][]string) (string, string) {
+	// See https://www.robustperception.io/life-of-a-label
+	scheme := labels.Get("__scheme__")
+	if len(scheme) == 0 {
+		scheme = "http"
+	}
+	metricsPath := labels.Get("__metrics_path__")
+	if len(metricsPath) == 0 {
+		metricsPath = "/metrics"
+	}
+	address := labels.Get("__address__")
+	if len(address) == 0 {
+		return "", ""
+	}
+	// Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__.
+	// Prometheus silently drops targets with __address__ containing scheme or metricsPath
+	// according to https://www.robustperception.io/life-of-a-label/ .
+	if strings.HasPrefix(address, "http://") {
+		scheme = "http"
+		address = address[len("http://"):]
+	} else if strings.HasPrefix(address, "https://") {
+		scheme = "https"
+		address = address[len("https://"):]
+	}
+	if n := strings.IndexByte(address, '/'); n >= 0 {
+		metricsPath = address[n:]
+		address = address[:n]
+	}
+	address = addMissingPort(address, scheme == "https")
+
+	if !strings.HasPrefix(metricsPath, "/") {
+		metricsPath = "/" + metricsPath
+	}
+	params := getParamsFromLabels(labels, extraParams)
+	optionalQuestion := ""
+	if len(params) > 0 {
+		optionalQuestion = "?"
+		if strings.Contains(metricsPath, "?") {
+			optionalQuestion = "&"
+		}
+	}
+	paramsStr := url.Values(params).Encode()
+	scrapeURL := buildScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr)
+	return scrapeURL, address
+}
+
+func getParamsFromLabels(labels *promutils.Labels, extraParams map[string][]string) map[string][]string {
+	// See https://www.robustperception.io/life-of-a-label
+	var m map[string][]string
+	for _, label := range labels.GetLabels() {
+		if !strings.HasPrefix(label.Name, "__param_") {
+			continue
+		}
+		name := label.Name[len("__param_"):]
+		values := []string{label.Value}
+		if p := extraParams[name]; len(p) > 1 {
+			values = append(values, p[1:]...)
+		}
+		if m == nil {
+			m = make(map[string][]string)
+		}
+		m[name] = values
+	}
+	return m
+}
+
+func buildScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr string) string {
+	bb := bbPool.Get()
+	b := bb.B[:0]
+	b = append(b, scheme...)
+	b = append(b, "://"...)
+	b = append(b, address...)
+	b = append(b, metricsPath...)
+	b = append(b, optionalQuestion...)
+	b = append(b, paramsStr...)
+	s := bytesutil.InternBytes(b)
+	bb.B = b
+	bbPool.Put(bb)
+	return s
+}
+
+func addMissingPort(addr string, isTLS bool) string {
+	if strings.Contains(addr, ":") {
+		return addr
+	}
+	if isTLS {
+		return concatTwoStrings(addr, ":443")
+	}
+	return concatTwoStrings(addr, ":80")
+}
+
+func concatTwoStrings(x, y string) string {
+	bb := bbPool.Get()
+	b := bb.B[:0]
+	b = append(b, x...)
+	b = append(b, y...)
+	s := bytesutil.InternBytes(b)
+	bb.B = b
+	bbPool.Put(bb)
+	return s
+}
+
+var bbPool bytesutil.ByteBufferPool

lib/promrelabel/scrape_url_test.go

@@ -0,0 +1,54 @@
+package promrelabel
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestGetScrapeURL(t *testing.T) {
+	f := func(labelsStr, expectedScrapeURL, expectedAddress string) {
+		t.Helper()
+		labels := promutils.MustNewLabelsFromString(labelsStr)
+		scrapeURL, address := GetScrapeURL(labels, nil)
+		if scrapeURL != expectedScrapeURL {
+			t.Fatalf("unexpected scrapeURL; got %q; want %q", scrapeURL, expectedScrapeURL)
+		}
+		if address != expectedAddress {
+			t.Fatalf("unexpected address; got %q; want %q", address, expectedAddress)
+		}
+	}
+
+	// Missing __address__
+	f("{}", "", "")
+	f(`{foo="bar"}`, "", "")
+
+	// __address__ without port
+	f(`{__address__="foo"}`, "http://foo:80/metrics", "foo:80")
+
+	// __address__ with explicit port
+	f(`{__address__="foo:1234"}`, "http://foo:1234/metrics", "foo:1234")
+
+	// explicit __scheme__
+	f(`{__address__="foo",__scheme__="https"}`, "https://foo:443/metrics", "foo:443")
+	f(`{__address__="foo:1234",__scheme__="https"}`, "https://foo:1234/metrics", "foo:1234")
+
+	// explicit __metrics_path__
+	f(`{__address__="foo",__metrics_path__="abc"}`, "http://foo:80/abc", "foo:80")
+	f(`{__address__="foo",__metrics_path__="/abc"}`, "http://foo:80/abc", "foo:80")
+	f(`{__address__="foo",__metrics_path__="/ab/c?d=ef&aa=bb"}`, "http://foo:80/ab/c?d=ef&aa=bb", "foo:80")
+
+	// explitit __param_*
+	f(`{__address__="foo",__param_x="y"}`, "http://foo:80/metrics?x=y", "foo:80")
+	f(`{__address__="foo",__param_x="y",__param_y="aa"}`, "http://foo:80/metrics?x=y&y=aa", "foo:80")
+	f(`{__address__="foo",__param_x="y",__metrics_path__="?abc=de"}`, "http://foo:80/?abc=de&x=y", "foo:80")
+	f(`{__address__="foo",__param_abc="y",__metrics_path__="?abc=de"}`, "http://foo:80/?abc=de&abc=y", "foo:80")
+
+	// __address__ with metrics path and/or scheme
+	f(`{__address__="foo/bar/baz?abc=de"}`, "http://foo:80/bar/baz?abc=de", "foo:80")
+	f(`{__address__="foo:784/bar/baz?abc=de"}`, "http://foo:784/bar/baz?abc=de", "foo:784")
+	f(`{__address__="foo:784/bar/baz?abc=de",__param_xx="yy"}`, "http://foo:784/bar/baz?abc=de&xx=yy", "foo:784")
+	f(`{__address__="foo:784/bar/baz?abc=de",__param_xx="yy",__scheme__="https"}`, "https://foo:784/bar/baz?abc=de&xx=yy", "foo:784")
+	f(`{__address__="http://foo/bar/baz?abc=de",__param_xx="yy"}`, "http://foo:80/bar/baz?abc=de&xx=yy", "foo:80")
+	f(`{__address__="https://foo/bar/baz?abc=de",__param_xx="yy"}`, "https://foo:443/bar/baz?abc=de&xx=yy", "foo:443")
+}

lib/promscrape/config.go

@@ -1258,36 +1258,15 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		droppedTargetsMap.Register(originalLabels, swc.relabelConfigs)
 		return nil, nil
 	}
-	// See https://www.robustperception.io/life-of-a-label
+	scrapeURL, address := promrelabel.GetScrapeURL(labels, swc.params)
-	scheme := labels.Get("__scheme__")
+	if scrapeURL == "" {
-	if len(scheme) == 0 {
+		// Drop target without URL.
-		scheme = "http"
-	}
-	metricsPath := labels.Get("__metrics_path__")
-	if len(metricsPath) == 0 {
-		metricsPath = "/metrics"
-	}
-	address := labels.Get("__address__")
-	if len(address) == 0 {
-		// Drop target without scrape address.
 		droppedTargetsMap.Register(originalLabels, swc.relabelConfigs)
 		return nil, nil
 	}
-	// Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__.
+	if _, err := url.Parse(scrapeURL); err != nil {
-	// Prometheus silently drops targets with __address__ containing scheme or metricsPath
+		return nil, fmt.Errorf("invalid target url=%q for job=%q: %w", scrapeURL, swc.jobName, err)
-	// according to https://www.robustperception.io/life-of-a-label/ .
-	if strings.HasPrefix(address, "http://") {
-		scheme = "http"
-		address = address[len("http://"):]
-	} else if strings.HasPrefix(address, "https://") {
-		scheme = "https"
-		address = address[len("https://"):]
 	}
-	if n := strings.IndexByte(address, '/'); n >= 0 {
-		metricsPath = address[n:]
-		address = address[:n]
-	}
-	address = addMissingPort(address, scheme == "https")
 
 	var at *auth.Token
 	tenantID := labels.Get("__tenant_id__")
@@ -1299,23 +1278,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		at = newToken
 	}
 
-	if !strings.HasPrefix(metricsPath, "/") {
-		metricsPath = "/" + metricsPath
-	}
-	params := getParamsFromLabels(labels, swc.params)
-	optionalQuestion := ""
-	if len(params) > 0 {
-		optionalQuestion = "?"
-		if strings.Contains(metricsPath, "?") {
-			optionalQuestion = "&"
-		}
-	}
-	paramsStr := url.Values(params).Encode()
-	scrapeURL := getScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr)
-	if _, err := url.Parse(scrapeURL); err != nil {
-		return nil, fmt.Errorf("invalid url %q for scheme=%q, target=%q, address=%q, metrics_path=%q for job=%q: %w",
-			scrapeURL, scheme, target, address, metricsPath, swc.jobName, err)
-	}
 	// Read __scrape_interval__ and __scrape_timeout__ from labels.
 	scrapeInterval := swc.scrapeInterval
 	if s := labels.Get("__scrape_interval__"); len(s) > 0 {
@@ -1398,41 +1360,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 	return sw, nil
 }
 
-func getScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr string) string {
-	bb := bbPool.Get()
-	b := bb.B[:0]
-	b = append(b, scheme...)
-	b = append(b, "://"...)
-	b = append(b, address...)
-	b = append(b, metricsPath...)
-	b = append(b, optionalQuestion...)
-	b = append(b, paramsStr...)
-	s := bytesutil.InternBytes(b)
-	bb.B = b
-	bbPool.Put(bb)
-	return s
-}
-
-func getParamsFromLabels(labels *promutils.Labels, paramsOrig map[string][]string) map[string][]string {
-	// See https://www.robustperception.io/life-of-a-label
-	var m map[string][]string
-	for _, label := range labels.GetLabels() {
-		if !strings.HasPrefix(label.Name, "__param_") {
-			continue
-		}
-		name := label.Name[len("__param_"):]
-		values := []string{label.Value}
-		if p := paramsOrig[name]; len(p) > 1 {
-			values = append(values, p[1:]...)
-		}
-		if m == nil {
-			m = make(map[string][]string)
-		}
-		m[name] = values
-	}
-	return m
-}
-
 func mergeLabels(dst *promutils.Labels, swc *scrapeWorkConfig, target string, extraLabels, metaLabels *promutils.Labels) {
 	if n := dst.Len(); n > 0 {
 		logger.Panicf("BUG: len(dst.Labels) must be 0; got %d", n)