lib/promscrape: implement target-level and metric-level relabel debugging

Target-level debugging is performed by clicking the 'debug' link at the corresponding target
on either http://vmagent:8429/targets page or on http://vmagent:8428/service-discovery page.

Metric-level debugging is perfromed at http://vmagent:8429/metric-relabel-debug page.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3407

See https://docs.victoriametrics.com/vmagent.html#relabel-debug

README.md

@@ -2303,8 +2303,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Supports an array of values separated by comma or specified via multiple flags.
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
-  -relabelDebug
-     Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -retentionFilter array
      Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.

app/victoria-metrics/main.go

@@ -101,6 +101,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"vmui", "Web UI"},
 			{"targets", "status for discovered active targets"},
 			{"service-discovery", "labels before and after relabeling for discovered targets"},
+			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},

app/vmagent/README.md

@@ -245,8 +245,6 @@ scrape_configs:
 * `scrape_align_interval: duration` for aligning scrapes to the given interval instead of using random offset
   in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time.
 * `scrape_offset: duration` for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`.
-* `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling).
-* `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling).
 
 See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options.
 
@@ -419,26 +417,25 @@ with [additional enhancements](#relabeling-enhancements). The relabeling can be
   This relabeling is used for modifying labels in discovered targets and for dropping unneded targets.
   See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details.
 
-  This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section.
+  This relabeling can be debugged by clicking the `debug` link at the corresponding target on the `http://vmagent:8429/targets` page
-  In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
+  or on the `http://vmagent:8429/service-discovery` page. See [these docs](#relabel-debug) for details.
 
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file.
   This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics.
   See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details.
 
-  This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
-  In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
 
 * At the `-remoteWrite.relabelConfig` file. This relabeling is used for modifying labels for all the collected metrics
-  (inluding [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics
+  (including [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics
   before sending them to all the configured `-remoteWrite.url` addresses.
-  This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`.
+
-  In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
 
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is used for modifying labels for metrics
   and for dropping unneeded metrics before sending them to a particular `-remoteWrite.url`.
-  This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`.
+
-  In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
 
 All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`,
 which are replaced by the corresponding environment variable values.
@@ -453,9 +450,6 @@ The following articles contain useful information about Prometheus relabeling:
 * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
-[This relabeler playground](https://relabeler.promlabs.com/) can help debugging issues related to relabeling.
-
-
 ## Relabeling enhancements
 
 `vmagent` provides the following enhancements on top of Prometheus-compatible relabeling:
@@ -597,6 +591,24 @@ Important notes about `action: graphite` relabeling rules:
 The `action: graphite` relabeling rules are easier to write and maintain than `action: replace` for labels extraction from Graphite-style metric names.
 Additionally, the `action: graphite` relabeling rules usually work much faster than the equivalent `action: replace` rules.
 
+## Relabel debug
+
+`vmagent` provides the following tools for debugging target-level and metric-level relabeling:
+
+- Target-level relabeling (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
+  can be performed by navigating to `http://vmagent:8429/targets` page and clicking the `debug` link at the target, which must be debugged.
+  The opened page will show step-by-step results for the actual relabeling rules applied to the target labels.
+
+  The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
+  is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page, find the dropped target
+  and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules,
+  which resulted to target drop.
+
+- Metric-level relabeling (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
+  and all the relabeling, which can be set up via `-relabelConfig`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`
+  command-line flags) can be performed by navigating to `http://vmagent:8429/metric-relabel-debug` page
+  and submitting there relabeling rules together with the metric to be relabeled.
+  The page will show step-by-step results for the entered relabeling rules executed against the entered metric.
 
 ## Prometheus staleness markers
 
@@ -1428,8 +1440,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
      Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
-  -remoteWrite.relabelDebug
-     Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.
@@ -1464,9 +1474,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.urlRelabelConfig array
      Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
      Supports an array of values separated by comma or specified via multiple flags.
-  -remoteWrite.urlRelabelDebug array
-     Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
-     Supports array of values separated by comma or specified via multiple flags.
   -sortLabels
      Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls

app/vmagent/main.go

@@ -207,6 +207,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
 			{"service-discovery", "labels before and after relabeling for discovered targets"},
+			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -325,6 +326,18 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrapeServiceDiscoveryRequests.Inc()
 		promscrape.WriteServiceDiscovery(w, r)
 		return true
+	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
+		promscrapeMetricRelabelDebugRequests.Inc()
+		promscrape.WriteMetricRelabelDebug(w, r)
+		return true
+	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
+		promscrapeTargetRelabelDebugRequests.Inc()
+		if err := promscrape.WriteTargetRelabelDebug(w, r); err != nil {
+			promscrapeTargetRelabelDebugErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -546,7 +559,13 @@ var (
 
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
-	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+
+	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`)
+
+	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`)
+	promscrapeTargetRelabelDebugErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target-relabel-debug"}`)
+
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)

app/vmagent/remotewrite/relabel.go

@@ -18,13 +18,8 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
 		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
-	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
-		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
 		"The path can point either to local file or to http url")
-	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
-		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
-		"This is useful for debugging the relabeling configs")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -42,7 +37,7 @@ func CheckRelabelConfigs() error {
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
-		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal, *relabelDebugGlobal)
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
@@ -58,7 +53,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 			// Skip empty relabel config.
 			continue
 		}
-		prc, err := promrelabel.LoadRelabelConfigs(path, relabelDebug.GetOptionalArg(i))
+		prc, err := promrelabel.LoadRelabelConfigs(path)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}

app/vmalert/notifier/alert_test.go

@@ -239,7 +239,7 @@ func TestAlert_toPromLabels(t *testing.T) {
   replacement: "aaa"
 - action: labeldrop
   regex: "env.*"
-`), false)
+`))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}

app/vmalert/notifier/config.go

@@ -83,12 +83,12 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if cfg.Timeout.Duration() == 0 {
 		cfg.Timeout = promutils.NewDuration(time.Second * 10)
 	}
-	rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false)
+	rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs)
 	if err != nil {
 		return fmt.Errorf("failed to parse relabeling config: %w", err)
 	}
 	cfg.parsedRelabelConfigs = rCfg
-	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs)
 	if err != nil {
 		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
 	}

app/vminsert/main.go

@@ -230,6 +230,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrapeServiceDiscoveryRequests.Inc()
 		promscrape.WriteServiceDiscovery(w, r)
 		return true
+	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
+		promscrapeMetricRelabelDebugRequests.Inc()
+		promscrape.WriteMetricRelabelDebug(w, r)
+		return true
+	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
+		promscrapeTargetRelabelDebugRequests.Inc()
+		if err := promscrape.WriteTargetRelabelDebug(w, r); err != nil {
+			promscrapeTargetRelabelDebugErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -333,7 +345,13 @@ var (
 
 	promscrapeTargetsRequests          = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vm_http_requests_total{path="/service-discovery"}`)
-	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
+
+	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vm_http_requests_total{path="/metric-relabel-debug"}`)
+
+	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vm_http_requests_total{path="/target-relabel-debug"}`)
+	promscrapeTargetRelabelDebugErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/target-relabel-debug"}`)
+
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vm_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/target_response"}`)

app/vminsert/relabel/relabel.go

@@ -19,8 +19,6 @@ var (
 	relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+
 		"The path can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal")
-	relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+
-		"then the metrics aren't sent to storage. This is useful for debugging the relabeling configs")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -77,7 +75,7 @@ func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) {
 	if len(*relabelConfig) == 0 {
 		return nil, nil
 	}
-	pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig, *relabelDebug)
+	pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error when reading -relabelConfig=%q: %w", *relabelConfig, err)
 	}

docs/CHANGELOG.md

@@ -19,6 +19,9 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 **Update note 2:** this release splits `type="indexdb"` metrics into `type="indexdb/inmemory"` and `type="indexdb/file"` metrics. This may break old dashboards and alerting rules, which contain [label filter](https://docs.victoriametrics.com/keyConcepts.html#filtering) on `{type="indexdb"}`. Such label filter must be substituted with `{type=~"indexdb.*"}`, so it matches `indexdb` from the previous releases and `indexdb/inmemory` + `indexdb/file` from new releases. It is recommended upgrading to the latest available dashboards and alerting rules mentioned in [these docs](https://docs.victoriametrics.com/#monitoring), since they already contain fixed label filters.
 
+**Update note 3:** this release deprecates `relabel_debug` and `metric_relabel_debug` config options in [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). The `-relabelDebug`, `-remoteWrite.relabelDebug` and `-remoteWrite.urlRelabelDebug` command-line options are also deprecated. Use more powerful target-level relabel debugging and metric-level relabel debugging instead as documented [here](https://docs.victoriametrics.com/vmagent.html#relabel-debug).
+
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): provide enhanced target-level and metric-level relabel debugging. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabel-debug) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3407).
 * FEATURE: leave a sample with the biggest value for identical timestamps per each `-dedup.minScrapeInterval` discrete interval when the [deduplication](https://docs.victoriametrics.com/#deduplication) is enabled. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333).
 * FEATURE: add `-inmemoryDataFlushInterval` command-line flag, which can be used for controlling the frequency of in-memory data flush to disk. The data flush frequency can be reduced when VictoriaMetrics stores data to low-end flash device with limited number of write cycles (for example, on Raspberry PI). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337).
 * FEATURE: expose additional metrics for `indexdb` and `storage` parts stored in memory and for `indexdb` parts stored in files (see [storage docs](https://docs.victoriametrics.com/#storage) for technical details):

docs/Cluster-VictoriaMetrics.md

@@ -870,8 +870,6 @@ Below is the output for `/path/to/vminsert -help`:
      Supports an array of values separated by comma or specified via multiple flags.
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
-  -relabelDebug
-     Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -replicationFactor int
      Replication factor for the ingested data, i.e. how many copies to make among distinct -storageNode instances. Note that vmselect must run with -dedup.minScrapeInterval=1ms for data de-duplication when replicationFactor is greater than 1. Higher values for -dedup.minScrapeInterval at vmselect is OK (default 1)
   -rpc.disableCompression

docs/README.md

@@ -2304,8 +2304,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Supports an array of values separated by comma or specified via multiple flags.
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
-  -relabelDebug
-     Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -retentionFilter array
      Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.

docs/Single-server-VictoriaMetrics.md

@@ -2307,8 +2307,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Supports an array of values separated by comma or specified via multiple flags.
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
-  -relabelDebug
-     Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -retentionFilter array
      Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.

docs/operator/api.MD

@@ -1371,8 +1371,6 @@ VMScrapeParams defines scrape target configuration that compatible only with Vic
 
 | Field | Description | Scheme | Required |
 | ----- | ----------- | ------ | -------- |
-| relabel_debug |  | *bool | false |
-| metric_relabel_debug |  | *bool | false |
 | disable_compression |  | *bool | false |
 | disable_keep_alive |  | *bool | false |
 | no_stale_markers |  | *bool | false |

docs/sd_configs.md

@@ -1181,14 +1181,6 @@ scrape_configs:
   # By default the limit is disabled.
   # sample_limit: <int>
 
-  # relabel_debug enables debugging for relabel_configs if set to true.
-  # See https://docs.victoriametrics.com/vmagent.html#relabeling
-  # relabel_debug: <boolean>
-
-  # metric_relabel_debug enables debugging for metric_relabel_configs if set to true.
-  # See https://docs.victoriametrics.com/vmagent.html#relabeling
-  # metric_relabel_debug: <boolean>
-
   # disable_compression allows disabling HTTP compression for responses received from scrape targets.
   # By default scrape targets are queried with `Accept-Encoding: gzip` http request header,
   # so targets could send compressed responses in order to save network bandwidth.

docs/vmagent.md

@@ -249,8 +249,6 @@ scrape_configs:
 * `scrape_align_interval: duration` for aligning scrapes to the given interval instead of using random offset
   in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time.
 * `scrape_offset: duration` for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`.
-* `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling).
-* `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling).
 
 See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options.
 
@@ -423,26 +421,25 @@ with [additional enhancements](#relabeling-enhancements). The relabeling can be
   This relabeling is used for modifying labels in discovered targets and for dropping unneded targets.
   See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details.
 
-  This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section.
+  This relabeling can be debugged by clicking the `debug` link at the corresponding target on the `http://vmagent:8429/targets` page
-  In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
+  or on the `http://vmagent:8429/service-discovery` page. See [these docs](#relabel-debug) for details.
 
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file.
   This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics.
   See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details.
 
-  This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
-  In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
 
 * At the `-remoteWrite.relabelConfig` file. This relabeling is used for modifying labels for all the collected metrics
-  (inluding [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics
+  (including [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics
   before sending them to all the configured `-remoteWrite.url` addresses.
-  This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`.
+
-  In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
 
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is used for modifying labels for metrics
   and for dropping unneeded metrics before sending them to a particular `-remoteWrite.url`.
-  This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`.
+
-  In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
+  This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details.
 
 All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`,
 which are replaced by the corresponding environment variable values.
@@ -457,9 +454,6 @@ The following articles contain useful information about Prometheus relabeling:
 * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
-[This relabeler playground](https://relabeler.promlabs.com/) can help debugging issues related to relabeling.
-
-
 ## Relabeling enhancements
 
 `vmagent` provides the following enhancements on top of Prometheus-compatible relabeling:
@@ -601,6 +595,24 @@ Important notes about `action: graphite` relabeling rules:
 The `action: graphite` relabeling rules are easier to write and maintain than `action: replace` for labels extraction from Graphite-style metric names.
 Additionally, the `action: graphite` relabeling rules usually work much faster than the equivalent `action: replace` rules.
 
+## Relabel debug
+
+`vmagent` provides the following tools for debugging target-level and metric-level relabeling:
+
+- Target-level relabeling (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
+  can be performed by navigating to `http://vmagent:8429/targets` page and clicking the `debug` link at the target, which must be debugged.
+  The opened page will show step-by-step results for the actual relabeling rules applied to the target labels.
+
+  The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
+  is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page, find the dropped target
+  and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules,
+  which resulted to target drop.
+
+- Metric-level relabeling (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
+  and all the relabeling, which can be set up via `-relabelConfig`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`
+  command-line flags) can be performed by navigating to `http://vmagent:8429/metric-relabel-debug` page
+  and submitting there relabeling rules together with the metric to be relabeled.
+  The page will show step-by-step results for the entered relabeling rules executed against the entered metric.
 
 ## Prometheus staleness markers
 
@@ -1432,8 +1444,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
      Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
-  -remoteWrite.relabelDebug
-     Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.
@@ -1468,9 +1478,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.urlRelabelConfig array
      Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
      Supports an array of values separated by comma or specified via multiple flags.
-  -remoteWrite.urlRelabelDebug array
-     Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
-     Supports array of values separated by comma or specified via multiple flags.
   -sortLabels
      Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls

lib/promrelabel/config.go

@@ -18,14 +18,14 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
 type RelabelConfig struct {
+	If           *IfExpression   `yaml:"if,omitempty"`
+	Action       string          `yaml:"action,omitempty"`
 	SourceLabels []string        `yaml:"source_labels,flow,omitempty"`
 	Separator    *string         `yaml:"separator,omitempty"`
 	TargetLabel  string          `yaml:"target_label,omitempty"`
 	Regex        *MultiLineRegex `yaml:"regex,omitempty"`
 	Modulus      uint64          `yaml:"modulus,omitempty"`
 	Replacement  *string         `yaml:"replacement,omitempty"`
-	Action       string          `yaml:"action,omitempty"`
-	If           *IfExpression   `yaml:"if,omitempty"`
 
 	// Match is used together with Labels for `action: graphite`. For example:
 	// - action: graphite
@@ -121,8 +121,7 @@ func (mlr *MultiLineRegex) MarshalYAML() (interface{}, error) {
 
 // ParsedConfigs represents parsed relabel configs.
 type ParsedConfigs struct {
-	prcs         []*parsedRelabelConfig
+	prcs []*parsedRelabelConfig
-	relabelDebug bool
 }
 
 // Len returns the number of relabel configs in pcs.
@@ -140,14 +139,23 @@ func (pcs *ParsedConfigs) String() string {
 	}
 	var a []string
 	for _, prc := range pcs.prcs {
-		s := "[" + prc.String() + "]"
+		s := prc.String()
+		lines := strings.Split(s, "\n")
+		lines[0] = "- " + lines[0]
+		for i := range lines[1:] {
+			line := &lines[1+i]
+			if len(*line) > 0 {
+				*line = "  " + *line
+			}
+		}
+		s = strings.Join(lines, "\n")
 		a = append(a, s)
 	}
-	return fmt.Sprintf("%s, relabelDebug=%v", strings.Join(a, ","), pcs.relabelDebug)
+	return strings.Join(a, "")
 }
 
 // LoadRelabelConfigs loads relabel configs from the given path.
-func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) {
+func LoadRelabelConfigs(path string) (*ParsedConfigs, error) {
 	data, err := fs.ReadFileOrHTTP(path)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err)
@@ -156,7 +164,7 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error)
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err)
 	}
-	pcs, err := ParseRelabelConfigsData(data, relabelDebug)
+	pcs, err := ParseRelabelConfigsData(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err)
 	}
@@ -164,16 +172,16 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error)
 }
 
 // ParseRelabelConfigsData parses relabel configs from the given data.
-func ParseRelabelConfigsData(data []byte, relabelDebug bool) (*ParsedConfigs, error) {
+func ParseRelabelConfigsData(data []byte) (*ParsedConfigs, error) {
 	var rcs []RelabelConfig
 	if err := yaml.UnmarshalStrict(data, &rcs); err != nil {
 		return nil, err
 	}
-	return ParseRelabelConfigs(rcs, relabelDebug)
+	return ParseRelabelConfigs(rcs)
 }
 
 // ParseRelabelConfigs parses rcs to dst.
-func ParseRelabelConfigs(rcs []RelabelConfig, relabelDebug bool) (*ParsedConfigs, error) {
+func ParseRelabelConfigs(rcs []RelabelConfig) (*ParsedConfigs, error) {
 	if len(rcs) == 0 {
 		return nil, nil
 	}
@@ -186,8 +194,7 @@ func ParseRelabelConfigs(rcs []RelabelConfig, relabelDebug bool) (*ParsedConfigs
 		prcs[i] = prc
 	}
 	return &ParsedConfigs{
-		prcs:         prcs,
+		prcs: prcs,
-		relabelDebug: relabelDebug,
 	}, nil
 }
 
@@ -350,7 +357,13 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 			return nil, fmt.Errorf("`labels` config cannot be applied to `action=%s`; it is applied only to `action=graphite`", action)
 		}
 	}
+	ruleOriginal, err := yaml.Marshal(rc)
+	if err != nil {
+		logger.Panicf("BUG: cannot marshal RelabelConfig: %s", err)
+	}
 	prc := &parsedRelabelConfig{
+		ruleOriginal: string(ruleOriginal),
+
 		SourceLabels:  sourceLabels,
 		Separator:     separator,
 		TargetLabel:   targetLabel,

lib/promrelabel/config_test.go

@@ -50,7 +50,7 @@ func TestRelabelConfigMarshalUnmarshal(t *testing.T) {
 	f(`
 - action: keep
   regex: foobar
-`, "- regex: foobar\n  action: keep\n")
+  `, "- action: keep\n  regex: foobar\n")
 	f(`
 - regex:
   - 'fo.+'
@@ -80,7 +80,7 @@ func TestRelabelConfigMarshalUnmarshal(t *testing.T) {
 
 func TestLoadRelabelConfigsSuccess(t *testing.T) {
 	path := "testdata/relabel_configs_valid.yml"
-	pcs, err := LoadRelabelConfigs(path, false)
+	pcs, err := LoadRelabelConfigs(path)
 	if err != nil {
 		t.Fatalf("cannot load relabel configs from %q: %s", path, err)
 	}
@@ -93,7 +93,7 @@ func TestLoadRelabelConfigsSuccess(t *testing.T) {
 func TestLoadRelabelConfigsFailure(t *testing.T) {
 	f := func(path string) {
 		t.Helper()
-		rcs, err := LoadRelabelConfigs(path, false)
+		rcs, err := LoadRelabelConfigs(path)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -112,7 +112,7 @@ func TestLoadRelabelConfigsFailure(t *testing.T) {
 func TestParsedConfigsString(t *testing.T) {
 	f := func(rcs []RelabelConfig, sExpected string) {
 		t.Helper()
-		pcs, err := ParseRelabelConfigs(rcs, false)
+		pcs, err := ParseRelabelConfigs(rcs)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -126,8 +126,7 @@ func TestParsedConfigsString(t *testing.T) {
 			TargetLabel:  "foo",
 			SourceLabels: []string{"aaa"},
 		},
-	}, "[SourceLabels=[aaa], Separator=;, TargetLabel=foo, Regex=.*, Modulus=0, Replacement=$1, Action=replace, If=, "+
+	}, "- source_labels: [aaa]\n  target_label: foo\n")
-		"graphiteMatchTemplate=<nil>, graphiteLabelRules=[]], relabelDebug=false")
 	var ie IfExpression
 	if err := ie.Parse("{foo=~'bar'}"); err != nil {
 		t.Fatalf("unexpected error when parsing if expression: %s", err)
@@ -141,8 +140,8 @@ func TestParsedConfigsString(t *testing.T) {
 			},
 			If: &ie,
 		},
-	}, "[SourceLabels=[], Separator=;, TargetLabel=, Regex=.*, Modulus=0, Replacement=$1, Action=graphite, If={foo=~'bar'}, "+
+	}, "- if: '{foo=~''bar''}'\n  action: graphite\n  match: foo.*.bar\n  labels:\n    job: $1-zz\n")
-		"graphiteMatchTemplate=foo.*.bar, graphiteLabelRules=[replaceTemplate=$1-zz, targetLabel=job]], relabelDebug=false")
+	replacement := "foo"
 	f([]RelabelConfig{
 		{
 			Action:       "replace",
@@ -150,19 +149,23 @@ func TestParsedConfigsString(t *testing.T) {
 			TargetLabel:  "x",
 			If:           &ie,
 		},
-	}, "[SourceLabels=[foo bar], Separator=;, TargetLabel=x, Regex=.*, Modulus=0, Replacement=$1, Action=replace, If={foo=~'bar'}, "+
+		{
-		"graphiteMatchTemplate=<nil>, graphiteLabelRules=[]], relabelDebug=false")
+			TargetLabel: "x",
+			Replacement: &replacement,
+		},
+	}, "- if: '{foo=~''bar''}'\n  action: replace\n  source_labels: [foo, bar]\n  target_label: x\n- target_label: x\n  replacement: foo\n")
 }
 
 func TestParseRelabelConfigsSuccess(t *testing.T) {
 	f := func(rcs []RelabelConfig, pcsExpected *ParsedConfigs) {
 		t.Helper()
-		pcs, err := ParseRelabelConfigs(rcs, false)
+		pcs, err := ParseRelabelConfigs(rcs)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if pcs != nil {
 			for _, prc := range pcs.prcs {
+				prc.ruleOriginal = ""
 				prc.stringReplacer = nil
 				prc.submatchReplacer = nil
 			}
@@ -198,7 +201,7 @@ func TestParseRelabelConfigsSuccess(t *testing.T) {
 func TestParseRelabelConfigsFailure(t *testing.T) {
 	f := func(rcs []RelabelConfig) {
 		t.Helper()
-		pcs, err := ParseRelabelConfigs(rcs, false)
+		pcs, err := ParseRelabelConfigs(rcs)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}

lib/promrelabel/if_expression_test.go

@@ -122,7 +122,7 @@ func TestIfExpressionMatch(t *testing.T) {
 		if err := yaml.UnmarshalStrict([]byte(ifExpr), &ie); err != nil {
 			t.Fatalf("unexpected error during unmarshal: %s", err)
 		}
-		labels := promutils.NewLabelsFromString(metricWithLabels)
+		labels := promutils.MustNewLabelsFromString(metricWithLabels)
 		if !ie.Match(labels.GetLabels()) {
 			t.Fatalf("unexpected mismatch of ifExpr=%s for %s", ifExpr, metricWithLabels)
 		}
@@ -156,7 +156,7 @@ func TestIfExpressionMismatch(t *testing.T) {
 		if err := yaml.UnmarshalStrict([]byte(ifExpr), &ie); err != nil {
 			t.Fatalf("unexpected error during unmarshal: %s", err)
 		}
-		labels := promutils.NewLabelsFromString(metricWithLabels)
+		labels := promutils.MustNewLabelsFromString(metricWithLabels)
 		if ie.Match(labels.GetLabels()) {
 			t.Fatalf("unexpected match of ifExpr=%s for %s", ifExpr, metricWithLabels)
 		}

lib/promrelabel/relabel.go

@@ -18,6 +18,9 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
 type parsedRelabelConfig struct {
+	// ruleOriginal contains the original relabeling rule for the given prasedRelabelConfig.
+	ruleOriginal string
+
 	SourceLabels  []string
 	Separator     string
 	TargetLabel   string
@@ -41,50 +44,78 @@ type parsedRelabelConfig struct {
 	submatchReplacer *bytesutil.FastStringTransformer
 }
 
+// DebugStep contains debug information about a single relabeling rule step
+type DebugStep struct {
+	// Rule contains string representation of the rule step
+	Rule string
+
+	// In contains the input labels before the exeuction of the rule step
+	In string
+
+	// Out contains the output labels after the execution of the rule step
+	Out string
+}
+
+// String returns human-readable representation for ds
+func (ds DebugStep) String() string {
+	return fmt.Sprintf("rule=%q, in=%s, out=%s", ds.Rule, ds.In, ds.Out)
+}
+
 // String returns human-readable representation for prc.
 func (prc *parsedRelabelConfig) String() string {
-	return fmt.Sprintf("SourceLabels=%s, Separator=%s, TargetLabel=%s, Regex=%s, Modulus=%d, Replacement=%s, Action=%s, If=%s, graphiteMatchTemplate=%s, graphiteLabelRules=%s",
+	return prc.ruleOriginal
-		prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.regexOriginal, prc.Modulus, prc.Replacement,
+}
-		prc.Action, prc.If, prc.graphiteMatchTemplate, prc.graphiteLabelRules)
+
+// ApplyDebug applies pcs to labels in debug mode.
+//
+// It returns DebugStep list - one entry per each applied relabeling step.
+func (pcs *ParsedConfigs) ApplyDebug(labels []prompbmarshal.Label) ([]prompbmarshal.Label, []DebugStep) {
+	labels, dss := pcs.applyInternal(labels, 0, true)
+	return labels, dss
 }
 
 // Apply applies pcs to labels starting from the labelsOffset.
 func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label {
-	var inStr string
+	labels, _ = pcs.applyInternal(labels, labelsOffset, false)
-	relabelDebug := false
+	return labels
+}
+
+func (pcs *ParsedConfigs) applyInternal(labels []prompbmarshal.Label, labelsOffset int, debug bool) ([]prompbmarshal.Label, []DebugStep) {
+	var dss []DebugStep
+	inStr := ""
+	if debug {
+		inStr = LabelsToString(labels[labelsOffset:])
+	}
 	if pcs != nil {
-		relabelDebug = pcs.relabelDebug
-		if relabelDebug {
-			inStr = labelsToString(labels[labelsOffset:])
-		}
 		for _, prc := range pcs.prcs {
-			tmp := prc.apply(labels, labelsOffset)
+			labels = prc.apply(labels, labelsOffset)
-			if len(tmp) == labelsOffset {
+			if debug {
-				// All the labels have been removed.
+				outStr := LabelsToString(labels[labelsOffset:])
-				if pcs.relabelDebug {
+				dss = append(dss, DebugStep{
-					logger.Infof("\nRelabel  In: %s\nRelabel Out: DROPPED - all labels removed", inStr)
+					Rule: prc.String(),
-				}
+					In:   inStr,
-				return tmp
+					Out:  outStr,
+				})
+				inStr = outStr
+			}
+			if len(labels) == labelsOffset {
+				// All the labels have been removed.
+				return labels, dss
 			}
-			labels = tmp
 		}
 	}
 	labels = removeEmptyLabels(labels, labelsOffset)
-	if relabelDebug {
+	if debug {
-		if len(labels) == labelsOffset {
+		outStr := LabelsToString(labels[labelsOffset:])
-			logger.Infof("\nRelabel  In: %s\nRelabel Out: DROPPED - all labels removed", inStr)
+		if outStr != inStr {
-			return labels
+			dss = append(dss, DebugStep{
+				Rule: "remove empty labels",
+				In:   inStr,
+				Out:  outStr,
+			})
 		}
-		outStr := labelsToString(labels[labelsOffset:])
-		if inStr == outStr {
-			logger.Infof("\nRelabel  In: %s\nRelabel Out: KEPT AS IS - no change", inStr)
-		} else {
-			logger.Infof("\nRelabel  In: %s\nRelabel Out: %s", inStr, outStr)
-		}
-		// Drop labels
-		labels = labels[:labelsOffset]
 	}
-	return labels
+	return labels, dss
 }
 
 func removeEmptyLabels(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label {
@@ -504,25 +535,27 @@ func CleanLabels(labels []prompbmarshal.Label) {
 	}
 }
 
-func labelsToString(labels []prompbmarshal.Label) string {
+// LabelsToString returns Prometheus string representation for the given labels.
+//
+// Labels in the returned string are sorted by name,
+// while the __name__ label is put in front of {} labels.
+func LabelsToString(labels []prompbmarshal.Label) string {
 	labelsCopy := append([]prompbmarshal.Label{}, labels...)
 	SortLabels(labelsCopy)
 	mname := ""
-	for _, label := range labelsCopy {
+	for i, label := range labelsCopy {
 		if label.Name == "__name__" {
 			mname = label.Value
+			labelsCopy = append(labelsCopy[:i], labelsCopy[i+1:]...)
 			break
 		}
 	}
-	if mname != "" && len(labelsCopy) <= 1 {
+	if mname != "" && len(labelsCopy) == 0 {
 		return mname
 	}
 	b := []byte(mname)
 	b = append(b, '{')
 	for i, label := range labelsCopy {
-		if label.Name == "__name__" {
-			continue
-		}
 		b = append(b, label.Name...)
 		b = append(b, '=')
 		b = strconv.AppendQuote(b, label.Value)

lib/promrelabel/relabel_test.go

@@ -2,6 +2,7 @@ package promrelabel
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -27,7 +28,7 @@ func TestSanitizeName(t *testing.T) {
 func TestLabelsToString(t *testing.T) {
 	f := func(labels []prompbmarshal.Label, sExpected string) {
 		t.Helper()
-		s := labelsToString(labels)
+		s := LabelsToString(labels)
 		if s != sExpected {
 			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", s, sExpected)
 		}
@@ -71,20 +72,95 @@ func TestLabelsToString(t *testing.T) {
 	}, `xxx{a="bc",foo="bar"}`)
 }
 
-func TestApplyRelabelConfigs(t *testing.T) {
+func TestParsedRelabelConfigsApplyDebug(t *testing.T) {
-	f := func(config, metric string, isFinalize bool, resultExpected string) {
+	f := func(config, metric string, dssExpected []DebugStep) {
 		t.Helper()
-		pcs, err := ParseRelabelConfigsData([]byte(config), false)
+		pcs, err := ParseRelabelConfigsData([]byte(config))
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", config, err)
 		}
-		labels := promutils.NewLabelsFromString(metric)
+		labels := promutils.MustNewLabelsFromString(metric)
+		_, dss := pcs.ApplyDebug(labels.GetLabels())
+		if !reflect.DeepEqual(dss, dssExpected) {
+			t.Fatalf("unexpected result; got\n%s\nwant\n%s", dss, dssExpected)
+		}
+	}
+
+	// empty relabel config
+	f(``, `foo`, nil)
+	// add label
+	f(`
+- target_label: abc
+  replacement: xyz
+`, `foo{bar="baz"}`, []DebugStep{
+		{
+			Rule: "target_label: abc\nreplacement: xyz\n",
+			In:   `foo{bar="baz"}`,
+			Out:  `foo{abc="xyz",bar="baz"}`,
+		},
+	})
+	// drop label
+	f(`
+- target_label: bar
+  replacement: ''
+`, `foo{bar="baz"}`, []DebugStep{
+		{
+			Rule: "target_label: bar\nreplacement: \"\"\n",
+			In:   `foo{bar="baz"}`,
+			Out:  `foo{bar=""}`,
+		},
+		{
+			Rule: "remove empty labels",
+			In:   `foo{bar=""}`,
+			Out:  `foo`,
+		},
+	})
+	// drop metric
+	f(`
+- action: drop
+  source_labels: [bar]
+  regex: baz
+`, `foo{bar="baz",abc="def"}`, []DebugStep{
+		{
+			Rule: "action: drop\nsource_labels: [bar]\nregex: baz\n",
+			In:   `foo{abc="def",bar="baz"}`,
+			Out:  `{}`,
+		},
+	})
+	// Multiple steps
+	f(`
+- action: labeldrop
+  regex: "foo.*"
+- target_label: foobar
+  replacement: "abc"
+`, `m{foo="x",foobc="123",a="b"}`, []DebugStep{
+		{
+			Rule: "action: labeldrop\nregex: foo.*\n",
+			In:   `m{a="b",foo="x",foobc="123"}`,
+			Out:  `m{a="b"}`,
+		},
+		{
+			Rule: "target_label: foobar\nreplacement: abc\n",
+			In:   `m{a="b"}`,
+			Out:  `m{a="b",foobar="abc"}`,
+		},
+	})
+}
+
+func TestParsedRelabelConfigsApply(t *testing.T) {
+	f := func(config, metric string, isFinalize bool, resultExpected string) {
+		t.Helper()
+		pcs, err := ParseRelabelConfigsData([]byte(config))
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", config, err)
+		}
+		labels := promutils.MustNewLabelsFromString(metric)
 		resultLabels := pcs.Apply(labels.GetLabels(), 0)
 		if isFinalize {
 			resultLabels = FinalizeLabels(resultLabels[:0], resultLabels)
 		}
 		SortLabels(resultLabels)
-		result := labelsToString(resultLabels)
+		result := LabelsToString(resultLabels)
 		if result != resultExpected {
 			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected)
 		}
@@ -726,9 +802,9 @@ func TestApplyRelabelConfigs(t *testing.T) {
 func TestFinalizeLabels(t *testing.T) {
 	f := func(metric, resultExpected string) {
 		t.Helper()
-		labels := promutils.NewLabelsFromString(metric)
+		labels := promutils.MustNewLabelsFromString(metric)
 		resultLabels := FinalizeLabels(nil, labels.GetLabels())
-		result := labelsToString(resultLabels)
+		result := LabelsToString(resultLabels)
 		if result != resultExpected {
 			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected)
 		}
@@ -742,7 +818,7 @@ func TestFinalizeLabels(t *testing.T) {
 func TestFillLabelReferences(t *testing.T) {
 	f := func(replacement, metric, resultExpected string) {
 		t.Helper()
-		labels := promutils.NewLabelsFromString(metric)
+		labels := promutils.MustNewLabelsFromString(metric)
 		result := fillLabelReferences(nil, replacement, labels.GetLabels())
 		if string(result) != resultExpected {
 			t.Fatalf("unexpected result; got\n%q\nwant\n%q", result, resultExpected)

lib/promrelabel/relabel_timing_test.go

@@ -1155,7 +1155,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 }
 
 func mustParseRelabelConfigs(config string) *ParsedConfigs {
-	pcs, err := ParseRelabelConfigsData([]byte(config), false)
+	pcs, err := ParseRelabelConfigsData([]byte(config))
 	if err != nil {
 		panic(fmt.Errorf("unexpected error: %w", err))
 	}

lib/promscrape/config.go

@@ -267,8 +267,6 @@ type ScrapeConfig struct {
 	YandexCloudSDConfigs  []yandexcloud.SDConfig  `yaml:"yandexcloud_sd_configs,omitempty"`
 
 	// These options are supported only by lib/promscrape.
-	RelabelDebug        bool                       `yaml:"relabel_debug,omitempty"`
-	MetricRelabelDebug  bool                       `yaml:"metric_relabel_debug,omitempty"`
 	DisableCompression  bool                       `yaml:"disable_compression,omitempty"`
 	DisableKeepAlive    bool                       `yaml:"disable_keepalive,omitempty"`
 	StreamParse         bool                       `yaml:"stream_parse,omitempty"`
@@ -928,11 +926,11 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config for `job_name` %q: %w", jobName, err)
 	}
-	relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs, sc.RelabelDebug)
+	relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
-	metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs, sc.MetricRelabelDebug)
+	metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
@@ -1188,7 +1186,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 	}
 	if labels.Len() == 0 {
 		// Drop target without labels.
-		droppedTargetsMap.Register(originalLabels)
+		droppedTargetsMap.Register(originalLabels, swc.relabelConfigs)
 		return nil, nil
 	}
 	// See https://www.robustperception.io/life-of-a-label
@@ -1203,7 +1201,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 	address := labels.Get("__address__")
 	if len(address) == 0 {
 		// Drop target without scrape address.
-		droppedTargetsMap.Register(originalLabels)
+		droppedTargetsMap.Register(originalLabels, swc.relabelConfigs)
 		return nil, nil
 	}
 	// Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__.
@@ -1314,6 +1312,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		ProxyURL:             swc.proxyURL,
 		ProxyAuthConfig:      swc.proxyAuthConfig,
 		AuthConfig:           swc.authConfig,
+		RelabelConfigs:       swc.relabelConfigs,
 		MetricRelabelConfigs: swc.metricRelabelConfigs,
 		SampleLimit:          swc.sampleLimit,
 		DisableCompression:   swc.disableCompression,

lib/promscrape/config_test.go

@@ -104,7 +104,6 @@ scrape_configs:
   static_configs:
   - targets:
     - foo
-  relabel_debug: true
   scrape_align_interval: 1h30m0s
   proxy_bearer_token_file: file.txt
   proxy_headers:
@@ -721,6 +720,7 @@ scrape_config_files:
 func resetNonEssentialFields(sws []*ScrapeWork) {
 	for _, sw := range sws {
 		sw.OriginalLabels = nil
+		sw.RelabelConfigs = nil
 		sw.MetricRelabelConfigs = nil
 	}
 }

lib/promscrape/relabel_debug.go

@@ -0,0 +1,133 @@
+package promscrape
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page
+func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
+	metric := r.FormValue("metric")
+	relabelConfigs := r.FormValue("relabel_configs")
+	if metric == "" {
+		metric = "{}"
+	}
+	labels, err := promutils.NewLabelsFromString(metric)
+	if err != nil {
+		err = fmt.Errorf("cannot parse metric: %s", err)
+		WriteMetricRelabelDebugSteps(w, nil, metric, relabelConfigs, err)
+		return
+	}
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfigs))
+	if err != nil {
+		err = fmt.Errorf("cannot parse relabel configs: %s", err)
+		WriteMetricRelabelDebugSteps(w, nil, metric, relabelConfigs, err)
+		return
+	}
+
+	// The metric relabeling below must be in sync with the code at scrapeWork.addRowToTimeseries
+
+	// Apply relabeling
+	labelsResult, dss := pcs.ApplyDebug(labels.GetLabels())
+
+	// Remove labels with __ prefix
+	inStr := promrelabel.LabelsToString(labelsResult)
+	labelsResult = promrelabel.FinalizeLabels(labelsResult[:0], labelsResult)
+	outStr := promrelabel.LabelsToString(labelsResult)
+	if inStr != outStr {
+		dss = append(dss, promrelabel.DebugStep{
+			Rule: "remove labels with __ prefix",
+			In:   inStr,
+			Out:  outStr,
+		})
+	}
+
+	// There is no need in labels' sorting, since promrelabel.LabelsToString() automatically sorts labels.
+
+	WriteMetricRelabelDebugSteps(w, dss, metric, relabelConfigs, nil)
+}
+
+// WriteTargetRelabelDebug generates response for /target-relabel-debug page
+func WriteTargetRelabelDebug(w http.ResponseWriter, r *http.Request) error {
+	targetID := r.FormValue("id")
+	relabelConfigs, labels, ok := getRelabelContextByTargetID(targetID)
+	if !ok {
+		return fmt.Errorf("cannot find target for id=%s", targetID)
+	}
+
+	// The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork
+
+	// Prevent from modifying the original labels
+	labels = labels.Clone()
+
+	// Apply relabeling
+	labelsResult, dss := relabelConfigs.ApplyDebug(labels.GetLabels())
+
+	// Remove labels with __meta_ prefix
+	inStr := promrelabel.LabelsToString(labelsResult)
+	labels.Labels = labelsResult
+	labels.RemoveMetaLabels()
+	outStr := promrelabel.LabelsToString(labels.Labels)
+	if inStr != outStr {
+		dss = append(dss, promrelabel.DebugStep{
+			Rule: "remove labels with __meta_ prefix",
+			In:   inStr,
+			Out:  outStr,
+		})
+	}
+
+	// Add missing instance label
+	if labels.Get("instance") == "" {
+		address := labels.Get("__address__")
+		if address != "" {
+			inStr = outStr
+			labels.Add("instance", address)
+			outStr = promrelabel.LabelsToString(labels.Labels)
+			dss = append(dss, promrelabel.DebugStep{
+				Rule: "add missing instance label from __address__ label",
+				In:   inStr,
+				Out:  outStr,
+			})
+		}
+	}
+
+	// Remove labels with __ prefix
+	inStr = outStr
+	labels.RemoveLabelsWithDoubleUnderscorePrefix()
+	outStr = promrelabel.LabelsToString(labels.Labels)
+	if inStr != outStr {
+		dss = append(dss, promrelabel.DebugStep{
+			Rule: "remove labels with __ prefix",
+			In:   inStr,
+			Out:  outStr,
+		})
+	}
+
+	// There is no need in labels' sorting, since promrelabel.LabelsToString() automatically sorts labels.
+
+	WriteTargetRelabelDebugSteps(w, dss)
+	return nil
+}
+
+func getChangedLabelNames(in, out *promutils.Labels) map[string]struct{} {
+	inMap := in.ToMap()
+	outMap := out.ToMap()
+	changed := make(map[string]struct{})
+	for k, v := range outMap {
+		inV, ok := inMap[k]
+		if !ok || inV != v {
+			changed[k] = struct{}{}
+		}
+	}
+	for k, v := range inMap {
+		outV, ok := outMap[k]
+		if !ok || outV != v {
+			changed[k] = struct{}{}
+		}
+	}
+	return changed
+}

lib/promscrape/relabel_debug.qtpl

@@ -0,0 +1,152 @@
+{% import (
+        "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+        "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+) %}
+
+{% stripspace %}
+
+{% func MetricRelabelDebugSteps(dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {%= commonHeader() %}
+    <title>Metric relabel debug</title>
+</head>
+<body>
+    {%= navbar() %}
+    <div class="container-fluid">
+        <a href="https://docs.victoriametrics.com/relabeling.html" target="_blank">Relabeling docs</a><br/>
+        {% if err != nil %}
+            {%= errorNotification(err) %}
+        {% endif %}
+
+        <div class="m-3">
+        <form method="POST">
+            <div>
+                Relabel configs:<br/>
+                <textarea name="relabel_configs" style="width: 50%; height: 10em">{%s relabelConfigs %}</textarea>
+            </div>
+
+            <div>
+                Metric with labels:<br/>
+                <textarea name="metric" style="width: 50%">{%s metric %}</textarea>
+            </div>
+            <input type="submit" value="Submit" />
+        </form>
+        </div>
+
+        <div class="row">
+            <main class="col-12">
+              {%= relabelDebugSteps(dss) %}
+            </main>
+        </div>
+    </div>
+</body>
+</html>
+{% endfunc %}
+
+{% func TargetRelabelDebugSteps(dss []promrelabel.DebugStep) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {%= commonHeader() %}
+    <title>Target relabel debug</title>
+</head>
+<body>
+    {%= navbar() %}
+    <div class="container-fluid">
+        <a href="https://docs.victoriametrics.com/relabeling.html" target="_blank">Relabeling docs</a><br/>
+        <div class="row">
+            <main class="col-12">
+                {%= relabelDebugSteps(dss) %}
+            </main>
+        </div>
+    </div>
+</body>
+</html>
+{% endfunc %}
+
+{% func relabelDebugSteps(dss []promrelabel.DebugStep) %}
+{% if len(dss) > 0 %}
+<div class="m-3">
+        <b>Original labels:</b> <samp>{%= mustFormatLabels(dss[0].In) %}</samp>
+</div>
+{% endif %}
+<table class="table table-striped table-hover table-bordered table-sm">
+  <thead>
+    <tr>
+      <th scope="col" style="width: 5%">Step</th>
+      <th scope="col" style="width: 25%">Relabeling Rule</th>
+      <th scope="col" style="width: 35%">Input Labels</th>
+      <th scope="col" stile="width: 35%">Output labels</a>
+    </tr>
+  </thead>
+  <tbody>
+        {% for i, ds := range dss %}
+        {% code
+                inLabels := promutils.MustNewLabelsFromString(ds.In)
+                outLabels := promutils.MustNewLabelsFromString(ds.Out)
+                changedLabels := getChangedLabelNames(inLabels, outLabels)
+        %}
+        <tr>
+                <td>{%d i %}</td>
+                <td><b><pre class="m-2">{%s ds.Rule %}</pre></b></td>
+                <td>
+                    <div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">
+                        {%= labelsWithHighlight(inLabels, changedLabels, "red") %}
+                    </div>
+                </td>
+                <td>
+                    <div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">
+                        {%= labelsWithHighlight(outLabels, changedLabels, "blue") %}
+                    </div>
+                </td>
+        </tr>
+        {% endfor %}
+  </tbody>
+</table>
+{% if len(dss) > 0 %}
+<div class="m-3">
+        <b>Resulting labels:</b> <samp>{%= mustFormatLabels(dss[len(dss)-1].Out) %}</samp>
+</div>
+{% endif %}
+{% endfunc %}
+
+{% func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) %}
+        {% code
+                labelsList := labels.GetLabels()
+                metricName := ""
+                for i, label := range labelsList {
+                        if label.Name == "__name__" {
+                                metricName = label.Value
+                                labelsList = append(labelsList[:i], labelsList[i+1:]...)
+                                break
+                        }
+                }
+        %}
+        {% if metricName != "" %}
+                {% if _, ok := highlight["__name__"]; ok %}
+                        <span style="font-weight:bold;color:{%s color %}">{%s metricName %}</span>
+                {% else %}
+                        {%s metricName %}
+                {% endif %}
+                {% if len(labelsList) == 0 %}{% return %}{% endif %}
+        {% endif %}
+{
+        {% for i, label := range labelsList %}
+                {% if _, ok := highlight[label.Name]; ok %}
+                        <span style="font-weight:bold;color:{%s color %}">{%s label.Name %}={%q label.Value %}</span>
+                {% else %}
+                        {%s label.Name %}={%q label.Value %}
+                {% endif %}
+                {% if i < len(labelsList)-1 %},{% space %}{% endif %}
+        {% endfor %}
+}
+{% endfunc %}
+
+{% func mustFormatLabels(s string) %}
+        {% code        labels := promutils.MustNewLabelsFromString(s) %}
+        {%= labelsWithHighlight(labels, nil, "") %}
+{% endfunc %}
+
+{% endstripspace %}

lib/promscrape/relabel_debug.qtpl.go

@@ -0,0 +1,360 @@
+// Code generated by qtc from "relabel_debug.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line lib/promscrape/relabel_debug.qtpl:1
+package promscrape
+
+//line lib/promscrape/relabel_debug.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+//line lib/promscrape/relabel_debug.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line lib/promscrape/relabel_debug.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line lib/promscrape/relabel_debug.qtpl:8
+func StreamMetricRelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) {
+//line lib/promscrape/relabel_debug.qtpl:8
+	qw422016.N().S(`<!DOCTYPE html><html lang="en"><head>`)
+//line lib/promscrape/relabel_debug.qtpl:12
+	streamcommonHeader(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:12
+	qw422016.N().S(`<title>Metric relabel debug</title></head><body>`)
+//line lib/promscrape/relabel_debug.qtpl:16
+	streamnavbar(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:16
+	qw422016.N().S(`<div class="container-fluid"><a href="https://docs.victoriametrics.com/relabeling.html" target="_blank">Relabeling docs</a><br/>`)
+//line lib/promscrape/relabel_debug.qtpl:19
+	if err != nil {
+//line lib/promscrape/relabel_debug.qtpl:20
+		streamerrorNotification(qw422016, err)
+//line lib/promscrape/relabel_debug.qtpl:21
+	}
+//line lib/promscrape/relabel_debug.qtpl:21
+	qw422016.N().S(`<div class="m-3"><form method="POST"><div>Relabel configs:<br/><textarea name="relabel_configs" style="width: 50%; height: 10em">`)
+//line lib/promscrape/relabel_debug.qtpl:27
+	qw422016.E().S(relabelConfigs)
+//line lib/promscrape/relabel_debug.qtpl:27
+	qw422016.N().S(`</textarea></div><div>Metric with labels:<br/><textarea name="metric" style="width: 50%">`)
+//line lib/promscrape/relabel_debug.qtpl:32
+	qw422016.E().S(metric)
+//line lib/promscrape/relabel_debug.qtpl:32
+	qw422016.N().S(`</textarea></div><input type="submit" value="Submit" /></form></div><div class="row"><main class="col-12">`)
+//line lib/promscrape/relabel_debug.qtpl:40
+	streamrelabelDebugSteps(qw422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:40
+	qw422016.N().S(`</main></div></div></body></html>`)
+//line lib/promscrape/relabel_debug.qtpl:46
+}
+
+//line lib/promscrape/relabel_debug.qtpl:46
+func WriteMetricRelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) {
+//line lib/promscrape/relabel_debug.qtpl:46
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/relabel_debug.qtpl:46
+	StreamMetricRelabelDebugSteps(qw422016, dss, metric, relabelConfigs, err)
+//line lib/promscrape/relabel_debug.qtpl:46
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:46
+}
+
+//line lib/promscrape/relabel_debug.qtpl:46
+func MetricRelabelDebugSteps(dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) string {
+//line lib/promscrape/relabel_debug.qtpl:46
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/relabel_debug.qtpl:46
+	WriteMetricRelabelDebugSteps(qb422016, dss, metric, relabelConfigs, err)
+//line lib/promscrape/relabel_debug.qtpl:46
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/relabel_debug.qtpl:46
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/relabel_debug.qtpl:46
+	return qs422016
+//line lib/promscrape/relabel_debug.qtpl:46
+}
+
+//line lib/promscrape/relabel_debug.qtpl:48
+func StreamTargetRelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep) {
+//line lib/promscrape/relabel_debug.qtpl:48
+	qw422016.N().S(`<!DOCTYPE html><html lang="en"><head>`)
+//line lib/promscrape/relabel_debug.qtpl:52
+	streamcommonHeader(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:52
+	qw422016.N().S(`<title>Target relabel debug</title></head><body>`)
+//line lib/promscrape/relabel_debug.qtpl:56
+	streamnavbar(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:56
+	qw422016.N().S(`<div class="container-fluid"><a href="https://docs.victoriametrics.com/relabeling.html" target="_blank">Relabeling docs</a><br/><div class="row"><main class="col-12">`)
+//line lib/promscrape/relabel_debug.qtpl:61
+	streamrelabelDebugSteps(qw422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:61
+	qw422016.N().S(`</main></div></div></body></html>`)
+//line lib/promscrape/relabel_debug.qtpl:67
+}
+
+//line lib/promscrape/relabel_debug.qtpl:67
+func WriteTargetRelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep) {
+//line lib/promscrape/relabel_debug.qtpl:67
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/relabel_debug.qtpl:67
+	StreamTargetRelabelDebugSteps(qw422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:67
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:67
+}
+
+//line lib/promscrape/relabel_debug.qtpl:67
+func TargetRelabelDebugSteps(dss []promrelabel.DebugStep) string {
+//line lib/promscrape/relabel_debug.qtpl:67
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/relabel_debug.qtpl:67
+	WriteTargetRelabelDebugSteps(qb422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:67
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/relabel_debug.qtpl:67
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/relabel_debug.qtpl:67
+	return qs422016
+//line lib/promscrape/relabel_debug.qtpl:67
+}
+
+//line lib/promscrape/relabel_debug.qtpl:69
+func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep) {
+//line lib/promscrape/relabel_debug.qtpl:70
+	if len(dss) > 0 {
+//line lib/promscrape/relabel_debug.qtpl:70
+		qw422016.N().S(`<div class="m-3"><b>Original labels:</b> <samp>`)
+//line lib/promscrape/relabel_debug.qtpl:72
+		streammustFormatLabels(qw422016, dss[0].In)
+//line lib/promscrape/relabel_debug.qtpl:72
+		qw422016.N().S(`</samp></div>`)
+//line lib/promscrape/relabel_debug.qtpl:74
+	}
+//line lib/promscrape/relabel_debug.qtpl:74
+	qw422016.N().S(`<table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Step</th><th scope="col" style="width: 25%">Relabeling Rule</th><th scope="col" style="width: 35%">Input Labels</th><th scope="col" stile="width: 35%">Output labels</a></tr></thead><tbody>`)
+//line lib/promscrape/relabel_debug.qtpl:85
+	for i, ds := range dss {
+//line lib/promscrape/relabel_debug.qtpl:87
+		inLabels := promutils.MustNewLabelsFromString(ds.In)
+		outLabels := promutils.MustNewLabelsFromString(ds.Out)
+		changedLabels := getChangedLabelNames(inLabels, outLabels)
+
+//line lib/promscrape/relabel_debug.qtpl:90
+		qw422016.N().S(`<tr><td>`)
+//line lib/promscrape/relabel_debug.qtpl:92
+		qw422016.N().D(i)
+//line lib/promscrape/relabel_debug.qtpl:92
+		qw422016.N().S(`</td><td><b><pre class="m-2">`)
+//line lib/promscrape/relabel_debug.qtpl:93
+		qw422016.E().S(ds.Rule)
+//line lib/promscrape/relabel_debug.qtpl:93
+		qw422016.N().S(`</pre></b></td><td><div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">`)
+//line lib/promscrape/relabel_debug.qtpl:96
+		streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "red")
+//line lib/promscrape/relabel_debug.qtpl:96
+		qw422016.N().S(`</div></td><td><div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
+//line lib/promscrape/relabel_debug.qtpl:101
+		streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "blue")
+//line lib/promscrape/relabel_debug.qtpl:101
+		qw422016.N().S(`</div></td></tr>`)
+//line lib/promscrape/relabel_debug.qtpl:105
+	}
+//line lib/promscrape/relabel_debug.qtpl:105
+	qw422016.N().S(`</tbody></table>`)
+//line lib/promscrape/relabel_debug.qtpl:108
+	if len(dss) > 0 {
+//line lib/promscrape/relabel_debug.qtpl:108
+		qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
+//line lib/promscrape/relabel_debug.qtpl:110
+		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
+//line lib/promscrape/relabel_debug.qtpl:110
+		qw422016.N().S(`</samp></div>`)
+//line lib/promscrape/relabel_debug.qtpl:112
+	}
+//line lib/promscrape/relabel_debug.qtpl:113
+}
+
+//line lib/promscrape/relabel_debug.qtpl:113
+func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep) {
+//line lib/promscrape/relabel_debug.qtpl:113
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/relabel_debug.qtpl:113
+	streamrelabelDebugSteps(qw422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:113
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:113
+}
+
+//line lib/promscrape/relabel_debug.qtpl:113
+func relabelDebugSteps(dss []promrelabel.DebugStep) string {
+//line lib/promscrape/relabel_debug.qtpl:113
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/relabel_debug.qtpl:113
+	writerelabelDebugSteps(qb422016, dss)
+//line lib/promscrape/relabel_debug.qtpl:113
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/relabel_debug.qtpl:113
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/relabel_debug.qtpl:113
+	return qs422016
+//line lib/promscrape/relabel_debug.qtpl:113
+}
+
+//line lib/promscrape/relabel_debug.qtpl:115
+func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+//line lib/promscrape/relabel_debug.qtpl:117
+	labelsList := labels.GetLabels()
+	metricName := ""
+	for i, label := range labelsList {
+		if label.Name == "__name__" {
+			metricName = label.Value
+			labelsList = append(labelsList[:i], labelsList[i+1:]...)
+			break
+		}
+	}
+
+//line lib/promscrape/relabel_debug.qtpl:127
+	if metricName != "" {
+//line lib/promscrape/relabel_debug.qtpl:128
+		if _, ok := highlight["__name__"]; ok {
+//line lib/promscrape/relabel_debug.qtpl:128
+			qw422016.N().S(`<span style="font-weight:bold;color:`)
+//line lib/promscrape/relabel_debug.qtpl:129
+			qw422016.E().S(color)
+//line lib/promscrape/relabel_debug.qtpl:129
+			qw422016.N().S(`">`)
+//line lib/promscrape/relabel_debug.qtpl:129
+			qw422016.E().S(metricName)
+//line lib/promscrape/relabel_debug.qtpl:129
+			qw422016.N().S(`</span>`)
+//line lib/promscrape/relabel_debug.qtpl:130
+		} else {
+//line lib/promscrape/relabel_debug.qtpl:131
+			qw422016.E().S(metricName)
+//line lib/promscrape/relabel_debug.qtpl:132
+		}
+//line lib/promscrape/relabel_debug.qtpl:133
+		if len(labelsList) == 0 {
+//line lib/promscrape/relabel_debug.qtpl:133
+			return
+//line lib/promscrape/relabel_debug.qtpl:133
+		}
+//line lib/promscrape/relabel_debug.qtpl:134
+	}
+//line lib/promscrape/relabel_debug.qtpl:134
+	qw422016.N().S(`{`)
+//line lib/promscrape/relabel_debug.qtpl:136
+	for i, label := range labelsList {
+//line lib/promscrape/relabel_debug.qtpl:137
+		if _, ok := highlight[label.Name]; ok {
+//line lib/promscrape/relabel_debug.qtpl:137
+			qw422016.N().S(`<span style="font-weight:bold;color:`)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.E().S(color)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.N().S(`">`)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.E().S(label.Name)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.N().S(`=`)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.E().Q(label.Value)
+//line lib/promscrape/relabel_debug.qtpl:138
+			qw422016.N().S(`</span>`)
+//line lib/promscrape/relabel_debug.qtpl:139
+		} else {
+//line lib/promscrape/relabel_debug.qtpl:140
+			qw422016.E().S(label.Name)
+//line lib/promscrape/relabel_debug.qtpl:140
+			qw422016.N().S(`=`)
+//line lib/promscrape/relabel_debug.qtpl:140
+			qw422016.E().Q(label.Value)
+//line lib/promscrape/relabel_debug.qtpl:141
+		}
+//line lib/promscrape/relabel_debug.qtpl:142
+		if i < len(labelsList)-1 {
+//line lib/promscrape/relabel_debug.qtpl:142
+			qw422016.N().S(`,`)
+//line lib/promscrape/relabel_debug.qtpl:142
+			qw422016.N().S(` `)
+//line lib/promscrape/relabel_debug.qtpl:142
+		}
+//line lib/promscrape/relabel_debug.qtpl:143
+	}
+//line lib/promscrape/relabel_debug.qtpl:143
+	qw422016.N().S(`}`)
+//line lib/promscrape/relabel_debug.qtpl:145
+}
+
+//line lib/promscrape/relabel_debug.qtpl:145
+func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) {
+//line lib/promscrape/relabel_debug.qtpl:145
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/relabel_debug.qtpl:145
+	streamlabelsWithHighlight(qw422016, labels, highlight, color)
+//line lib/promscrape/relabel_debug.qtpl:145
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:145
+}
+
+//line lib/promscrape/relabel_debug.qtpl:145
+func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string {
+//line lib/promscrape/relabel_debug.qtpl:145
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/relabel_debug.qtpl:145
+	writelabelsWithHighlight(qb422016, labels, highlight, color)
+//line lib/promscrape/relabel_debug.qtpl:145
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/relabel_debug.qtpl:145
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/relabel_debug.qtpl:145
+	return qs422016
+//line lib/promscrape/relabel_debug.qtpl:145
+}
+
+//line lib/promscrape/relabel_debug.qtpl:147
+func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
+//line lib/promscrape/relabel_debug.qtpl:148
+	labels := promutils.MustNewLabelsFromString(s)
+
+//line lib/promscrape/relabel_debug.qtpl:149
+	streamlabelsWithHighlight(qw422016, labels, nil, "")
+//line lib/promscrape/relabel_debug.qtpl:150
+}
+
+//line lib/promscrape/relabel_debug.qtpl:150
+func writemustFormatLabels(qq422016 qtio422016.Writer, s string) {
+//line lib/promscrape/relabel_debug.qtpl:150
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/relabel_debug.qtpl:150
+	streammustFormatLabels(qw422016, s)
+//line lib/promscrape/relabel_debug.qtpl:150
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/relabel_debug.qtpl:150
+}
+
+//line lib/promscrape/relabel_debug.qtpl:150
+func mustFormatLabels(s string) string {
+//line lib/promscrape/relabel_debug.qtpl:150
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/relabel_debug.qtpl:150
+	writemustFormatLabels(qb422016, s)
+//line lib/promscrape/relabel_debug.qtpl:150
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/relabel_debug.qtpl:150
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/relabel_debug.qtpl:150
+	return qs422016
+//line lib/promscrape/relabel_debug.qtpl:150
+}

lib/promscrape/scraper.go

@@ -363,9 +363,9 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) {
 					"make sure service discovery and relabeling is set up properly; "+
 					"see also https://docs.victoriametrics.com/vmagent.html#troubleshooting; "+
 					"original labels for target1: %s; original labels for target2: %s",
-					sw.ScrapeURL, sw.LabelsString(), originalLabels.String(), sw.OriginalLabels.String())
+					sw.ScrapeURL, sw.Labels.String(), originalLabels.String(), sw.OriginalLabels.String())
 			}
-			droppedTargetsMap.Register(sw.OriginalLabels)
+			droppedTargetsMap.Register(sw.OriginalLabels, sw.RelabelConfigs)
 			continue
 		}
 		swsMap[key] = sw.OriginalLabels

lib/promscrape/scrapework.go

@@ -101,6 +101,9 @@ type ScrapeWork struct {
 	// Auth config
 	AuthConfig *promauth.Config
 
+	// Optional `relabel_configs`.
+	RelabelConfigs *promrelabel.ParsedConfigs
+
 	// Optional `metric_relabel_configs`.
 	MetricRelabelConfigs *promrelabel.ParsedConfigs
 
@@ -147,15 +150,17 @@ func (sw *ScrapeWork) canSwitchToStreamParseMode() bool {
 // It can be used for comparing for equality for two ScrapeWork objects.
 func (sw *ScrapeWork) key() string {
 	// Do not take into account OriginalLabels, since they can be changed with relabeling.
+	// Do not take into account RelabelConfigs, since it is already applied to Labels.
 	// Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling.
 	key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+
 		"ExternalLabels=%s, "+
-		"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
+		"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+
+		"SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
 		"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v",
-		sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.LabelsString(),
+		sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(),
 		sw.ExternalLabels.String(),
-		sw.ProxyURL.String(), sw.ProxyAuthConfig.String(),
+		sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(),
-		sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,
+		sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,
 		sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers)
 	return key
 }
@@ -165,11 +170,6 @@ func (sw *ScrapeWork) Job() string {
 	return sw.Labels.Get("job")
 }
 
-// LabelsString returns labels in Prometheus format for the given sw.
-func (sw *ScrapeWork) LabelsString() string {
-	return sw.Labels.String()
-}
-
 type scrapeWork struct {
 	// Config for the scrape.
 	Config *ScrapeWork
@@ -281,7 +281,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{})
 		// scrapes replicated targets at different time offsets. This guarantees that the deduplication consistently leaves samples
 		// received from the same vmagent replica.
 		// See https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets
-		key := fmt.Sprintf("clusterName=%s, clusterMemberID=%d, ScrapeURL=%s, Labels=%s", *clusterName, clusterMemberID, sw.Config.ScrapeURL, sw.Config.LabelsString())
+		key := fmt.Sprintf("clusterName=%s, clusterMemberID=%d, ScrapeURL=%s, Labels=%s", *clusterName, clusterMemberID, sw.Config.ScrapeURL, sw.Config.Labels.String())
 		h := xxhash.Sum64(bytesutil.ToUnsafeBytes(key))
 		randSleep = uint64(float64(scrapeInterval) * (float64(h) / (1 << 64)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval)
@@ -348,7 +348,7 @@ func (sw *scrapeWork) logError(s string) {
 	if !*suppressScrapeErrors {
 		logger.ErrorfSkipframes(1, "error when scraping %q from job %q with labels %s: %s; "+
 			"scrape errors can be disabled by -promscrape.suppressScrapeErrors command-line flag",
-			sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.LabelsString(), s)
+			sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.Labels.String(), s)
 	}
 }
 
@@ -362,7 +362,7 @@ func (sw *scrapeWork) scrapeAndLogError(scrapeTimestamp, realTimestamp int64) {
 		sw.errsSuppressedCount++
 		return
 	}
-	err = fmt.Errorf("cannot scrape %q (job %q, labels %s): %w", sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.LabelsString(), err)
+	err = fmt.Errorf("cannot scrape %q (job %q, labels %s): %w", sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.Labels.String(), err)
 	if sw.errsSuppressedCount > 0 {
 		err = fmt.Errorf("%w; %d similar errors suppressed during the last %.1f seconds", err, sw.errsSuppressedCount, d.Seconds())
 	}

lib/promscrape/scrapework_test.go

@@ -40,8 +40,8 @@ func TestIsAutoMetric(t *testing.T) {
 func TestAppendExtraLabels(t *testing.T) {
 	f := func(sourceLabels, extraLabels string, honorLabels bool, resultExpected string) {
 		t.Helper()
-		src := promutils.NewLabelsFromString(sourceLabels)
+		src := promutils.MustNewLabelsFromString(sourceLabels)
-		extra := promutils.NewLabelsFromString(extraLabels)
+		extra := promutils.MustNewLabelsFromString(extraLabels)
 		var labels promutils.Labels
 		labels.Labels = appendExtraLabels(src.GetLabels(), extra.GetLabels(), 0, honorLabels)
 		result := labels.String()
@@ -794,7 +794,7 @@ func timeseriesToString(ts *prompbmarshal.TimeSeries) string {
 }
 
 func mustParseRelabelConfigs(config string) *promrelabel.ParsedConfigs {
-	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config), false)
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config))
 	if err != nil {
 		panic(fmt.Errorf("cannot parse %q: %w", config, err))
 	}

lib/promscrape/targetstatus.go

@@ -147,15 +147,16 @@ func (tsm *targetStatusMap) getScrapeWorkByTargetID(targetID string) *scrapeWork
 	tsm.mu.Lock()
 	defer tsm.mu.Unlock()
 	for sw := range tsm.m {
-		if getTargetID(sw) == targetID {
+		// The target is uniquely identified by a pointer to its original labels.
+		if getLabelsID(sw.Config.OriginalLabels) == targetID {
 			return sw
 		}
 	}
 	return nil
 }
 
-func getTargetID(sw *scrapeWork) string {
+func getLabelsID(labels *promutils.Labels) string {
-	return fmt.Sprintf("%016x", uintptr(unsafe.Pointer(sw)))
+	return fmt.Sprintf("%016x", uintptr(unsafe.Pointer(labels)))
 }
 
 // StatusByGroup returns the number of targets with status==up
@@ -254,36 +255,36 @@ type droppedTargets struct {
 
 type droppedTarget struct {
 	originalLabels *promutils.Labels
+	relabelConfigs *promrelabel.ParsedConfigs
 	deadline       uint64
 }
 
-func (dt *droppedTargets) getTargetsLabels() []*promutils.Labels {
+func (dt *droppedTargets) getTargetsList() []droppedTarget {
 	dt.mu.Lock()
-	dtls := make([]*promutils.Labels, 0, len(dt.m))
+	dts := make([]droppedTarget, 0, len(dt.m))
 	for _, v := range dt.m {
-		dtls = append(dtls, v.originalLabels)
+		dts = append(dts, v)
 	}
 	dt.mu.Unlock()
 	// Sort discovered targets by __address__ label, so they stay in consistent order across calls
-	sort.Slice(dtls, func(i, j int) bool {
+	sort.Slice(dts, func(i, j int) bool {
-		addr1 := dtls[i].Get("__address__")
+		addr1 := dts[i].originalLabels.Get("__address__")
-		addr2 := dtls[j].Get("__address__")
+		addr2 := dts[j].originalLabels.Get("__address__")
 		return addr1 < addr2
 	})
-	return dtls
+	return dts
 }
 
-func (dt *droppedTargets) Register(originalLabels *promutils.Labels) {
+func (dt *droppedTargets) Register(originalLabels *promutils.Labels, relabelConfigs *promrelabel.ParsedConfigs) {
-	// It is better to have hash collisions instead of spending additional CPU on promLabelsString() call.
+	// It is better to have hash collisions instead of spending additional CPU on originalLabels.String() call.
 	key := labelsHash(originalLabels)
 	currentTime := fasttime.UnixTimestamp()
 	dt.mu.Lock()
-	if k, ok := dt.m[key]; ok {
+	_, ok := dt.m[key]
-		k.deadline = currentTime + 10*60
+	if ok || len(dt.m) < *maxDroppedTargets {
-		dt.m[key] = k
-	} else if len(dt.m) < *maxDroppedTargets {
 		dt.m[key] = droppedTarget{
 			originalLabels: originalLabels,
+			relabelConfigs: relabelConfigs,
 			deadline:       currentTime + 10*60,
 		}
 	}
@@ -318,13 +319,13 @@ var xxhashPool = &sync.Pool{
 
 // WriteDroppedTargetsJSON writes `droppedTargets` contents to w according to https://prometheus.io/docs/prometheus/latest/querying/api/#targets
 func (dt *droppedTargets) WriteDroppedTargetsJSON(w io.Writer) {
-	dtls := dt.getTargetsLabels()
+	dts := dt.getTargetsList()
 	fmt.Fprintf(w, `[`)
-	for i, labels := range dtls {
+	for i, dt := range dts {
 		fmt.Fprintf(w, `{"discoveredLabels":`)
-		writeLabelsJSON(w, labels)
+		writeLabelsJSON(w, dt.originalLabels)
 		fmt.Fprintf(w, `}`)
-		if i+1 < len(dtls) {
+		if i+1 < len(dts) {
 			fmt.Fprintf(w, `,`)
 		}
 	}
@@ -385,12 +386,12 @@ func (tsm *targetStatusMap) getTargetsStatusByJob(filter *requestFilter) *target
 		// Do not show empty jobs if target filters are set.
 		emptyJobs = nil
 	}
-	dtls := droppedTargetsMap.getTargetsLabels()
+	dts := droppedTargetsMap.getTargetsList()
 	return &targetsStatusResult{
-		jobTargetsStatuses:   jts,
+		jobTargetsStatuses: jts,
-		droppedTargetsLabels: dtls,
+		droppedTargets:     dts,
-		emptyJobs:            emptyJobs,
+		emptyJobs:          emptyJobs,
-		err:                  err,
+		err:                err,
 	}
 }
 
@@ -497,16 +498,16 @@ func getRequestFilter(r *http.Request) *requestFilter {
 }
 
 type targetsStatusResult struct {
-	jobTargetsStatuses   []*jobTargetsStatuses
+	jobTargetsStatuses []*jobTargetsStatuses
-	droppedTargetsLabels []*promutils.Labels
+	droppedTargets     []droppedTarget
-	emptyJobs            []string
+	emptyJobs          []string
-	err                  error
+	err                error
 }
 
 type targetLabels struct {
-	up               bool
+	up             bool
-	discoveredLabels *promutils.Labels
+	originalLabels *promutils.Labels
-	labels           *promutils.Labels
+	labels         *promutils.Labels
 }
 type targetLabelsByJob struct {
 	jobName        string
@@ -515,6 +516,43 @@ type targetLabelsByJob struct {
 	droppedTargets int
 }
 
+func getRelabelContextByTargetID(targetID string) (*promrelabel.ParsedConfigs, *promutils.Labels, bool) {
+	var relabelConfigs *promrelabel.ParsedConfigs
+	var labels *promutils.Labels
+	found := false
+
+	// Search for relabel context in tsmGlobal (aka active targets)
+	tsmGlobal.mu.Lock()
+	for sw := range tsmGlobal.m {
+		// The target is uniquely identified by a pointer to its original labels.
+		if getLabelsID(sw.Config.OriginalLabels) == targetID {
+			relabelConfigs = sw.Config.RelabelConfigs
+			labels = sw.Config.OriginalLabels
+			found = true
+			break
+		}
+	}
+	tsmGlobal.mu.Unlock()
+
+	if found {
+		return relabelConfigs, labels, true
+	}
+
+	// Search for relabel context in droppedTargetsMap (aka deleted targets)
+	droppedTargetsMap.mu.Lock()
+	for _, dt := range droppedTargetsMap.m {
+		if getLabelsID(dt.originalLabels) == targetID {
+			relabelConfigs = dt.relabelConfigs
+			labels = dt.originalLabels
+			found = true
+			break
+		}
+	}
+	droppedTargetsMap.mu.Unlock()
+
+	return relabelConfigs, labels, found
+}
+
 func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob {
 	byJob := make(map[string]*targetLabelsByJob)
 	for _, jts := range tsr.jobTargetsStatuses {
@@ -529,14 +567,14 @@ func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob {
 			}
 			m.activeTargets++
 			m.targets = append(m.targets, targetLabels{
-				up:               ts.up,
+				up:             ts.up,
-				discoveredLabels: ts.sw.Config.OriginalLabels,
+				originalLabels: ts.sw.Config.OriginalLabels,
-				labels:           ts.sw.Config.Labels,
+				labels:         ts.sw.Config.Labels,
 			})
 		}
 	}
-	for _, labels := range tsr.droppedTargetsLabels {
+	for _, dt := range tsr.droppedTargets {
-		jobName := labels.Get("job")
+		jobName := dt.originalLabels.Get("job")
 		m := byJob[jobName]
 		if m == nil {
 			m = &targetLabelsByJob{
@@ -546,7 +584,7 @@ func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob {
 		}
 		m.droppedTargets++
 		m.targets = append(m.targets, targetLabels{
-			discoveredLabels: labels,
+			originalLabels: dt.originalLabels,
 		})
 	}
 	a := make([]*targetLabelsByJob, 0, len(byJob))

lib/promscrape/targetstatus.qtpl

@@ -103,7 +103,7 @@
 {% func navbar() %}
     <div class="navbar navbar-dark bg-dark box-shadow">
         <div class="d-flex justify-content-between">
-            <a href="#" class="navbar-brand d-flex align-items-center ms-3" title="The High Performance Open Source Time Series Database &amp; Monitoring Solution ">
+            <a href="/" class="navbar-brand d-flex align-items-center ms-3" title="The High Performance Open Source Time Series Database &amp; Monitoring Solution ">
                 <svg xmlns="http://www.w3.org/2000/svg" id="VM_logo" viewBox="0 0 464.61 533.89" width="20" height="20" class="me-1"><defs><style>.cls-1{fill:#fff;}</style></defs><path class="cls-1" d="M459.86,467.77c9,7.67,24.12,13.49,39.3,13.69v0h1.68v0c15.18-.2,30.31-6,39.3-13.69,47.43-40.45,184.65-166.24,184.65-166.24,36.84-34.27-65.64-68.28-223.95-68.47h-1.68c-158.31.19-260.79,34.2-224,68.47C275.21,301.53,412.43,427.32,459.86,467.77Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,535.88c-9,7.67-24.12,13.5-39.3,13.7h-1.6c-15.18-.2-30.31-6-39.3-13.7-32.81-28-148.56-132.93-192.16-172.7v60.74c0,6.67,2.55,15.52,7.09,19.68,29.64,27.18,143.94,131.8,185.07,166.88,9,7.67,24.12,13.49,39.3,13.69v0h1.6v0c15.18-.2,30.31-6,39.3-13.69,41.13-35.08,155.43-139.7,185.07-166.88,4.54-4.16,7.09-13,7.09-19.68V363.18C688.66,403,572.91,507.9,540.1,535.88Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,678.64c-9,7.67-24.12,13.49-39.3,13.69v0h-1.6v0c-15.18-.2-30.31-6-39.3-13.69-32.81-28-148.56-132.94-192.16-172.7v60.73c0,6.67,2.55,15.53,7.09,19.69,29.64,27.17,143.94,131.8,185.07,166.87,9,7.67,24.12,13.5,39.3,13.7h1.6c15.18-.2,30.31-6,39.3-13.7,41.13-35.07,155.43-139.7,185.07-166.87,4.54-4.16,7.09-13,7.09-19.69V505.94C688.66,545.7,572.91,650.66,540.1,678.64Z" transform="translate(-267.7 -233.05)"/></svg>
                 <strong>VictoriaMetrics</strong>
             </a>
@@ -225,6 +225,7 @@
                             <th scope="col">Endpoint</th>
                             <th scope="col">State</th>
                             <th scope="col" title="target labels">Labels</th>
+                            <th scope="col" title="debug relabeling">Debug relabeling</th>
                             <th scope="col" title="total scrapes">Scrapes</th>
                             <th scope="col" title="total scrape errors">Errors</th>
                             <th scope="col" title="the time of the last scrape">Last Scrape</th>
@@ -237,7 +238,8 @@
                     {% for _, ts := range jts.targetsStatus %}
                     {% code
                         endpoint := ts.sw.Config.ScrapeURL
-                        targetID := getTargetID(ts.sw)
+                        // The target is uniquely identified by a pointer to its original labels.
+                        targetID := getLabelsID(ts.sw.Config.OriginalLabels)
                         lastScrapeDuration := ts.getDurationFromLastScrape()
                     %}
                         <tr {% if !ts.up %}{%space%}class="alert alert-danger" role="alert" {% endif %}>
@@ -263,6 +265,9 @@
                                     {%= formatLabels(ts.sw.Config.OriginalLabels) %}
                                 </div>
                             </td>
+                            <td>
+                                <a href="target-relabel-debug?id={%s targetID %}" target="_blank">debug</a>
+                            </td>
                             <td>{%d ts.scrapesTotal %}</td>
                             <td>{%d ts.scrapesFailed %}</td>
                             <td>
@@ -304,8 +309,9 @@
             <thead>
                 <tr>
                     <th scope="col" style="width: 5%">Status</th>
-                    <th scope="col" style="width: 65%">Discovered Labels</th>
+                    <th scope="col" style="width: 60%">Discovered Labels</th>
                     <th scope="col" style="width: 30%">Target Labels</th>
+                    <th scope="col" stile="width: 5%">Debug relabeling</a>
                 </tr>
             </thead>
             <tbody>
@@ -330,11 +336,15 @@
                         {% endif %}
                     </td>
                     <td class="labels">
-                        {%= formatLabels(t.discoveredLabels) %}
+                        {%= formatLabels(t.originalLabels) %}
                     </td>
                     <td class="labels">
                         {%= formatLabels(t.labels) %}
                     </td>
+                    <td>
+                        {% code targetID := getLabelsID(t.originalLabels) %}
+                        <a href="target-relabel-debug?id={%s targetID %}" target="_blank">debug</a>
+                    </td>
                 </tr>
             {% endfor %}
             </tbody>

lib/promscrape/targetstatus.qtpl.go

@@ -355,7 +355,7 @@ func commonHeader() string {
 //line lib/promscrape/targetstatus.qtpl:103
 func streamnavbar(qw422016 *qt422016.Writer) {
 //line lib/promscrape/targetstatus.qtpl:103
-	qw422016.N().S(`<div class="navbar navbar-dark bg-dark box-shadow"><div class="d-flex justify-content-between"><a href="#" class="navbar-brand d-flex align-items-center ms-3" title="The High Performance Open Source Time Series Database &amp; Monitoring Solution "><svg xmlns="http://www.w3.org/2000/svg" id="VM_logo" viewBox="0 0 464.61 533.89" width="20" height="20" class="me-1"><defs><style>.cls-1{fill:#fff;}</style></defs><path class="cls-1" d="M459.86,467.77c9,7.67,24.12,13.49,39.3,13.69v0h1.68v0c15.18-.2,30.31-6,39.3-13.69,47.43-40.45,184.65-166.24,184.65-166.24,36.84-34.27-65.64-68.28-223.95-68.47h-1.68c-158.31.19-260.79,34.2-224,68.47C275.21,301.53,412.43,427.32,459.86,467.77Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,535.88c-9,7.67-24.12,13.5-39.3,13.7h-1.6c-15.18-.2-30.31-6-39.3-13.7-32.81-28-148.56-132.93-192.16-172.7v60.74c0,6.67,2.55,15.52,7.09,19.68,29.64,27.18,143.94,131.8,185.07,166.88,9,7.67,24.12,13.49,39.3,13.69v0h1.6v0c15.18-.2,30.31-6,39.3-13.69,41.13-35.08,155.43-139.7,185.07-166.88,4.54-4.16,7.09-13,7.09-19.68V363.18C688.66,403,572.91,507.9,540.1,535.88Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,678.64c-9,7.67-24.12,13.49-39.3,13.69v0h-1.6v0c-15.18-.2-30.31-6-39.3-13.69-32.81-28-148.56-132.94-192.16-172.7v60.73c0,6.67,2.55,15.53,7.09,19.69,29.64,27.17,143.94,131.8,185.07,166.87,9,7.67,24.12,13.5,39.3,13.7h1.6c15.18-.2,30.31-6,39.3-13.7,41.13-35.07,155.43-139.7,185.07-166.87,4.54-4.16,7.09-13,7.09-19.69V505.94C688.66,545.7,572.91,650.66,540.1,678.64Z" transform="translate(-267.7 -233.05)"/></svg><strong>VictoriaMetrics</strong></a></div></div>`)
+	qw422016.N().S(`<div class="navbar navbar-dark bg-dark box-shadow"><div class="d-flex justify-content-between"><a href="/" class="navbar-brand d-flex align-items-center ms-3" title="The High Performance Open Source Time Series Database &amp; Monitoring Solution "><svg xmlns="http://www.w3.org/2000/svg" id="VM_logo" viewBox="0 0 464.61 533.89" width="20" height="20" class="me-1"><defs><style>.cls-1{fill:#fff;}</style></defs><path class="cls-1" d="M459.86,467.77c9,7.67,24.12,13.49,39.3,13.69v0h1.68v0c15.18-.2,30.31-6,39.3-13.69,47.43-40.45,184.65-166.24,184.65-166.24,36.84-34.27-65.64-68.28-223.95-68.47h-1.68c-158.31.19-260.79,34.2-224,68.47C275.21,301.53,412.43,427.32,459.86,467.77Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,535.88c-9,7.67-24.12,13.5-39.3,13.7h-1.6c-15.18-.2-30.31-6-39.3-13.7-32.81-28-148.56-132.93-192.16-172.7v60.74c0,6.67,2.55,15.52,7.09,19.68,29.64,27.18,143.94,131.8,185.07,166.88,9,7.67,24.12,13.49,39.3,13.69v0h1.6v0c15.18-.2,30.31-6,39.3-13.69,41.13-35.08,155.43-139.7,185.07-166.88,4.54-4.16,7.09-13,7.09-19.68V363.18C688.66,403,572.91,507.9,540.1,535.88Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,678.64c-9,7.67-24.12,13.49-39.3,13.69v0h-1.6v0c-15.18-.2-30.31-6-39.3-13.69-32.81-28-148.56-132.94-192.16-172.7v60.73c0,6.67,2.55,15.53,7.09,19.69,29.64,27.17,143.94,131.8,185.07,166.87,9,7.67,24.12,13.5,39.3,13.7h1.6c15.18-.2,30.31-6,39.3-13.7,41.13-35.07,155.43-139.7,185.07-166.87,4.54-4.16,7.09-13,7.09-19.69V505.94C688.66,545.7,572.91,650.66,540.1,678.64Z" transform="translate(-267.7 -233.05)"/></svg><strong>VictoriaMetrics</strong></a></div></div>`)
 //line lib/promscrape/targetstatus.qtpl:112
 }
 
@@ -633,336 +633,350 @@ func streamscrapeJobTargets(qw422016 *qt422016.Writer, num int, jts *jobTargetsS
 //line lib/promscrape/targetstatus.qtpl:221
 	qw422016.N().D(num)
 //line lib/promscrape/targetstatus.qtpl:221
-	qw422016.N().S(`" class="scrape-job table-responsive"><table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col">Endpoint</th><th scope="col">State</th><th scope="col" title="target labels">Labels</th><th scope="col" title="total scrapes">Scrapes</th><th scope="col" title="total scrape errors">Errors</th><th scope="col" title="the time of the last scrape">Last Scrape</th><th scope="col" title="the duration of the last scrape">Duration</th><th scope="col" title="the number of metrics scraped during the last scrape">Samples</th><th scope="col" title="error from the last scrape (if any)">Last error</th></tr></thead><tbody>`)
+	qw422016.N().S(`" class="scrape-job table-responsive"><table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col">Endpoint</th><th scope="col">State</th><th scope="col" title="target labels">Labels</th><th scope="col" title="debug relabeling">Debug relabeling</th><th scope="col" title="total scrapes">Scrapes</th><th scope="col" title="total scrape errors">Errors</th><th scope="col" title="the time of the last scrape">Last Scrape</th><th scope="col" title="the duration of the last scrape">Duration</th><th scope="col" title="the number of metrics scraped during the last scrape">Samples</th><th scope="col" title="error from the last scrape (if any)">Last error</th></tr></thead><tbody>`)
-//line lib/promscrape/targetstatus.qtpl:237
+//line lib/promscrape/targetstatus.qtpl:238
 	for _, ts := range jts.targetsStatus {
-//line lib/promscrape/targetstatus.qtpl:239
+//line lib/promscrape/targetstatus.qtpl:240
 		endpoint := ts.sw.Config.ScrapeURL
-		targetID := getTargetID(ts.sw)
+		// The target is uniquely identified by a pointer to its original labels.
+		targetID := getLabelsID(ts.sw.Config.OriginalLabels)
 		lastScrapeDuration := ts.getDurationFromLastScrape()
 
-//line lib/promscrape/targetstatus.qtpl:242
+//line lib/promscrape/targetstatus.qtpl:244
 		qw422016.N().S(`<tr`)
-//line lib/promscrape/targetstatus.qtpl:243
+//line lib/promscrape/targetstatus.qtpl:245
 		if !ts.up {
-//line lib/promscrape/targetstatus.qtpl:243
+//line lib/promscrape/targetstatus.qtpl:245
 			qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:243
+//line lib/promscrape/targetstatus.qtpl:245
 			qw422016.N().S(`class="alert alert-danger" role="alert"`)
-//line lib/promscrape/targetstatus.qtpl:243
+//line lib/promscrape/targetstatus.qtpl:245
 		}
-//line lib/promscrape/targetstatus.qtpl:243
+//line lib/promscrape/targetstatus.qtpl:245
 		qw422016.N().S(`><td class="endpoint"><a href="`)
-//line lib/promscrape/targetstatus.qtpl:245
+//line lib/promscrape/targetstatus.qtpl:247
 		qw422016.E().S(endpoint)
-//line lib/promscrape/targetstatus.qtpl:245
+//line lib/promscrape/targetstatus.qtpl:247
 		qw422016.N().S(`" target="_blank">`)
-//line lib/promscrape/targetstatus.qtpl:245
+//line lib/promscrape/targetstatus.qtpl:247
 		qw422016.E().S(endpoint)
-//line lib/promscrape/targetstatus.qtpl:245
+//line lib/promscrape/targetstatus.qtpl:247
 		qw422016.N().S(`</a> (<a href="target_response?id=`)
-//line lib/promscrape/targetstatus.qtpl:246
+//line lib/promscrape/targetstatus.qtpl:248
 		qw422016.E().S(targetID)
-//line lib/promscrape/targetstatus.qtpl:246
+//line lib/promscrape/targetstatus.qtpl:248
 		qw422016.N().S(`" target="_blank"title="click to fetch target response on behalf of the scraper">response</a>)</td><td>`)
-//line lib/promscrape/targetstatus.qtpl:251
+//line lib/promscrape/targetstatus.qtpl:253
 		if ts.up {
-//line lib/promscrape/targetstatus.qtpl:251
+//line lib/promscrape/targetstatus.qtpl:253
 			qw422016.N().S(`<span class="badge bg-success">UP</span>`)
-//line lib/promscrape/targetstatus.qtpl:253
+//line lib/promscrape/targetstatus.qtpl:255
 		} else {
-//line lib/promscrape/targetstatus.qtpl:253
+//line lib/promscrape/targetstatus.qtpl:255
 			qw422016.N().S(`<span class="badge bg-danger">DOWN</span>`)
-//line lib/promscrape/targetstatus.qtpl:255
+//line lib/promscrape/targetstatus.qtpl:257
 		}
-//line lib/promscrape/targetstatus.qtpl:255
+//line lib/promscrape/targetstatus.qtpl:257
 		qw422016.N().S(`</td><td class="labels"><div title="click to show original labels"onclick="document.getElementById('original-labels-`)
-//line lib/promscrape/targetstatus.qtpl:259
+//line lib/promscrape/targetstatus.qtpl:261
 		qw422016.E().S(targetID)
-//line lib/promscrape/targetstatus.qtpl:259
+//line lib/promscrape/targetstatus.qtpl:261
 		qw422016.N().S(`').style.display='block'">`)
-//line lib/promscrape/targetstatus.qtpl:260
+//line lib/promscrape/targetstatus.qtpl:262
 		streamformatLabels(qw422016, ts.sw.Config.Labels)
-//line lib/promscrape/targetstatus.qtpl:260
+//line lib/promscrape/targetstatus.qtpl:262
 		qw422016.N().S(`</div><div style="display:none" id="original-labels-`)
-//line lib/promscrape/targetstatus.qtpl:262
+//line lib/promscrape/targetstatus.qtpl:264
 		qw422016.E().S(targetID)
-//line lib/promscrape/targetstatus.qtpl:262
+//line lib/promscrape/targetstatus.qtpl:264
 		qw422016.N().S(`">`)
-//line lib/promscrape/targetstatus.qtpl:263
+//line lib/promscrape/targetstatus.qtpl:265
 		streamformatLabels(qw422016, ts.sw.Config.OriginalLabels)
-//line lib/promscrape/targetstatus.qtpl:263
+//line lib/promscrape/targetstatus.qtpl:265
-		qw422016.N().S(`</div></td><td>`)
+		qw422016.N().S(`</div></td><td><a href="target-relabel-debug?id=`)
-//line lib/promscrape/targetstatus.qtpl:266
-		qw422016.N().D(ts.scrapesTotal)
-//line lib/promscrape/targetstatus.qtpl:266
-		qw422016.N().S(`</td><td>`)
-//line lib/promscrape/targetstatus.qtpl:267
-		qw422016.N().D(ts.scrapesFailed)
-//line lib/promscrape/targetstatus.qtpl:267
-		qw422016.N().S(`</td><td>`)
 //line lib/promscrape/targetstatus.qtpl:269
-		if lastScrapeDuration < 365*24*time.Hour {
+		qw422016.E().S(targetID)
-//line lib/promscrape/targetstatus.qtpl:270
+//line lib/promscrape/targetstatus.qtpl:269
-			qw422016.N().D(int(lastScrapeDuration.Milliseconds()))
+		qw422016.N().S(`" target="_blank">debug</a></td><td>`)
-//line lib/promscrape/targetstatus.qtpl:270
-			qw422016.N().S(`ms ago`)
 //line lib/promscrape/targetstatus.qtpl:271
-		} else {
+		qw422016.N().D(ts.scrapesTotal)
 //line lib/promscrape/targetstatus.qtpl:271
-			qw422016.N().S(`none`)
-//line lib/promscrape/targetstatus.qtpl:273
-		}
-//line lib/promscrape/targetstatus.qtpl:273
-		qw422016.N().S(`<td>`)
-//line lib/promscrape/targetstatus.qtpl:274
-		qw422016.N().D(int(ts.scrapeDuration))
-//line lib/promscrape/targetstatus.qtpl:274
-		qw422016.N().S(`ms</td><td>`)
-//line lib/promscrape/targetstatus.qtpl:275
-		qw422016.N().D(ts.samplesScraped)
-//line lib/promscrape/targetstatus.qtpl:275
 		qw422016.N().S(`</td><td>`)
+//line lib/promscrape/targetstatus.qtpl:272
+		qw422016.N().D(ts.scrapesFailed)
+//line lib/promscrape/targetstatus.qtpl:272
+		qw422016.N().S(`</td><td>`)
+//line lib/promscrape/targetstatus.qtpl:274
+		if lastScrapeDuration < 365*24*time.Hour {
+//line lib/promscrape/targetstatus.qtpl:275
+			qw422016.N().D(int(lastScrapeDuration.Milliseconds()))
+//line lib/promscrape/targetstatus.qtpl:275
+			qw422016.N().S(`ms ago`)
 //line lib/promscrape/targetstatus.qtpl:276
-		if ts.err != nil {
+		} else {
-//line lib/promscrape/targetstatus.qtpl:276
-			qw422016.E().S(ts.err.Error())
 //line lib/promscrape/targetstatus.qtpl:276
+			qw422016.N().S(`none`)
+//line lib/promscrape/targetstatus.qtpl:278
 		}
-//line lib/promscrape/targetstatus.qtpl:276
+//line lib/promscrape/targetstatus.qtpl:278
+		qw422016.N().S(`<td>`)
+//line lib/promscrape/targetstatus.qtpl:279
+		qw422016.N().D(int(ts.scrapeDuration))
+//line lib/promscrape/targetstatus.qtpl:279
+		qw422016.N().S(`ms</td><td>`)
+//line lib/promscrape/targetstatus.qtpl:280
+		qw422016.N().D(ts.samplesScraped)
+//line lib/promscrape/targetstatus.qtpl:280
+		qw422016.N().S(`</td><td>`)
+//line lib/promscrape/targetstatus.qtpl:281
+		if ts.err != nil {
+//line lib/promscrape/targetstatus.qtpl:281
+			qw422016.E().S(ts.err.Error())
+//line lib/promscrape/targetstatus.qtpl:281
+		}
+//line lib/promscrape/targetstatus.qtpl:281
 		qw422016.N().S(`</td></tr>`)
-//line lib/promscrape/targetstatus.qtpl:278
+//line lib/promscrape/targetstatus.qtpl:283
 	}
-//line lib/promscrape/targetstatus.qtpl:278
+//line lib/promscrape/targetstatus.qtpl:283
 	qw422016.N().S(`</tbody></table></div></div></div>`)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 }
 
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 func writescrapeJobTargets(qq422016 qtio422016.Writer, num int, jts *jobTargetsStatuses) {
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	streamscrapeJobTargets(qw422016, num, jts)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 }
 
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 func scrapeJobTargets(num int, jts *jobTargetsStatuses) string {
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	writescrapeJobTargets(qb422016, num, jts)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:284
+//line lib/promscrape/targetstatus.qtpl:289
 }
 
-//line lib/promscrape/targetstatus.qtpl:286
+//line lib/promscrape/targetstatus.qtpl:291
 func streamdiscoveredTargets(qw422016 *qt422016.Writer, tsr *targetsStatusResult) {
-//line lib/promscrape/targetstatus.qtpl:287
+//line lib/promscrape/targetstatus.qtpl:292
 	tljs := tsr.getTargetLabelsByJob()
 
-//line lib/promscrape/targetstatus.qtpl:287
+//line lib/promscrape/targetstatus.qtpl:292
 	qw422016.N().S(`<div class="row mt-4"><div class="col-12">`)
-//line lib/promscrape/targetstatus.qtpl:290
+//line lib/promscrape/targetstatus.qtpl:295
 	for i, tlj := range tljs {
-//line lib/promscrape/targetstatus.qtpl:291
+//line lib/promscrape/targetstatus.qtpl:296
 		streamdiscoveredJobTargets(qw422016, i, tlj)
-//line lib/promscrape/targetstatus.qtpl:292
+//line lib/promscrape/targetstatus.qtpl:297
 	}
-//line lib/promscrape/targetstatus.qtpl:292
+//line lib/promscrape/targetstatus.qtpl:297
 	qw422016.N().S(`</div></div>`)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 }
 
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 func writediscoveredTargets(qq422016 qtio422016.Writer, tsr *targetsStatusResult) {
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	streamdiscoveredTargets(qw422016, tsr)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 }
 
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 func discoveredTargets(tsr *targetsStatusResult) string {
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	writediscoveredTargets(qb422016, tsr)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:295
+//line lib/promscrape/targetstatus.qtpl:300
 }
 
-//line lib/promscrape/targetstatus.qtpl:297
+//line lib/promscrape/targetstatus.qtpl:302
 func streamdiscoveredJobTargets(qw422016 *qt422016.Writer, num int, tlj *targetLabelsByJob) {
-//line lib/promscrape/targetstatus.qtpl:297
+//line lib/promscrape/targetstatus.qtpl:302
 	qw422016.N().S(`<h4><span class="me-2">`)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.E().S(tlj.jobName)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().S(`(`)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().D(tlj.activeTargets)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().S(`/`)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().D(tlj.activeTargets + tlj.droppedTargets)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:299
+//line lib/promscrape/targetstatus.qtpl:304
 	qw422016.N().S(`active)</span>`)
-//line lib/promscrape/targetstatus.qtpl:300
+//line lib/promscrape/targetstatus.qtpl:305
 	streamshowHideScrapeJobButtons(qw422016, num)
-//line lib/promscrape/targetstatus.qtpl:300
+//line lib/promscrape/targetstatus.qtpl:305
 	qw422016.N().S(`</h4><div id="scrape-job-`)
-//line lib/promscrape/targetstatus.qtpl:302
+//line lib/promscrape/targetstatus.qtpl:307
 	qw422016.N().D(num)
-//line lib/promscrape/targetstatus.qtpl:302
+//line lib/promscrape/targetstatus.qtpl:307
-	qw422016.N().S(`" class="scrape-job table-responsive"><table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Status</th><th scope="col" style="width: 65%">Discovered Labels</th><th scope="col" style="width: 30%">Target Labels</th></tr></thead><tbody>`)
+	qw422016.N().S(`" class="scrape-job table-responsive"><table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Status</th><th scope="col" style="width: 60%">Discovered Labels</th><th scope="col" style="width: 30%">Target Labels</th><th scope="col" stile="width: 5%">Debug relabeling</a></tr></thead><tbody>`)
-//line lib/promscrape/targetstatus.qtpl:312
+//line lib/promscrape/targetstatus.qtpl:318
 	for _, t := range tlj.targets {
-//line lib/promscrape/targetstatus.qtpl:312
+//line lib/promscrape/targetstatus.qtpl:318
 		qw422016.N().S(`<tr`)
-//line lib/promscrape/targetstatus.qtpl:314
-		if !t.up {
-//line lib/promscrape/targetstatus.qtpl:315
-			qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:315
-			qw422016.N().S(`role="alert"`)
-//line lib/promscrape/targetstatus.qtpl:315
-			qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:316
-			if t.labels.Len() > 0 {
-//line lib/promscrape/targetstatus.qtpl:316
-				qw422016.N().S(`class="alert alert-danger"`)
-//line lib/promscrape/targetstatus.qtpl:318
-			} else {
-//line lib/promscrape/targetstatus.qtpl:318
-				qw422016.N().S(`class="alert alert-warning"`)
 //line lib/promscrape/targetstatus.qtpl:320
+		if !t.up {
+//line lib/promscrape/targetstatus.qtpl:321
+			qw422016.N().S(` `)
+//line lib/promscrape/targetstatus.qtpl:321
+			qw422016.N().S(`role="alert"`)
+//line lib/promscrape/targetstatus.qtpl:321
+			qw422016.N().S(` `)
+//line lib/promscrape/targetstatus.qtpl:322
+			if t.labels.Len() > 0 {
+//line lib/promscrape/targetstatus.qtpl:322
+				qw422016.N().S(`class="alert alert-danger"`)
+//line lib/promscrape/targetstatus.qtpl:324
+			} else {
+//line lib/promscrape/targetstatus.qtpl:324
+				qw422016.N().S(`class="alert alert-warning"`)
+//line lib/promscrape/targetstatus.qtpl:326
 			}
-//line lib/promscrape/targetstatus.qtpl:321
+//line lib/promscrape/targetstatus.qtpl:327
 		}
-//line lib/promscrape/targetstatus.qtpl:321
+//line lib/promscrape/targetstatus.qtpl:327
 		qw422016.N().S(`><td>`)
-//line lib/promscrape/targetstatus.qtpl:324
+//line lib/promscrape/targetstatus.qtpl:330
 		if t.up {
-//line lib/promscrape/targetstatus.qtpl:324
+//line lib/promscrape/targetstatus.qtpl:330
 			qw422016.N().S(`<span class="badge bg-success">UP</span>`)
-//line lib/promscrape/targetstatus.qtpl:326
+//line lib/promscrape/targetstatus.qtpl:332
 		} else if t.labels.Len() > 0 {
-//line lib/promscrape/targetstatus.qtpl:326
+//line lib/promscrape/targetstatus.qtpl:332
 			qw422016.N().S(`<span class="badge bg-danger">DOWN</span>`)
-//line lib/promscrape/targetstatus.qtpl:328
+//line lib/promscrape/targetstatus.qtpl:334
 		} else {
-//line lib/promscrape/targetstatus.qtpl:328
+//line lib/promscrape/targetstatus.qtpl:334
 			qw422016.N().S(`<span class="badge bg-warning">DROPPED</span>`)
-//line lib/promscrape/targetstatus.qtpl:330
+//line lib/promscrape/targetstatus.qtpl:336
 		}
-//line lib/promscrape/targetstatus.qtpl:330
-		qw422016.N().S(`</td><td class="labels">`)
-//line lib/promscrape/targetstatus.qtpl:333
-		streamformatLabels(qw422016, t.discoveredLabels)
-//line lib/promscrape/targetstatus.qtpl:333
-		qw422016.N().S(`</td><td class="labels">`)
 //line lib/promscrape/targetstatus.qtpl:336
+		qw422016.N().S(`</td><td class="labels">`)
+//line lib/promscrape/targetstatus.qtpl:339
+		streamformatLabels(qw422016, t.originalLabels)
+//line lib/promscrape/targetstatus.qtpl:339
+		qw422016.N().S(`</td><td class="labels">`)
+//line lib/promscrape/targetstatus.qtpl:342
 		streamformatLabels(qw422016, t.labels)
-//line lib/promscrape/targetstatus.qtpl:336
+//line lib/promscrape/targetstatus.qtpl:342
-		qw422016.N().S(`</td></tr>`)
+		qw422016.N().S(`</td><td>`)
-//line lib/promscrape/targetstatus.qtpl:339
+//line lib/promscrape/targetstatus.qtpl:345
+		targetID := getLabelsID(t.originalLabels)
+
+//line lib/promscrape/targetstatus.qtpl:345
+		qw422016.N().S(`<a href="target-relabel-debug?id=`)
+//line lib/promscrape/targetstatus.qtpl:346
+		qw422016.E().S(targetID)
+//line lib/promscrape/targetstatus.qtpl:346
+		qw422016.N().S(`" target="_blank">debug</a></td></tr>`)
+//line lib/promscrape/targetstatus.qtpl:349
 	}
-//line lib/promscrape/targetstatus.qtpl:339
+//line lib/promscrape/targetstatus.qtpl:349
 	qw422016.N().S(`</tbody></table></div>`)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 }
 
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 func writediscoveredJobTargets(qq422016 qtio422016.Writer, num int, tlj *targetLabelsByJob) {
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	streamdiscoveredJobTargets(qw422016, num, tlj)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 }
 
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 func discoveredJobTargets(num int, tlj *targetLabelsByJob) string {
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	writediscoveredJobTargets(qb422016, num, tlj)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:343
+//line lib/promscrape/targetstatus.qtpl:353
 }
 
-//line lib/promscrape/targetstatus.qtpl:345
+//line lib/promscrape/targetstatus.qtpl:355
 func streamshowHideScrapeJobButtons(qw422016 *qt422016.Writer, num int) {
-//line lib/promscrape/targetstatus.qtpl:345
+//line lib/promscrape/targetstatus.qtpl:355
 	qw422016.N().S(`<button type="button" class="btn btn-primary btn-sm me-1"onclick="document.getElementById('scrape-job-`)
-//line lib/promscrape/targetstatus.qtpl:347
+//line lib/promscrape/targetstatus.qtpl:357
 	qw422016.N().D(num)
-//line lib/promscrape/targetstatus.qtpl:347
+//line lib/promscrape/targetstatus.qtpl:357
 	qw422016.N().S(`').style.display='none'">collapse</button><button type="button" class="btn btn-secondary btn-sm me-1"onclick="document.getElementById('scrape-job-`)
-//line lib/promscrape/targetstatus.qtpl:351
+//line lib/promscrape/targetstatus.qtpl:361
 	qw422016.N().D(num)
-//line lib/promscrape/targetstatus.qtpl:351
+//line lib/promscrape/targetstatus.qtpl:361
 	qw422016.N().S(`').style.display='block'">expand</button>`)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 }
 
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 func writeshowHideScrapeJobButtons(qq422016 qtio422016.Writer, num int) {
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	streamshowHideScrapeJobButtons(qw422016, num)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 }
 
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 func showHideScrapeJobButtons(num int) string {
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	writeshowHideScrapeJobButtons(qb422016, num)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:354
+//line lib/promscrape/targetstatus.qtpl:364
 }
 
-//line lib/promscrape/targetstatus.qtpl:356
+//line lib/promscrape/targetstatus.qtpl:366
 func streamqueryArgs(qw422016 *qt422016.Writer, filter *requestFilter, override map[string]string) {
-//line lib/promscrape/targetstatus.qtpl:358
+//line lib/promscrape/targetstatus.qtpl:368
 	showOnlyUnhealthy := "false"
 	if filter.showOnlyUnhealthy {
 		showOnlyUnhealthy = "true"
@@ -980,126 +994,126 @@ func streamqueryArgs(qw422016 *qt422016.Writer, filter *requestFilter, override
 		qa[k] = []string{v}
 	}
 
-//line lib/promscrape/targetstatus.qtpl:375
+//line lib/promscrape/targetstatus.qtpl:385
 	qw422016.E().S(qa.Encode())
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 }
 
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 func writequeryArgs(qq422016 qtio422016.Writer, filter *requestFilter, override map[string]string) {
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	streamqueryArgs(qw422016, filter, override)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 }
 
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 func queryArgs(filter *requestFilter, override map[string]string) string {
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	writequeryArgs(qb422016, filter, override)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:376
+//line lib/promscrape/targetstatus.qtpl:386
 }
 
-//line lib/promscrape/targetstatus.qtpl:378
+//line lib/promscrape/targetstatus.qtpl:388
 func streamformatLabels(qw422016 *qt422016.Writer, labels *promutils.Labels) {
-//line lib/promscrape/targetstatus.qtpl:379
+//line lib/promscrape/targetstatus.qtpl:389
 	labelsList := labels.GetLabels()
 
-//line lib/promscrape/targetstatus.qtpl:379
+//line lib/promscrape/targetstatus.qtpl:389
 	qw422016.N().S(`{`)
-//line lib/promscrape/targetstatus.qtpl:381
+//line lib/promscrape/targetstatus.qtpl:391
 	for i, label := range labelsList {
-//line lib/promscrape/targetstatus.qtpl:382
+//line lib/promscrape/targetstatus.qtpl:392
 		qw422016.E().S(label.Name)
-//line lib/promscrape/targetstatus.qtpl:382
+//line lib/promscrape/targetstatus.qtpl:392
 		qw422016.N().S(`=`)
-//line lib/promscrape/targetstatus.qtpl:382
+//line lib/promscrape/targetstatus.qtpl:392
 		qw422016.E().Q(label.Value)
-//line lib/promscrape/targetstatus.qtpl:383
+//line lib/promscrape/targetstatus.qtpl:393
 		if i+1 < len(labelsList) {
-//line lib/promscrape/targetstatus.qtpl:383
+//line lib/promscrape/targetstatus.qtpl:393
 			qw422016.N().S(`,`)
-//line lib/promscrape/targetstatus.qtpl:383
+//line lib/promscrape/targetstatus.qtpl:393
 			qw422016.N().S(` `)
-//line lib/promscrape/targetstatus.qtpl:383
+//line lib/promscrape/targetstatus.qtpl:393
 		}
-//line lib/promscrape/targetstatus.qtpl:384
+//line lib/promscrape/targetstatus.qtpl:394
 	}
-//line lib/promscrape/targetstatus.qtpl:384
+//line lib/promscrape/targetstatus.qtpl:394
 	qw422016.N().S(`}`)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 }
 
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 func writeformatLabels(qq422016 qtio422016.Writer, labels *promutils.Labels) {
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	streamformatLabels(qw422016, labels)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 }
 
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 func formatLabels(labels *promutils.Labels) string {
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	writeformatLabels(qb422016, labels)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:386
+//line lib/promscrape/targetstatus.qtpl:396
 }
 
-//line lib/promscrape/targetstatus.qtpl:388
+//line lib/promscrape/targetstatus.qtpl:398
 func streamerrorNotification(qw422016 *qt422016.Writer, err error) {
-//line lib/promscrape/targetstatus.qtpl:388
+//line lib/promscrape/targetstatus.qtpl:398
 	qw422016.N().S(`<div class="alert alert-danger d-flex align-items-center" role="alert"><svg class="bi flex-shrink-0 me-2" width="24" height="24" role="img" aria-label="Danger:"><use xlink:href="#exclamation-triangle-fill"/></svg><div>`)
-//line lib/promscrape/targetstatus.qtpl:393
+//line lib/promscrape/targetstatus.qtpl:403
 	qw422016.E().S(err.Error())
-//line lib/promscrape/targetstatus.qtpl:393
+//line lib/promscrape/targetstatus.qtpl:403
 	qw422016.N().S(`</div></div>`)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 }
 
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 func writeerrorNotification(qq422016 qtio422016.Writer, err error) {
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	streamerrorNotification(qw422016, err)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 }
 
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 func errorNotification(err error) string {
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	writeerrorNotification(qb422016, err)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 	return qs422016
-//line lib/promscrape/targetstatus.qtpl:396
+//line lib/promscrape/targetstatus.qtpl:406
 }

lib/promutils/labels.go

@@ -35,7 +35,7 @@ func NewLabelsFromMap(m map[string]string) *Labels {
 
 // MarshalYAML implements yaml.Marshaler interface.
 func (x *Labels) MarshalYAML() (interface{}, error) {
-	m := x.toMap()
+	m := x.ToMap()
 	return m, nil
 }
 
@@ -51,7 +51,7 @@ func (x *Labels) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 // MarshalJSON returns JSON respresentation for x.
 func (x *Labels) MarshalJSON() ([]byte, error) {
-	m := x.toMap()
+	m := x.ToMap()
 	return json.Marshal(m)
 }
 
@@ -74,7 +74,8 @@ func (x *Labels) InitFromMap(m map[string]string) {
 	x.Sort()
 }
 
-func (x *Labels) toMap() map[string]string {
+// ToMap returns a map for the given labels x.
+func (x *Labels) ToMap() map[string]string {
 	labels := x.GetLabels()
 	m := make(map[string]string, len(labels))
 	for _, label := range labels {
@@ -293,10 +294,21 @@ func PutLabels(x *Labels) {
 
 var labelsPool sync.Pool
 
+// MustNewLabelsFromString creates labels from s, which can have the form `metric{labels}`.
+//
+// This function must be used only in tests. Use NewLabelsFromString in production code.
+func MustNewLabelsFromString(metricWithLabels string) *Labels {
+	labels, err := NewLabelsFromString(metricWithLabels)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse %q: %s", metricWithLabels, err)
+	}
+	return labels
+}
+
 // NewLabelsFromString creates labels from s, which can have the form `metric{labels}`.
 //
 // This function must be used only in tests
-func NewLabelsFromString(metricWithLabels string) *Labels {
+func NewLabelsFromString(metricWithLabels string) (*Labels, error) {
 	stripDummyMetric := false
 	if strings.HasPrefix(metricWithLabels, "{") {
 		// Add a dummy metric name, since the parser needs it
@@ -311,10 +323,10 @@ func NewLabelsFromString(metricWithLabels string) *Labels {
 		err = fmt.Errorf("error during metric parse: %s", s)
 	})
 	if err != nil {
-		logger.Panicf("BUG: cannot parse %q: %s", metricWithLabels, err)
+		return nil, err
 	}
 	if len(rows.Rows) != 1 {
-		logger.Panicf("BUG: unexpected number of rows parsed; got %d; want 1", len(rows.Rows))
+		return nil, fmt.Errorf("unexpected number of rows parsed; got %d; want 1", len(rows.Rows))
 	}
 	r := rows.Rows[0]
 	var x Labels
@@ -324,5 +336,5 @@ func NewLabelsFromString(metricWithLabels string) *Labels {
 	for _, tag := range r.Tags {
 		x.Add(tag.Key, tag.Value)
 	}
-	return &x
+	return &x, nil
 }

lib/promutils/labels_test.go

@@ -147,7 +147,7 @@ func TestLabelsAddFrom(t *testing.T) {
 func TestLabelsRemoveMetaLabels(t *testing.T) {
 	f := func(metric, resultExpected string) {
 		t.Helper()
-		labels := NewLabelsFromString(metric)
+		labels := MustNewLabelsFromString(metric)
 		labels.RemoveMetaLabels()
 		result := labels.String()
 		if result != resultExpected {
@@ -163,7 +163,7 @@ func TestLabelsRemoveMetaLabels(t *testing.T) {
 func TestLabelsRemoveLabelsWithDoubleUnderscorePrefix(t *testing.T) {
 	f := func(metric, resultExpected string) {
 		t.Helper()
-		labels := NewLabelsFromString(metric)
+		labels := MustNewLabelsFromString(metric)
 		labels.RemoveLabelsWithDoubleUnderscorePrefix()
 		result := labels.String()
 		if result != resultExpected {