new feature: debug relabeling (#1344)

* new feature: relabel logging

Use scrape_configs[x].relabel_debug = true to log metric names inkl.
labels before and after relabeling. After relabeling related metrics
get dropped, i.e. not submitted to servers.

* vminsert wants relabel logging, too.

app/vmagent/remotewrite/relabel.go

@@ -18,6 +18,7 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+	relabelDebug =  flag.Bool("remoteWrite.relabelDebug", false, "Show relabel results of via -remoteWrite.relabelConfig specified rules and skip its submission.")
 )
 
 var labelsGlobal []prompbmarshal.Label
@@ -32,6 +33,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
 		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global.RelabelDebug = *relabelDebug
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
@@ -51,6 +53,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}
+		prc.RelabelDebug = *relabelDebug
 		rcs.perURL[i] = prc
 	}
 	return &rcs, nil
@@ -100,7 +103,11 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 				labels = append(labels, *extraLabel)
 			}
 		}
-		labels = pcs.Apply(labels, labelsLen, true)
+		labels = pcs.Apply(labels, labelsLen, true, pcs.RelabelDebug)
+		if (pcs.RelabelDebug) {
+			// simulate "all labels dropped" to avoid submission
+			labels = labels[:labelsLen]
+		}
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue

app/vminsert/relabel/relabel.go

@@ -101,7 +101,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 	}
 
 	// Apply relabeling
-	tmpLabels = pcs.Apply(tmpLabels, 0, true)
+	tmpLabels = pcs.Apply(tmpLabels, 0, true, pcs.RelabelDebug)
 	ctx.tmpLabels = tmpLabels
 	if len(tmpLabels) == 0 {
 		metricsDropped.Inc()

docs/vmagent.md

@@ -225,6 +225,7 @@ The relabeling can be defined in the following places:
 
 * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels.
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`.
+* By setting the `relabel_debug` property of a `scrape_configs` target section in the `-promscrape.config` file to `true`, one is able to instruct the vmagent to just log the metric before and after relabeling and skip its submission to servers. This way it is much easier to understand and check scraped relabeled metrics before poisoning servers with it.
 * At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
 

lib/promrelabel/config.go

@@ -26,6 +26,7 @@ type RelabelConfig struct {
 // ParsedConfigs represents parsed relabel configs.
 type ParsedConfigs struct {
 	prcs []*parsedRelabelConfig
+	RelabelDebug bool
 }
 
 // Len returns the number of relabel configs in pcs.

lib/promrelabel/relabel.go

@@ -35,23 +35,65 @@ func (prc *parsedRelabelConfig) String() string {
 		prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.Regex.String(), prc.Modulus, prc.Replacement, prc.Action)
 }
 
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b []byte
+	b = append(b, '{')
+	mname := ""
+	for _, label := range labels {
+		if (label.Name == "__name__") {
+			mname = label.Value
+			continue
+		}
+		b = append(b, label.Name...)
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		b = append(b, ',')
+	}
+	if b[len(b)-1] == ',' {
+		b[len(b)-1] = '}'
+		return mname + string(b)
+	}
+	return mname
+}
+
 // Apply applies pcs to labels starting from the labelsOffset.
 //
 // If isFinalize is set, then FinalizeLabels is called on the labels[labelsOffset:].
 //
 // The returned labels at labels[labelsOffset:] are sorted.
-func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool) []prompbmarshal.Label {
+func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool, relabelDebug bool) []prompbmarshal.Label {
+	var inStr, outStr string
 	if pcs != nil {
+		if relabelDebug {
+			inStr = labelsToString(labels[labelsOffset:])
+		}
 		for _, prc := range pcs.prcs {
 			tmp := prc.apply(labels, labelsOffset)
 			if len(tmp) == labelsOffset {
 				// All the labels have been removed.
+				if relabelDebug {
+					logger.Infof("\nRelabel  In: %s\nRelabel Out: DROPPED - all labels removed.", inStr)
+				}
 				return tmp
 			}
 			labels = tmp
 		}
 	}
 	labels = removeEmptyLabels(labels, labelsOffset)
+	if relabelDebug {
+		outStr = labelsToString(labels[labelsOffset:])
+		if inStr == outStr {
+			logger.Infof("\nRelabel  In: %s\nRelabel Out: KEPT AS IS - no change.", inStr)
+		} else {
+			logger.Infof("\nRelabel  In: %s\nRelabel Out: %s", inStr, outStr)
+		}
+		// lib/promscrape/config.go::getScrapeWork() would drop the entire
+		// target, if this func does not return a single label. So we can't do
+		// a simple 'labels = labels[:labelsOffset]' here.
+		// BTW: Other callees would just drop the related ts, yes, e.g.
+		// app/vmagent/remotewrite/relabel.go::applyRelabeling() and
+		// lib/promscrape/scrapework.go::addRowToTimeseries()
+	}
 	if isFinalize {
 		labels = FinalizeLabels(labels[:labelsOffset], labels[labelsOffset:])
 	}

lib/promscrape/config.go

@@ -104,6 +104,7 @@ type ScrapeConfig struct {
 	ProxyURL             proxy.URL                   `yaml:"proxy_url,omitempty"`
 	RelabelConfigs       []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
 	MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
+	RelabelDebug         bool                        `yaml:"relabel_debug,omitempty"`
 	SampleLimit          int                         `yaml:"sample_limit,omitempty"`
 
 	StaticConfigs        []StaticConfig         `yaml:"static_configs,omitempty"`
@@ -581,6 +582,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
+	metricRelabelConfigs.RelabelDebug = sc.RelabelDebug
 	swc := &scrapeWorkConfig{
 		scrapeInterval:       scrapeInterval,
 		scrapeTimeout:        scrapeTimeout,
@@ -820,7 +822,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		// Reduce memory usage by interning all the strings in originalLabels.
 		internLabelStrings(originalLabels)
 	}
-	labels = swc.relabelConfigs.Apply(labels, 0, false)
+	labels = swc.relabelConfigs.Apply(labels, 0, false, swc.metricRelabelConfigs.RelabelDebug)
 	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
 	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
 	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.

lib/promscrape/scrapework.go

@@ -512,7 +512,10 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim
 	labelsLen := len(wc.labels)
 	wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels)
 	if needRelabel {
-		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true)
+		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true, sw.Config.MetricRelabelConfigs.RelabelDebug)
+		if sw.Config.MetricRelabelConfigs.RelabelDebug {
+			return	// avoid submission
+		}
 	} else {
 		wc.labels = promrelabel.FinalizeLabels(wc.labels[:labelsLen], wc.labels[labelsLen:])
 		promrelabel.SortLabels(wc.labels[labelsLen:])