From 09f796e2ab819b94512772705d7f0e3f8374c495 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 14 Apr 2020 12:21:10 +0300
Subject: [PATCH] lib/promscrape: remove labels starting with `__meta_` after
 applying `relabel_configs` as Prometheus does

This should reduce CPU load during scraping when target discovery generates
big number of `__meta_*` labels (for instance, k8s discovery).

See https://www.robustperception.io/life-of-a-label for details.
---
 lib/promrelabel/relabel.go      | 14 ++++++++++
 lib/promrelabel/relabel_test.go | 47 +++++++++++++++++++++++++++++++++
 lib/promscrape/config.go        |  6 +++--
 lib/promscrape/config_test.go   | 24 ++++++++---------
 4 files changed, 77 insertions(+), 14 deletions(-)

diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index 960eb69e7..ec2db39b3 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -69,6 +69,20 @@ func removeEmptyLabels(labels []prompbmarshal.Label, labelsOffset int) []prompbm
 	return dst
 }
 
+// RemoveMetaLabels removes all the `__meta_` labels from src and puts the rest of labels to dst.
+//
+// See https://www.robustperception.io/life-of-a-label fo details.
+func RemoveMetaLabels(dst, src []prompbmarshal.Label) []prompbmarshal.Label {
+	for i := range src {
+		label := &src[i]
+		if strings.HasPrefix(label.Name, "__meta_") {
+			continue
+		}
+		dst = append(dst, *label)
+	}
+	return dst
+}
+
 // FinalizeLabels finalizes labels according to relabel_config rules.
 //
 // It renames `__address__` to `instance` and removes labels with "__" in the beginning.
diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go
index 46037aa16..dedbebd99 100644
--- a/lib/promrelabel/relabel_test.go
+++ b/lib/promrelabel/relabel_test.go
@@ -628,3 +628,50 @@ func TestFinalizeLabels(t *testing.T) {
 		},
 	})
 }
+
+func TestRemoveMetaLabels(t *testing.T) {
+	f := func(labels, resultExpected []prompbmarshal.Label) {
+		t.Helper()
+		result := RemoveMetaLabels(nil, labels)
+		if !reflect.DeepEqual(result, resultExpected) {
+			t.Fatalf("unexpected result of RemoveMetaLabels;\ngot\n%v\nwant\n%v", result, resultExpected)
+		}
+	}
+	f(nil, nil)
+	f([]prompbmarshal.Label{
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+	}, []prompbmarshal.Label{
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+	})
+	f([]prompbmarshal.Label{
+		{
+			Name:  "__meta_foo",
+			Value: "bar",
+		},
+	}, nil)
+	f([]prompbmarshal.Label{
+		{
+			Name:  "__meta_foo",
+			Value: "bdffr",
+		},
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+		{
+			Name:  "__meta_xxx",
+			Value: "basd",
+		},
+	}, []prompbmarshal.Label{
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+	})
+}
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index a9950af73..d4117cbe0 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -190,9 +190,9 @@ func (cfg *Config) getFileSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
 	swPrev := make(map[string][]ScrapeWork)
 	for i := range prev {
 		sw := &prev[i]
-		label := promrelabel.GetLabelByName(sw.Labels, "__meta_filepath")
+		label := promrelabel.GetLabelByName(sw.Labels, "__vm_filepath")
 		if label == nil {
-			logger.Panicf("BUG: missing `__meta_filepath` label")
+			logger.Panicf("BUG: missing `__vm_filepath` label")
 		} else {
 			swPrev[label.Value] = append(swPrev[label.Value], *sw)
 		}
@@ -401,6 +401,7 @@ func (sdc *FileSDConfig) appendScrapeWork(dst []ScrapeWork, swPrev map[string][]
 			}
 			metaLabels := map[string]string{
 				"__meta_filepath": pathShort,
+				"__vm_filepath":   pathShort, // This label is needed for internal promscrape logic
 			}
 			for i := range stcs {
 				dst = stcs[i].appendScrapeWork(dst, swc, metaLabels)
@@ -431,6 +432,7 @@ func (stc *StaticConfig) appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfi
 func appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) ([]ScrapeWork, error) {
 	labels := mergeLabels(swc.jobName, swc.scheme, target, swc.metricsPath, extraLabels, swc.externalLabels, metaLabels, swc.params)
 	labels = promrelabel.ApplyRelabelConfigs(labels, 0, swc.relabelConfigs, false)
+	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
 	if len(labels) == 0 {
 		// Drop target without labels.
 		return dst, nil
diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go
index 517b93ec0..b4bef5eb1 100644
--- a/lib/promscrape/config_test.go
+++ b/lib/promscrape/config_test.go
@@ -408,10 +408,6 @@ scrape_configs:
 					Name:  "__address__",
 					Value: "host1",
 				},
-				{
-					Name:  "__meta_filepath",
-					Value: "testdata/file_sd.json",
-				},
 				{
 					Name:  "__metrics_path__",
 					Value: "/abc/de",
@@ -420,6 +416,10 @@ scrape_configs:
 					Name:  "__scheme__",
 					Value: "http",
 				},
+				{
+					Name:  "__vm_filepath",
+					Value: "testdata/file_sd.json",
+				},
 				{
 					Name:  "job",
 					Value: "foo",
@@ -442,10 +442,6 @@ scrape_configs:
 					Name:  "__address__",
 					Value: "host2",
 				},
-				{
-					Name:  "__meta_filepath",
-					Value: "testdata/file_sd.json",
-				},
 				{
 					Name:  "__metrics_path__",
 					Value: "/abc/de",
@@ -454,6 +450,10 @@ scrape_configs:
 					Name:  "__scheme__",
 					Value: "http",
 				},
+				{
+					Name:  "__vm_filepath",
+					Value: "testdata/file_sd.json",
+				},
 				{
 					Name:  "job",
 					Value: "foo",
@@ -476,10 +476,6 @@ scrape_configs:
 					Name:  "__address__",
 					Value: "localhost:9090",
 				},
-				{
-					Name:  "__meta_filepath",
-					Value: "testdata/file_sd_1.yml",
-				},
 				{
 					Name:  "__metrics_path__",
 					Value: "/abc/de",
@@ -488,6 +484,10 @@ scrape_configs:
 					Name:  "__scheme__",
 					Value: "http",
 				},
+				{
+					Name:  "__vm_filepath",
+					Value: "testdata/file_sd_1.yml",
+				},
 				{
 					Name:  "job",
 					Value: "foo",