lib/promscrape: allow overriding external_labels as Prometheus does

Prometheus docs at https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config say:

> In communication with external systems, they are always applied only
> when a time series does not have a given label yet and are ignored otherwise.

Though this may result in consistency chaos when scrape targets override `external_labels`,
let's stick with Prometheus behavior for the sake of backwards compatibility.

There is last resort in vmagent with `-remoteWrite.label`, which consistently
sets the configured labels to all the metrics before sending them to remote storage.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/366

lib/promscrape/config.go

@@ -509,39 +509,26 @@ func getParamsFromLabels(labels []prompbmarshal.Label, paramsOrig map[string][]s
 
 func mergeLabels(job, scheme, target, metricsPath string, labels, externalLabels, metaLabels map[string]string, params map[string][]string) ([]prompbmarshal.Label, error) {
 	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
-	m := map[string]string{
+	m := make(map[string]string)
-		"job":              job,
-		"__address__":      target,
-		"__scheme__":       scheme,
-		"__metrics_path__": metricsPath,
-	}
 	for k, v := range externalLabels {
-		if vOrig, ok := m[k]; ok {
-			return nil, fmt.Errorf("external label `%q: %q` clashes with the previously set label with value %q", k, v, vOrig)
-		}
-		m[k] = v
-	}
-	for k, v := range metaLabels {
-		if vOrig, ok := m[k]; ok {
-			return nil, fmt.Errorf("meta label `%q: %q` clashes with the previously set label with value %q", k, v, vOrig)
-		}
-		m[k] = v
-	}
-	for k, v := range labels {
-		if vOrig, ok := m[k]; ok {
-			return nil, fmt.Errorf("label `%q: %q` clashes with the previously set label with value %q", k, v, vOrig)
-		}
 		m[k] = v
 	}
+	m["job"] = job
+	m["__address__"] = target
+	m["__scheme__"] = scheme
+	m["__metrics_path__"] = metricsPath
 	for k, args := range params {
 		if len(args) == 0 {
 			continue
 		}
 		k = "__param_" + k
 		v := args[0]
-		if vOrig, ok := m[k]; ok {
+		m[k] = v
-			return nil, fmt.Errorf("param `%q: %q` claches with the previously set label with value %q", k, v, vOrig)
+	}
-		}
+	for k, v := range labels {
+		m[k] = v
+	}
+	for k, v := range metaLabels {
 		m[k] = v
 	}
 	result := make([]prompbmarshal.Label, 0, len(m))

lib/promscrape/config_test.go

@@ -270,43 +270,6 @@ scrape_configs:
   - targets: ["a"]
 `)
 
-	// Clash of external_label with job or instance
-	f(`
-global:
-  external_labels:
-    job: foobar
-scrape_configs:
-- job_name: aaa
-  static_configs:
-  - targets: ["a"]
-`)
-
-	// Clash of external_label with static_configs label
-	f(`
-global:
-  external_labels:
-    xxx: foobar
-scrape_configs:
-- job_name: aaa
-  static_configs:
-  - targets: ["a"]
-    labels:
-      xxx: yyy
-`)
-
-	// Clash of param with external_labels
-	f(`
-global:
-  external_labels:
-    __param_xxx: foobar
-scrape_configs:
-- job_name: aaa
-  params:
-    xxx: [abcd]
-  static_configs:
-  - targets: ["a"]
-`)
-
 	// non-existing ca_file
 	f(`
 scrape_configs:
@@ -1091,6 +1054,62 @@ scrape_configs:
 		},
 	})
 	f(`
+global:
+  external_labels:
+    job: foobar
+    foo: xx
+    q: qwe
+    __address__: aaasdf
+    __param_a: jlfd
+scrape_configs:
+- job_name: aaa
+  params:
+    a: [b, xy]
+  static_configs:
+  - targets: ["a"]
+    labels:
+      foo: bar
+      __param_a: c
+      __address__: pp
+      job: yyy
+`, []ScrapeWork{
+		{
+			ScrapeURL:      "http://pp:80/metrics?a=c&a=xy",
+			ScrapeInterval: defaultScrapeInterval,
+			ScrapeTimeout:  defaultScrapeTimeout,
+			Labels: []prompbmarshal.Label{
+				{
+					Name:  "__address__",
+					Value: "pp",
+				},
+				{
+					Name:  "__metrics_path__",
+					Value: "/metrics",
+				},
+				{
+					Name:  "__param_a",
+					Value: "c",
+				},
+				{
+					Name:  "__scheme__",
+					Value: "http",
+				},
+				{
+					Name:  "foo",
+					Value: "bar",
+				},
+				{
+					Name:  "job",
+					Value: "yyy",
+				},
+				{
+					Name:  "q",
+					Value: "qwe",
+				},
+			},
+		},
+	})
+	f(`
 scrape_configs:
   - job_name: 'snmp'
     static_configs: