From b186b63e0768c88e3bba520982d644d740c7b0a9 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Wed, 6 Jul 2022 23:18:55 +0300
Subject: [PATCH] lib/promscrape/discovery/kubernetes: allow attaching
 node-level labels to `role: endpoints` and `role: endpointlice` targets in
 the same way as Prometheus does

See https://github.com/prometheus/prometheus/pull/10759
---
 docs/CHANGELOG.md                             |   1 +
 .../discovery/kubernetes/api_watcher.go       |   6 +-
 .../discovery/kubernetes/endpoints_test.go    | 396 ++++++++++--------
 3 files changed, 226 insertions(+), 177 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ab488bb0a2..b32de5d253 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -23,6 +23,7 @@ If you use alerting rules or Grafana dashboards, which rely on this metric, then
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add an UI for [query tracing](https://docs.victoriametrics.com/#query-tracing). It can be enabled by clicking `enable query tracing` checkbox and re-running the query. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2703).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.headers` command-line option for specifying optional HTTP headers to send to the configured `-remoteWrite.url`. For example, `-remoteWrite.headers='Foo:Bar^^Baz:x'` would send `Foo: Bar` and `Baz: x` HTTP headers with every request to `-remoteWrite.url`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2805).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): push per-target `scrape_samples_limit` metric to the cofigured `-remoteWrite.url` if `sample_limit` option is set for this target in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). See [this feature request](https://github.com/VictoriaMetrics/operator/issues/497).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): attach node-level labels to [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) targets if `attach_metadata: {"node": true}` is set for `role: endpoints` and `role: endpointslice`. This is a feature backport from Prometheus 2.37 - see [this pull request](https://github.com/prometheus/prometheus/pull/10759).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`:
 
 ```yaml
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index 63a266afb7..a4afcd200a 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -345,7 +345,7 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 		gw.startWatchersForRole("pod", nil)
 		gw.startWatchersForRole("service", nil)
 	}
-	if gw.attachNodeMetadata && role == "pod" {
+	if gw.attachNodeMetadata && (role == "pod" || role == "endpoints" || role == "endpointslice") {
 		gw.startWatchersForRole("node", nil)
 	}
 	paths := getAPIPathsWithNamespaces(role, gw.namespaces, gw.selectors)
@@ -803,8 +803,8 @@ func (uw *urlWatcher) maybeUpdateDependedScrapeWorksLocked() {
 			uwx.needRecreateScrapeWorks = true
 			continue
 		}
-		if attachNodeMetadata && role == "node" && uwx.role == "pod" {
-			// pod objects depend on node objects if attachNodeMetadata is set
+		if attachNodeMetadata && role == "node" && (uwx.role == "pod" || uwx.role == "endpoints" || uwx.role == "endpointslice") {
+			// pod, endpoints and enpointslices objects depend on node objects if attachNodeMetadata is set
 			uwx.needRecreateScrapeWorks = true
 			continue
 		}
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index 845d920015..7f13fa75f8 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -119,197 +119,245 @@ func TestGetEndpointLabels(t *testing.T) {
 		containerPorts map[string][]ContainerPort
 		endpointPorts  []EndpointPort
 	}
-	f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) {
-		t.Run(name, func(t *testing.T) {
-			eps := Endpoints{
-				Metadata: ObjectMeta{
-					Name:      "test-eps",
-					Namespace: "default",
-				},
-				Subsets: []EndpointSubset{
-					{
-						Ports: args.endpointPorts,
-						Addresses: []EndpointAddress{
-							{
-								IP: "10.13.15.15",
-								TargetRef: ObjectReference{
-									Kind:      "Pod",
-									Namespace: "default",
-									Name:      "test-pod",
-								},
+	f := func(t *testing.T, args testArgs, wantLabels [][]prompbmarshal.Label) {
+		t.Helper()
+		eps := Endpoints{
+			Metadata: ObjectMeta{
+				Name:      "test-eps",
+				Namespace: "default",
+			},
+			Subsets: []EndpointSubset{
+				{
+					Ports: args.endpointPorts,
+					Addresses: []EndpointAddress{
+						{
+							IP: "10.13.15.15",
+							TargetRef: ObjectReference{
+								Kind:      "Pod",
+								Namespace: "default",
+								Name:      "test-pod",
 							},
 						},
 					},
 				},
-			}
-			svc := Service{
-				Metadata: ObjectMeta{
-					Name:      "test-eps",
-					Namespace: "default",
-				},
-				Spec: ServiceSpec{
-					Ports: []ServicePort{
-						{
-							Name: "test-port",
-							Port: 8081,
-						},
+			},
+		}
+		svc := Service{
+			Metadata: ObjectMeta{
+				Name:      "test-eps",
+				Namespace: "default",
+			},
+			Spec: ServiceSpec{
+				ClusterIP: "1.2.3.4",
+				Type:      "service-type",
+				Ports: []ServicePort{
+					{
+						Name: "test-port",
+						Port: 8081,
 					},
 				},
-			}
-			pod := Pod{
-				Metadata: ObjectMeta{
-					Name:      "test-pod",
-					Namespace: "default",
-				},
-				Status: PodStatus{PodIP: "192.168.15.1"},
-			}
-			for cn, ports := range args.containerPorts {
-				pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
-			}
-			var gw groupWatcher
-			gw.m = map[string]*urlWatcher{
-				"pod": {
-					role: "pod",
-					objectsByKey: map[string]object{
-						"default/test-pod": &pod,
+			},
+		}
+		pod := Pod{
+			Metadata: ObjectMeta{
+				UID:       "pod-uid",
+				Name:      "test-pod",
+				Namespace: "default",
+			},
+			Spec: PodSpec{
+				NodeName: "test-node",
+			},
+			Status: PodStatus{
+				Phase:  "abc",
+				PodIP:  "192.168.15.1",
+				HostIP: "4.5.6.7",
+			},
+		}
+		node := Node{
+			Metadata: ObjectMeta{
+				Labels: []prompbmarshal.Label{
+					{
+						Name:  "node-label",
+						Value: "xyz",
 					},
 				},
-				"service": {
-					role: "service",
-					objectsByKey: map[string]object{
-						"default/test-eps": &svc,
-					},
+			},
+		}
+		for cn, ports := range args.containerPorts {
+			pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
+		}
+		var gw groupWatcher
+		gw.m = map[string]*urlWatcher{
+			"pod": {
+				role: "pod",
+				objectsByKey: map[string]object{
+					"default/test-pod": &pod,
 				},
-			}
-			var sortedLabelss [][]prompbmarshal.Label
-			gotLabels := eps.getTargetLabels(&gw)
-			for _, lbs := range gotLabels {
-				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
-			}
-			if !areEqualLabelss(sortedLabelss, wantLabels) {
-				t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
-
-			}
-		})
+			},
+			"service": {
+				role: "service",
+				objectsByKey: map[string]object{
+					"default/test-eps": &svc,
+				},
+			},
+			"node": {
+				role: "node",
+				objectsByKey: map[string]object{
+					"/test-node": &node,
+				},
+			},
+		}
+		gw.attachNodeMetadata = true
+		var sortedLabelss [][]prompbmarshal.Label
+		gotLabels := eps.getTargetLabels(&gw)
+		for _, lbs := range gotLabels {
+			sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
+		}
+		if !areEqualLabelss(sortedLabelss, wantLabels) {
+			t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
+		}
 	}
 
-	f("1 port from endpoint", testArgs{
-		endpointPorts: []EndpointPort{
-			{
-				Name: "web",
-				Port: 8081,
+	t.Run("1 port from endpoint", func(t *testing.T) {
+		f(t, testArgs{
+			endpointPorts: []EndpointPort{
+				{
+					Name:     "web",
+					Port:     8081,
+					Protocol: "foobar",
+				},
 			},
-		},
-	}, [][]prompbmarshal.Label{
-		discoveryutils.GetSortedLabels(map[string]string{
-			"__address__": "10.13.15.15:8081",
-			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
-			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
-			"__meta_kubernetes_endpoint_port_name":           "web",
-			"__meta_kubernetes_endpoint_port_protocol":       "",
-			"__meta_kubernetes_endpoint_ready":               "true",
-			"__meta_kubernetes_endpoints_name":               "test-eps",
-			"__meta_kubernetes_namespace":                    "default",
-			"__meta_kubernetes_pod_host_ip":                  "",
-			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
-			"__meta_kubernetes_pod_name":                     "test-pod",
-			"__meta_kubernetes_pod_node_name":                "",
-			"__meta_kubernetes_pod_phase":                    "",
-			"__meta_kubernetes_pod_ready":                    "unknown",
-			"__meta_kubernetes_pod_uid":                      "",
-			"__meta_kubernetes_service_cluster_ip":           "",
-			"__meta_kubernetes_service_name":                 "test-eps",
-			"__meta_kubernetes_service_type":                 "",
-		}),
+		}, [][]prompbmarshal.Label{
+			discoveryutils.GetSortedLabels(map[string]string{
+				"__address__": "10.13.15.15:8081",
+				"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+				"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+				"__meta_kubernetes_endpoint_port_name":           "web",
+				"__meta_kubernetes_endpoint_port_protocol":       "foobar",
+				"__meta_kubernetes_endpoint_ready":               "true",
+				"__meta_kubernetes_endpoints_name":               "test-eps",
+				"__meta_kubernetes_namespace":                    "default",
+				"__meta_kubernetes_node_label_node_label":        "xyz",
+				"__meta_kubernetes_node_labelpresent_node_label": "true",
+				"__meta_kubernetes_node_name":                    "test-node",
+				"__meta_kubernetes_pod_host_ip":                  "4.5.6.7",
+				"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+				"__meta_kubernetes_pod_name":                     "test-pod",
+				"__meta_kubernetes_pod_node_name":                "test-node",
+				"__meta_kubernetes_pod_phase":                    "abc",
+				"__meta_kubernetes_pod_ready":                    "unknown",
+				"__meta_kubernetes_pod_uid":                      "pod-uid",
+				"__meta_kubernetes_service_cluster_ip":           "1.2.3.4",
+				"__meta_kubernetes_service_name":                 "test-eps",
+				"__meta_kubernetes_service_type":                 "service-type",
+			}),
+		})
 	})
 
-	f("1 port from endpoint and 1 from pod", testArgs{
-		containerPorts: map[string][]ContainerPort{"metrics": {{
-			Name:          "http-metrics",
-			ContainerPort: 8428,
-		}}},
-		endpointPorts: []EndpointPort{
-			{
-				Name: "web",
-				Port: 8081,
+	t.Run("1 port from endpoint and 1 from pod", func(t *testing.T) {
+		f(t, testArgs{
+			containerPorts: map[string][]ContainerPort{"metrics": {{
+				Name:          "http-metrics",
+				ContainerPort: 8428,
+				Protocol:      "foobar",
+			}}},
+			endpointPorts: []EndpointPort{
+				{
+					Name:     "web",
+					Port:     8081,
+					Protocol: "https",
+				},
 			},
-		},
-	}, [][]prompbmarshal.Label{
-		discoveryutils.GetSortedLabels(map[string]string{
-			"__address__": "10.13.15.15:8081",
-			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
-			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
-			"__meta_kubernetes_endpoint_port_name":           "web",
-			"__meta_kubernetes_endpoint_port_protocol":       "",
-			"__meta_kubernetes_endpoint_ready":               "true",
-			"__meta_kubernetes_endpoints_name":               "test-eps",
-			"__meta_kubernetes_namespace":                    "default",
-			"__meta_kubernetes_pod_host_ip":                  "",
-			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
-			"__meta_kubernetes_pod_name":                     "test-pod",
-			"__meta_kubernetes_pod_node_name":                "",
-			"__meta_kubernetes_pod_phase":                    "",
-			"__meta_kubernetes_pod_ready":                    "unknown",
-			"__meta_kubernetes_pod_uid":                      "",
-			"__meta_kubernetes_service_cluster_ip":           "",
-			"__meta_kubernetes_service_name":                 "test-eps",
-			"__meta_kubernetes_service_type":                 "",
-		}),
-		discoveryutils.GetSortedLabels(map[string]string{
-			"__address__":                                   "192.168.15.1:8428",
-			"__meta_kubernetes_namespace":                   "default",
-			"__meta_kubernetes_pod_container_name":          "metrics",
-			"__meta_kubernetes_pod_container_port_name":     "http-metrics",
-			"__meta_kubernetes_pod_container_port_number":   "8428",
-			"__meta_kubernetes_pod_container_port_protocol": "",
-			"__meta_kubernetes_pod_host_ip":                 "",
-			"__meta_kubernetes_pod_ip":                      "192.168.15.1",
-			"__meta_kubernetes_pod_name":                    "test-pod",
-			"__meta_kubernetes_pod_node_name":               "",
-			"__meta_kubernetes_pod_phase":                   "",
-			"__meta_kubernetes_pod_ready":                   "unknown",
-			"__meta_kubernetes_pod_uid":                     "",
-			"__meta_kubernetes_service_cluster_ip":          "",
-			"__meta_kubernetes_service_name":                "test-eps",
-			"__meta_kubernetes_service_type":                "",
-		}),
+		}, [][]prompbmarshal.Label{
+			discoveryutils.GetSortedLabels(map[string]string{
+				"__address__": "10.13.15.15:8081",
+				"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+				"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+				"__meta_kubernetes_endpoint_port_name":           "web",
+				"__meta_kubernetes_endpoint_port_protocol":       "https",
+				"__meta_kubernetes_endpoint_ready":               "true",
+				"__meta_kubernetes_endpoints_name":               "test-eps",
+				"__meta_kubernetes_namespace":                    "default",
+				"__meta_kubernetes_node_label_node_label":        "xyz",
+				"__meta_kubernetes_node_labelpresent_node_label": "true",
+				"__meta_kubernetes_node_name":                    "test-node",
+				"__meta_kubernetes_pod_host_ip":                  "4.5.6.7",
+				"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+				"__meta_kubernetes_pod_name":                     "test-pod",
+				"__meta_kubernetes_pod_node_name":                "test-node",
+				"__meta_kubernetes_pod_phase":                    "abc",
+				"__meta_kubernetes_pod_ready":                    "unknown",
+				"__meta_kubernetes_pod_uid":                      "pod-uid",
+				"__meta_kubernetes_service_cluster_ip":           "1.2.3.4",
+				"__meta_kubernetes_service_name":                 "test-eps",
+				"__meta_kubernetes_service_type":                 "service-type",
+			}),
+			discoveryutils.GetSortedLabels(map[string]string{
+				"__address__":                                    "192.168.15.1:8428",
+				"__meta_kubernetes_namespace":                    "default",
+				"__meta_kubernetes_node_label_node_label":        "xyz",
+				"__meta_kubernetes_node_labelpresent_node_label": "true",
+				"__meta_kubernetes_node_name":                    "test-node",
+				"__meta_kubernetes_pod_container_name":           "metrics",
+				"__meta_kubernetes_pod_container_port_name":      "http-metrics",
+				"__meta_kubernetes_pod_container_port_number":    "8428",
+				"__meta_kubernetes_pod_container_port_protocol":  "foobar",
+				"__meta_kubernetes_pod_host_ip":                  "4.5.6.7",
+				"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+				"__meta_kubernetes_pod_name":                     "test-pod",
+				"__meta_kubernetes_pod_node_name":                "test-node",
+				"__meta_kubernetes_pod_phase":                    "abc",
+				"__meta_kubernetes_pod_ready":                    "unknown",
+				"__meta_kubernetes_pod_uid":                      "pod-uid",
+				"__meta_kubernetes_service_cluster_ip":           "1.2.3.4",
+				"__meta_kubernetes_service_name":                 "test-eps",
+				"__meta_kubernetes_service_type":                 "service-type",
+			}),
+		})
 	})
 
-	f("1 port from endpoint", testArgs{
-		containerPorts: map[string][]ContainerPort{"metrics": {{
-			Name:          "web",
-			ContainerPort: 8428,
-		}}},
-		endpointPorts: []EndpointPort{
-			{
-				Name: "web",
-				Port: 8428,
+	t.Run("1 port from endpoint", func(t *testing.T) {
+		f(t, testArgs{
+			containerPorts: map[string][]ContainerPort{"metrics": {{
+				Name:          "web",
+				ContainerPort: 8428,
+				Protocol:      "sdc",
+			}}},
+			endpointPorts: []EndpointPort{
+				{
+					Name:     "web",
+					Port:     8428,
+					Protocol: "xabc",
+				},
 			},
-		},
-	}, [][]prompbmarshal.Label{
-		discoveryutils.GetSortedLabels(map[string]string{
-			"__address__": "10.13.15.15:8428",
-			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
-			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
-			"__meta_kubernetes_endpoint_port_name":           "web",
-			"__meta_kubernetes_endpoint_port_protocol":       "",
-			"__meta_kubernetes_endpoint_ready":               "true",
-			"__meta_kubernetes_endpoints_name":               "test-eps",
-			"__meta_kubernetes_namespace":                    "default",
-			"__meta_kubernetes_pod_container_name":           "metrics",
-			"__meta_kubernetes_pod_container_port_name":      "web",
-			"__meta_kubernetes_pod_container_port_number":    "8428",
-			"__meta_kubernetes_pod_container_port_protocol":  "",
-			"__meta_kubernetes_pod_host_ip":                  "",
-			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
-			"__meta_kubernetes_pod_name":                     "test-pod",
-			"__meta_kubernetes_pod_node_name":                "",
-			"__meta_kubernetes_pod_phase":                    "",
-			"__meta_kubernetes_pod_ready":                    "unknown",
-			"__meta_kubernetes_pod_uid":                      "",
-			"__meta_kubernetes_service_cluster_ip":           "",
-			"__meta_kubernetes_service_name":                 "test-eps",
-			"__meta_kubernetes_service_type":                 "",
-		}),
+		}, [][]prompbmarshal.Label{
+			discoveryutils.GetSortedLabels(map[string]string{
+				"__address__": "10.13.15.15:8428",
+				"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+				"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+				"__meta_kubernetes_endpoint_port_name":           "web",
+				"__meta_kubernetes_endpoint_port_protocol":       "xabc",
+				"__meta_kubernetes_endpoint_ready":               "true",
+				"__meta_kubernetes_endpoints_name":               "test-eps",
+				"__meta_kubernetes_namespace":                    "default",
+				"__meta_kubernetes_node_label_node_label":        "xyz",
+				"__meta_kubernetes_node_labelpresent_node_label": "true",
+				"__meta_kubernetes_node_name":                    "test-node",
+				"__meta_kubernetes_pod_container_name":           "metrics",
+				"__meta_kubernetes_pod_container_port_name":      "web",
+				"__meta_kubernetes_pod_container_port_number":    "8428",
+				"__meta_kubernetes_pod_container_port_protocol":  "sdc",
+				"__meta_kubernetes_pod_host_ip":                  "4.5.6.7",
+				"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+				"__meta_kubernetes_pod_name":                     "test-pod",
+				"__meta_kubernetes_pod_node_name":                "test-node",
+				"__meta_kubernetes_pod_phase":                    "abc",
+				"__meta_kubernetes_pod_ready":                    "unknown",
+				"__meta_kubernetes_pod_uid":                      "pod-uid",
+				"__meta_kubernetes_service_cluster_ip":           "1.2.3.4",
+				"__meta_kubernetes_service_name":                 "test-eps",
+				"__meta_kubernetes_service_type":                 "service-type",
+			}),
+		})
 	})
 }