From c4cc45d7f850038889377a8105867b8a76bb2229 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 6 Jul 2022 23:18:55 +0300 Subject: [PATCH] lib/promscrape/discovery/kubernetes: allow attaching node-level labels to `role: endpoints` and `role: endpointlice` targets in the same way as Prometheus does See https://github.com/prometheus/prometheus/pull/10759 --- docs/CHANGELOG.md | 1 + .../discovery/kubernetes/api_watcher.go | 6 +- .../discovery/kubernetes/endpoints_test.go | 396 ++++++++++-------- 3 files changed, 226 insertions(+), 177 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ab488bb0a..b32de5d25 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,6 +23,7 @@ If you use alerting rules or Grafana dashboards, which rely on this metric, then * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add an UI for [query tracing](https://docs.victoriametrics.com/#query-tracing). It can be enabled by clicking `enable query tracing` checkbox and re-running the query. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2703). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.headers` command-line option for specifying optional HTTP headers to send to the configured `-remoteWrite.url`. For example, `-remoteWrite.headers='Foo:Bar^^Baz:x'` would send `Foo: Bar` and `Baz: x` HTTP headers with every request to `-remoteWrite.url`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2805). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): push per-target `scrape_samples_limit` metric to the cofigured `-remoteWrite.url` if `sample_limit` option is set for this target in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). See [this feature request](https://github.com/VictoriaMetrics/operator/issues/497). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): attach node-level labels to [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) targets if `attach_metadata: {"node": true}` is set for `role: endpoints` and `role: endpointslice`. This is a feature backport from Prometheus 2.37 - see [this pull request](https://github.com/prometheus/prometheus/pull/10759). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`: ```yaml diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go index 63a266afb..a4afcd200 100644 --- a/lib/promscrape/discovery/kubernetes/api_watcher.go +++ b/lib/promscrape/discovery/kubernetes/api_watcher.go @@ -345,7 +345,7 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) { gw.startWatchersForRole("pod", nil) gw.startWatchersForRole("service", nil) } - if gw.attachNodeMetadata && role == "pod" { + if gw.attachNodeMetadata && (role == "pod" || role == "endpoints" || role == "endpointslice") { gw.startWatchersForRole("node", nil) } paths := getAPIPathsWithNamespaces(role, gw.namespaces, gw.selectors) @@ -803,8 +803,8 @@ func (uw *urlWatcher) maybeUpdateDependedScrapeWorksLocked() { uwx.needRecreateScrapeWorks = true continue } - if attachNodeMetadata && role == "node" && uwx.role == "pod" { - // pod objects depend on node objects if attachNodeMetadata is set + if attachNodeMetadata && role == "node" && (uwx.role == "pod" || uwx.role == "endpoints" || uwx.role == "endpointslice") { + // pod, endpoints and enpointslices objects depend on node objects if attachNodeMetadata is set uwx.needRecreateScrapeWorks = true continue } diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go index 845d92001..7f13fa75f 100644 --- a/lib/promscrape/discovery/kubernetes/endpoints_test.go +++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go @@ -119,197 +119,245 @@ func TestGetEndpointLabels(t *testing.T) { containerPorts map[string][]ContainerPort endpointPorts []EndpointPort } - f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) { - t.Run(name, func(t *testing.T) { - eps := Endpoints{ - Metadata: ObjectMeta{ - Name: "test-eps", - Namespace: "default", - }, - Subsets: []EndpointSubset{ - { - Ports: args.endpointPorts, - Addresses: []EndpointAddress{ - { - IP: "10.13.15.15", - TargetRef: ObjectReference{ - Kind: "Pod", - Namespace: "default", - Name: "test-pod", - }, + f := func(t *testing.T, args testArgs, wantLabels [][]prompbmarshal.Label) { + t.Helper() + eps := Endpoints{ + Metadata: ObjectMeta{ + Name: "test-eps", + Namespace: "default", + }, + Subsets: []EndpointSubset{ + { + Ports: args.endpointPorts, + Addresses: []EndpointAddress{ + { + IP: "10.13.15.15", + TargetRef: ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "test-pod", }, }, }, }, - } - svc := Service{ - Metadata: ObjectMeta{ - Name: "test-eps", - Namespace: "default", - }, - Spec: ServiceSpec{ - Ports: []ServicePort{ - { - Name: "test-port", - Port: 8081, - }, + }, + } + svc := Service{ + Metadata: ObjectMeta{ + Name: "test-eps", + Namespace: "default", + }, + Spec: ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: "service-type", + Ports: []ServicePort{ + { + Name: "test-port", + Port: 8081, }, }, - } - pod := Pod{ - Metadata: ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Status: PodStatus{PodIP: "192.168.15.1"}, - } - for cn, ports := range args.containerPorts { - pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports}) - } - var gw groupWatcher - gw.m = map[string]*urlWatcher{ - "pod": { - role: "pod", - objectsByKey: map[string]object{ - "default/test-pod": &pod, + }, + } + pod := Pod{ + Metadata: ObjectMeta{ + UID: "pod-uid", + Name: "test-pod", + Namespace: "default", + }, + Spec: PodSpec{ + NodeName: "test-node", + }, + Status: PodStatus{ + Phase: "abc", + PodIP: "192.168.15.1", + HostIP: "4.5.6.7", + }, + } + node := Node{ + Metadata: ObjectMeta{ + Labels: []prompbmarshal.Label{ + { + Name: "node-label", + Value: "xyz", }, }, - "service": { - role: "service", - objectsByKey: map[string]object{ - "default/test-eps": &svc, - }, + }, + } + for cn, ports := range args.containerPorts { + pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports}) + } + var gw groupWatcher + gw.m = map[string]*urlWatcher{ + "pod": { + role: "pod", + objectsByKey: map[string]object{ + "default/test-pod": &pod, }, - } - var sortedLabelss [][]prompbmarshal.Label - gotLabels := eps.getTargetLabels(&gw) - for _, lbs := range gotLabels { - sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs)) - } - if !areEqualLabelss(sortedLabelss, wantLabels) { - t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels) - - } - }) + }, + "service": { + role: "service", + objectsByKey: map[string]object{ + "default/test-eps": &svc, + }, + }, + "node": { + role: "node", + objectsByKey: map[string]object{ + "/test-node": &node, + }, + }, + } + gw.attachNodeMetadata = true + var sortedLabelss [][]prompbmarshal.Label + gotLabels := eps.getTargetLabels(&gw) + for _, lbs := range gotLabels { + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs)) + } + if !areEqualLabelss(sortedLabelss, wantLabels) { + t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels) + } } - f("1 port from endpoint", testArgs{ - endpointPorts: []EndpointPort{ - { - Name: "web", - Port: 8081, + t.Run("1 port from endpoint", func(t *testing.T) { + f(t, testArgs{ + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8081, + Protocol: "foobar", + }, }, - }, - }, [][]prompbmarshal.Label{ - discoveryutils.GetSortedLabels(map[string]string{ - "__address__": "10.13.15.15:8081", - "__meta_kubernetes_endpoint_address_target_kind": "Pod", - "__meta_kubernetes_endpoint_address_target_name": "test-pod", - "__meta_kubernetes_endpoint_port_name": "web", - "__meta_kubernetes_endpoint_port_protocol": "", - "__meta_kubernetes_endpoint_ready": "true", - "__meta_kubernetes_endpoints_name": "test-eps", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_pod_host_ip": "", - "__meta_kubernetes_pod_ip": "192.168.15.1", - "__meta_kubernetes_pod_name": "test-pod", - "__meta_kubernetes_pod_node_name": "", - "__meta_kubernetes_pod_phase": "", - "__meta_kubernetes_pod_ready": "unknown", - "__meta_kubernetes_pod_uid": "", - "__meta_kubernetes_service_cluster_ip": "", - "__meta_kubernetes_service_name": "test-eps", - "__meta_kubernetes_service_type": "", - }), + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8081", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "foobar", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_node_label_node_label": "xyz", + "__meta_kubernetes_node_labelpresent_node_label": "true", + "__meta_kubernetes_node_name": "test-node", + "__meta_kubernetes_pod_host_ip": "4.5.6.7", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "test-node", + "__meta_kubernetes_pod_phase": "abc", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "pod-uid", + "__meta_kubernetes_service_cluster_ip": "1.2.3.4", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "service-type", + }), + }) }) - f("1 port from endpoint and 1 from pod", testArgs{ - containerPorts: map[string][]ContainerPort{"metrics": {{ - Name: "http-metrics", - ContainerPort: 8428, - }}}, - endpointPorts: []EndpointPort{ - { - Name: "web", - Port: 8081, + t.Run("1 port from endpoint and 1 from pod", func(t *testing.T) { + f(t, testArgs{ + containerPorts: map[string][]ContainerPort{"metrics": {{ + Name: "http-metrics", + ContainerPort: 8428, + Protocol: "foobar", + }}}, + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8081, + Protocol: "https", + }, }, - }, - }, [][]prompbmarshal.Label{ - discoveryutils.GetSortedLabels(map[string]string{ - "__address__": "10.13.15.15:8081", - "__meta_kubernetes_endpoint_address_target_kind": "Pod", - "__meta_kubernetes_endpoint_address_target_name": "test-pod", - "__meta_kubernetes_endpoint_port_name": "web", - "__meta_kubernetes_endpoint_port_protocol": "", - "__meta_kubernetes_endpoint_ready": "true", - "__meta_kubernetes_endpoints_name": "test-eps", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_pod_host_ip": "", - "__meta_kubernetes_pod_ip": "192.168.15.1", - "__meta_kubernetes_pod_name": "test-pod", - "__meta_kubernetes_pod_node_name": "", - "__meta_kubernetes_pod_phase": "", - "__meta_kubernetes_pod_ready": "unknown", - "__meta_kubernetes_pod_uid": "", - "__meta_kubernetes_service_cluster_ip": "", - "__meta_kubernetes_service_name": "test-eps", - "__meta_kubernetes_service_type": "", - }), - discoveryutils.GetSortedLabels(map[string]string{ - "__address__": "192.168.15.1:8428", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_pod_container_name": "metrics", - "__meta_kubernetes_pod_container_port_name": "http-metrics", - "__meta_kubernetes_pod_container_port_number": "8428", - "__meta_kubernetes_pod_container_port_protocol": "", - "__meta_kubernetes_pod_host_ip": "", - "__meta_kubernetes_pod_ip": "192.168.15.1", - "__meta_kubernetes_pod_name": "test-pod", - "__meta_kubernetes_pod_node_name": "", - "__meta_kubernetes_pod_phase": "", - "__meta_kubernetes_pod_ready": "unknown", - "__meta_kubernetes_pod_uid": "", - "__meta_kubernetes_service_cluster_ip": "", - "__meta_kubernetes_service_name": "test-eps", - "__meta_kubernetes_service_type": "", - }), + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8081", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "https", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_node_label_node_label": "xyz", + "__meta_kubernetes_node_labelpresent_node_label": "true", + "__meta_kubernetes_node_name": "test-node", + "__meta_kubernetes_pod_host_ip": "4.5.6.7", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "test-node", + "__meta_kubernetes_pod_phase": "abc", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "pod-uid", + "__meta_kubernetes_service_cluster_ip": "1.2.3.4", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "service-type", + }), + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "192.168.15.1:8428", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_node_label_node_label": "xyz", + "__meta_kubernetes_node_labelpresent_node_label": "true", + "__meta_kubernetes_node_name": "test-node", + "__meta_kubernetes_pod_container_name": "metrics", + "__meta_kubernetes_pod_container_port_name": "http-metrics", + "__meta_kubernetes_pod_container_port_number": "8428", + "__meta_kubernetes_pod_container_port_protocol": "foobar", + "__meta_kubernetes_pod_host_ip": "4.5.6.7", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "test-node", + "__meta_kubernetes_pod_phase": "abc", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "pod-uid", + "__meta_kubernetes_service_cluster_ip": "1.2.3.4", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "service-type", + }), + }) }) - f("1 port from endpoint", testArgs{ - containerPorts: map[string][]ContainerPort{"metrics": {{ - Name: "web", - ContainerPort: 8428, - }}}, - endpointPorts: []EndpointPort{ - { - Name: "web", - Port: 8428, + t.Run("1 port from endpoint", func(t *testing.T) { + f(t, testArgs{ + containerPorts: map[string][]ContainerPort{"metrics": {{ + Name: "web", + ContainerPort: 8428, + Protocol: "sdc", + }}}, + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8428, + Protocol: "xabc", + }, }, - }, - }, [][]prompbmarshal.Label{ - discoveryutils.GetSortedLabels(map[string]string{ - "__address__": "10.13.15.15:8428", - "__meta_kubernetes_endpoint_address_target_kind": "Pod", - "__meta_kubernetes_endpoint_address_target_name": "test-pod", - "__meta_kubernetes_endpoint_port_name": "web", - "__meta_kubernetes_endpoint_port_protocol": "", - "__meta_kubernetes_endpoint_ready": "true", - "__meta_kubernetes_endpoints_name": "test-eps", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_pod_container_name": "metrics", - "__meta_kubernetes_pod_container_port_name": "web", - "__meta_kubernetes_pod_container_port_number": "8428", - "__meta_kubernetes_pod_container_port_protocol": "", - "__meta_kubernetes_pod_host_ip": "", - "__meta_kubernetes_pod_ip": "192.168.15.1", - "__meta_kubernetes_pod_name": "test-pod", - "__meta_kubernetes_pod_node_name": "", - "__meta_kubernetes_pod_phase": "", - "__meta_kubernetes_pod_ready": "unknown", - "__meta_kubernetes_pod_uid": "", - "__meta_kubernetes_service_cluster_ip": "", - "__meta_kubernetes_service_name": "test-eps", - "__meta_kubernetes_service_type": "", - }), + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8428", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "xabc", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_node_label_node_label": "xyz", + "__meta_kubernetes_node_labelpresent_node_label": "true", + "__meta_kubernetes_node_name": "test-node", + "__meta_kubernetes_pod_container_name": "metrics", + "__meta_kubernetes_pod_container_port_name": "web", + "__meta_kubernetes_pod_container_port_number": "8428", + "__meta_kubernetes_pod_container_port_protocol": "sdc", + "__meta_kubernetes_pod_host_ip": "4.5.6.7", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "test-node", + "__meta_kubernetes_pod_phase": "abc", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "pod-uid", + "__meta_kubernetes_service_cluster_ip": "1.2.3.4", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "service-type", + }), + }) }) }