lib/promscrape/discovery/kubernetes: add common labels to all ports discovered from endpoints (#4235)

* lib/promscrape/discovery/kubernetes: add common labels to all ports discovered from endpoints

Sets
`__meta_kubernetes_endpoints_name` and `__meta_kubernetes_namespace` labels to all ports of pod.
Prometheus sets those labels to all ports in pod (0ab9553611/discovery/kubernetes/endpoints.go (L267C15-L269)) even if port is not matching any service.

See: #4154

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

* lib/promscrape/discovery/kubernetes: fix test for updated discovery logic

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

---------

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

docs/CHANGELOG.md

@@ -22,6 +22,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): prevent from possible panic when the number of vmstorage nodes increases when [automatic vmstorage discovery](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery) is enabled.
 * BUGFIX: properly limit the number of [OpenTSDB HTTP](https://docs.victoriametrics.com/#sending-opentsdb-data-via-http-apiput-requests) concurrent requests specified via `-maxConcurrentInserts` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4204). Thanks to @zouxiang1993 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4208).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` and `/api/v1/alerts` API handlers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_endpoints_name` label for all ports discovered from endpoint. Previously, ports not matched by `Service` did not have this label. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4154) for details.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).
 
 ## [v1.87.5](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.5)

lib/promscrape/discovery/kubernetes/endpoints.go

@@ -114,6 +114,10 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 		logger.Warnf(`the number of targets for "role: endpoints" %q exceeds 1000 and will be truncated in the next k8s releases; please use "role: endpointslice" instead`, eps.Metadata.key())
 	}
 
+	// Prometheus sets endpoints_name and namespace labels for all endpoints
+	// Even if port is not matching service port.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4154
+	commonEpLabels := eps.getCommonLabels()
 	// Append labels for skipped ports on seen pods.
 	portSeen := func(port int, ports []int) bool {
 		for _, p := range ports {
@@ -134,6 +138,7 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 				m.Add("__address__", addr)
 				p.appendCommonLabels(m, gw)
 				p.appendContainerLabels(m, c, &cp)
+				m.AddFrom(commonEpLabels)
 				if svc != nil {
 					svc.appendCommonLabels(m)
 				}
@@ -145,6 +150,13 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 	return ms
 }
 
+func (eps *Endpoints) getCommonLabels() *promutils.Labels {
+	m := promutils.GetLabels()
+	m.Add("__meta_kubernetes_namespace", eps.Metadata.Namespace)
+	m.Add("__meta_kubernetes_endpoints_name", eps.Metadata.Name)
+	return m
+}
+
 func appendEndpointLabelsForAddresses(ms []*promutils.Labels, gw *groupWatcher, podPortsSeen map[*Pod][]int, eps *Endpoints,
 	eas []EndpointAddress, epp EndpointPort, svc *Service, ready string) []*promutils.Labels {
 	for _, ea := range eas {

lib/promscrape/discovery/kubernetes/endpoints_test.go

@@ -293,6 +293,7 @@ func TestGetEndpointsLabels(t *testing.T) {
 			}),
 			promutils.NewLabelsFromMap(map[string]string{
 				"__address__":                                    "192.168.15.1:8428",
+				"__meta_kubernetes_endpoints_name":               "test-eps",
 				"__meta_kubernetes_namespace":                    "default",
 				"__meta_kubernetes_node_label_node_label":        "xyz",
 				"__meta_kubernetes_node_labelpresent_node_label": "true",