From ab57b92932604158f1cbe228439e6b2cf1d91467 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 27 Jan 2023 16:33:02 -0800
Subject: [PATCH] lib/promscrape/discovery/kubernetes: add support for
__meta_kubernetes_pod_container_id
See https://github.com/prometheus/prometheus/issues/11843
and https://github.com/prometheus/prometheus/pull/11844
---
docs/CHANGELOG.md | 1 +
docs/sd_configs.md | 3 +-
lib/promscrape/discovery/kubernetes/pod.go | 51 +++++++++++++++----
.../discovery/kubernetes/pod_test.go | 1 +
4 files changed, 46 insertions(+), 10 deletions(-)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7a25885a2c..4889b4a9b1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduce memory usage when sending stale markers for targets, which expose big number of metrics. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3668) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3675) issues.
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow limiting the number of concurrent requests sent to `vmauth` via `-maxConcurrentRequests` command-line flag. This allows controlling memory usage of `vmauth` and the resource usage of backends behind `vmauth`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346). Thanks to @dmitryk-dk for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3486).
* FEATURE: allow using VictoriaMetrics components behind proxies, which communicate with the backend via [proxy protocol](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3335). For example, [vmauth](https://docs.victoriametrics.com/vmauth.html) accepts proxy protocol connections when it starts with `-httpListenAddr.useProxyProtocol` command-line flag.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_pod_container_id` meta-label to the targets discovered via [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). This label has been added in Prometheus starting from `v2.42.0`. See [this feature request](https://github.com/prometheus/prometheus/issues/11843).
* FEATURE: add `-internStringMaxLen` command-line flag, which can be used for fine-tuning RAM vs CPU usage in certain workloads. For example, if the stored time series contain long labels, then it may be useful reducing the `-internStringMaxLen` in order to reduce memory usage at the cost of increased CPU usage. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692).
* BUGFIX: fix a bug, which could prevent background merges for the previous partitions until restart if the storage didn't have enough disk space for final deduplication and down-sampling.
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index 05eab86777..a1f99ae35c 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -788,7 +788,8 @@ One of the following `role` types can be configured to discover targets:
* `__meta_kubernetes_pod_labelpresent_<labelname>`: "true" for each label from the pod object.
* `__meta_kubernetes_pod_annotation_<annotationname>`: Each annotation from the pod object.
* `__meta_kubernetes_pod_annotationpresent_<annotationname>`: "true" for each annotation from the pod object.
- * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer
+ * `__meta_kubernetes_pod_container_id`: ID of the container in the form `<type>://<container_id>`.
+ * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer.
* `__meta_kubernetes_pod_container_image`: Container image the target address points to.
* `__meta_kubernetes_pod_container_name`: Name of the container the target address points to.
* `__meta_kubernetes_pod_container_port_name`: Name of the container port.
diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go
index 77c82f1df9..7256c8b579 100644
--- a/lib/promscrape/discovery/kubernetes/pod.go
+++ b/lib/promscrape/discovery/kubernetes/pod.go
@@ -82,10 +82,12 @@ type ContainerPort struct {
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podstatus-v1-core
type PodStatus struct {
- Phase string
- PodIP string
- HostIP string
- Conditions []PodCondition
+ Phase string
+ PodIP string
+ HostIP string
+ Conditions []PodCondition
+ ContainerStatuses []ContainerStatus
+ InitContainerStatuses []ContainerStatus
}
// PodCondition implements k8s pod condition.
@@ -96,6 +98,27 @@ type PodCondition struct {
Status string
}
+// ContainerStatus implements k8s container status.
+//
+// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core
+type ContainerStatus struct {
+ Name string
+ ContainerID string
+}
+
+func getContainerID(p *Pod, containerName string, isInit bool) string {
+ css := p.Status.ContainerStatuses
+ if isInit {
+ css = p.Status.InitContainerStatuses
+ }
+ for _, cs := range css {
+ if cs.Name == containerName {
+ return cs.ContainerID
+ }
+ }
+ return ""
+}
+
// getTargetLabels returns labels for each port of the given p.
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
@@ -105,12 +128,12 @@ func (p *Pod) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
return nil
}
var ms []*promutils.Labels
- ms = appendPodLabels(ms, gw, p, p.Spec.Containers, "false")
- ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, "true")
+ ms = appendPodLabels(ms, gw, p, p.Spec.Containers, false)
+ ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, true)
return ms
}
-func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit string) []*promutils.Labels {
+func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit bool) []*promutils.Labels {
for _, c := range cs {
for _, cp := range c.Ports {
ms = appendPodLabelsInternal(ms, gw, p, c, &cp, isInit)
@@ -122,14 +145,24 @@ func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Cont
return ms
}
-func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit string) []*promutils.Labels {
+func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit bool) []*promutils.Labels {
addr := p.Status.PodIP
if cp != nil {
addr = discoveryutils.JoinHostPort(addr, cp.ContainerPort)
}
m := promutils.GetLabels()
m.Add("__address__", addr)
- m.Add("__meta_kubernetes_pod_container_init", isInit)
+ isInitStr := "false"
+ if isInit {
+ isInitStr = "true"
+ }
+ m.Add("__meta_kubernetes_pod_container_init", isInitStr)
+
+ containerID := getContainerID(p, c.Name, isInit)
+ if containerID != "" {
+ m.Add("__meta_kubernetes_pod_container_id", containerID)
+ }
+
p.appendCommonLabels(m, gw)
p.appendContainerLabels(m, c, cp)
return append(ms, m)
diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go
index 4311c29e64..2891a79b05 100644
--- a/lib/promscrape/discovery/kubernetes/pod_test.go
+++ b/lib/promscrape/discovery/kubernetes/pod_test.go
@@ -262,6 +262,7 @@ func TestParsePodListSuccess(t *testing.T) {
"__meta_kubernetes_pod_controller_kind": "Node",
"__meta_kubernetes_pod_controller_name": "m01",
"__meta_kubernetes_pod_container_init": "false",
+ "__meta_kubernetes_pod_container_id": "docker://a28f0800855008485376c1eece1cf61de97cb7026b9188d138b0d55d92fc2f5c",
"__meta_kubernetes_pod_label_component": "etcd",
"__meta_kubernetes_pod_label_tier": "control-plane",