diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7a25885a2c..4889b4a9b1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduce memory usage when sending stale markers for targets, which expose big number of metrics. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3668) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3675) issues. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow limiting the number of concurrent requests sent to `vmauth` via `-maxConcurrentRequests` command-line flag. This allows controlling memory usage of `vmauth` and the resource usage of backends behind `vmauth`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346). Thanks to @dmitryk-dk for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3486). * FEATURE: allow using VictoriaMetrics components behind proxies, which communicate with the backend via [proxy protocol](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3335). For example, [vmauth](https://docs.victoriametrics.com/vmauth.html) accepts proxy protocol connections when it starts with `-httpListenAddr.useProxyProtocol` command-line flag. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_pod_container_id` meta-label to the targets discovered via [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). This label has been added in Prometheus starting from `v2.42.0`. See [this feature request](https://github.com/prometheus/prometheus/issues/11843). * FEATURE: add `-internStringMaxLen` command-line flag, which can be used for fine-tuning RAM vs CPU usage in certain workloads. For example, if the stored time series contain long labels, then it may be useful reducing the `-internStringMaxLen` in order to reduce memory usage at the cost of increased CPU usage. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692). * BUGFIX: fix a bug, which could prevent background merges for the previous partitions until restart if the storage didn't have enough disk space for final deduplication and down-sampling. diff --git a/docs/sd_configs.md b/docs/sd_configs.md index 05eab86777..a1f99ae35c 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -788,7 +788,8 @@ One of the following `role` types can be configured to discover targets: * `__meta_kubernetes_pod_labelpresent_`: "true" for each label from the pod object. * `__meta_kubernetes_pod_annotation_`: Each annotation from the pod object. * `__meta_kubernetes_pod_annotationpresent_`: "true" for each annotation from the pod object. - * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer + * `__meta_kubernetes_pod_container_id`: ID of the container in the form `://`. + * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer. * `__meta_kubernetes_pod_container_image`: Container image the target address points to. * `__meta_kubernetes_pod_container_name`: Name of the container the target address points to. * `__meta_kubernetes_pod_container_port_name`: Name of the container port. diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go index 77c82f1df9..7256c8b579 100644 --- a/lib/promscrape/discovery/kubernetes/pod.go +++ b/lib/promscrape/discovery/kubernetes/pod.go @@ -82,10 +82,12 @@ type ContainerPort struct { // // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podstatus-v1-core type PodStatus struct { - Phase string - PodIP string - HostIP string - Conditions []PodCondition + Phase string + PodIP string + HostIP string + Conditions []PodCondition + ContainerStatuses []ContainerStatus + InitContainerStatuses []ContainerStatus } // PodCondition implements k8s pod condition. @@ -96,6 +98,27 @@ type PodCondition struct { Status string } +// ContainerStatus implements k8s container status. +// +// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core +type ContainerStatus struct { + Name string + ContainerID string +} + +func getContainerID(p *Pod, containerName string, isInit bool) string { + css := p.Status.ContainerStatuses + if isInit { + css = p.Status.InitContainerStatuses + } + for _, cs := range css { + if cs.Name == containerName { + return cs.ContainerID + } + } + return "" +} + // getTargetLabels returns labels for each port of the given p. // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod @@ -105,12 +128,12 @@ func (p *Pod) getTargetLabels(gw *groupWatcher) []*promutils.Labels { return nil } var ms []*promutils.Labels - ms = appendPodLabels(ms, gw, p, p.Spec.Containers, "false") - ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, "true") + ms = appendPodLabels(ms, gw, p, p.Spec.Containers, false) + ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, true) return ms } -func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit string) []*promutils.Labels { +func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit bool) []*promutils.Labels { for _, c := range cs { for _, cp := range c.Ports { ms = appendPodLabelsInternal(ms, gw, p, c, &cp, isInit) @@ -122,14 +145,24 @@ func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Cont return ms } -func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit string) []*promutils.Labels { +func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit bool) []*promutils.Labels { addr := p.Status.PodIP if cp != nil { addr = discoveryutils.JoinHostPort(addr, cp.ContainerPort) } m := promutils.GetLabels() m.Add("__address__", addr) - m.Add("__meta_kubernetes_pod_container_init", isInit) + isInitStr := "false" + if isInit { + isInitStr = "true" + } + m.Add("__meta_kubernetes_pod_container_init", isInitStr) + + containerID := getContainerID(p, c.Name, isInit) + if containerID != "" { + m.Add("__meta_kubernetes_pod_container_id", containerID) + } + p.appendCommonLabels(m, gw) p.appendContainerLabels(m, c, cp) return append(ms, m) diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go index 4311c29e64..2891a79b05 100644 --- a/lib/promscrape/discovery/kubernetes/pod_test.go +++ b/lib/promscrape/discovery/kubernetes/pod_test.go @@ -262,6 +262,7 @@ func TestParsePodListSuccess(t *testing.T) { "__meta_kubernetes_pod_controller_kind": "Node", "__meta_kubernetes_pod_controller_name": "m01", "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a28f0800855008485376c1eece1cf61de97cb7026b9188d138b0d55d92fc2f5c", "__meta_kubernetes_pod_label_component": "etcd", "__meta_kubernetes_pod_label_tier": "control-plane",