github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape/discovery/kubernetes: support kubernetes native sidecars (#7324)

Browse source 
This commit adds Kubernetes Native Sidecar support. 

It's the special type of init containers, that have restartPolicy == "Always" and continue to run after container initialization. 


related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287
This commit is contained in:
Andrii Chubatiuk 2024-10-24 18:04:12 +03:00 committed by GitHub
parent 837d0d136d
commit fc537bea00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 241 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/changelog/CHANGELOG.md
View file

@ -19,6 +19,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
## tip
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support scraping from Kubernetes Native Sidecars. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287).
* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add a separate cache type for storing sparse entries when performing large index scans. This significantly reduces memory usage when applying [downsampling filters](https://docs.victoriametrics.com/#downsampling) and [retention filters](https://docs.victoriametrics.com/#retention-filters) during background merge. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7182) for the details.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly set `group_name` and `file` fields for recording rules in `/api/v1/rules`.

45
lib/promscrape/discovery/kubernetes/endpoints.go
View file

@ -37,7 +37,7 @@ func parseEndpoints(data []byte) (object, error) {
// EndpointsList implements k8s endpoints list.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslist-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointslist-v1-core
type EndpointsList struct {
Metadata ListMeta
Items []*Endpoints
@ -45,7 +45,7 @@ type EndpointsList struct {
// Endpoints implements k8s endpoints.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoints-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpoints-v1-core
type Endpoints struct {
Metadata ObjectMeta
Subsets []EndpointSubset
@ -53,7 +53,7 @@ type Endpoints struct {
// EndpointSubset implements k8s endpoint subset.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointsubset-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointsubset-v1-core
type EndpointSubset struct {
Addresses []EndpointAddress
NotReadyAddresses []EndpointAddress
@ -62,7 +62,7 @@ type EndpointSubset struct {
// EndpointAddress implements k8s endpoint address.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointaddress-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointaddress-v1-core
type EndpointAddress struct {
Hostname string
IP string
@ -72,7 +72,7 @@ type EndpointAddress struct {
// ObjectReference implements k8s object reference.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#objectreference-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectreference-v1-core
type ObjectReference struct {
Kind string
Name string
@ -81,7 +81,7 @@ type ObjectReference struct {
// EndpointPort implements k8s endpoint port.
//
// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointport-v1-discovery-k8s-io
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointport-v1-discovery-k8s-io
type EndpointPort struct {
AppProtocol string
Name string
@ -123,17 +123,16 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
}
return false
}
for p, ports := range podPortsSeen {
for _, c := range p.Spec.Containers {
appendPodMetadata := func(p *Pod, c *Container, seen []int, isInit bool) {
for _, cp := range c.Ports {
if portSeen(cp.ContainerPort, ports) {
if portSeen(cp.ContainerPort, seen) {
continue
}
addr := discoveryutils.JoinHostPort(p.Status.PodIP, cp.ContainerPort)
m := promutils.GetLabels()
m.Add("__address__", addr)
p.appendCommonLabels(m, gw)
p.appendContainerLabels(m, &c, &cp)
p.appendContainerLabels(m, c, &cp, isInit)
// Prometheus sets endpoints_name and namespace labels for all endpoints
// Even if port is not matching service port.
@ -147,6 +146,17 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
ms = append(ms, m)
}
}
for p, ports := range podPortsSeen {
for _, c := range p.Spec.Containers {
appendPodMetadata(p, &c, ports, false)
}
for _, c := range p.Spec.InitContainers {
// Defines native sidecar https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/#what-are-sidecar-containers-in-1-28
if c.RestartPolicy != "Always" {
continue
}
appendPodMetadata(p, &c, ports, true)
}
}
return ms
}
@ -189,7 +199,20 @@ func getEndpointLabelsForAddressAndPort(gw *groupWatcher, podPortsSeen map[*Pod]
for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port {
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, &c, &cp)
p.appendContainerLabels(m, &c, &cp, false)
break
}
}
}
for _, c := range p.Spec.InitContainers {
// Defines native sidecar https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/#what-are-sidecar-containers-in-1-28
if c.RestartPolicy != "Always" {
continue
}
for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port {
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, &c, &cp, true)
break
}
}

58
lib/promscrape/discovery/kubernetes/endpoints_test.go
View file

@ -116,6 +116,7 @@ func TestParseEndpointsListSuccess(t *testing.T) {
func TestGetEndpointsLabels(t *testing.T) {
type testArgs struct {
containerPorts map[string][]ContainerPort
initContainerPorts map[string][]ContainerPort
endpointPorts []EndpointPort
}
f := func(t *testing.T, args testArgs, wantLabels []*promutils.Labels) {
@ -177,6 +178,14 @@ func TestGetEndpointsLabels(t *testing.T) {
Labels: promutils.NewLabelsFromMap(map[string]string{"node-label": "xyz"}),
},
}
for cn, ports := range args.initContainerPorts {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, Container{
Name: cn,
Image: "test-init-image",
Ports: ports,
RestartPolicy: "Always",
})
}
for cn, ports := range args.containerPorts {
pod.Spec.Containers = append(pod.Spec.Containers, Container{
Name: cn,
@ -301,6 +310,7 @@ func TestGetEndpointsLabels(t *testing.T) {
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-image",
"__meta_kubernetes_pod_container_init": "false",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "http-metrics",
"__meta_kubernetes_pod_container_port_number": "8428",
@ -347,6 +357,54 @@ func TestGetEndpointsLabels(t *testing.T) {
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-image",
"__meta_kubernetes_pod_container_init": "false",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "sdc",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "service-type",
}),
})
})
t.Run("1 init container port from endpoint", func(t *testing.T) {
f(t, testArgs{
initContainerPorts: map[string][]ContainerPort{"metrics": {{
Name: "web",
ContainerPort: 8428,
Protocol: "sdc",
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8428,
Protocol: "xabc",
},
},
}, []*promutils.Labels{
promutils.NewLabelsFromMap(map[string]string{
"__address__": "10.13.15.15:8428",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "xabc",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-init-image",
"__meta_kubernetes_pod_container_init": "true",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",

40
lib/promscrape/discovery/kubernetes/endpointslice.go
View file

@ -73,17 +73,16 @@ func (eps *EndpointSlice) getTargetLabels(gw *groupWatcher) []*promutils.Labels
}
return false
}
for p, ports := range podPortsSeen {
for _, c := range p.Spec.Containers {
appendPodMetadata := func(p *Pod, c *Container, seen []int, isInit bool) {
for _, cp := range c.Ports {
if portSeen(cp.ContainerPort, ports) {
if portSeen(cp.ContainerPort, seen) {
continue
}
addr := discoveryutils.JoinHostPort(p.Status.PodIP, cp.ContainerPort)
m := promutils.GetLabels()
m.Add("__address__", addr)
p.appendCommonLabels(m, gw)
p.appendContainerLabels(m, &c, &cp)
p.appendContainerLabels(m, c, &cp, isInit)
// Prometheus sets endpoints_name and namespace labels for all endpoints
// Even if port is not matching service port.
@ -92,11 +91,23 @@ func (eps *EndpointSlice) getTargetLabels(gw *groupWatcher) []*promutils.Labels
if svc != nil {
svc.appendCommonLabels(m)
}
// Remove possible duplicate labels, which can appear after appendCommonLabels() calls
// Remove possible duplicate labels, which can appear after appendCommonLabels() call
m.RemoveDuplicates()
ms = append(ms, m)
}
}
for p, ports := range podPortsSeen {
for _, c := range p.Spec.Containers {
appendPodMetadata(p, &c, ports, false)
}
for _, c := range p.Spec.InitContainers {
// Defines native sidecar https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/#what-are-sidecar-containers-in-1-28
if c.RestartPolicy != "Always" {
continue
}
appendPodMetadata(p, &c, ports, true)
}
}
return ms
}
@ -127,7 +138,16 @@ func getEndpointSliceLabelsForAddressAndPort(gw *groupWatcher, podPortsSeen map[
for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port {
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, &c, &cp)
p.appendContainerLabels(m, &c, &cp, false)
break
}
}
}
for _, c := range p.Spec.InitContainers {
for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port {
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, &c, &cp, true)
break
}
}
@ -167,7 +187,7 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En
// EndpointSliceList - implements kubernetes endpoint slice list object, that groups service endpoints slices.
//
// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslicelist-v1-discovery-k8s-io
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointslicelist-v1-discovery-k8s-io
type EndpointSliceList struct {
Metadata ListMeta
Items []*EndpointSlice
@ -175,7 +195,7 @@ type EndpointSliceList struct {
// EndpointSlice - implements kubernetes endpoint slice.
//
// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslice-v1-discovery-k8s-io
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointslice-v1-discovery-k8s-io
type EndpointSlice struct {
Metadata ObjectMeta
Endpoints []Endpoint
@ -185,7 +205,7 @@ type EndpointSlice struct {
// Endpoint implements kubernetes object endpoint for endpoint slice.
//
// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpoint-v1-discovery-k8s-io
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpoint-v1-discovery-k8s-io
type Endpoint struct {
Addresses []string
Conditions EndpointConditions
@ -196,7 +216,7 @@ type Endpoint struct {
// EndpointConditions implements kubernetes endpoint condition.
//
// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointconditions-v1-discovery-k8s-io
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#endpointconditions-v1-discovery-k8s-io
type EndpointConditions struct {
Ready bool
}

65
lib/promscrape/discovery/kubernetes/endpointslice_test.go
View file

@ -225,6 +225,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
func TestGetEndpointsliceLabels(t *testing.T) {
type testArgs struct {
initContainerPorts map[string][]ContainerPort
containerPorts map[string][]ContainerPort
endpointPorts []EndpointPort
}
@ -296,6 +297,14 @@ func TestGetEndpointsliceLabels(t *testing.T) {
Labels: promutils.NewLabelsFromMap(map[string]string{"node-label": "xyz"}),
},
}
for cn, ports := range args.initContainerPorts {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, Container{
Name: cn,
Image: "test-init-image",
Ports: ports,
RestartPolicy: "Always",
})
}
for cn, ports := range args.containerPorts {
pod.Spec.Containers = append(pod.Spec.Containers, Container{
Name: cn,
@ -437,6 +446,7 @@ func TestGetEndpointsliceLabels(t *testing.T) {
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-image",
"__meta_kubernetes_pod_container_init": "false",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "http-metrics",
"__meta_kubernetes_pod_container_port_number": "8428",
@ -490,6 +500,61 @@ func TestGetEndpointsliceLabels(t *testing.T) {
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-image",
"__meta_kubernetes_pod_container_init": "false",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "sdc",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-svc",
"__meta_kubernetes_service_type": "service-type",
}),
})
})
t.Run("1 init container port from endpoint", func(t *testing.T) {
f(t, testArgs{
initContainerPorts: map[string][]ContainerPort{"metrics": {{
Name: "web",
ContainerPort: 8428,
Protocol: "sdc",
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8428,
Protocol: "xabc",
},
},
}, []*promutils.Labels{
promutils.NewLabelsFromMap(map[string]string{
"__address__": "10.13.15.15:8428",
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
"__meta_kubernetes_endpointslice_address_target_name": "test-pod",
"__meta_kubernetes_endpointslice_address_type": "foobar",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_endpoint_hostname": "foo.bar",
"__meta_kubernetes_endpointslice_endpoint_topology_present_x": "true",
"__meta_kubernetes_endpointslice_endpoint_topology_x": "y",
"__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "test-svc",
"__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true",
"__meta_kubernetes_endpointslice_name": "test-eps",
"__meta_kubernetes_endpointslice_port": "8428",
"__meta_kubernetes_endpointslice_port_name": "web",
"__meta_kubernetes_endpointslice_port_protocol": "xabc",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_image": "test-init-image",
"__meta_kubernetes_pod_container_init": "true",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",

27
lib/promscrape/discovery/kubernetes/pod.go
View file

@ -38,7 +38,7 @@ func parsePod(data []byte) (object, error) {
// PodList implements k8s pod list.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podlist-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podlist-v1-core
type PodList struct {
Metadata ListMeta
Items []*Pod
@ -46,7 +46,7 @@ type PodList struct {
// Pod implements k8s pod.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#pod-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#pod-v1-core
type Pod struct {
Metadata ObjectMeta
Spec PodSpec
@ -55,7 +55,7 @@ type Pod struct {
// PodSpec implements k8s pod spec.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podspec-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podspec-v1-core
type PodSpec struct {
NodeName string
Containers []Container
@ -64,11 +64,12 @@ type PodSpec struct {
// Container implements k8s container.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#container-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#container-v1-core
type Container struct {
Name string
Image string
Ports []ContainerPort
RestartPolicy string
}
// ContainerPort implements k8s container port.
@ -80,7 +81,7 @@ type ContainerPort struct {
// PodStatus implements k8s pod status.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podstatus-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podstatus-v1-core
type PodStatus struct {
Phase string
PodIP string
@ -92,7 +93,7 @@ type PodStatus struct {
// PodCondition implements k8s pod condition.
//
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podcondition-v1-core
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podcondition-v1-core
type PodCondition struct {
Type string
Status string
@ -198,11 +199,6 @@ func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c
}
m := promutils.GetLabels()
m.Add("__address__", addr)
isInitStr := "false"
if isInit {
isInitStr = "true"
}
m.Add("__meta_kubernetes_pod_container_init", isInitStr)
containerID := getContainerID(p, c.Name, isInit)
if containerID != "" {
@ -210,13 +206,18 @@ func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c
}
p.appendCommonLabels(m, gw)
p.appendContainerLabels(m, c, cp)
p.appendContainerLabels(m, c, cp, isInit)
return append(ms, m)
}
func (p *Pod) appendContainerLabels(m *promutils.Labels, c *Container, cp *ContainerPort) {
func (p *Pod) appendContainerLabels(m *promutils.Labels, c *Container, cp *ContainerPort, isInit bool) {
m.Add("__meta_kubernetes_pod_container_image", c.Image)
m.Add("__meta_kubernetes_pod_container_name", c.Name)
isInitStr := "false"
if isInit {
isInitStr = "true"
}
m.Add("__meta_kubernetes_pod_container_init", isInitStr)
if cp != nil {
m.Add("__meta_kubernetes_pod_container_port_name", cp.Name)
m.Add("__meta_kubernetes_pod_container_port_number", bytesutil.Itoa(cp.ContainerPort))