github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-11 15:34:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

fixes service discovery for kubernetes (#2173)

Browse source 
* fixes service discovery for kubernetes
now it must take in account all pods that belong to the discovered endpoint and endpointslice
adds simple test for endpoints
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134

* wip

* docs/CHANGELOG.md: document the change

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Nikolay 2022-02-11 14:34:22 +03:00 committed by Aliaksandr Valialkin
parent 895c9f4f11
commit 265938a385
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
4 changed files with 213 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -38,6 +38,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix possible UI freeze after querying `node_uname_info` time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2115). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix possible UI freeze after querying `node_uname_info` time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2115).
* BUGFIX: show the original location of the warning or error message when logging throttled messages. Previously the location inside `lib/logger/throttler.go` was shown. This could increase the complexity of debugging. * BUGFIX: show the original location of the warning or error message when logging throttled messages. Previously the location inside `lib/logger/throttler.go` was shown. This could increase the complexity of debugging.
* BUGFIX: vmalert: fix links at web UI. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2167). * BUGFIX: vmalert: fix links at web UI. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2167).
* BUGFIX: vmagent: properly discover pods without exposed ports for the given service for `role: endpoints` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134).
## [v1.72.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.72.0) ## [v1.72.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.72.0)

7
lib/promscrape/discovery/kubernetes/endpoints.go
View file

@ -169,11 +169,16 @@ func getEndpointLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, eps *Endpoi
return m return m
} }
p.appendCommonLabels(m) p.appendCommonLabels(m)
// always add pod targetRef, even if epp port doesn't match container port
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134
if _, ok := podPortsSeen[p]; !ok {
podPortsSeen[p] = []int{}
}
for _, c := range p.Spec.Containers { for _, c := range p.Spec.Containers {
for _, cp := range c.Ports { for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port { if cp.ContainerPort == epp.Port {
p.appendContainerLabels(m, c, &cp)
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort) podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, c, &cp)
break break
} }
} }

200
lib/promscrape/discovery/kubernetes/endpoints_test.go
View file

@ -113,3 +113,203 @@ func TestParseEndpointsListSuccess(t *testing.T) {
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss) t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss)
} }
} }
func TestGetEndpointLabels(t *testing.T) {
type testArgs struct {
containerPorts map[string][]ContainerPort
endpointPorts []EndpointPort
}
f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) {
t.Run(name, func(t *testing.T) {
eps := Endpoints{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Subsets: []EndpointSubset{
{
Ports: args.endpointPorts,
Addresses: []EndpointAddress{
{
IP: "10.13.15.15",
TargetRef: ObjectReference{
Kind: "Pod",
Namespace: "default",
Name: "test-pod",
},
},
},
},
},
}
svc := Service{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Spec: ServiceSpec{
Ports: []ServicePort{
{
Name: "test-port",
Port: 8081,
},
},
},
}
pod := Pod{
Metadata: ObjectMeta{
Name: "test-pod",
Namespace: "default",
},
Status: PodStatus{PodIP: "192.168.15.1"},
}
for cn, ports := range args.containerPorts {
pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
}
var gw groupWatcher
gw.m = map[string]*urlWatcher{
"pod": &urlWatcher{
role: "pod",
objectsByKey: map[string]object{
"default/test-pod": &pod,
},
},
"service": &urlWatcher{
role: "service",
objectsByKey: map[string]object{
"default/test-eps": &svc,
},
},
}
var sortedLabelss [][]prompbmarshal.Label
gotLabels := eps.getTargetLabels(&gw)
for _, lbs := range gotLabels {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
}
if !areEqualLabelss(sortedLabelss, wantLabels) {
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
}
})
}
f("1 port from endpoint", testArgs{
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
})
f("1 port from endpoint and 1 from pod", testArgs{
containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{
Name: "http-metrics",
ContainerPort: 8428,
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "192.168.15.1:8428",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "http-metrics",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
})
f("1 port from endpoint", testArgs{
containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{
Name: "web",
ContainerPort: 8428,
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8428,
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8428",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
})
}

8
lib/promscrape/discovery/kubernetes/endpointslice.go
View file

@ -101,12 +101,17 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s
if ea.TargetRef.Kind != "Pod" || p == nil { if ea.TargetRef.Kind != "Pod" || p == nil {
return m return m
} }
// always add pod targetRef, even if epp port doesn't match container port.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134
if _, ok := podPortsSeen[p]; !ok {
podPortsSeen[p] = []int{}
}
p.appendCommonLabels(m) p.appendCommonLabels(m)
for _, c := range p.Spec.Containers { for _, c := range p.Spec.Containers {
for _, cp := range c.Ports { for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port { if cp.ContainerPort == epp.Port {
p.appendContainerLabels(m, c, &cp)
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort) podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
p.appendContainerLabels(m, c, &cp)
break break
} }
} }
@ -117,7 +122,6 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s
// //getEndpointSliceLabels builds labels for given EndpointSlice // //getEndpointSliceLabels builds labels for given EndpointSlice
func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp EndpointPort) map[string]string { func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp EndpointPort) map[string]string {
addr = discoveryutils.JoinHostPort(addr, epp.Port) addr = discoveryutils.JoinHostPort(addr, epp.Port)
m := map[string]string{ m := map[string]string{
"__address__": addr, "__address__": addr,