Added endpointslices discovery to k8s api (#760)

This is similar to https://github.com/prometheus/prometheus/pull/6838 , which will be added in Prometheus v2.21.
See https://github.com/prometheus/prometheus/releases/tag/v2.21.0-rc.1

* Added endpointslices discovery to k8s api

Started from 1.17 k8s version endpointslices is beta,
it allows to query k8s api for endpoints more efficient.
It presents at scrape_config.yaml as separate role for kubernetes_sd_config.
kubernetes_sd_config:
- role: endpointslices

* fixed typos, changed EndpointConditions signature - with values instead of pointers
This commit is contained in:
Nikolay Khramchikhin 2020-09-11 12:16:45 +03:00 committed by GitHub
parent 204ec415b4
commit 6c80ae0da8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 660 additions and 0 deletions

View file

@ -0,0 +1,212 @@
package kubernetes
import (
"encoding/json"
"fmt"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// getEndpointSlicesLabels returns labels for k8s endpointSlices obtained from the given cfg.
func getEndpointSlicesLabels(cfg *apiConfig) ([]map[string]string, error) {
eps, err := getEndpointSlices(cfg)
if err != nil {
return nil, err
}
pods, err := getPods(cfg)
if err != nil {
return nil, err
}
svcs, err := getServices(cfg)
if err != nil {
return nil, err
}
var ms []map[string]string
for _, ep := range eps {
ms = ep.appendTargetLabels(ms, pods, svcs)
}
return ms, nil
}
// getEndpointSlices retrieves endpointSlice with given apiConfig
func getEndpointSlices(cfg *apiConfig) ([]EndpointSlice, error) {
if len(cfg.namespaces) == 0 {
return getEndpointSlicesByPath(cfg, "/apis/discovery.k8s.io/v1beta1/endpointslices")
}
// Query /api/v1/namespaces/* for each namespace.
// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
cfgCopy := *cfg
namespaces := cfgCopy.namespaces
cfgCopy.namespaces = nil
cfg = &cfgCopy
var result []EndpointSlice
for _, ns := range namespaces {
path := fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/namespaces/%s/endpointslices", ns)
eps, err := getEndpointSlicesByPath(cfg, path)
if err != nil {
return nil, err
}
result = append(result, eps...)
}
return result, nil
}
// getEndpointSlicesByPath retrieves endpointSlices from k8s api by given path
func getEndpointSlicesByPath(cfg *apiConfig, path string) ([]EndpointSlice, error) {
data, err := getAPIResponse(cfg, "endpointslices", path)
if err != nil {
return nil, fmt.Errorf("cannot obtain endpointslices data from API server: %w", err)
}
epl, err := parseEndpointSlicesList(data)
if err != nil {
return nil, fmt.Errorf("cannot parse endpointslices response from API server: %w", err)
}
return epl.Items, nil
}
// parseEndpointsList parses EndpointSliceList from data.
func parseEndpointSlicesList(data []byte) (*EndpointSliceList, error) {
var esl EndpointSliceList
if err := json.Unmarshal(data, &esl); err != nil {
return nil, fmt.Errorf("cannot unmarshal EndpointSliceList from %q: %w", data, err)
}
return &esl, nil
}
// appendTargetLabels injects labels for endPointSlice to slice map
// follows TargetRef for enrich labels with pod and service metadata
func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, pods []Pod, svcs []Service) []map[string]string {
svc := getService(svcs, eps.Metadata.Namespace, eps.Metadata.Name)
podPortsSeen := make(map[*Pod][]int)
for _, ess := range eps.Endpoints {
pod := getPod(pods, ess.TargetRef.Namespace, ess.TargetRef.Name)
for _, epp := range eps.Ports {
for _, addr := range ess.Addresses {
ms = append(ms, getEndpointSliceLabelsForAddressAndPort(podPortsSeen, addr, eps, ess, epp, pod, svc))
}
}
}
// Append labels for skipped ports on seen pods.
portSeen := func(port int, ports []int) bool {
for _, p := range ports {
if p == port {
return true
}
}
return false
}
for p, ports := range podPortsSeen {
for _, c := range p.Spec.Containers {
for _, cp := range c.Ports {
if portSeen(cp.ContainerPort, ports) {
continue
}
addr := discoveryutils.JoinHostPort(p.Status.PodIP, cp.ContainerPort)
m := map[string]string{
"__address__": addr,
}
p.appendCommonLabels(m)
p.appendContainerLabels(m, c, &cp)
ms = append(ms, m)
}
}
}
return ms
}
// getEndpointSliceLabelsForAddressAndPort gets labels for endpointSlice
// from address, Endpoint and EndpointPort
// enriches labels with TargetRef
// pod appended to seen Ports
// if TargetRef matches
func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr string, eps *EndpointSlice, ea Endpoint, epp EndpointPort, p *Pod, svc *Service) map[string]string {
m := getEndpointSliceLabels(eps, addr, ea, epp)
if svc != nil {
svc.appendCommonLabels(m)
}
if ea.TargetRef.Kind != "Pod" || p == nil {
return m
}
p.appendCommonLabels(m)
for _, c := range p.Spec.Containers {
for _, cp := range c.Ports {
if cp.ContainerPort == epp.Port {
p.appendContainerLabels(m, c, &cp)
podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
break
}
}
}
return m
}
// //getEndpointSliceLabels builds labels for given EndpointSlice
func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp EndpointPort) map[string]string {
addr = discoveryutils.JoinHostPort(addr, epp.Port)
m := map[string]string{
"__address__": addr,
"__meta_kubernetes_namespace": eps.Metadata.Namespace,
"__meta_kubernetes_endpointslice_name": eps.Metadata.Name,
"__meta_kubernetes_endpointslice_address_type": eps.AddressType,
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": strconv.FormatBool(ea.Conditions.Ready),
"__meta_kubernetes_endpointslice_port_name": epp.Name,
"__meta_kubernetes_endpointslice_port_protocol": epp.Protocol,
"__meta_kubernetes_endpointslice_port": strconv.FormatUint(uint64(epp.Port), 10),
}
if epp.AppProtocol != "" {
m["__meta_kubernetes_endpointslice_port_app_protocol"] = epp.AppProtocol
}
if ea.TargetRef.Kind != "" {
m["__meta_kubernetes_endpointslice_address_target_kind"] = ea.TargetRef.Kind
m["__meta_kubernetes_endpointslice_address_target_name"] = ea.TargetRef.Name
}
if ea.Hostname != "" {
m["__meta_kubernetes_endpointslice_endpoint_hostname"] = ea.Hostname
}
for k, v := range ea.Topology {
m["__meta_kubernetes_endpointslice_endpoint_topology_"+discoveryutils.SanitizeLabelName(k)] = v
m["__meta_kubernetes_endpointslice_endpoint_topology_present_"+discoveryutils.SanitizeLabelName(k)] = "true"
}
return m
}
// EndpointSliceList - implements kubernetes endpoint slice list object,
// that groups service endpoints slices.
// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
type EndpointSliceList struct {
Items []EndpointSlice
}
// EndpointSlice - implements kubernetes endpoint slice.
// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
type EndpointSlice struct {
Metadata ObjectMeta
Endpoints []Endpoint
AddressType string
Ports []EndpointPort
}
// Endpoint implements kubernetes object endpoint for endpoint slice.
// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io
type Endpoint struct {
Addresses []string
Conditions EndpointConditions
Hostname string
TargetRef ObjectReference
Topology map[string]string
}
// EndpointConditions implements kubernetes endpoint condition.
// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointconditions-v1beta1-discovery-k8s-io
type EndpointConditions struct {
Ready bool
}

View file

@ -0,0 +1,446 @@
package kubernetes
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_parseEndpointSlicesListFail(t *testing.T) {
f := func(data string) {
eslList, err := parseEndpointSlicesList([]byte(data))
if err == nil {
t.Errorf("unexpected result, test must fail! data: %s", data)
}
if eslList != nil {
t.Errorf("endpointSliceList must be nil, got: %v", eslList)
}
}
f(``)
f(`{"items": [1,2,3]`)
f(`{"items": [
{
"metadata": {
"name": "kubernetes"}]}`)
}
func Test_parseEndpointSlicesListSuccess(t *testing.T) {
data := `{
"kind": "EndpointSliceList",
"apiVersion": "discovery.k8s.io/v1beta1",
"metadata": {
"selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices",
"resourceVersion": "1177"
},
"items": [
{
"metadata": {
"name": "kubernetes",
"namespace": "default",
"selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/default/endpointslices/kubernetes",
"uid": "a60d9173-5fe4-4bc3-87a6-269daee71f8a",
"resourceVersion": "159",
"generation": 1,
"creationTimestamp": "2020-09-07T14:27:22Z",
"labels": {
"kubernetes.io/service-name": "kubernetes"
},
"managedFields": [
{
"manager": "kube-apiserver",
"operation": "Update",
"apiVersion": "discovery.k8s.io/v1beta1",
"time": "2020-09-07T14:27:22Z",
"fieldsType": "FieldsV1",
"fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:labels":{".":{},"f:kubernetes.io/service-name":{}}},"f:ports":{}}
}
]
},
"addressType": "IPv4",
"endpoints": [
{
"addresses": [
"172.18.0.2"
],
"conditions": {
"ready": true
}
}
],
"ports": [
{
"name": "https",
"protocol": "TCP",
"port": 6443
}
]
},
{
"metadata": {
"name": "kube-dns-22mvb",
"generateName": "kube-dns-",
"namespace": "kube-system",
"selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/kube-system/endpointslices/kube-dns-22mvb",
"uid": "7c95c854-f34c-48e1-86f5-bb8269113c11",
"resourceVersion": "604",
"generation": 5,
"creationTimestamp": "2020-09-07T14:27:39Z",
"labels": {
"endpointslice.kubernetes.io/managed-by": "endpointslice-controller.k8s.io",
"kubernetes.io/service-name": "kube-dns"
},
"annotations": {
"endpoints.kubernetes.io/last-change-trigger-time": "2020-09-07T14:28:35Z"
},
"ownerReferences": [
{
"apiVersion": "v1",
"kind": "Service",
"name": "kube-dns",
"uid": "509e80d8-6d05-487b-bfff-74f5768f1024",
"controller": true,
"blockOwnerDeletion": true
}
],
"managedFields": [
{
"manager": "kube-controller-manager",
"operation": "Update",
"apiVersion": "discovery.k8s.io/v1beta1",
"time": "2020-09-07T14:28:35Z",
"fieldsType": "FieldsV1",
"fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:annotations":{".":{},"f:endpoints.kubernetes.io/last-change-trigger-time":{}},"f:generateName":{},"f:labels":{".":{},"f:endpointslice.kubernetes.io/managed-by":{},"f:kubernetes.io/service-name":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"509e80d8-6d05-487b-bfff-74f5768f1024\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:ports":{}}
}
]
},
"addressType": "IPv4",
"endpoints": [
{
"addresses": [
"10.244.0.3"
],
"conditions": {
"ready": true
},
"targetRef": {
"kind": "Pod",
"namespace": "kube-system",
"name": "coredns-66bff467f8-z8czk",
"uid": "36a545ff-dbba-4192-a5f6-1dbb0c21c73d",
"resourceVersion": "603"
},
"topology": {
"kubernetes.io/hostname": "kind-control-plane"
}
},
{
"addresses": [
"10.244.0.4"
],
"conditions": {
"ready": true
},
"targetRef": {
"kind": "Pod",
"namespace": "kube-system",
"name": "coredns-66bff467f8-kpbhk",
"uid": "db38d8b4-847a-4e82-874c-fe444fba2718",
"resourceVersion": "576"
},
"topology": {
"kubernetes.io/hostname": "kind-control-plane"
}
}
],
"ports": [
{
"name": "dns-tcp",
"protocol": "TCP",
"port": 53
},
{
"name": "metrics",
"protocol": "TCP",
"port": 9153
},
{
"name": "dns",
"protocol": "UDP",
"port": 53
}
]
}
]
}`
esl, err := parseEndpointSlicesList([]byte(data))
if err != nil {
t.Errorf("cannot parse data for EndpointSliceList: %v", err)
return
}
if len(esl.Items) != 2 {
t.Fatalf("expected 2 items at endpointSliceList, got: %d", len(esl.Items))
}
firstEsl := esl.Items[0]
got := firstEsl.appendTargetLabels(nil, nil, nil)
sortedLables := [][]prompbmarshal.Label{}
for _, labels := range got {
sortedLables = append(sortedLables, discoveryutils.GetSortedLabels(labels))
}
expectedLabels := [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "172.18.0.2:6443",
"__meta_kubernetes_endpointslice_address_type": "IPv4",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_name": "kubernetes",
"__meta_kubernetes_endpointslice_port": "6443",
"__meta_kubernetes_endpointslice_port_name": "https",
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
"__meta_kubernetes_namespace": "default",
})}
if !reflect.DeepEqual(sortedLables, expectedLabels) {
t.Fatalf("unexpected labels,\ngot:\n%v,\nwant:\n%v", sortedLables, expectedLabels)
}
}
func TestEndpointSlice_appendTargetLabels(t *testing.T) {
type fields struct {
Metadata ObjectMeta
Endpoints []Endpoint
AddressType string
Ports []EndpointPort
}
type args struct {
ms []map[string]string
pods []Pod
svcs []Service
}
tests := []struct {
name string
fields fields
args args
want [][]prompbmarshal.Label
}{
{
name: "simple eps",
args: args{},
fields: fields{
Metadata: ObjectMeta{
Name: "fake-esl",
Namespace: "default",
},
AddressType: "ipv4",
Endpoints: []Endpoint{
{Addresses: []string{"127.0.0.1"},
Hostname: "node-1",
Topology: map[string]string{"kubernetes.topoligy.io/zone": "gce-1"},
Conditions: EndpointConditions{Ready: true},
TargetRef: ObjectReference{
Kind: "Pod",
Namespace: "default",
Name: "main-pod",
},
},
},
Ports: []EndpointPort{
{
Name: "http",
Port: 8085,
AppProtocol: "http",
Protocol: "tcp",
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "127.0.0.1:8085",
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
"__meta_kubernetes_endpointslice_address_target_name": "main-pod",
"__meta_kubernetes_endpointslice_address_type": "ipv4",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_topoligy_io_zone": "gce-1",
"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_topoligy_io_zone": "true",
"__meta_kubernetes_endpointslice_endpoint_hostname": "node-1",
"__meta_kubernetes_endpointslice_name": "fake-esl",
"__meta_kubernetes_endpointslice_port": "8085",
"__meta_kubernetes_endpointslice_port_app_protocol": "http",
"__meta_kubernetes_endpointslice_port_name": "http",
"__meta_kubernetes_endpointslice_port_protocol": "tcp",
"__meta_kubernetes_namespace": "default",
}),
},
},
{
name: "eps with pods and services",
args: args{
pods: []Pod{
{
Metadata: ObjectMeta{
UID: "some-pod-uuid",
Namespace: "monitoring",
Name: "main-pod",
Labels: discoveryutils.GetSortedLabels(map[string]string{
"pod-label-1": "pod-value-1",
"pod-label-2": "pod-value-2",
}),
Annotations: discoveryutils.GetSortedLabels(map[string]string{
"pod-annotations-1": "annotation-value-1",
}),
},
Status: PodStatus{PodIP: "192.168.11.5", HostIP: "172.15.1.1"},
Spec: PodSpec{NodeName: "node-2", Containers: []Container{
{
Name: "container-1",
Ports: []ContainerPort{
{
ContainerPort: 8085,
Protocol: "tcp",
Name: "http",
},
{
ContainerPort: 8011,
Protocol: "udp",
Name: "dns",
},
},
},
}},
},
},
svcs: []Service{
{
Spec: ServiceSpec{Type: "ClusterIP", Ports: []ServicePort{
{
Name: "http",
Protocol: "tcp",
Port: 8085,
},
}},
Metadata: ObjectMeta{
Name: "custom-esl",
Namespace: "monitoring",
Labels: discoveryutils.GetSortedLabels(map[string]string{
"service-label-1": "value-1",
"service-label-2": "value-2",
}),
},
},
},
},
fields: fields{
Metadata: ObjectMeta{
Name: "custom-esl",
Namespace: "monitoring",
},
AddressType: "ipv4",
Endpoints: []Endpoint{
{Addresses: []string{"127.0.0.1"},
Hostname: "node-1",
Topology: map[string]string{"kubernetes.topoligy.io/zone": "gce-1"},
Conditions: EndpointConditions{Ready: true},
TargetRef: ObjectReference{
Kind: "Pod",
Namespace: "monitoring",
Name: "main-pod",
},
},
},
Ports: []EndpointPort{
{
Name: "http",
Port: 8085,
AppProtocol: "http",
Protocol: "tcp",
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "127.0.0.1:8085",
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
"__meta_kubernetes_endpointslice_address_target_name": "main-pod",
"__meta_kubernetes_endpointslice_address_type": "ipv4",
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_topoligy_io_zone": "gce-1",
"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_topoligy_io_zone": "true",
"__meta_kubernetes_endpointslice_endpoint_hostname": "node-1",
"__meta_kubernetes_endpointslice_name": "custom-esl",
"__meta_kubernetes_endpointslice_port": "8085",
"__meta_kubernetes_endpointslice_port_app_protocol": "http",
"__meta_kubernetes_endpointslice_port_name": "http",
"__meta_kubernetes_endpointslice_port_protocol": "tcp",
"__meta_kubernetes_namespace": "monitoring",
"__meta_kubernetes_pod_annotation_pod_annotations_1": "annotation-value-1",
"__meta_kubernetes_pod_annotationpresent_pod_annotations_1": "true",
"__meta_kubernetes_pod_container_name": "container-1",
"__meta_kubernetes_pod_container_port_name": "http",
"__meta_kubernetes_pod_container_port_number": "8085",
"__meta_kubernetes_pod_container_port_protocol": "tcp",
"__meta_kubernetes_pod_host_ip": "172.15.1.1",
"__meta_kubernetes_pod_ip": "192.168.11.5",
"__meta_kubernetes_pod_label_pod_label_1": "pod-value-1",
"__meta_kubernetes_pod_label_pod_label_2": "pod-value-2",
"__meta_kubernetes_pod_labelpresent_pod_label_1": "true",
"__meta_kubernetes_pod_labelpresent_pod_label_2": "true",
"__meta_kubernetes_pod_name": "main-pod",
"__meta_kubernetes_pod_node_name": "node-2",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "some-pod-uuid",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_label_service_label_1": "value-1",
"__meta_kubernetes_service_label_service_label_2": "value-2",
"__meta_kubernetes_service_labelpresent_service_label_1": "true",
"__meta_kubernetes_service_labelpresent_service_label_2": "true",
"__meta_kubernetes_service_name": "custom-esl",
"__meta_kubernetes_service_type": "ClusterIP",
}),
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "192.168.11.5:8011",
"__meta_kubernetes_namespace": "monitoring",
"__meta_kubernetes_pod_annotation_pod_annotations_1": "annotation-value-1",
"__meta_kubernetes_pod_annotationpresent_pod_annotations_1": "true",
"__meta_kubernetes_pod_container_name": "container-1",
"__meta_kubernetes_pod_container_port_name": "dns",
"__meta_kubernetes_pod_container_port_number": "8011",
"__meta_kubernetes_pod_container_port_protocol": "udp",
"__meta_kubernetes_pod_host_ip": "172.15.1.1",
"__meta_kubernetes_pod_ip": "192.168.11.5",
"__meta_kubernetes_pod_label_pod_label_1": "pod-value-1",
"__meta_kubernetes_pod_label_pod_label_2": "pod-value-2",
"__meta_kubernetes_pod_labelpresent_pod_label_1": "true",
"__meta_kubernetes_pod_labelpresent_pod_label_2": "true",
"__meta_kubernetes_pod_name": "main-pod",
"__meta_kubernetes_pod_node_name": "node-2",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "some-pod-uuid",
}),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
eps := &EndpointSlice{
Metadata: tt.fields.Metadata,
Endpoints: tt.fields.Endpoints,
AddressType: tt.fields.AddressType,
Ports: tt.fields.Ports,
}
got := eps.appendTargetLabels(tt.args.ms, tt.args.pods, tt.args.svcs)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("got unxpected labels: \ngot:\n %v, \nexpect:\n %v", sortedLabelss, tt.want)
}
})
}
}

View file

@ -50,6 +50,8 @@ func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
return getPodsLabels(cfg)
case "endpoints":
return getEndpointsLabels(cfg)
case "endpointslices":
return getEndpointSlicesLabels(cfg)
case "ingress":
return getIngressesLabels(cfg)
default: