From 3408a05d124c05b63853ec0fe2a80fbed6c51053 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Tue, 19 Oct 2021 13:19:18 +0300
Subject: [PATCH] lib/promscrape/discovery/kubernetes: log a warning if `role:
 endpoints` discovers more than 1000 targets per a single endpoint

In this case `role: endpointslice` must be used instead.

See the following references:

* https://kubernetes.io/docs/reference/labels-annotations-taints/#endpoints-kubernetes-io-over-capacity
* https://github.com/kubernetes/kubernetes/pull/99975
* https://github.com/prometheus/prometheus/issues/7572#issuecomment-934779398
---
 lib/promscrape/discovery/kubernetes/endpoints.go |  9 +++++++++
 lib/promscrape/discoveryutils/utils.go           | 10 ++++++++++
 2 files changed, 19 insertions(+)

diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index a4a9b21b9e..8c1fe6b9d3 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -103,6 +104,14 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []map[string]string {
 			ms = appendEndpointLabelsForAddresses(ms, gw, podPortsSeen, eps, ess.NotReadyAddresses, epp, svc, "false")
 		}
 	}
+	// See https://kubernetes.io/docs/reference/labels-annotations-taints/#endpoints-kubernetes-io-over-capacity
+	// and https://github.com/kubernetes/kubernetes/pull/99975
+	switch eps.Metadata.Annotations.GetByName("endpoints.kubernetes.io/over-capacity") {
+	case "truncated":
+		logger.Warnf(`the number of targets for "role: endpoints" %q exceeds 1000 and has been truncated; please use "role: endpointslice" instead`, eps.Metadata.key())
+	case "warning":
+		logger.Warnf(`the number of targets for "role: endpoints" %q exceeds 1000 and will be truncated in the next k8s releases; please use "role: endpointslice" instead`, eps.Metadata.key())
+	}
 
 	// Append labels for skipped ports on seen pods.
 	portSeen := func(port int, ports []int) bool {
diff --git a/lib/promscrape/discoveryutils/utils.go b/lib/promscrape/discoveryutils/utils.go
index 18c7d00e00..cd0d103c9f 100644
--- a/lib/promscrape/discoveryutils/utils.go
+++ b/lib/promscrape/discoveryutils/utils.go
@@ -33,6 +33,16 @@ func JoinHostPort(host string, port int) string {
 // SortedLabels represents sorted labels.
 type SortedLabels []prompbmarshal.Label
 
+// GetByName returns the label with the given name from sls.
+func (sls *SortedLabels) GetByName(name string) string {
+	for _, lb := range *sls {
+		if lb.Name == name {
+			return lb.Value
+		}
+	}
+	return ""
+}
+
 // UnmarshalJSON unmarshals JSON from data.
 func (sls *SortedLabels) UnmarshalJSON(data []byte) error {
 	var m map[string]string