From b43ba6d85ff27c13a25fb3861ac131250f252e38 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sat, 1 May 2021 09:27:53 +0300
Subject: [PATCH] lib/storage: log dropped labels if the number of labels in a
 metric exceeds `-maxLabelsPerTimeseries` command-line flag value

This should improve debuggability for this case.
---
 docs/CHANGELOG.md          |  1 +
 lib/storage/metric_name.go | 45 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 162ca7c719..030c5c423c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -12,6 +12,7 @@ Thanks to @johnseekins!
 * FEATURE: improved new time series registration speed on systems with many CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1244). Thanks to @waldoweng for the idea and [draft implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1243).
 * FEATURE: vmagent: list user-visible endpoints at `http://vmagent:8429/`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1251).
 * FEATURE: vmalert: use the same technique as Grafana for determining evaluation timestamps for recording rules. This should make consistent graphs for series generated by recording rules compared to graphs generated for queries from recording rules in Grafana. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232).
+* FEATURE: log metrics with dropped labels if the number of labels in the ingested metric exceeds `-maxLabelsPerTimeseries`. This should simplify debugging for this case.
 
 * BUGFIX: vmagent: properly update `role: endpoints` and `role: endpointslices` scrape targets if the underlying service changes in `kubernetes_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240).
 * BUGFIX: vmagent: apply `scrape_timeout` on receiving the first response byte from `stream_parse: true` scrape targets. Previously it was applied to receiving and *processing* the full response stream. This could result in false timeout errors when scrape target exposes millions of metrics as described [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047).
diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go
index 8dbd4b13a2..50845eef0b 100644
--- a/lib/storage/metric_name.go
+++ b/lib/storage/metric_name.go
@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -509,7 +511,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
 	dstSize := dstLen + 8
 	for i := range labels {
 		if i >= maxLabelsPerTimeseries {
-			atomic.AddUint64(&MetricsWithDroppedLabels, 1)
+			trackDroppedLabels(labels, labels[i:])
 			break
 		}
 		label := &labels[i]
@@ -563,6 +565,47 @@ var (
 	TooLongLabelValues uint64
 )
 
+func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
+	atomic.AddUint64(&MetricsWithDroppedLabels, 1)
+	ct := fasttime.UnixTimestamp()
+	if ct < atomic.LoadUint64(&droppedLabelsLogNextTimestamp) {
+		return
+	}
+	droppedLabelsLogOnce.Do(func() {
+		atomic.StoreUint64(&droppedLabelsLogNextTimestamp, ct+5)
+		logger.Infof("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
+			"or increase -maxLabelsPerTimeseries=%d command-line flag value",
+			len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
+		droppedLabelsLogOnce = &sync.Once{}
+	})
+}
+
+var droppedLabelsLogOnce = &sync.Once{}
+var droppedLabelsLogNextTimestamp uint64
+
+func labelsToString(labels []prompb.Label) string {
+	labelsCopy := append([]prompb.Label{}, labels...)
+	sort.Slice(labelsCopy, func(i, j int) bool {
+		return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
+	})
+	var b []byte
+	b = append(b, '{')
+	for i, label := range labelsCopy {
+		if len(label.Name) == 0 {
+			b = append(b, "__name__"...)
+		} else {
+			b = append(b, label.Name...)
+		}
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, string(label.Value))
+		if i < len(labels)-1 {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, '}')
+	return string(b)
+}
+
 // MarshalMetricLabelRaw marshals label to dst.
 func MarshalMetricLabelRaw(dst []byte, label *prompb.Label) []byte {
 	dst = marshalBytesFast(dst, label.Name)