From 4f673a520164453489df691c3b3d15ecd224a12c Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 19 Jun 2020 01:10:18 +0300
Subject: [PATCH] app/vminsert: export metrics for determining ingested rows
 with dropped or truncated labels

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/565
---
 app/vminsert/main.go                  | 11 +++++++++++
 docs/Single-server-VictoriaMetrics.md |  4 ++++
 lib/storage/metric_name.go            | 15 +++++++++++++++
 3 files changed, 30 insertions(+)

diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index abcb51d4f..e31e7d8aa 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
@@ -208,4 +209,14 @@ var (
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/influx/", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+
+	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+	})
 )
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index cca34ca2f..9af260c7d 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -987,6 +987,10 @@ The most interesting metrics are:
   VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
   while `topN` equals to 10.
 
+* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.
+  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+
 
 ### Backfilling
 
diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go
index bc24a6ec6..6d789f1fa 100644
--- a/lib/storage/metric_name.go
+++ b/lib/storage/metric_name.go
@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -430,13 +431,16 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
 	dstSize := dstLen + 8
 	for i := range labels {
 		if i >= maxLabelsPerTimeseries {
+			atomic.AddUint64(&MetricsWithDroppedLabels, 1)
 			break
 		}
 		label := &labels[i]
 		if len(label.Name) > maxLabelNameLen {
+			atomic.AddUint64(&TooLongLabelNames, 1)
 			label.Name = label.Name[:maxLabelNameLen]
 		}
 		if len(label.Value) > maxLabelValueLen {
+			atomic.AddUint64(&TooLongLabelValues, 1)
 			label.Value = label.Value[:maxLabelValueLen]
 		}
 		if len(label.Value) == 0 {
@@ -470,6 +474,17 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
 	return dst
 }
 
+var (
+	// MetricsWithDroppedLabels is the number of metrics with at least a single dropped label
+	MetricsWithDroppedLabels uint64
+
+	// TooLongLabelNames is the number of too long label names
+	TooLongLabelNames uint64
+
+	// TooLongLabelValues is the number of too long label values
+	TooLongLabelValues uint64
+)
+
 // MarshalMetricLabelRaw marshals label to dst.
 func MarshalMetricLabelRaw(dst []byte, label *prompb.Label) []byte {
 	dst = marshalBytesFast(dst, label.Name)