app/{vminsert,vmagent}: drop series on exceeding -maxLabelsPerTimeseries, -maxLabelValueLen flags and maxLabelNameLen=256 limits

app/victoria-metrics/self_scraper.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -48,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
 	var mrs []storage.MetricRow
-	var labels []prompb.Label
+	var labels []prompbmarshal.Label
 	t := time.NewTicker(scrapeInterval)
 	f := func(currentTime time.Time, sendStaleMarkers bool) {
 		currentTimestamp := currentTime.UnixNano() / 1e6
@@ -99,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) {
 	}
 }
 
-func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
 	if len(dst) < cap(dst) {
 		dst = dst[:len(dst)+1]
 	} else {
-		dst = append(dst, prompb.Label{})
+		dst = append(dst, prompbmarshal.Label{})
 	}
 	lb := &dst[len(dst)-1]
 	lb.Name = key

app/vmagent/remotewrite/remotewrite.go

@@ -7,13 +7,10 @@ import (
 	"net/url"
 	"path/filepath"
 	"slices"
-	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -21,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -29,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
+	storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
@@ -485,6 +484,14 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			rowsCountAfterRelabel := getRowsCount(tssBlock)
 			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
+		idxDiff := 0
+		for i, ts := range tssBlock {
+			if storagelimits.ExceedingLabels(ts.Labels) {
+				idxDiff++
+			} else if idxDiff > 0 {
+				tss[i-idxDiff] = tss[i]
+			}
+		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
 		if sas.IsEnabled() {
@@ -729,29 +736,14 @@ func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue i
 	select {
 	case <-logSkippedSeriesTicker.C:
 		// Do not use logger.WithThrottler() here, since this will increase CPU usage
-		// because every call to logSkippedSeries will result to a call to labelsToString.
+		// because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString.
-		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
+		logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue)
 	default:
 	}
 }
 
 var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
 
-func labelsToString(labels []prompbmarshal.Label) string {
-	var b []byte
-	b = append(b, '{')
-	for i, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, '=')
-		b = strconv.AppendQuote(b, label.Value)
-		if i+1 < len(labels) {
-			b = append(b, ',')
-		}
-	}
-	b = append(b, '}')
-	return string(b)
-}
-
 var (
 	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
 	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")

app/vmctl/vm_native_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -214,15 +214,15 @@ func processFlags() {
 func fillStorage(series []vm.TimeSeries) error {
 	var mrs []storage.MetricRow
 	for _, series := range series {
-		var labels []prompb.Label
+		var labels []prompbmarshal.Label
 		for _, lp := range series.LabelPairs {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  lp.Name,
 				Value: lp.Value,
 			})
 		}
 		if series.Name != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: series.Name,
 			})

app/vminsert/common/insert_ctx.go

@@ -8,9 +8,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits"
 )
 
 // InsertCtx contains common bits for data points insertion.
@@ -30,7 +31,7 @@ type InsertCtx struct {
 func (ctx *InsertCtx) Reset(rowsLen int) {
 	labels := ctx.Labels
 	for i := range labels {
-		labels[i] = prompb.Label{}
+		labels[i] = prompbmarshal.Label{}
 	}
 	ctx.Labels = labels[:0]
 
@@ -51,7 +52,7 @@ func cleanMetricRow(mr *storage.MetricRow) {
 	mr.MetricNameRaw = nil
 }
 
-func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
+func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompbmarshal.Label) []byte {
 	start := len(ctx.metricNamesBuf)
 	ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
 	ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
@@ -60,7 +61,10 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
 }
 
 // WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
-func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) error {
+func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error {
+	if storagelimits.ExceedingLabels(labels) {
+		return nil
+	}
 	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
 	return ctx.addRow(metricNameRaw, timestamp, value)
 }
@@ -68,7 +72,10 @@ func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, times
 // WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer.
 //
 // It returns metricNameRaw for the given labels if len(metricNameRaw) == 0.
-func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) ([]byte, error) {
+func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompbmarshal.Label, timestamp int64, value float64) ([]byte, error) {
+	if storagelimits.ExceedingLabels(labels) {
+		return metricNameRaw, nil
+	}
 	if len(metricNameRaw) == 0 {
 		metricNameRaw = ctx.marshalMetricNameRaw(nil, labels)
 	}
@@ -106,7 +113,7 @@ func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
 		// Do not skip labels with empty name, since they are equal to __name__.
 		return
 	}
-	ctx.Labels = append(ctx.Labels, prompb.Label{
+	ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
 		// Do not copy name and value contents for performance reasons.
 		// This reduces GC overhead on the number of objects and allocations.
 		Name:  bytesutil.ToUnsafeString(name),
@@ -124,7 +131,7 @@ func (ctx *InsertCtx) AddLabel(name, value string) {
 		// Do not skip labels with empty name, since they are equal to __name__.
 		return
 	}
-	ctx.Labels = append(ctx.Labels, prompb.Label{
+	ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
 		// Do not copy name and value contents for performance reasons.
 		// This reduces GC overhead on the number of objects and allocations.
 		Name:  name,

app/vminsert/common/sort_labels.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"sort"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to storage. `+
@@ -19,7 +19,7 @@ func (ctx *InsertCtx) SortLabelsIfNeeded() {
 	}
 }
 
-type sortedLabels []prompb.Label
+type sortedLabels []prompbmarshal.Label
 
 func (sl *sortedLabels) Len() int { return len(*sl) }
 func (sl *sortedLabels) Less(i, j int) bool {

app/vminsert/influx/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
@@ -150,7 +149,7 @@ type pushCtx struct {
 	Common         common.InsertCtx
 	metricNameBuf  []byte
 	metricGroupBuf []byte
-	originLabels   []prompb.Label
+	originLabels   []prompbmarshal.Label
 }
 
 func (ctx *pushCtx) reset() {
@@ -160,7 +159,7 @@ func (ctx *pushCtx) reset() {
 
 	originLabels := ctx.originLabels
 	for i := range originLabels {
-		originLabels[i] = prompb.Label{}
+		originLabels[i] = prompbmarshal.Label{}
 	}
 	ctx.originLabels = originLabels[:0]
 }

app/vminsert/main.go

@@ -41,7 +41,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
 
@@ -67,8 +66,6 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )
 
 var (
@@ -87,8 +84,6 @@ var staticServer = http.FileServer(http.FS(staticFiles))
 func Init() {
 	relabel.Init()
 	vminsertCommon.InitStreamAggr()
-	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
-	storage.SetMaxLabelValueLen(*maxLabelValueLen)
 	common.StartUnmarshalWorkers()
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
@@ -439,14 +434,4 @@ var (
 	promscrapeStatusConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
-
-	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
-		return float64(storage.MetricsWithDroppedLabels.Load())
-	})
-	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
-		return float64(storage.TooLongLabelNames.Load())
-	})
-	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
-		return float64(storage.TooLongLabelValues.Load())
-	})
 )

app/vminsert/relabel/relabel.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/metrics"
@@ -108,7 +107,7 @@ func (ctx *Ctx) Reset() {
 // ApplyRelabeling applies relabeling to the given labels and returns the result.
 //
 // The returned labels are valid until the next call to ApplyRelabeling.
-func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
+func (ctx *Ctx) ApplyRelabeling(labels []prompbmarshal.Label) []prompbmarshal.Label {
 	pcs := pcsGlobal.Load()
 	if pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// There are no relabeling rules.
@@ -159,7 +158,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 			name = ""
 		}
 		value := label.Value
-		dst = append(dst, prompb.Label{
+		dst = append(dst, prompbmarshal.Label{
 			Name:  name,
 			Value: value,
 		})

app/vmselect/graphite/tags_api.go

@@ -14,7 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
@@ -95,7 +95,7 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 	_ = deadline // TODO: use the deadline as in the cluster branch
 	paths := r.Form["path"]
 	var row graphiteparser.Row
-	var labels []prompb.Label
+	var labels []prompbmarshal.Label
 	var b []byte
 	var tagsPool []graphiteparser.Tag
 	mrs := make([]storage.MetricRow, len(paths))
@@ -122,12 +122,12 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 		canonicalPaths[i] = string(b)
 
 		// Convert parsed metric and tags to labels.
-		labels = append(labels[:0], prompb.Label{
+		labels = append(labels[:0], prompbmarshal.Label{
 			Name:  "__name__",
 			Value: row.Metric,
 		})
 		for _, tag := range row.Tags {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})

dashboards/victoriametrics-cluster.json

@@ -3674,12 +3674,12 @@
                 "uid": "$ds"
               },
               "exemplar": true,
-              "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
+              "expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
               "format": "time_series",
               "hide": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "metrics with dropped labels",
+              "legendFormat": "dropped series, which a hitting {{reason}}",
               "refId": "A"
             }
           ],

dashboards/victoriametrics.json

@@ -4190,12 +4190,12 @@
                 "uid": "$ds"
               },
               "exemplar": false,
-              "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)",
+              "expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance,reason)",
               "format": "time_series",
               "hide": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}} - limit exceeded",
+              "legendFormat": "{{instance}} - {{reason}}",
               "range": true,
               "refId": "A"
             }

dashboards/vm/victoriametrics-cluster.json

@@ -3675,12 +3675,12 @@
                 "uid": "$ds"
               },
               "exemplar": true,
-              "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
+              "expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
               "format": "time_series",
               "hide": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "metrics with dropped labels",
+              "legendFormat": "dropped series, which a hitting {{reason}}",
               "refId": "A"
             }
           ],

dashboards/vm/victoriametrics.json

@@ -4191,12 +4191,12 @@
                 "uid": "$ds"
               },
               "exemplar": false,
-              "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)",
+              "expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, reason)",
               "format": "time_series",
               "hide": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}} - limit exceeded",
+              "legendFormat": "{{instance}} - {{reason}}",
               "range": true,
               "refId": "A"
             }

deployment/docker/rules/alerts-cluster.yml

@@ -141,13 +141,13 @@ groups:
             See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
 
       - alert: LabelsLimitExceededOnIngestion
-        expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0
+        expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=116&var-instance={{ $labels.instance }}"
-          summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding labels limit"
+          summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`"
           description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
            This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
            correctly or that clients which send these metrics aren't misbehaving."

deployment/docker/rules/alerts-health.yml

@@ -120,10 +120,11 @@ groups:
             `-maxLabelValueLen` command-line flags.
 
       - alert: TooLongLabelValues
-        expr: increase(vm_too_long_label_values_total[5m]) > 0
+        expr: increase(vm_series_dropped_total{reason="too_long_label_value"}[5m]) > 0
         labels:
           severity: critical
         annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
           summary: "VictoriaMetrics truncates too long label values"
           description: | 
             The maximum length of a label value is limited via `-maxLabelValueLen` cmd-line flag. 
@@ -132,10 +133,11 @@ groups:
             either reduce the size of label values or increase `-maxLabelValueLen`.
 
       - alert: TooLongLabelNames
-        expr: increase(vm_too_long_label_names_total[5m]) > 0
+        expr: increase(vm_series_dropped_total{reason="too_long_label_name"}[5m]) > 0
         labels:
           severity: critical
         annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
           summary: "VictoriaMetrics truncates too long label names"
           description: > 
            The maximum length of a label name is limited by 256 bytes. 

deployment/docker/rules/alerts.yml

@@ -121,13 +121,13 @@ groups:
             See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
 
       - alert: LabelsLimitExceededOnIngestion
-        expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0
+        expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
-          summary: "Metrics ingested in ({{ $labels.instance }}) are exceeding labels limit"
+          summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`"
           description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
-           This prevents ingestion of metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
+           This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
            correctly or that clients which send these metrics aren't misbehaving."

docs/Cluster-VictoriaMetrics.md

@@ -1243,7 +1243,7 @@ Below is the output for `/path/to/vminsert -help`:
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096)
+     The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096)
   -maxLabelsPerTimeseries int
      The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size

docs/README.md

@@ -2462,9 +2462,10 @@ and [cardinality explorer docs](#cardinality-explorer).
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
 * VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag
-  and drops superfluous labels. This prevents from ingesting metrics with too many labels.
+  and drops series with superfluous labels. This prevents from ingesting metrics with too many labels.
-  It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  It is recommended [monitoring](#monitoring) `vm_series_dropped_total{reason="too_many_labels"}`
   metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+  Alternatively you can use [relabeling](https://docs.victoriametrics.com/single-server-victoriametrics/#relabeling) to change metric target labels.
 
 * If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `{__graphite__="foo.*.baz"}` filter can be used for selecting such metrics.
   See [these docs](#selecting-graphite-metrics) for details. You can also query Graphite metrics with [Graphite querying API](#graphite-render-api-usage).
@@ -2960,9 +2961,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096)
+     The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096)
   -maxLabelsPerTimeseries int
-     The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
+     The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

docs/changelog/CHANGELOG.md

@@ -20,12 +20,15 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 * SECURITY: upgrade Go builder from Go1.23.1 to Go1.23.3. See the list of issues addressed in [Go1.23.2](https://github.com/golang/go/issues?q=milestone%3AGo1.23.2+label%3ACherryPickApproved) and [Go1.23.3](https://github.com/golang/go/issues?q=milestone%3AGo1.23.3+label%3ACherryPickApproved).
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent) added `-maxLabelsPerTimeseries` and `-maxLabelValueLen` flags, which limits amount of labels, label name and value length for pushed to/scraped by vmagent data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928).
+
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068).
 * BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly inherit [`drop_src_path_prefix_parts`](https://docs.victoriametrics.com/vmauth/#dropping-request-path-prefix), [`load_balancing_policy`](https://docs.victoriametrics.com/vmauth/#high-availability), [`retry_status_codes`](https://docs.victoriametrics.com/vmauth/#load-balancing) and [`discover_backend_ips`](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) options by `url_map` entries if `url_prefix` option isn't set at the [user config level](https://docs.victoriametrics.com/vmauth/#auth-config). These options were inherited only when the `url_prefix` option was set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7519).
 * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): add `file` label filter to vmalert dashboard panels. Previously, metrics from groups with the same name but different rule files could be mixed in the results.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) changed a meaning of `-maxLabelsPerTimeseries` and `-maxLabelValueLen`. Previously excessive labels, label names and values were truncated. With this change all series, which are hitting these limits will be dropped. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928)
 
 ## [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0)
 

docs/vmagent.md

@@ -1893,6 +1893,10 @@ See the docs at https://docs.victoriametrics.com/vmagent/ .
   -maxInsertRequestSize size
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
+  -maxLabelValueLen int
+     The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096).
+  -maxLabelsPerTimeseries int
+     The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

lib/prompbmarshal/types.pb.go

@@ -6,6 +6,8 @@ package prompbmarshal
 import (
 	"encoding/binary"
 	"math"
+	"sort"
+	"strconv"
 )
 
 type Sample struct {
@@ -124,3 +126,27 @@ func (m *Label) Size() (n int) {
 	}
 	return n
 }
+
+// LabelsToString converts labels to Prometheus-compatible string
+func LabelsToString(labels []Label) string {
+	labelsCopy := append([]Label{}, labels...)
+	sort.Slice(labelsCopy, func(i, j int) bool {
+		return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
+	})
+	var b []byte
+	b = append(b, '{')
+	for i, label := range labelsCopy {
+		if len(label.Name) == 0 {
+			b = append(b, "__name__"...)
+		} else {
+			b = append(b, label.Name...)
+		}
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		if i < len(labels)-1 {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, '}')
+	return string(b)
+}

lib/storage/limits/limits.go

@@ -0,0 +1,113 @@
+package limits
+
+import (
+	"flag"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+const maxLabelNameLen = 256
+
+var (
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels per time series to be accepted. Timeseries with superfluous labels are dropped. In this case the vm_series_dropped_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
+	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Metrics with longer label value are dropped. In this case the vm_series_dropped_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+)
+
+var (
+	droppedSeriesWithTooManyLabelsLogTicker     = time.NewTicker(5 * time.Second)
+	droppedSeriesWithTooLongLabelNameLogTicker  = time.NewTicker(5 * time.Second)
+	droppedSeriesWithTooLongLabelValueLogTicker = time.NewTicker(5 * time.Second)
+)
+
+var (
+	// droppedSeriesWithTooManyLabels is the number of dropped series with too many labels
+	droppedSeriesWithTooManyLabels atomic.Uint64
+
+	// droppedSeriesWithTooLongLabelName is the number of dropped series which contain labels with too long names
+	droppedSeriesWithTooLongLabelName atomic.Uint64
+
+	// droppedSeriesWithTooLongLabelValue is the number of dropped series which contain labels with too long values
+	droppedSeriesWithTooLongLabelValue atomic.Uint64
+)
+
+var (
+	_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_many_labels"}`, func() float64 {
+		return float64(droppedSeriesWithTooManyLabels.Load())
+	})
+	_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_name"}`, func() float64 {
+		return float64(droppedSeriesWithTooLongLabelName.Load())
+	})
+	_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_value"}`, func() float64 {
+		return float64(droppedSeriesWithTooLongLabelValue.Load())
+	})
+)
+
+func trackDroppedSeriesWithTooManyLabels(labels []prompbmarshal.Label) {
+	droppedSeriesWithTooManyLabels.Add(1)
+	select {
+	case <-droppedSeriesWithTooManyLabelsLogTicker.C:
+		// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
+		// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooManyLabels call.
+		logger.Warnf("dropping series with %d labels for %s; either reduce the number of labels for this metric "+
+			"or increase -maxLabelsPerTimeseries=%d command-line flag value",
+			len(labels), prompbmarshal.LabelsToString(labels), *maxLabelsPerTimeseries)
+	default:
+	}
+}
+
+func trackDroppedSeriesWithTooLongLabelValue(l *prompbmarshal.Label, labels []prompbmarshal.Label) {
+	droppedSeriesWithTooLongLabelValue.Add(1)
+	select {
+	case <-droppedSeriesWithTooLongLabelValueLogTicker.C:
+		label := *l
+		// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
+		// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelValue call.
+		logger.Warnf("drop series with a value %s for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+
+			"original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value",
+			label.Value, label.Name, len(label.Value), *maxLabelValueLen, prompbmarshal.LabelsToString(labels))
+	default:
+	}
+}
+
+func trackDroppedSeriesWithTooLongLabelName(l *prompbmarshal.Label, labels []prompbmarshal.Label) {
+	droppedSeriesWithTooLongLabelName.Add(1)
+	select {
+	case <-droppedSeriesWithTooLongLabelNameLogTicker.C:
+		label := *l
+		// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
+		// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelName call.
+		logger.Warnf("drop series with a value for label %s because its length=%d exceeds %d; "+
+			"original labels: %s; consider reducing the label name length",
+			label.Name, len(label.Name), maxLabelNameLen, prompbmarshal.LabelsToString(labels))
+	default:
+	}
+}
+
+// ExceedingLabels checks if passed labels exceed one of the limits:
+// * Maximum allowed labels limit
+// * Maximum allowed label name length limit
+// * Maximum allowed label value length limit
+//
+// increments metrics and shows warning in logs
+func ExceedingLabels(labels []prompbmarshal.Label) bool {
+	if len(labels) > *maxLabelsPerTimeseries {
+		trackDroppedSeriesWithTooManyLabels(labels)
+		return true
+	}
+	for _, l := range labels {
+		if len(l.Name) > maxLabelNameLen {
+			trackDroppedSeriesWithTooLongLabelName(&l, labels)
+			return true
+		}
+		if len(l.Value) > *maxLabelValueLen {
+			trackDroppedSeriesWithTooLongLabelValue(&l, labels)
+			return true
+		}
+	}
+	return false
+}

lib/storage/metric_name.go

@@ -5,16 +5,12 @@ import (
 	"fmt"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 )
 
@@ -467,63 +463,15 @@ func (mn *MetricName) Unmarshal(src []byte) error {
 	return nil
 }
 
-// The maximum length of label name.
-//
-// Longer names are truncated.
-const maxLabelNameLen = 256
-
-// The maximum length of label value.
-//
-// Longer values are truncated.
-var maxLabelValueLen = 1024
-
-// SetMaxLabelValueLen sets the limit on the label value length.
-//
-// This function can be called before using the storage package.
-//
-// Label values with longer length are truncated.
-func SetMaxLabelValueLen(n int) {
-	if n > 0 {
-		maxLabelValueLen = n
-	}
-}
-
-// The maximum number of labels per each timeseries.
-var maxLabelsPerTimeseries = 30
-
-// SetMaxLabelsPerTimeseries sets the limit on the number of labels
-// per each time series.
-//
-// This function can be called before using the storage package.
-//
-// Superfluous labels are dropped.
-func SetMaxLabelsPerTimeseries(maxLabels int) {
-	if maxLabels > 0 {
-		maxLabelsPerTimeseries = maxLabels
-	}
-}
-
 // MarshalMetricNameRaw marshals labels to dst and returns the result.
 //
 // The result must be unmarshaled with MetricName.UnmarshalRaw
-func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
+func MarshalMetricNameRaw(dst []byte, labels []prompbmarshal.Label) []byte {
 	// Calculate the required space for dst.
 	dstLen := len(dst)
 	dstSize := dstLen
 	for i := range labels {
-		if i >= maxLabelsPerTimeseries {
-			trackDroppedLabels(labels, labels[i:])
-			break
-		}
 		label := &labels[i]
-		if len(label.Name) > maxLabelNameLen {
-			TooLongLabelNames.Add(1)
-			label.Name = label.Name[:maxLabelNameLen]
-		}
-		if len(label.Value) > maxLabelValueLen {
-			trackTruncatedLabels(labels, label)
-			label.Value = label.Value[:maxLabelValueLen]
-		}
 		if len(label.Value) == 0 {
 			// Skip labels without values, since they have no sense in prometheus.
 			continue
@@ -539,9 +487,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 
 	// Marshal labels to dst.
 	for i := range labels {
-		if i >= maxLabelsPerTimeseries {
-			break
-		}
 		label := &labels[i]
 		if len(label.Value) == 0 {
 			// Skip labels without values, since they have no sense in prometheus.
@@ -553,71 +498,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 	return dst
 }
 
-var (
-	// MetricsWithDroppedLabels is the number of metrics with at least a single dropped label
-	MetricsWithDroppedLabels atomic.Uint64
-
-	// TooLongLabelNames is the number of too long label names
-	TooLongLabelNames atomic.Uint64
-
-	// TooLongLabelValues is the number of too long label values
-	TooLongLabelValues atomic.Uint64
-)
-
-func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
-	MetricsWithDroppedLabels.Add(1)
-	select {
-	case <-droppedLabelsLogTicker.C:
-		// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
-		// because labelsToString() will be called with each trackDroppedLabels call.
-		logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
-			"or increase -maxLabelsPerTimeseries=%d command-line flag value",
-			len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
-	default:
-	}
-}
-
-func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) {
-	TooLongLabelValues.Add(1)
-	select {
-	case <-truncatedLabelsLogTicker.C:
-		// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
-		// because labelsToString() will be called with each trackTruncatedLabels call.
-		logger.Warnf("truncate value for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+
-			"original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value",
-			truncated.Name, len(truncated.Value), maxLabelValueLen, labelsToString(labels))
-	default:
-	}
-}
-
-var (
-	droppedLabelsLogTicker   = time.NewTicker(5 * time.Second)
-	truncatedLabelsLogTicker = time.NewTicker(5 * time.Second)
-)
-
-func labelsToString(labels []prompb.Label) string {
-	labelsCopy := append([]prompb.Label{}, labels...)
-	sort.Slice(labelsCopy, func(i, j int) bool {
-		return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
-	})
-	var b []byte
-	b = append(b, '{')
-	for i, label := range labelsCopy {
-		if len(label.Name) == 0 {
-			b = append(b, "__name__"...)
-		} else {
-			b = append(b, label.Name...)
-		}
-		b = append(b, '=')
-		b = strconv.AppendQuote(b, string(label.Value))
-		if i < len(labels)-1 {
-			b = append(b, ',')
-		}
-	}
-	b = append(b, '}')
-	return string(b)
-}
-
 // marshalRaw marshals mn to dst and returns the result.
 //
 // The results may be unmarshaled with MetricName.UnmarshalRaw.