From a921ad738015770a278b7beddf706d3cbc61cfc8 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Fri, 15 Nov 2024 14:05:14 +0200 Subject: [PATCH] app/{vminsert,vmagent}: drop series on exceeding -maxLabelsPerTimeseries, -maxLabelValueLen flags and maxLabelNameLen=256 limits --- app/victoria-metrics/self_scraper.go | 8 +- app/vmagent/remotewrite/remotewrite.go | 32 ++---- app/vmctl/vm_native_test.go | 8 +- app/vminsert/common/insert_ctx.go | 21 ++-- app/vminsert/common/sort_labels.go | 4 +- app/vminsert/influx/request_handler.go | 5 +- app/vminsert/main.go | 19 +--- app/vminsert/relabel/relabel.go | 5 +- app/vmselect/graphite/tags_api.go | 8 +- dashboards/victoriametrics-cluster.json | 4 +- dashboards/victoriametrics.json | 6 +- dashboards/vm/victoriametrics-cluster.json | 4 +- dashboards/vm/victoriametrics.json | 6 +- deployment/docker/rules/alerts-cluster.yml | 4 +- deployment/docker/rules/alerts-health.yml | 8 +- deployment/docker/rules/alerts.yml | 8 +- docs/Cluster-VictoriaMetrics.md | 2 +- docs/README.md | 9 +- docs/changelog/CHANGELOG.md | 3 + docs/vmagent.md | 4 + lib/prompbmarshal/types.pb.go | 26 +++++ lib/storage/limits/limits.go | 113 +++++++++++++++++++ lib/storage/metric_name.go | 124 +-------------------- 23 files changed, 221 insertions(+), 210 deletions(-) create mode 100644 lib/storage/limits/limits.go diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go index cc405884d..f1e7a4d3a 100644 --- a/app/victoria-metrics/self_scraper.go +++ b/app/victoria-metrics/self_scraper.go @@ -10,7 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) @@ -48,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) { var bb bytesutil.ByteBuffer var rows prometheus.Rows var mrs []storage.MetricRow - var labels []prompb.Label + var labels []prompbmarshal.Label t := time.NewTicker(scrapeInterval) f := func(currentTime time.Time, sendStaleMarkers bool) { currentTimestamp := currentTime.UnixNano() / 1e6 @@ -99,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) { } } -func addLabel(dst []prompb.Label, key, value string) []prompb.Label { +func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label { if len(dst) < cap(dst) { dst = dst[:len(dst)+1] } else { - dst = append(dst, prompb.Label{}) + dst = append(dst, prompbmarshal.Label{}) } lb := &dst[len(dst)-1] lb.Name = key diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index ea74c4985..bd5d01a2c 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -7,13 +7,10 @@ import ( "net/url" "path/filepath" "slices" - "strconv" "sync" "sync/atomic" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" @@ -21,6 +18,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" "github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue" @@ -29,6 +27,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter" + storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits" "github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr" "github.com/VictoriaMetrics/metrics" "github.com/cespare/xxhash/v2" @@ -485,6 +484,14 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF rowsCountAfterRelabel := getRowsCount(tssBlock) rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel) } + idxDiff := 0 + for i, ts := range tssBlock { + if storagelimits.ExceedingLabels(ts.Labels) { + idxDiff++ + } else if idxDiff > 0 { + tss[i-idxDiff] = tss[i] + } + } sortLabelsIfNeeded(tssBlock) tssBlock = limitSeriesCardinality(tssBlock) if sas.IsEnabled() { @@ -729,29 +736,14 @@ func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue i select { case <-logSkippedSeriesTicker.C: // Do not use logger.WithThrottler() here, since this will increase CPU usage - // because every call to logSkippedSeries will result to a call to labelsToString. - logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue) + // because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString. + logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue) default: } } var logSkippedSeriesTicker = time.NewTicker(5 * time.Second) -func labelsToString(labels []prompbmarshal.Label) string { - var b []byte - b = append(b, '{') - for i, label := range labels { - b = append(b, label.Name...) - b = append(b, '=') - b = strconv.AppendQuote(b, label.Value) - if i+1 < len(labels) { - b = append(b, ',') - } - } - b = append(b, '}') - return string(b) -} - var ( globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total") rowsDroppedByGlobalRelabel = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total") diff --git a/app/vmctl/vm_native_test.go b/app/vmctl/vm_native_test.go index fa7bf4b68..f94ca7cb2 100644 --- a/app/vmctl/vm_native_test.go +++ b/app/vmctl/vm_native_test.go @@ -17,7 +17,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) @@ -214,15 +214,15 @@ func processFlags() { func fillStorage(series []vm.TimeSeries) error { var mrs []storage.MetricRow for _, series := range series { - var labels []prompb.Label + var labels []prompbmarshal.Label for _, lp := range series.LabelPairs { - labels = append(labels, prompb.Label{ + labels = append(labels, prompbmarshal.Label{ Name: lp.Name, Value: lp.Value, }) } if series.Name != "" { - labels = append(labels, prompb.Label{ + labels = append(labels, prompbmarshal.Label{ Name: "__name__", Value: series.Name, }) diff --git a/app/vminsert/common/insert_ctx.go b/app/vminsert/common/insert_ctx.go index f7a06960b..22abcb545 100644 --- a/app/vminsert/common/insert_ctx.go +++ b/app/vminsert/common/insert_ctx.go @@ -8,9 +8,10 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" + storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits" ) // InsertCtx contains common bits for data points insertion. @@ -30,7 +31,7 @@ type InsertCtx struct { func (ctx *InsertCtx) Reset(rowsLen int) { labels := ctx.Labels for i := range labels { - labels[i] = prompb.Label{} + labels[i] = prompbmarshal.Label{} } ctx.Labels = labels[:0] @@ -51,7 +52,7 @@ func cleanMetricRow(mr *storage.MetricRow) { mr.MetricNameRaw = nil } -func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte { +func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompbmarshal.Label) []byte { start := len(ctx.metricNamesBuf) ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...) ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels) @@ -60,7 +61,10 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) } // WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer. -func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) error { +func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error { + if storagelimits.ExceedingLabels(labels) { + return nil + } metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels) return ctx.addRow(metricNameRaw, timestamp, value) } @@ -68,7 +72,10 @@ func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, times // WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer. // // It returns metricNameRaw for the given labels if len(metricNameRaw) == 0. -func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) ([]byte, error) { +func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompbmarshal.Label, timestamp int64, value float64) ([]byte, error) { + if storagelimits.ExceedingLabels(labels) { + return metricNameRaw, nil + } if len(metricNameRaw) == 0 { metricNameRaw = ctx.marshalMetricNameRaw(nil, labels) } @@ -106,7 +113,7 @@ func (ctx *InsertCtx) AddLabelBytes(name, value []byte) { // Do not skip labels with empty name, since they are equal to __name__. return } - ctx.Labels = append(ctx.Labels, prompb.Label{ + ctx.Labels = append(ctx.Labels, prompbmarshal.Label{ // Do not copy name and value contents for performance reasons. // This reduces GC overhead on the number of objects and allocations. Name: bytesutil.ToUnsafeString(name), @@ -124,7 +131,7 @@ func (ctx *InsertCtx) AddLabel(name, value string) { // Do not skip labels with empty name, since they are equal to __name__. return } - ctx.Labels = append(ctx.Labels, prompb.Label{ + ctx.Labels = append(ctx.Labels, prompbmarshal.Label{ // Do not copy name and value contents for performance reasons. // This reduces GC overhead on the number of objects and allocations. Name: name, diff --git a/app/vminsert/common/sort_labels.go b/app/vminsert/common/sort_labels.go index 16fa88cc0..5ca933708 100644 --- a/app/vminsert/common/sort_labels.go +++ b/app/vminsert/common/sort_labels.go @@ -4,7 +4,7 @@ import ( "flag" "sort" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to storage. `+ @@ -19,7 +19,7 @@ func (ctx *InsertCtx) SortLabelsIfNeeded() { } } -type sortedLabels []prompb.Label +type sortedLabels []prompbmarshal.Label func (sl *sortedLabels) Len() int { return len(*sl) } func (sl *sortedLabels) Less(i, j int) bool { diff --git a/app/vminsert/influx/request_handler.go b/app/vminsert/influx/request_handler.go index 632501814..2ad0e4e94 100644 --- a/app/vminsert/influx/request_handler.go +++ b/app/vminsert/influx/request_handler.go @@ -9,7 +9,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx" @@ -150,7 +149,7 @@ type pushCtx struct { Common common.InsertCtx metricNameBuf []byte metricGroupBuf []byte - originLabels []prompb.Label + originLabels []prompbmarshal.Label } func (ctx *pushCtx) reset() { @@ -160,7 +159,7 @@ func (ctx *pushCtx) reset() { originLabels := ctx.originLabels for i := range originLabels { - originLabels[i] = prompb.Label{} + originLabels[i] = prompbmarshal.Label{} } ctx.originLabels = originLabels[:0] } diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 9b02dc18d..fdffcb92c 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -41,7 +41,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil" ) @@ -65,10 +64,8 @@ var ( "See also -opentsdbHTTPListenAddr.useProxyProtocol") opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+ "at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt") - configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*") - reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") - maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented") - maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented") + configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*") + reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") ) var ( @@ -87,8 +84,6 @@ var staticServer = http.FileServer(http.FS(staticFiles)) func Init() { relabel.Init() vminsertCommon.InitStreamAggr() - storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries) - storage.SetMaxLabelValueLen(*maxLabelValueLen) common.StartUnmarshalWorkers() if len(*graphiteListenAddr) > 0 { graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler) @@ -439,14 +434,4 @@ var ( promscrapeStatusConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/config"}`) promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`) - - _ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 { - return float64(storage.MetricsWithDroppedLabels.Load()) - }) - _ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 { - return float64(storage.TooLongLabelNames.Load()) - }) - _ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 { - return float64(storage.TooLongLabelValues.Load()) - }) ) diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go index 17a969445..444e05d0b 100644 --- a/app/vminsert/relabel/relabel.go +++ b/app/vminsert/relabel/relabel.go @@ -8,7 +8,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/metrics" @@ -108,7 +107,7 @@ func (ctx *Ctx) Reset() { // ApplyRelabeling applies relabeling to the given labels and returns the result. // // The returned labels are valid until the next call to ApplyRelabeling. -func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label { +func (ctx *Ctx) ApplyRelabeling(labels []prompbmarshal.Label) []prompbmarshal.Label { pcs := pcsGlobal.Load() if pcs.Len() == 0 && !*usePromCompatibleNaming { // There are no relabeling rules. @@ -159,7 +158,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label { name = "" } value := label.Value - dst = append(dst, prompb.Label{ + dst = append(dst, prompbmarshal.Label{ Name: name, Value: value, }) diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index d558388a0..393478d69 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -14,7 +14,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" @@ -95,7 +95,7 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request _ = deadline // TODO: use the deadline as in the cluster branch paths := r.Form["path"] var row graphiteparser.Row - var labels []prompb.Label + var labels []prompbmarshal.Label var b []byte var tagsPool []graphiteparser.Tag mrs := make([]storage.MetricRow, len(paths)) @@ -122,12 +122,12 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request canonicalPaths[i] = string(b) // Convert parsed metric and tags to labels. - labels = append(labels[:0], prompb.Label{ + labels = append(labels[:0], prompbmarshal.Label{ Name: "__name__", Value: row.Metric, }) for _, tag := range row.Tags { - labels = append(labels, prompb.Label{ + labels = append(labels, prompbmarshal.Label{ Name: tag.Key, Value: tag.Value, }) diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json index bead53552..4dd61c832 100644 --- a/dashboards/victoriametrics-cluster.json +++ b/dashboards/victoriametrics-cluster.json @@ -3674,12 +3674,12 @@ "uid": "$ds" }, "exemplar": true, - "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "metrics with dropped labels", + "legendFormat": "dropped series, which a hitting {{reason}}", "refId": "A" } ], diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json index 674eb3180..0b7ce6b31 100644 --- a/dashboards/victoriametrics.json +++ b/dashboards/victoriametrics.json @@ -4190,12 +4190,12 @@ "uid": "$ds" }, "exemplar": false, - "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance,reason)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}} - limit exceeded", + "legendFormat": "{{instance}} - {{reason}}", "range": true, "refId": "A" } @@ -6133,4 +6133,4 @@ "uid": "wNf0q_kZk", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/dashboards/vm/victoriametrics-cluster.json b/dashboards/vm/victoriametrics-cluster.json index bbdcd6e47..e536e4bd4 100644 --- a/dashboards/vm/victoriametrics-cluster.json +++ b/dashboards/vm/victoriametrics-cluster.json @@ -3675,12 +3675,12 @@ "uid": "$ds" }, "exemplar": true, - "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "metrics with dropped labels", + "legendFormat": "dropped series, which a hitting {{reason}}", "refId": "A" } ], diff --git a/dashboards/vm/victoriametrics.json b/dashboards/vm/victoriametrics.json index 7bf01e0f2..0bd583665 100644 --- a/dashboards/vm/victoriametrics.json +++ b/dashboards/vm/victoriametrics.json @@ -4191,12 +4191,12 @@ "uid": "$ds" }, "exemplar": false, - "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, reason)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}} - limit exceeded", + "legendFormat": "{{instance}} - {{reason}}", "range": true, "refId": "A" } @@ -6134,4 +6134,4 @@ "uid": "wNf0q_kZk_vm", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/deployment/docker/rules/alerts-cluster.yml b/deployment/docker/rules/alerts-cluster.yml index 0d96e19f1..f89fa57a8 100644 --- a/deployment/docker/rules/alerts-cluster.yml +++ b/deployment/docker/rules/alerts-cluster.yml @@ -141,13 +141,13 @@ groups: See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183" - alert: LabelsLimitExceededOnIngestion - expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0 + expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0 for: 15m labels: severity: warning annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=116&var-instance={{ $labels.instance }}" - summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding labels limit" + summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`" description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving." diff --git a/deployment/docker/rules/alerts-health.yml b/deployment/docker/rules/alerts-health.yml index 98713d9d1..fc93f025d 100644 --- a/deployment/docker/rules/alerts-health.yml +++ b/deployment/docker/rules/alerts-health.yml @@ -120,10 +120,11 @@ groups: `-maxLabelValueLen` command-line flags. - alert: TooLongLabelValues - expr: increase(vm_too_long_label_values_total[5m]) > 0 + expr: increase(vm_series_dropped_total{reason="too_long_label_value"}[5m]) > 0 labels: severity: critical annotations: + dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}" summary: "VictoriaMetrics truncates too long label values" description: | The maximum length of a label value is limited via `-maxLabelValueLen` cmd-line flag. @@ -132,11 +133,12 @@ groups: either reduce the size of label values or increase `-maxLabelValueLen`. - alert: TooLongLabelNames - expr: increase(vm_too_long_label_names_total[5m]) > 0 + expr: increase(vm_series_dropped_total{reason="too_long_label_name"}[5m]) > 0 labels: severity: critical annotations: + dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}" summary: "VictoriaMetrics truncates too long label names" description: > The maximum length of a label name is limited by 256 bytes. - Longer label names are truncated and may result into time series overlapping. \ No newline at end of file + Longer label names are truncated and may result into time series overlapping. diff --git a/deployment/docker/rules/alerts.yml b/deployment/docker/rules/alerts.yml index 7c7832c6f..429d0d4ec 100644 --- a/deployment/docker/rules/alerts.yml +++ b/deployment/docker/rules/alerts.yml @@ -121,13 +121,13 @@ groups: See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183" - alert: LabelsLimitExceededOnIngestion - expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0 + expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0 for: 15m labels: severity: warning annotations: dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}" - summary: "Metrics ingested in ({{ $labels.instance }}) are exceeding labels limit" + summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`" description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n - This prevents ingestion of metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured - correctly or that clients which send these metrics aren't misbehaving." \ No newline at end of file + This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured + correctly or that clients which send these metrics aren't misbehaving." diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 992973bd7..a5ab6707f 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -1243,7 +1243,7 @@ Below is the output for `/path/to/vminsert -help`: The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) -maxLabelValueLen int - The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096) + The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096) -maxLabelsPerTimeseries int The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30) -memory.allowedBytes size diff --git a/docs/README.md b/docs/README.md index 333b85cf0..27a8a4b70 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2462,9 +2462,10 @@ and [cardinality explorer docs](#cardinality-explorer). * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics. * VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag - and drops superfluous labels. This prevents from ingesting metrics with too many labels. - It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total` + and drops series with superfluous labels. This prevents from ingesting metrics with too many labels. + It is recommended [monitoring](#monitoring) `vm_series_dropped_total{reason="too_many_labels"}` metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload. + Alternatively you can use [relabeling](https://docs.victoriametrics.com/single-server-victoriametrics/#relabeling) to change metric target labels. * If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `{__graphite__="foo.*.baz"}` filter can be used for selecting such metrics. See [these docs](#selecting-graphite-metrics) for details. You can also query Graphite metrics with [Graphite querying API](#graphite-render-api-usage). @@ -2960,9 +2961,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) -maxLabelValueLen int - The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096) + The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096) -maxLabelsPerTimeseries int - The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30) + The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30) -memory.allowedBytes size Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 315e3da9e..45d41951a 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -20,12 +20,15 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * SECURITY: upgrade Go builder from Go1.23.1 to Go1.23.3. See the list of issues addressed in [Go1.23.2](https://github.com/golang/go/issues?q=milestone%3AGo1.23.2+label%3ACherryPickApproved) and [Go1.23.3](https://github.com/golang/go/issues?q=milestone%3AGo1.23.3+label%3ACherryPickApproved). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent) added `-maxLabelsPerTimeseries` and `-maxLabelValueLen` flags, which limits amount of labels, label name and value length for pushed to/scraped by vmagent data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928). + * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test. * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068). * BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details. * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details. * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly inherit [`drop_src_path_prefix_parts`](https://docs.victoriametrics.com/vmauth/#dropping-request-path-prefix), [`load_balancing_policy`](https://docs.victoriametrics.com/vmauth/#high-availability), [`retry_status_codes`](https://docs.victoriametrics.com/vmauth/#load-balancing) and [`discover_backend_ips`](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) options by `url_map` entries if `url_prefix` option isn't set at the [user config level](https://docs.victoriametrics.com/vmauth/#auth-config). These options were inherited only when the `url_prefix` option was set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7519). * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): add `file` label filter to vmalert dashboard panels. Previously, metrics from groups with the same name but different rule files could be mixed in the results. +* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) changed a meaning of `-maxLabelsPerTimeseries` and `-maxLabelValueLen`. Previously excessive labels, label names and values were truncated. With this change all series, which are hitting these limits will be dropped. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928) ## [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0) diff --git a/docs/vmagent.md b/docs/vmagent.md index 2ecb7c524..b297b79d8 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1893,6 +1893,10 @@ See the docs at https://docs.victoriametrics.com/vmagent/ . -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) + -maxLabelValueLen int + The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096). + -maxLabelsPerTimeseries int + The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30) -memory.allowedBytes size Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) diff --git a/lib/prompbmarshal/types.pb.go b/lib/prompbmarshal/types.pb.go index ca2b62f1b..c3e25e2c2 100644 --- a/lib/prompbmarshal/types.pb.go +++ b/lib/prompbmarshal/types.pb.go @@ -6,6 +6,8 @@ package prompbmarshal import ( "encoding/binary" "math" + "sort" + "strconv" ) type Sample struct { @@ -124,3 +126,27 @@ func (m *Label) Size() (n int) { } return n } + +// LabelsToString converts labels to Prometheus-compatible string +func LabelsToString(labels []Label) string { + labelsCopy := append([]Label{}, labels...) + sort.Slice(labelsCopy, func(i, j int) bool { + return string(labelsCopy[i].Name) < string(labelsCopy[j].Name) + }) + var b []byte + b = append(b, '{') + for i, label := range labelsCopy { + if len(label.Name) == 0 { + b = append(b, "__name__"...) + } else { + b = append(b, label.Name...) + } + b = append(b, '=') + b = strconv.AppendQuote(b, label.Value) + if i < len(labels)-1 { + b = append(b, ',') + } + } + b = append(b, '}') + return string(b) +} diff --git a/lib/storage/limits/limits.go b/lib/storage/limits/limits.go new file mode 100644 index 000000000..a0d6b4aa3 --- /dev/null +++ b/lib/storage/limits/limits.go @@ -0,0 +1,113 @@ +package limits + +import ( + "flag" + "sync/atomic" + "time" + + "github.com/VictoriaMetrics/metrics" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" +) + +const maxLabelNameLen = 256 + +var ( + maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels per time series to be accepted. Timeseries with superfluous labels are dropped. In this case the vm_series_dropped_total{reason=\"too_many_labels\"} metric at /metrics page is incremented") + maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Metrics with longer label value are dropped. In this case the vm_series_dropped_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented") +) + +var ( + droppedSeriesWithTooManyLabelsLogTicker = time.NewTicker(5 * time.Second) + droppedSeriesWithTooLongLabelNameLogTicker = time.NewTicker(5 * time.Second) + droppedSeriesWithTooLongLabelValueLogTicker = time.NewTicker(5 * time.Second) +) + +var ( + // droppedSeriesWithTooManyLabels is the number of dropped series with too many labels + droppedSeriesWithTooManyLabels atomic.Uint64 + + // droppedSeriesWithTooLongLabelName is the number of dropped series which contain labels with too long names + droppedSeriesWithTooLongLabelName atomic.Uint64 + + // droppedSeriesWithTooLongLabelValue is the number of dropped series which contain labels with too long values + droppedSeriesWithTooLongLabelValue atomic.Uint64 +) + +var ( + _ = metrics.NewGauge(`vm_series_dropped_total{reason="too_many_labels"}`, func() float64 { + return float64(droppedSeriesWithTooManyLabels.Load()) + }) + _ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_name"}`, func() float64 { + return float64(droppedSeriesWithTooLongLabelName.Load()) + }) + _ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_value"}`, func() float64 { + return float64(droppedSeriesWithTooLongLabelValue.Load()) + }) +) + +func trackDroppedSeriesWithTooManyLabels(labels []prompbmarshal.Label) { + droppedSeriesWithTooManyLabels.Add(1) + select { + case <-droppedSeriesWithTooManyLabelsLogTicker.C: + // Do not call logger.WithThrottler() here, since this will result in increased CPU usage + // because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooManyLabels call. + logger.Warnf("dropping series with %d labels for %s; either reduce the number of labels for this metric "+ + "or increase -maxLabelsPerTimeseries=%d command-line flag value", + len(labels), prompbmarshal.LabelsToString(labels), *maxLabelsPerTimeseries) + default: + } +} + +func trackDroppedSeriesWithTooLongLabelValue(l *prompbmarshal.Label, labels []prompbmarshal.Label) { + droppedSeriesWithTooLongLabelValue.Add(1) + select { + case <-droppedSeriesWithTooLongLabelValueLogTicker.C: + label := *l + // Do not call logger.WithThrottler() here, since this will result in increased CPU usage + // because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelValue call. + logger.Warnf("drop series with a value %s for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+ + "original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value", + label.Value, label.Name, len(label.Value), *maxLabelValueLen, prompbmarshal.LabelsToString(labels)) + default: + } +} + +func trackDroppedSeriesWithTooLongLabelName(l *prompbmarshal.Label, labels []prompbmarshal.Label) { + droppedSeriesWithTooLongLabelName.Add(1) + select { + case <-droppedSeriesWithTooLongLabelNameLogTicker.C: + label := *l + // Do not call logger.WithThrottler() here, since this will result in increased CPU usage + // because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelName call. + logger.Warnf("drop series with a value for label %s because its length=%d exceeds %d; "+ + "original labels: %s; consider reducing the label name length", + label.Name, len(label.Name), maxLabelNameLen, prompbmarshal.LabelsToString(labels)) + default: + } +} + +// ExceedingLabels checks if passed labels exceed one of the limits: +// * Maximum allowed labels limit +// * Maximum allowed label name length limit +// * Maximum allowed label value length limit +// +// increments metrics and shows warning in logs +func ExceedingLabels(labels []prompbmarshal.Label) bool { + if len(labels) > *maxLabelsPerTimeseries { + trackDroppedSeriesWithTooManyLabels(labels) + return true + } + for _, l := range labels { + if len(l.Name) > maxLabelNameLen { + trackDroppedSeriesWithTooLongLabelName(&l, labels) + return true + } + if len(l.Value) > *maxLabelValueLen { + trackDroppedSeriesWithTooLongLabelValue(&l, labels) + return true + } + } + return false +} diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index bbf262c3e..86c8bc793 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -5,16 +5,12 @@ import ( "fmt" "runtime" "sort" - "strconv" "strings" "sync" - "sync/atomic" - "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil" ) @@ -467,63 +463,15 @@ func (mn *MetricName) Unmarshal(src []byte) error { return nil } -// The maximum length of label name. -// -// Longer names are truncated. -const maxLabelNameLen = 256 - -// The maximum length of label value. -// -// Longer values are truncated. -var maxLabelValueLen = 1024 - -// SetMaxLabelValueLen sets the limit on the label value length. -// -// This function can be called before using the storage package. -// -// Label values with longer length are truncated. -func SetMaxLabelValueLen(n int) { - if n > 0 { - maxLabelValueLen = n - } -} - -// The maximum number of labels per each timeseries. -var maxLabelsPerTimeseries = 30 - -// SetMaxLabelsPerTimeseries sets the limit on the number of labels -// per each time series. -// -// This function can be called before using the storage package. -// -// Superfluous labels are dropped. -func SetMaxLabelsPerTimeseries(maxLabels int) { - if maxLabels > 0 { - maxLabelsPerTimeseries = maxLabels - } -} - // MarshalMetricNameRaw marshals labels to dst and returns the result. // // The result must be unmarshaled with MetricName.UnmarshalRaw -func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte { +func MarshalMetricNameRaw(dst []byte, labels []prompbmarshal.Label) []byte { // Calculate the required space for dst. dstLen := len(dst) dstSize := dstLen for i := range labels { - if i >= maxLabelsPerTimeseries { - trackDroppedLabels(labels, labels[i:]) - break - } label := &labels[i] - if len(label.Name) > maxLabelNameLen { - TooLongLabelNames.Add(1) - label.Name = label.Name[:maxLabelNameLen] - } - if len(label.Value) > maxLabelValueLen { - trackTruncatedLabels(labels, label) - label.Value = label.Value[:maxLabelValueLen] - } if len(label.Value) == 0 { // Skip labels without values, since they have no sense in prometheus. continue @@ -539,9 +487,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte { // Marshal labels to dst. for i := range labels { - if i >= maxLabelsPerTimeseries { - break - } label := &labels[i] if len(label.Value) == 0 { // Skip labels without values, since they have no sense in prometheus. @@ -553,71 +498,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte { return dst } -var ( - // MetricsWithDroppedLabels is the number of metrics with at least a single dropped label - MetricsWithDroppedLabels atomic.Uint64 - - // TooLongLabelNames is the number of too long label names - TooLongLabelNames atomic.Uint64 - - // TooLongLabelValues is the number of too long label values - TooLongLabelValues atomic.Uint64 -) - -func trackDroppedLabels(labels, droppedLabels []prompb.Label) { - MetricsWithDroppedLabels.Add(1) - select { - case <-droppedLabelsLogTicker.C: - // Do not call logger.WithThrottler() here, since this will result in increased CPU usage - // because labelsToString() will be called with each trackDroppedLabels call. - logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ - "or increase -maxLabelsPerTimeseries=%d command-line flag value", - len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) - default: - } -} - -func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) { - TooLongLabelValues.Add(1) - select { - case <-truncatedLabelsLogTicker.C: - // Do not call logger.WithThrottler() here, since this will result in increased CPU usage - // because labelsToString() will be called with each trackTruncatedLabels call. - logger.Warnf("truncate value for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+ - "original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value", - truncated.Name, len(truncated.Value), maxLabelValueLen, labelsToString(labels)) - default: - } -} - -var ( - droppedLabelsLogTicker = time.NewTicker(5 * time.Second) - truncatedLabelsLogTicker = time.NewTicker(5 * time.Second) -) - -func labelsToString(labels []prompb.Label) string { - labelsCopy := append([]prompb.Label{}, labels...) - sort.Slice(labelsCopy, func(i, j int) bool { - return string(labelsCopy[i].Name) < string(labelsCopy[j].Name) - }) - var b []byte - b = append(b, '{') - for i, label := range labelsCopy { - if len(label.Name) == 0 { - b = append(b, "__name__"...) - } else { - b = append(b, label.Name...) - } - b = append(b, '=') - b = strconv.AppendQuote(b, string(label.Value)) - if i < len(labels)-1 { - b = append(b, ',') - } - } - b = append(b, '}') - return string(b) -} - // marshalRaw marshals mn to dst and returns the result. // // The results may be unmarshaled with MetricName.UnmarshalRaw.