From a5d101304207590fe1669f73dd11f388a6f25153 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 22 May 2024 21:53:53 +0200 Subject: [PATCH] lib/storage: change default value for maxLabelValueLen to 1024 (#6313) * It must reduce memory usage for misbehaving clients. Since VictoriaMetrics stores sparse index inmemory. * Reduce disk space usage for indexdb. * Prevent possible indexDB items drops. * It may trigger slow insert and new timeseries registration due to default value for flag change https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6176 --------- Co-authored-by: Aliaksandr Valialkin --- README.md | 2 +- app/vminsert/main.go | 2 +- docs/CHANGELOG.md | 2 ++ docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- docs/keyConcepts.md | 2 +- lib/storage/metric_name.go | 9 ++++++--- 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c5d3fd6c2..31fddc283 100644 --- a/README.md +++ b/README.md @@ -2912,7 +2912,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) -maxLabelValueLen int - The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384) + The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 1024) -maxLabelsPerTimeseries int The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30) -memory.allowedBytes size diff --git a/app/vminsert/main.go b/app/vminsert/main.go index ec27c95d2..181d00b83 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -76,7 +76,7 @@ var ( configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg") reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented") - maxLabelValueLen = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented") + maxLabelValueLen = flag.Int("maxLabelValueLen", 1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented") ) var ( diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 34282c1ef..7cdb15198 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). **Update note 1: the `-remoteWrite.multitenantURL` command-line flag at `vmagent` was removed starting from this release. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy).** **Update note 2: the `-streamAggr.dropInputLabels` command-line flag at `vmagent` was renamed to `-remoteWrite.streamAggr.dropInputLabels`. `-streamAggr.dropInputLabels` is now used for global streaming aggregation.** +**Update note 3: the `-maxLabelValueLen` command-line flag default value was changed from 16kB to 1kB. It may lead to truncating of labels with enormous values.** * SECURITY: upgrade Go builder from Go1.22.2 to Go1.22.3. See [the list of issues addressed in Go1.22.3](https://github.com/golang/go/issues?q=milestone%3AGo1.22.3+label%3ACherryPickApproved). @@ -62,6 +63,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): prevent from excessive resource usage when stream aggregation config file is empty. * BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/changelog_2022/#v1830). * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s. +* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): reduce the default value for `-maxLabelValueLen` command-line flag from `16KiB` to `1KiB`. This should prevent from issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6176) when time series with too long labels are ingested into VictoriaMetrics. * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247). * BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): fix `AnnotationQueryRunner` error in Grafana when executing annotations query against Prometheus backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6309) for details. diff --git a/docs/README.md b/docs/README.md index 01311e67b..1def54aaf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2915,7 +2915,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) -maxLabelValueLen int - The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384) + The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 1024) -maxLabelsPerTimeseries int The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30) -memory.allowedBytes size diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 3870ed511..8e2634852 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2923,7 +2923,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) -maxLabelValueLen int - The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384) + The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 1024) -maxLabelsPerTimeseries int The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30) -memory.allowedBytes size diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md index 20e08e87c..53bcb8403 100644 --- a/docs/keyConcepts.md +++ b/docs/keyConcepts.md @@ -349,7 +349,7 @@ This limit can be changed via `-maxLabelsPerTimeseries` command-line flag if nec Every label value can contain an arbitrary string value. The good practice is to use short and meaningful label values to describe the attribute of the metric, not to tell the story about it. For example, label-value pair `environment="prod"` is ok, but `log_message="long log message with a lot of details..."` is not ok. By default, -VictoriaMetrics limits label's value size with 16kB. This limit can be changed via `-maxLabelValueLen` command-line flag. +VictoriaMetrics limits label's value size with 1kB. This limit can be changed via `-maxLabelValueLen` command-line flag. It is very important to keep under control the number of unique label values, since every unique label value leads to a new [time series](#time-series). Try to avoid using volatile label values such as session ID or query ID in order to diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index 4f102bd79..bbf262c3e 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -475,7 +475,7 @@ const maxLabelNameLen = 256 // The maximum length of label value. // // Longer values are truncated. -var maxLabelValueLen = 16 * 1024 +var maxLabelValueLen = 1024 // SetMaxLabelValueLen sets the limit on the label value length. // @@ -590,8 +590,10 @@ func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) { } } -var droppedLabelsLogTicker = time.NewTicker(5 * time.Second) -var truncatedLabelsLogTicker = time.NewTicker(5 * time.Second) +var ( + droppedLabelsLogTicker = time.NewTicker(5 * time.Second) + truncatedLabelsLogTicker = time.NewTicker(5 * time.Second) +) func labelsToString(labels []prompb.Label) string { labelsCopy := append([]prompb.Label{}, labels...) @@ -767,6 +769,7 @@ func (ts *canonicalTagsSort) Less(i, j int) bool { x := *ts return string(x[i].key) < string(x[j].key) } + func (ts *canonicalTagsSort) Swap(i, j int) { x := *ts x[i], x[j] = x[j], x[i]