From 0dc93cca7f4b99642e58fe491e2da53a8a6a21d1 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 1 Oct 2022 18:39:34 +0300 Subject: [PATCH] app/vmagent/remotewrite: allow specifying per-`-remoteWrite.url` disk limits for persistent queue with pending data This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071 Related issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970 --- app/vmagent/README.md | 5 +++-- app/vmagent/remotewrite/remotewrite.go | 5 +++-- docs/CHANGELOG.md | 1 + docs/vmagent.md | 5 +++-- lib/flagutil/array.go | 16 ++++++---------- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 96082dd167..eaabb89f71 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -1378,9 +1378,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608) -remoteWrite.maxDailySeries int The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter - -remoteWrite.maxDiskUsagePerURL size + -remoteWrite.maxDiskUsagePerURL array The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0 - Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0) + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB. + Supports array of values separated by comma or specified via multiple flags. -remoteWrite.maxHourlySeries int The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter -remoteWrite.maxRowsPerBlock int diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 995139502d..a7326f66ed 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -38,7 +38,7 @@ var ( "isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs") showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+ "It is hidden by default, since it can contain sensitive info such as auth key") - maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+ + maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+ "for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+ "Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+ "Disk usage is unlimited if the value is set to 0") @@ -436,7 +436,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI pqURL.Fragment = "" h := xxhash.Sum64([]byte(pqURL.String())) queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h) - fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N) + maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0) + fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes) _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 { return float64(fq.GetPendingBytes()) }) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9ff5fe9326..83b31f7066 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -47,6 +47,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_ec2_region` label for [ec2_sd_config](https://docs.victoriametrics.com/sd_configs.html#ec2_sd_configs) in the same way as [Prometheus 2.39 does](https://github.com/prometheus/prometheus/pull/11326). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): accept data ingestion requests via paths starting from `/prometheus` prefix in the same way as [VictoriaMetrics does](https://docs.victoriametrics.com/#how-to-import-time-series-data). For example, `vmagent` now accepts Prometheus `remote_write` data via both `/api/v1/write` and `/prometheus/api/v1/write`. This simplifies switching between single-node VictoriaMetrics and `vmagent`. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `external_labels` from `global` section at `-promscrape.config` after the [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) is applied to scraped metrics. This aligns with Prometheus behaviour. Previously the `external_labels` were added to scrape targets, so they could be modified during relabeling. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3137). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying per-`-remoteWrite.url` limits for on-disk size for pending data via `-remoteWrite.maxDiskUsagePerURL` command-line flag. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071). * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): properly handle request paths ending with `/` such as `/vmui/`. Previously `vmui` was dropping the traling `/`, which could prevent from using `vmui` via `vmauth`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171). diff --git a/docs/vmagent.md b/docs/vmagent.md index c61295874b..ec4af24fd3 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1382,9 +1382,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608) -remoteWrite.maxDailySeries int The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter - -remoteWrite.maxDiskUsagePerURL size + -remoteWrite.maxDiskUsagePerURL array The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0 - Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0) + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB. + Supports array of values separated by comma or specified via multiple flags. -remoteWrite.maxHourlySeries int The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter -remoteWrite.maxRowsPerBlock int diff --git a/lib/flagutil/array.go b/lib/flagutil/array.go index 4dc4d4dc9e..b691e160ca 100644 --- a/lib/flagutil/array.go +++ b/lib/flagutil/array.go @@ -10,8 +10,7 @@ import ( // NewArrayString returns new ArrayString with the given name and description. func NewArrayString(name, description string) *ArrayString { - description += "\nSupports an `array` of values separated by comma" + - " or specified via multiple flags." + description += "\nSupports an `array` of values separated by comma or specified via multiple flags." var a ArrayString flag.Var(&a, name, description) return &a @@ -19,8 +18,7 @@ func NewArrayString(name, description string) *ArrayString { // NewArrayDuration returns new ArrayDuration with the given name and description. func NewArrayDuration(name, description string) *ArrayDuration { - description += "\nSupports `array` of values separated by comma" + - " or specified via multiple flags." + description += "\nSupports `array` of values separated by comma or specified via multiple flags." var a ArrayDuration flag.Var(&a, name, description) return &a @@ -28,8 +26,7 @@ func NewArrayDuration(name, description string) *ArrayDuration { // NewArrayBool returns new ArrayBool with the given name and description. func NewArrayBool(name, description string) *ArrayBool { - description += "\nSupports `array` of values separated by comma" + - " or specified via multiple flags." + description += "\nSupports `array` of values separated by comma or specified via multiple flags." var a ArrayBool flag.Var(&a, name, description) return &a @@ -37,8 +34,7 @@ func NewArrayBool(name, description string) *ArrayBool { // NewArrayInt returns new ArrayInt with the given name and description. func NewArrayInt(name, description string) *ArrayInt { - description += "\nSupports `array` of values separated by comma" + - " or specified via multiple flags." + description += "\nSupports `array` of values separated by comma or specified via multiple flags." var a ArrayInt flag.Var(&a, name, description) return &a @@ -46,8 +42,8 @@ func NewArrayInt(name, description string) *ArrayInt { // NewArrayBytes returns new ArrayBytes with the given name and description. func NewArrayBytes(name, description string) *ArrayBytes { - description += "\nSupports `array` of values separated by comma" + - " or specified via multiple flags." + description += "\nSupports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB." + description += "\nSupports `array` of values separated by comma or specified via multiple flags." var a ArrayBytes flag.Var(&a, name, description) return &a