mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
app/vmagent/remotewrite: allow specifying per--remoteWrite.url disk limits for persistent queue with pending data
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071 Related issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970
This commit is contained in:
Aliaksandr Valialkin
parent
c1fa9828b3
commit
0dc93cca7f
5 changed files with 16 additions and 16 deletions
|
|
@ -1378,9 +1378,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
|
|||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
|
||||
-remoteWrite.maxDailySeries int
|
||||
The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
|
||||
-remoteWrite.maxDiskUsagePerURL size
|
||||
-remoteWrite.maxDiskUsagePerURL array
|
||||
The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB.
|
||||
Supports array of values separated by comma or specified via multiple flags.
|
||||
-remoteWrite.maxHourlySeries int
|
||||
The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
|
||||
-remoteWrite.maxRowsPerBlock int
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ var (
|
|||
"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
|
||||
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
|
||||
"It is hidden by default, since it can contain sensitive info such as auth key")
|
||||
maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
|
||||
maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
|
||||
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
|
||||
"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
|
||||
"Disk usage is unlimited if the value is set to 0")
|
||||
|
|
@ -436,7 +436,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
|
|||
pqURL.Fragment = ""
|
||||
h := xxhash.Sum64([]byte(pqURL.String()))
|
||||
queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
|
||||
fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
|
||||
maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
|
||||
fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
|
||||
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
|
||||
return float64(fq.GetPendingBytes())
|
||||
})
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
|
|||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_ec2_region` label for [ec2_sd_config](https://docs.victoriametrics.com/sd_configs.html#ec2_sd_configs) in the same way as [Prometheus 2.39 does](https://github.com/prometheus/prometheus/pull/11326).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): accept data ingestion requests via paths starting from `/prometheus` prefix in the same way as [VictoriaMetrics does](https://docs.victoriametrics.com/#how-to-import-time-series-data). For example, `vmagent` now accepts Prometheus `remote_write` data via both `/api/v1/write` and `/prometheus/api/v1/write`. This simplifies switching between single-node VictoriaMetrics and `vmagent`.
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `external_labels` from `global` section at `-promscrape.config` after the [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) is applied to scraped metrics. This aligns with Prometheus behaviour. Previously the `external_labels` were added to scrape targets, so they could be modified during relabeling. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3137).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying per-`-remoteWrite.url` limits for on-disk size for pending data via `-remoteWrite.maxDiskUsagePerURL` command-line flag. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071).
|
||||
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): properly handle request paths ending with `/` such as `/vmui/`. Previously `vmui` was dropping the traling `/`, which could prevent from using `vmui` via `vmauth`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171).
|
||||
|
|
|
|||
|
|
@ -1382,9 +1382,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
|
|||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
|
||||
-remoteWrite.maxDailySeries int
|
||||
The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
|
||||
-remoteWrite.maxDiskUsagePerURL size
|
||||
-remoteWrite.maxDiskUsagePerURL array
|
||||
The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB.
|
||||
Supports array of values separated by comma or specified via multiple flags.
|
||||
-remoteWrite.maxHourlySeries int
|
||||
The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
|
||||
-remoteWrite.maxRowsPerBlock int
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ import (
|
|||
|
||||
// NewArrayString returns new ArrayString with the given name and description.
|
||||
func NewArrayString(name, description string) *ArrayString {
|
||||
description += "\nSupports an `array` of values separated by comma" +
|
||||
" or specified via multiple flags."
|
||||
description += "\nSupports an `array` of values separated by comma or specified via multiple flags."
|
||||
var a ArrayString
|
||||
flag.Var(&a, name, description)
|
||||
return &a
|
||||
|
|
@ -19,8 +18,7 @@ func NewArrayString(name, description string) *ArrayString {
|
|||
|
||||
// NewArrayDuration returns new ArrayDuration with the given name and description.
|
||||
func NewArrayDuration(name, description string) *ArrayDuration {
|
||||
description += "\nSupports `array` of values separated by comma" +
|
||||
" or specified via multiple flags."
|
||||
description += "\nSupports `array` of values separated by comma or specified via multiple flags."
|
||||
var a ArrayDuration
|
||||
flag.Var(&a, name, description)
|
||||
return &a
|
||||
|
|
@ -28,8 +26,7 @@ func NewArrayDuration(name, description string) *ArrayDuration {
|
|||
|
||||
// NewArrayBool returns new ArrayBool with the given name and description.
|
||||
func NewArrayBool(name, description string) *ArrayBool {
|
||||
description += "\nSupports `array` of values separated by comma" +
|
||||
" or specified via multiple flags."
|
||||
description += "\nSupports `array` of values separated by comma or specified via multiple flags."
|
||||
var a ArrayBool
|
||||
flag.Var(&a, name, description)
|
||||
return &a
|
||||
|
|
@ -37,8 +34,7 @@ func NewArrayBool(name, description string) *ArrayBool {
|
|||
|
||||
// NewArrayInt returns new ArrayInt with the given name and description.
|
||||
func NewArrayInt(name, description string) *ArrayInt {
|
||||
description += "\nSupports `array` of values separated by comma" +
|
||||
" or specified via multiple flags."
|
||||
description += "\nSupports `array` of values separated by comma or specified via multiple flags."
|
||||
var a ArrayInt
|
||||
flag.Var(&a, name, description)
|
||||
return &a
|
||||
|
|
@ -46,8 +42,8 @@ func NewArrayInt(name, description string) *ArrayInt {
|
|||
|
||||
// NewArrayBytes returns new ArrayBytes with the given name and description.
|
||||
func NewArrayBytes(name, description string) *ArrayBytes {
|
||||
description += "\nSupports `array` of values separated by comma" +
|
||||
" or specified via multiple flags."
|
||||
description += "\nSupports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB."
|
||||
description += "\nSupports `array` of values separated by comma or specified via multiple flags."
|
||||
var a ArrayBytes
|
||||
flag.Var(&a, name, description)
|
||||
return &a
|
||||
|
|
|
|||
Loading…
Reference in a new issue