From c0e4ccb7b50c3f0473888a7b45c3c3ba6b4a5cd7 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Mon, 22 Apr 2024 14:52:04 +0300 Subject: [PATCH 01/58] lib/streamaggr: add option to ignore first N aggregation intervals (#6137) Stream aggregation may yield inaccurate results if it processes incomplete data. This issue can arise when data is sourced from clients that maintain a queue of unsent data, such as Prometheus or vmagent. If the queue isn't fully cleared within the aggregation interval, only a portion of the time series may be included in that period, leading to distorted calculations. To mitigate this we add an option to ignore first N aggregation intervals. It is expected, that client queues will be cleared during the time while aggregation ignores first N intervals and all subsequent aggregations will be correct. --- app/vmagent/remotewrite/remotewrite.go | 10 ++++--- app/vminsert/common/streamaggr.go | 24 +++++++++------- docs/stream-aggregation.md | 12 ++++++++ lib/streamaggr/streamaggr.go | 39 +++++++++++++++++++++----- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 41f41dc5f7..63d0b91d7b 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -105,7 +105,8 @@ var ( "with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication") streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+ "for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples") - streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ + streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent.") + streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ "before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels") disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+ @@ -857,9 +858,10 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in ignoreOldSamples := streamAggrIgnoreOldSamples.GetOptionalArg(argIdx) if sasFile != "" { opts := &streamaggr.Options{ - DedupInterval: dedupInterval, - DropInputLabels: *streamAggrDropInputLabels, - IgnoreOldSamples: ignoreOldSamples, + DedupInterval: dedupInterval, + DropInputLabels: *streamAggrDropInputLabels, + IgnoreOldSamples: ignoreOldSamples, + IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals, } sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts) if err != nil { diff --git a/app/vminsert/common/streamaggr.go b/app/vminsert/common/streamaggr.go index 680956428c..e1cd6146ad 100644 --- a/app/vminsert/common/streamaggr.go +++ b/app/vminsert/common/streamaggr.go @@ -32,7 +32,8 @@ var ( "See also -streamAggr.dropInputLabels and -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication") streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ "before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels") - streamAggrIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the current aggregation interval. "+ + streamAggrIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent.") + streamAggrIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the current aggregation interval. "+ "See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples") ) @@ -56,9 +57,10 @@ func CheckStreamAggrConfig() error { } pushNoop := func(_ []prompbmarshal.TimeSeries) {} opts := &streamaggr.Options{ - DedupInterval: *streamAggrDedupInterval, - DropInputLabels: *streamAggrDropInputLabels, - IgnoreOldSamples: *streamAggrIgnoreOldSamples, + DedupInterval: *streamAggrDedupInterval, + DropInputLabels: *streamAggrDropInputLabels, + IgnoreOldSamples: *streamAggrIgnoreOldSamples, + IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals, } sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushNoop, opts) if err != nil { @@ -84,9 +86,10 @@ func InitStreamAggr() { sighupCh := procutil.NewSighupChan() opts := &streamaggr.Options{ - DedupInterval: *streamAggrDedupInterval, - DropInputLabels: *streamAggrDropInputLabels, - IgnoreOldSamples: *streamAggrIgnoreOldSamples, + DedupInterval: *streamAggrDedupInterval, + DropInputLabels: *streamAggrDropInputLabels, + IgnoreOldSamples: *streamAggrIgnoreOldSamples, + IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals, } sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, opts) if err != nil { @@ -117,9 +120,10 @@ func reloadStreamAggrConfig() { saCfgReloads.Inc() opts := &streamaggr.Options{ - DedupInterval: *streamAggrDedupInterval, - DropInputLabels: *streamAggrDropInputLabels, - IgnoreOldSamples: *streamAggrIgnoreOldSamples, + DedupInterval: *streamAggrDedupInterval, + DropInputLabels: *streamAggrDropInputLabels, + IgnoreOldSamples: *streamAggrIgnoreOldSamples, + IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals, } sasNew, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, opts) if err != nil { diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md index 5b7027c2fe..c100c059d6 100644 --- a/docs/stream-aggregation.md +++ b/docs/stream-aggregation.md @@ -92,6 +92,18 @@ must be ignored, then the following options can be used: - To set `ignore_old_samples: true` option at the particular [aggregation config](#stream-aggregation-config). This enables ignoring old samples for that particular aggregation config. +## Ignore aggregation intervals on start + +Stream aggregation may yield inaccurate results if it processes incomplete data. This issue can arise when data is sourced from clients that maintain a queue of unsent data, such as Prometheus or vmagent. If the queue isn't fully cleared within the aggregation interval, only a portion of the time series may be included in that period, leading to distorted calculations. To mitigate this, consider the following options: + +- Set `-remoteWrite.streamAggr.ignoreFirstIntervals=` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent/) + or `-streamAggr.ignoreFirstIntervals=` command-line flag to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to skip first `` [aggregation intervals](#stream-aggregation-config) + from persisting to the storage. It is expected that all incomplete or queued data will be processed during + specified `` and all subsequent aggregation intervals will produce correct data. + +- To set `ignore_first_intervals: ` option at the particular [aggregation config](#stream-aggregation-config). + This enables ignoring first `` aggregation intervals for that particular aggregation config. + ## Flush time alignment By default the time for aggregated data flush is aligned by the `interval` option specified in [aggregate config](#stream-aggregation-config). diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index 06d281f696..055af0436b 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -111,6 +111,13 @@ type Options struct { // // This option can be overridden individually per each aggregation via ignore_old_samples option. IgnoreOldSamples bool + + // IgnoreFirstIntervals sets amount of intervals to ignore on start + // + // By default no intervals will be ignored. + // + // This option can be overridden individually per each aggregation via ignore_intervals_on_start option. + IgnoreFirstIntervals int } // Config is a configuration for a single stream aggregation. @@ -175,6 +182,9 @@ type Config struct { // IgnoreOldSamples instructs to ignore samples with old timestamps outside the current aggregation interval. IgnoreOldSamples *bool `yaml:"ignore_old_samples,omitempty"` + // IgnoreFirstIntervals sets number of aggregation intervals to be ignored on start. + IgnoreFirstIntervals *int `yaml:"ignore_first_intervals,omitempty"` + // By is an optional list of labels for grouping input series. // // See also Without. @@ -479,6 +489,12 @@ func newAggregator(cfg *Config, pushFunc PushFunc, ms *metrics.Set, opts *Option ignoreOldSamples = *v } + // check cfg.IgnoreFirstIntervals + ignoreFirstIntervals := opts.IgnoreFirstIntervals + if v := cfg.IgnoreFirstIntervals; v != nil { + ignoreFirstIntervals = *v + } + // initialize outputs list if len(cfg.Outputs) == 0 { return nil, fmt.Errorf("`outputs` list must contain at least a single entry from the list %s; "+ @@ -600,14 +616,14 @@ func newAggregator(cfg *Config, pushFunc PushFunc, ms *metrics.Set, opts *Option a.wg.Add(1) go func() { - a.runFlusher(pushFunc, alignFlushToInterval, skipIncompleteFlush, interval, dedupInterval) + a.runFlusher(pushFunc, alignFlushToInterval, skipIncompleteFlush, interval, dedupInterval, ignoreFirstIntervals) a.wg.Done() }() return a, nil } -func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipIncompleteFlush bool, interval, dedupInterval time.Duration) { +func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipIncompleteFlush bool, interval, dedupInterval time.Duration, ignoreFirstIntervals int) { alignedSleep := func(d time.Duration) { if !alignFlushToInterval { return @@ -642,7 +658,12 @@ func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipInc } for tickerWait(t) { - a.flush(pushFunc, interval, true) + pf := pushFunc + if ignoreFirstIntervals > 0 { + pf = nil + ignoreFirstIntervals-- + } + a.flush(pf, interval, true) if alignFlushToInterval { select { @@ -663,13 +684,17 @@ func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipInc ct := time.Now() if ct.After(flushDeadline) { + pf := pushFunc + if ignoreFirstIntervals > 0 { + pf = nil + ignoreFirstIntervals-- + } // It is time to flush the aggregated state if alignFlushToInterval && skipIncompleteFlush && !isSkippedFirstFlush { - a.flush(nil, interval, true) + pf = nil isSkippedFirstFlush = true - } else { - a.flush(pushFunc, interval, true) } + a.flush(pf, interval, true) for ct.After(flushDeadline) { flushDeadline = flushDeadline.Add(interval) } @@ -684,7 +709,7 @@ func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipInc } } - if !skipIncompleteFlush { + if !skipIncompleteFlush && ignoreFirstIntervals == 0 { a.dedupFlush(dedupInterval) a.flush(pushFunc, interval, true) } From bae3874e6aecf767e470fde66355604e901d978d Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 22 Apr 2024 14:22:59 +0200 Subject: [PATCH 02/58] app/streamaggr: follow-up after https://github.com/VictoriaMetrics/VictoriaMetrics/commit/c0e4ccb7b50c3f0473888a7b45c3c3ba6b4a5cd7 * rm vmagent mentions from vminsert flags * improve documentation wording, add links to related sections * mention `ignore_first_intervals` in the stream aggr options * update flags description * add basic test for config parsing validation Signed-off-by: hagen1778 --- README.md | 2 ++ app/vmagent/remotewrite/remotewrite.go | 5 ++-- app/vminsert/common/streamaggr.go | 5 ++-- docs/README.md | 2 ++ docs/Single-server-VictoriaMetrics.md | 2 ++ docs/stream-aggregation.md | 35 +++++++++++++++++--------- docs/vmagent.md | 2 ++ lib/streamaggr/streamaggr.go | 6 ++--- lib/streamaggr/streamaggr_test.go | 8 ++++++ 9 files changed, 48 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d9b02c5689..87d5ceb80b 100644 --- a/README.md +++ b/README.md @@ -3159,6 +3159,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li An optional list of labels to drop from samples before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -streamAggr.ignoreFirstIntervals int + Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start -streamAggr.ignoreOldSamples Whether to ignore input samples with old timestamps outside the current aggregation interval. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples -streamAggr.keepInput diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 63d0b91d7b..7cd8415cb1 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -105,8 +105,9 @@ var ( "with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication") streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+ "for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples") - streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent.") - streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ + streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. "+ + "See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start") + streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ "before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels") disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+ diff --git a/app/vminsert/common/streamaggr.go b/app/vminsert/common/streamaggr.go index e1cd6146ad..7a8ae0e118 100644 --- a/app/vminsert/common/streamaggr.go +++ b/app/vminsert/common/streamaggr.go @@ -32,8 +32,9 @@ var ( "See also -streamAggr.dropInputLabels and -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication") streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+ "before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels") - streamAggrIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent.") - streamAggrIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the current aggregation interval. "+ + streamAggrIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. "+ + "See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start") + streamAggrIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the current aggregation interval. "+ "See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples") ) diff --git a/docs/README.md b/docs/README.md index ceb58995ac..486003f451 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3162,6 +3162,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li An optional list of labels to drop from samples before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -streamAggr.ignoreFirstIntervals int + Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start -streamAggr.ignoreOldSamples Whether to ignore input samples with old timestamps outside the current aggregation interval. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples -streamAggr.keepInput diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index e83be4aa39..c98f0420b1 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -3170,6 +3170,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li An optional list of labels to drop from samples before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -streamAggr.ignoreFirstIntervals int + Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start -streamAggr.ignoreOldSamples Whether to ignore input samples with old timestamps outside the current aggregation interval. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples -streamAggr.keepInput diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md index c100c059d6..bd418dd142 100644 --- a/docs/stream-aggregation.md +++ b/docs/stream-aggregation.md @@ -19,7 +19,7 @@ The aggregation is applied to all the metrics received via any [supported data i and/or scraped from [Prometheus-compatible targets](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter) after applying all the configured [relabeling stages](https://docs.victoriametrics.com/vmagent/#relabeling). -By default stream aggregation ignores timestamps associated with the input [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples). +By default, stream aggregation ignores timestamps associated with the input [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples). It expects that the ingested samples have timestamps close to the current time. See [how to ignore old samples](#ignoring-old-samples). Stream aggregation can be configured via the following command-line flags: @@ -82,7 +82,7 @@ The online de-duplication uses the same logic as [`-dedup.minScrapeInterval` com ## Ignoring old samples -By default all the input samples are taken into account during stream aggregation. If samples with old timestamps outside the current [aggregation interval](#stream-aggregation-config) +By default, all the input samples are taken into account during stream aggregation. If samples with old timestamps outside the current [aggregation interval](#stream-aggregation-config) must be ignored, then the following options can be used: - To pass `-remoteWrite.streamAggr.ignoreOldSamples` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent/) @@ -94,19 +94,23 @@ must be ignored, then the following options can be used: ## Ignore aggregation intervals on start -Stream aggregation may yield inaccurate results if it processes incomplete data. This issue can arise when data is sourced from clients that maintain a queue of unsent data, such as Prometheus or vmagent. If the queue isn't fully cleared within the aggregation interval, only a portion of the time series may be included in that period, leading to distorted calculations. To mitigate this, consider the following options: +Stream aggregation may yield inaccurate results if it processes incomplete data. This issue can arise when data is +received from clients that maintain a queue of unsent data, such as Prometheus or vmagent. If the queue isn't fully +cleared within the aggregation `interval`, only a portion of the time series may be processed, leading to distorted +calculations. To mitigate this, consider the following options: - Set `-remoteWrite.streamAggr.ignoreFirstIntervals=` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent/) - or `-streamAggr.ignoreFirstIntervals=` command-line flag to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to skip first `` [aggregation intervals](#stream-aggregation-config) - from persisting to the storage. It is expected that all incomplete or queued data will be processed during - specified `` and all subsequent aggregation intervals will produce correct data. + or `-streamAggr.ignoreFirstIntervals=` command-line flag to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) + to skip first `` [aggregation intervals](#stream-aggregation-config) + from persisting to the storage. It is expected that all incomplete or queued data will be processed during + specified `` and all subsequent aggregation intervals will produce correct data. -- To set `ignore_first_intervals: ` option at the particular [aggregation config](#stream-aggregation-config). +- Set `ignore_first_intervals: ` option individually per [aggregation config](#stream-aggregation-config). This enables ignoring first `` aggregation intervals for that particular aggregation config. ## Flush time alignment -By default the time for aggregated data flush is aligned by the `interval` option specified in [aggregate config](#stream-aggregation-config). +By default, the time for aggregated data flush is aligned by the `interval` option specified in [aggregate config](#stream-aggregation-config). For example: - if `interval: 1m` is set, then the aggregated data is flushed to the storage at the end of every minute - if `interval: 1h` is set, then the aggregated data is flushed to the storage at the end of every hour @@ -887,7 +891,7 @@ at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server- # Samples are de-duplicated on a per-series basis. See https://docs.victoriametrics.com/keyconcepts/#time-series # and https://docs.victoriametrics.com/#deduplication # The deduplication is performed after input_relabel_configs relabeling is applied. - # By default the deduplication is disabled unless -remoteWrite.streamAggr.dedupInterval or -streamAggr.dedupInterval + # By default, the deduplication is disabled unless -remoteWrite.streamAggr.dedupInterval or -streamAggr.dedupInterval # command-line flags are set. # # dedup_interval: 30s @@ -904,7 +908,7 @@ at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server- # staleness_interval: 2m # no_align_flush_to_interval disables aligning of flush times for the aggregated data to multiples of interval. - # By default flush times for the aggregated data is aligned to multiples of interval. + # By default, flush times for the aggregated data is aligned to multiples of interval. # For example: # - if `interval: 1m` is set, then flushes happen at the end of every minute, # - if `interval: 1h` is set, then flushes happen at the end of every hour @@ -934,16 +938,23 @@ at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server- # keep_metric_names instructs keeping the original metric names for the aggregated samples. # This option can be set only if outputs list contains only a single output. - # By default a special suffix is added to original metric names in the aggregated samples. + # By default, a special suffix is added to original metric names in the aggregated samples. # See https://docs.victoriametrics.com/stream-aggregation/#output-metric-names # # keep_metric_names: false # ignore_old_samples instructs ignoring input samples with old timestamps outside the current aggregation interval. + # See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples # See also -streamAggr.ignoreOldSamples command-line flag. # # ignore_old_samples: false + # ignore_first_intervals instructs ignoring first N aggregation intervals after process start. + # See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start + # See also -remoteWrite.streamAggr.ignoreFirstIntervals or -streamAggr.ignoreFirstIntervals + # + # ignore_first_intervals: false + # drop_input_labels instructs dropping the given labels from input samples. # The labels' dropping is performed before input_relabel_configs are applied. # This also means that the labels are dropped before de-duplication ( https://docs.victoriametrics.com/stream-aggregation/#deduplication ) @@ -1016,7 +1027,7 @@ These issues can be be fixed in the following ways: - By increasing the `interval` option at [stream aggregation config](#stream-aggregation-config), so it covers the expected delays in data ingestion pipelines. - By specifying the `staleness_interval` option at [stream aggregation config](#stream-aggregation-config), so it covers the expected - delays in data ingestion pipelines. By default the `staleness_interval` equals to `2 x interval`. + delays in data ingestion pipelines. By default, the `staleness_interval` equals to `2 x interval`. ### High resource usage diff --git a/docs/vmagent.md b/docs/vmagent.md index ecc2f0e07b..227c6e45d0 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -2166,6 +2166,8 @@ See the docs at https://docs.victoriametrics.com/vmagent/ . Whether to drop all the input samples after the aggregation with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/ Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -remoteWrite.streamAggr.ignoreFirstIntervals int + Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start -remoteWrite.streamAggr.ignoreOldSamples array Whether to ignore input samples with old timestamps outside the current aggregation interval for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples Supports array of values separated by comma or specified via multiple flags. diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index 055af0436b..5cb480aad2 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -112,11 +112,11 @@ type Options struct { // This option can be overridden individually per each aggregation via ignore_old_samples option. IgnoreOldSamples bool - // IgnoreFirstIntervals sets amount of intervals to ignore on start + // IgnoreFirstIntervals sets amount of aggregation intervals to ignore on start. // - // By default no intervals will be ignored. + // By default, no intervals will be ignored. // - // This option can be overridden individually per each aggregation via ignore_intervals_on_start option. + // This option can be overridden individually per each aggregation via ignore_first_intervals option. IgnoreFirstIntervals int } diff --git a/lib/streamaggr/streamaggr_test.go b/lib/streamaggr/streamaggr_test.go index 5a079979bd..c4731673bb 100644 --- a/lib/streamaggr/streamaggr_test.go +++ b/lib/streamaggr/streamaggr_test.go @@ -199,6 +199,14 @@ func TestAggregatorsEqual(t *testing.T) { interval: 5m flush_on_shutdown: false `, false) + f(` +- outputs: [total] + interval: 5m + ignore_first_intervals: 2 +`, ` +- outputs: [total] + interval: 5m + ignore_first_intervals: 4`, false) } func TestAggregatorsSuccess(t *testing.T) { From 9bedbcfa2f2a97875ec3425c0d73dd2002e27304 Mon Sep 17 00:00:00 2001 From: Denys Holius <5650611+denisgolius@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:53:26 +0300 Subject: [PATCH 03/58] docs: removed code-style highlighting for commanad-line flags of VM components (#6147) Using `sh` or `console` formatting doesn't do word-breaking on render. This makes flags description harder to read, as users need to scroll the web page horizontally. Removing the formatting renders the description with normal word-breaking. --- docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- docs/vmagent.md | 2 +- docs/vmalert.md | 2 +- docs/vmauth.md | 2 +- docs/vmbackup.md | 2 +- docs/vmbackupmanager.md | 2 +- docs/vmgateway.md | 2 +- docs/vmrestore.md | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index 486003f451..dcd35ae519 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2679,7 +2679,7 @@ Files included in each folder: Pass `-help` to VictoriaMetrics in order to see the list of supported command-line flags with their description: -```sh +``` -bigMergeConcurrency int Deprecated: this flag does nothing -blockcache.missesBeforeCaching int diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index c98f0420b1..e3329834db 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2687,7 +2687,7 @@ Files included in each folder: Pass `-help` to VictoriaMetrics in order to see the list of supported command-line flags with their description: -```sh +``` -bigMergeConcurrency int Deprecated: this flag does nothing -blockcache.missesBeforeCaching int diff --git a/docs/vmagent.md b/docs/vmagent.md index 227c6e45d0..752efb903d 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1623,7 +1623,7 @@ It is safe sharing the collected profiles from security point of view, since the `vmagent` can be fine-tuned with various command-line flags. Run `./vmagent -help` in order to see the full list of these flags with their descriptions and default values: -```sh +``` ./vmagent -help vmagent collects metrics data via popular data ingestion protocols and routes them to VictoriaMetrics. diff --git a/docs/vmalert.md b/docs/vmalert.md index 1534ee835e..a5c03c3f35 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -965,7 +965,7 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: -```sh +``` -clusterMode If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert/#multitenancy . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/enterprise/ -configCheckInterval duration diff --git a/docs/vmauth.md b/docs/vmauth.md index 56d5367cc8..b69dd8d485 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -1121,7 +1121,7 @@ It is safe sharing the collected profiles from security point of view, since the Pass `-help` command-line arg to `vmauth` in order to see all the configuration options: -```sh +``` ./vmauth -help vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics. diff --git a/docs/vmbackup.md b/docs/vmbackup.md index 628e364bf0..465e2f8a5e 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -304,7 +304,7 @@ Refer to the respective documentation for your object storage provider for more Run `vmbackup -help` in order to see all the available options: -```sh +``` -concurrency int The number of concurrent workers. Higher concurrency may reduce backup duration (default 10) -configFilePath string diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index 2bcd7dbdb6..dcef2ab9c5 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -415,7 +415,7 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: -```text +``` vmbackupmanager performs regular backups according to the provided configs. subcommands: diff --git a/docs/vmgateway.md b/docs/vmgateway.md index 09445aee8b..cc263e66a8 100644 --- a/docs/vmgateway.md +++ b/docs/vmgateway.md @@ -273,7 +273,7 @@ Example usage for tokens issued by Google: Below is the list of configuration flags (it can be viewed by running `./vmgateway -help`): -```sh +``` -auth.httpHeader string HTTP header name to look for JWT authorization token (default "Authorization") -auth.jwksEndpoints array diff --git a/docs/vmrestore.md b/docs/vmrestore.md index a8dec2ce13..f640539a92 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -91,7 +91,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q * Run `vmrestore -help` in order to see all the available options: -```sh +``` -concurrency int The number of concurrent workers. Higher concurrency may reduce restore duration (default 10) -configFilePath string From 6dbb4c1671cd7ef2da55cb9e4da2d2aa85876ece Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Mon, 22 Apr 2024 20:55:41 +0800 Subject: [PATCH 04/58] Automatic update operator docs from VictoriaMetrics/operator@7e2ba6f (#6165) --- docs/operator/CHANGELOG.md | 7 +++++++ docs/operator/vars.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index a7d0d10a68..562524157c 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -16,6 +16,13 @@ aliases: ## Next release + + +## [v0.43.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.43.2) - 22 Apr 2024 + +- [vmagent](./api.md#vmagent): fixes bug with `ServiceAccount` not found with `ingestOnlyMode`. +- [vmagent](./api.md#vmagent): fixes `unknown long flag '--rules-dir'` for prometheus-config-reloader. + ## [v0.43.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.43.1) - 18 Apr 2024 diff --git a/docs/operator/vars.md b/docs/operator/vars.md index f293cefe34..9dc57bf305 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ menu: # Auto Generated vars for package config - updated at Thu Apr 18 19:07:23 UTC 2024 + updated at Mon Apr 22 10:10:22 UTC 2024 | varible name | variable default value | variable required | variable description | From 5f487c7090e99654a55fc8429ffd9dc514c5bc46 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 22 Apr 2024 15:02:10 +0200 Subject: [PATCH 05/58] app/vmalert: fix links with anchors in vmalert's UI (#6146) Starting from v1.99.0 vmalert could ignore anchors pointing to specific rule groups if `search` param was present in URL. This change makes anchors compatible with `search` param in UI. Signed-off-by: hagen1778 --- app/vmalert/static/js/custom.js | 18 ++++++++++++++++-- docs/CHANGELOG.md | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/vmalert/static/js/custom.js b/app/vmalert/static/js/custom.js index 25adebd2ba..c79d342d23 100644 --- a/app/vmalert/static/js/custom.js +++ b/app/vmalert/static/js/custom.js @@ -13,6 +13,20 @@ function collapseAll() { $('.collapse').removeClass('show'); } +function showByID(id) { + if (!id) { + return + } + let parent = $("#" + id).parent(); + if (!parent) { + return + } + let target = $("#" + parent.attr("data-bs-target")); + if (target.length > 0) { + target.addClass('show'); + } +} + function toggleByID(id) { if (id) { let el = $("#" + id); @@ -61,7 +75,7 @@ function search() { function setParamURL(key, value) { let url = new URL(location.href) url.searchParams.set(key, value); - window.history.replaceState(null, null, `?${url.searchParams.toString()}`); + window.history.replaceState(null, null, `?${url.searchParams.toString()}${url.hash}`); } function getParamURL(key) { @@ -141,7 +155,7 @@ $(document).ready(function () { search() let hash = window.location.hash.substr(1); - toggleByID(hash); + showByID(hash); }); $(document).ready(function () { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7ea0e2715c..296d0be4ea 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -44,6 +44,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): avoid blocking `/api/v1/rules`, `/api/v1/alerts`, `/metrics` APIs when alerting rule uses template functions `query`, which could takes a while to execute. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079). +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix links with anchors in vmalert's UI. Starting from [v1.99.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.99.0) vmalert could ignore anchors pointing to specific rule groups if `search` param was present in URL. * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): don't treat concurrency limit hit as an error of the backend. Previously, hitting the concurrency limit would increment both `vmauth_concurrent_requests_limit_reached_total` and `vmauth_user_request_backend_errors_total` counters. Now, only concurrency limit counter is incremented. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565). ## [v1.100.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.100.1) From bde4693a9038e94644ac9f82d32b18ec86e328af Mon Sep 17 00:00:00 2001 From: Denys Holius <5650611+denisgolius@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:07:45 +0300 Subject: [PATCH 06/58] deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml: update fluentbit version from v2.1.4 to v3.0.2 (#6120) see also https://fluentbit.io/announcements/v3.0.0/ --- .../docker/victorialogs/fluentbit-docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml index 12c57cfc5a..343aa3d102 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: fluentbit: - image: cr.fluentbit.io/fluent/fluent-bit:2.1.4 + image: cr.fluentbit.io/fluent/fluent-bit:3.0.2 volumes: - /var/lib/docker/containers:/var/lib/docker/containers:ro - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf From 7958f3886467aef1a386e3111136f1d3c59f7149 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Tue, 23 Apr 2024 13:12:10 +0300 Subject: [PATCH 07/58] deployment/docker: allow cross-platform building on arm64 platform (#6158) Added x86_64 libraries to allow building cross-platform images on arm64 --- deployment/docker/Makefile | 13 ++++++------- deployment/docker/builder/Dockerfile | 11 ++++++++--- docs/Release-Guide.md | 5 ----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index d9d2de7bd8..00a73a48c5 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -102,24 +102,23 @@ run-via-docker: package-via-docker app-via-docker-goos-goarch: APP_SUFFIX='-$(GOOS)-$(GOARCH)' \ - DOCKER_OPTS='--env CGO_ENABLED=$(CGO_ENABLED) --env GOOS=$(GOOS) --env GOARCH=$(GOARCH)' \ + DOCKER_OPTS='--env CGO_ENABLED=$(CGO_ENABLED) --env GOOS=$(GOOS) --env GOARCH=$(GOARCH) $(foreach v,$(EXTRA_ENVS),--env $(v))' \ $(MAKE) app-via-docker app-via-docker-pure: APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker app-via-docker-linux-amd64: + EXTRA_ENVS='CC=/opt/cross-builder/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc' \ CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-via-docker-goos-goarch app-via-docker-linux-arm: - APP_SUFFIX='-linux-arm' \ - DOCKER_OPTS='--env CGO_ENABLED=0 --env GOOS=linux --env GOARCH=arm --env GOARM=5' \ - $(MAKE) app-via-docker + EXTRA_ENVS='GOARM=5' \ + CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-via-docker-goos-goarch app-via-docker-linux-arm64: - APP_SUFFIX='-linux-arm64' \ - DOCKER_OPTS='--env CGO_ENABLED=1 --env GOOS=linux --env GOARCH=arm64 --env CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc' \ - $(MAKE) app-via-docker + CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 $(MAKE) app-via-docker-goos-goarch app-via-docker-linux-ppc64le: CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-via-docker-goos-goarch diff --git a/deployment/docker/builder/Dockerfile b/deployment/docker/builder/Dockerfile index 0efb33b914..20543b7d56 100644 --- a/deployment/docker/builder/Dockerfile +++ b/deployment/docker/builder/Dockerfile @@ -3,7 +3,12 @@ FROM $go_builder_image STOPSIGNAL SIGINT RUN apk add git gcc musl-dev make wget --no-cache && \ mkdir /opt/cross-builder && \ - wget https://musl.cc/aarch64-linux-musl-cross.tgz -O /opt/cross-builder/aarch64-musl.tgz --no-verbose && \ cd /opt/cross-builder && \ - tar zxf aarch64-musl.tgz -C ./ && \ - rm /opt/cross-builder/aarch64-musl.tgz + for arch in aarch64 x86_64; do \ + wget \ + https://musl.cc/${arch}-linux-musl-cross.tgz \ + -O /opt/cross-builder/${arch}-musl.tgz \ + --no-verbose && \ + tar zxf ${arch}-musl.tgz -C ./ && \ + rm /opt/cross-builder/${arch}-musl.tgz; \ + done diff --git a/docs/Release-Guide.md b/docs/Release-Guide.md index f73bf55b37..e19d9b4648 100644 --- a/docs/Release-Guide.md +++ b/docs/Release-Guide.md @@ -38,11 +38,6 @@ docker buildx create --use --name=qemu docker buildx inspect --bootstrap ``` -For ARM arch (M1/M2 processors) additionally configure docker with preferred platform: -``` -export DOCKER_DEFAULT_PLATFORM=linux/amd64 -``` - By default, docker on MacOS has limited amount of resources (CPU, mem) to use. Bumping the limits may significantly improve build speed. From 301bd387d450fc1f918bd964aa5a76323ad2c3a8 Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:12:35 +0800 Subject: [PATCH 08/58] Automatic update operator docs from VictoriaMetrics/operator@e343a8e (#6168) --- docs/operator/api.md | 2 +- docs/operator/vars.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operator/api.md b/docs/operator/api.md index 2a34beb262..c470b0c9dc 100644 --- a/docs/operator/api.md +++ b/docs/operator/api.md @@ -951,7 +951,7 @@ ServiceSpec defines additional service for CRD with user-defined params. by defa | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| useAsDefault | UseAsDefault applies changes from given service definition to the main object Service Chaning from headless service to clusterIP or loadbalancer may break cross-component communication | bool | false | +| useAsDefault | UseAsDefault applies changes from given service definition to the main object Service Changing from headless service to clusterIP or loadbalancer may break cross-component communication | bool | false | | metadata | EmbeddedObjectMetadata defines objectMeta for additional service. | [EmbeddedObjectMetadata](#embeddedobjectmetadata) | false | | spec | ServiceSpec describes the attributes that a user creates on a service. More info: https://kubernetes.io/docs/concepts/services-networking/service/ | v1.ServiceSpec | true | diff --git a/docs/operator/vars.md b/docs/operator/vars.md index 9dc57bf305..118ed1855f 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ menu: # Auto Generated vars for package config - updated at Mon Apr 22 10:10:22 UTC 2024 + updated at Mon Apr 22 16:35:37 UTC 2024 | varible name | variable default value | variable required | variable description | From 42512927088c530acf356aaedaa557e89c790553 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 23 Apr 2024 14:49:45 +0200 Subject: [PATCH 09/58] app/vmagent: mention corner case with dangling queues and identical URLs See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140 We don't cover this corner case as it has low chance for reproduction. Precisely, the requirements are following: 1. vmagent need to be configured with multiple identical `remoteWrite.url` flags; 2. At least one of the persistent queues need to be non-empty, which already signalizes about issues with setup; 3. vmagent need to be restarted with removing of one of `remoteWrite.url` flags. We do not document this case in vmagent.md as it seems to be a rare corner case and its explanation will require too much of explanation and confuse users. Signed-off-by: hagen1778 --- app/vmagent/remotewrite/remotewrite.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 7cd8415cb1..93e41b50d4 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -259,6 +259,9 @@ func dropDanglingQueues() { // This is required for the case when the number of queues has been changed or URL have been changed. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014 // + // In case if there were many persistent queues with identical *remoteWriteURLs + // the queue with the last index will be dropped. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140 existingQueues := make(map[string]struct{}, len(rwctxsDefault)) for _, rwctx := range rwctxsDefault { existingQueues[rwctx.fq.Dirname()] = struct{}{} From 22cfab1ea24807b71f555221c64b8fc2d9bcf731 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 23 Apr 2024 16:34:35 +0200 Subject: [PATCH 10/58] docs/changelog: mention icnreased CPU usage in 1.100.1 release for ENT distributions Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 296d0be4ea..99f4b4b0cc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -51,7 +51,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). Released at 2024-04-11 -**Update note 1: When upgrading to this release from [v1.99.0](https://docs.victoriametrics.com/changelog/#v1990) or [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) it is recommended to reset caches stored on disk according to [these](https://docs.victoriametrics.com/single-server-victoriametrics/#cache-removal) docs.** +**Update note 1: This release contains the issue which could lead to extra CPU usage of storage component in ENT distribution of VictoriaMetrics. The issue is caused by [downsampling per distinct sets of time series](https://docs.victoriametrics.com/#downsampling) feature introduced in 1.100.0 ENT version. Please, rollback [v1.98.0](https://docs.victoriametrics.com/changelog/#v1980) ENT version if you're affected.** +**Update note 2: When upgrading to this release from [v1.99.0](https://docs.victoriametrics.com/changelog/#v1990) or [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) it is recommended to reset caches stored on disk according to [these](https://docs.victoriametrics.com/single-server-victoriametrics/#cache-removal) docs.** * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): allow specifying custom backup interval via `-backupInterval` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5966). From 035de57e5e3f84a5e587659092d7f25cf1936c54 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 23 Apr 2024 16:41:48 +0200 Subject: [PATCH 11/58] dashboards: show max number of active merges instead of cumulative The cumulative number of active merges could be red herring as it its value depends on the number of vmstorages. For example, vmstorage could be added or removed and this will affect the panel. Or, each vmstorage could start a merging process (i.e. for downsampling) and visiually it could look like a massive change. Signed-off-by: hagen1778 --- dashboards/victoriametrics-cluster.json | 4 ++-- dashboards/vm/victoriametrics-cluster.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json index ffb70b10bf..b6f850119a 100644 --- a/dashboards/victoriametrics-cluster.json +++ b/dashboards/victoriametrics-cluster.json @@ -5521,7 +5521,7 @@ "type": "prometheus", "uid": "$ds" }, - "description": "The number of on-going merges in storage nodes. It is expected to have high numbers for `storage/small` metric.", + "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", "fieldConfig": { "defaults": { "color": { @@ -5610,7 +5610,7 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", "legendFormat": "__auto", "range": true, "refId": "A" diff --git a/dashboards/vm/victoriametrics-cluster.json b/dashboards/vm/victoriametrics-cluster.json index ca906f49f0..5cc8d84589 100644 --- a/dashboards/vm/victoriametrics-cluster.json +++ b/dashboards/vm/victoriametrics-cluster.json @@ -5522,7 +5522,7 @@ "type": "victoriametrics-datasource", "uid": "$ds" }, - "description": "The number of on-going merges in storage nodes. It is expected to have high numbers for `storage/small` metric.", + "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", "fieldConfig": { "defaults": { "color": { @@ -5611,7 +5611,7 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", "legendFormat": "__auto", "range": true, "refId": "A" From aeba3f39dbc606c0fe3a61cc088675a187e44fdb Mon Sep 17 00:00:00 2001 From: Artem Navoiev Date: Tue, 23 Apr 2024 21:47:17 +0200 Subject: [PATCH 12/58] docs: vmalert remove new lines in configuration section Signed-off-by: Artem Navoiev --- docs/vmalert.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/vmalert.md b/docs/vmalert.md index a5c03c3f35..fc2f05c51f 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1362,14 +1362,17 @@ The shortlist of configuration flags is the following: -rule="http:///path/to/rules". HTTP URL to a page with alerting rules. -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". -rule="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively. - Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. - + Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Enterprise version of vmalert supports S3 and GCS paths to rules. For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches all files with prefix rule_ in folder dir. +<<<<<<< Updated upstream See https://docs.victoriametrics.com/vmalert/#reading-rules-from-object-storage +======= + See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage +>>>>>>> Stashed changes Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -rule.evalDelay time @@ -1386,8 +1389,7 @@ The shortlist of configuration flags is the following: -rule.templates="/path/to/file". Path to a single file with go templates -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, absolute path to all .tpl files in root. - -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively. - + -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively. Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -rule.updateEntriesLimit int From 592a9fe9f2ccefef4f1017a1def80631afc83af4 Mon Sep 17 00:00:00 2001 From: Artem Navoiev Date: Tue, 23 Apr 2024 22:30:31 +0200 Subject: [PATCH 13/58] docs: vmalert remove new lines in configuration section Signed-off-by: Artem Navoiev --- docs/vmalert.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/vmalert.md b/docs/vmalert.md index fc2f05c51f..fbb78bd995 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1367,12 +1367,6 @@ The shortlist of configuration flags is the following: For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches all files with prefix rule_ in folder dir. -<<<<<<< Updated upstream - See https://docs.victoriametrics.com/vmalert/#reading-rules-from-object-storage - -======= - See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage ->>>>>>> Stashed changes Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -rule.evalDelay time From a14acd4fccd9c42f95b1d174981791ff0996b7a8 Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:41:23 +0800 Subject: [PATCH 14/58] Automatic update operator docs from VictoriaMetrics/operator@6e1a876 (#6179) --- docs/operator/CHANGELOG.md | 7 +++++++ docs/operator/resources/vmauth.md | 2 +- docs/operator/vars.md | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index 562524157c..dd47c46f39 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -16,6 +16,13 @@ aliases: ## Next release + + +## [v0.43.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.43.2) - 23 Apr 2024 + +- [operator](./README.md): fix conversion from `ServiceMonitor` to `VMServiceScrape`, `bearerTokenSecret` is dropped mistakenly since [v0.43.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.43.0). See [this issue](https://github.com/VictoriaMetrics/operator/issues/932). +- [operator](./README.md): fix selector match for config resources like VMUser, VMRule... , before it could be ignored when update resource labels. + ## [v0.43.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.43.2) - 22 Apr 2024 diff --git a/docs/operator/resources/vmauth.md b/docs/operator/resources/vmauth.md index cac3df05cd..3a9920d4e3 100644 --- a/docs/operator/resources/vmauth.md +++ b/docs/operator/resources/vmauth.md @@ -33,7 +33,7 @@ The CRD specifies which `VMUser`s should be covered by the deployed `VMAuth` ins The Operator then generates a configuration based on the included `VMUser`s and updates the `Configmaps` containing the configuration. It continuously does so for all changes that are made to `VMUser`s or to the `VMAuth` resource itself. -[VMUser](./vmrule.md) objects generate part of `VMAuth` configuration. +[VMUser](./vmuser.md) objects generate part of `VMAuth` configuration. For filtering users `VMAuth` uses selectors `userNamespaceSelector` and `userSelector`. It allows configuring rules access control across namespaces and different environments. diff --git a/docs/operator/vars.md b/docs/operator/vars.md index 118ed1855f..5d2ec52d57 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ menu: # Auto Generated vars for package config - updated at Mon Apr 22 16:35:37 UTC 2024 + updated at Wed Apr 24 02:53:53 UTC 2024 | varible name | variable default value | variable required | variable description | From 029060af60774bb8f3638d4ed09f88d29e7ca919 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Wed, 24 Apr 2024 10:57:54 +0200 Subject: [PATCH 15/58] app/vmbackup: introduce new flag type URL (#6152) The new flag type is supposed to be used for specifying URL values which could contain sensitive information such as auth tokens in GET params or HTTP basic authentication. The URL flag also allows loading its value from files if `file://` prefix is specified. As example, the new flag type was used in app/vmbackup as it requires specifying `authKey` param for making the snapshot. See related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973 Thanks to @wasim-nihal for initial implementation https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6060 --------- Signed-off-by: hagen1778 --- app/vmbackup/main.go | 33 +++----- docs/CHANGELOG.md | 2 + lib/flagutil/url.go | 159 +++++++++++++++++++++++++++++++++++++++ lib/flagutil/url_test.go | 45 +++++++++++ lib/snapshot/snapshot.go | 12 +-- 5 files changed, 224 insertions(+), 27 deletions(-) create mode 100644 lib/flagutil/url.go create mode 100644 lib/flagutil/url_test.go diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go index 3603f21050..9f18cb41bc 100644 --- a/app/vmbackup/main.go +++ b/app/vmbackup/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "net/url" "os" "path/filepath" "strings" @@ -27,9 +26,9 @@ var ( httpListenAddr = flag.String("httpListenAddr", ":8420", "TCP address for exporting metrics at /metrics page") storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage") snapshotName = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set") - snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+ + snapshotCreateURL = flagutil.NewURL("snapshot.createURL", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+ "Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set") - snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+ + snapshotDeleteURL = flagutil.NewURL("snapshot.deleteURL", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+ "All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete") dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+ "Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup/dir\n"+ @@ -55,31 +54,23 @@ func main() { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055 deleteSnapshot := func() {} - if len(*snapshotCreateURL) > 0 { - // create net/url object - createURL, err := url.Parse(*snapshotCreateURL) - if err != nil { - logger.Fatalf("cannot parse snapshotCreateURL: %s", err) - } + if len(snapshotCreateURL.String()) > 0 { if len(*snapshotName) > 0 { logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case") } - logger.Infof("Snapshot create url %s", createURL.Redacted()) - if len(*snapshotDeleteURL) <= 0 { - err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1)) + + logger.Infof("Snapshot create url %s", snapshotCreateURL.String()) + if len(snapshotDeleteURL.String()) <= 0 { + err := flag.Set("snapshot.deleteURL", strings.Replace(snapshotCreateURL.Get(), "/create", "/delete", 1)) if err != nil { logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err) } } - deleteURL, err := url.Parse(*snapshotDeleteURL) - if err != nil { - logger.Fatalf("cannot parse snapshotDeleteURL: %s", err) - } - logger.Infof("Snapshot delete url %s", deleteURL.Redacted()) - name, err := snapshot.Create(createURL.String()) + logger.Infof("Snapshot delete url %s", snapshotDeleteURL) + name, err := snapshot.Create(snapshotCreateURL.Get()) if err != nil { - logger.Fatalf("cannot create snapshot: %s", err) + logger.Fatalf("cannot create snapshot via %q: %s", snapshotCreateURL, err) } err = flag.Set("snapshotName", name) if err != nil { @@ -87,9 +78,9 @@ func main() { } deleteSnapshot = func() { - err := snapshot.Delete(deleteURL.String(), name) + err := snapshot.Delete(snapshotDeleteURL.Get(), name) if err != nil { - logger.Fatalf("cannot delete snapshot: %s", err) + logger.Fatalf("cannot delete snapshot via %q: %s", snapshotDeleteURL, err) } } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 99f4b4b0cc..6450cdd4ef 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -41,6 +41,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): support regex matching when routing incoming requests based on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6070). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): optimize auto-suggestion performance for metric names when the database contains big number of unique time series. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): in the Select component, user-entered values are now preserved on blur if they match options in the list. +* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup): support loading values for flags `-snapshot.createURL` and `-snapshot.deleteURL` from files for security reasons. To load value from file use the following format `-snapshot.createURL=file:///abs/path/to/file` or `-snapshot.deleteURL=file://./relative/path/to/file`. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973). +* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup): hide sensitive information for flags `-snapshot.createURL` and `-snapshot.deleteURL` when printing logs. The change masks HTTP basic authentication user and password values, as well as GET params matching `auth`, `pass`, `key`, `secret`, `token` words. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973). Thanks to @wasim-nihal for [initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6060). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): avoid blocking `/api/v1/rules`, `/api/v1/alerts`, `/metrics` APIs when alerting rule uses template functions `query`, which could takes a while to execute. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079). diff --git a/lib/flagutil/url.go b/lib/flagutil/url.go new file mode 100644 index 0000000000..f8dcf8f65d --- /dev/null +++ b/lib/flagutil/url.go @@ -0,0 +1,159 @@ +package flagutil + +import ( + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + "sync/atomic" + "unicode" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" +) + +// NewURL returns new `url` flag with the given name and description. +// +// The url value is redacted when calling URL.String() in the following way: +// 1. Basic Auth username and password are replaced with "xxxxx" +// 2. Values of GET params matching `secretWordsRe` expression are replaced with "xxxxx". +// +// Call URL.Get() for obtaining original URL address. +func NewURL(name, description string) *URL { + description += fmt.Sprintf("\nFlag value can be read from the given file when using -%s=file:///abs/path/to/file or -%s=file://./relative/path/to/file . ", name, name) + u := &URL{ + flagname: name, + } + ru := &redactedURL{ + URL: &url.URL{}, + redacted: "", + } + u.value.Store(&ru) + flag.Var(u, name, description) + return u +} + +// URL is a flag holding URL address +// +// If the flag value is file:///path/to/file, +// then its contents is automatically re-read from the given file on disk. +type URL struct { + nextRefreshTimestamp atomic.Uint64 + + value atomic.Pointer[*redactedURL] + + // flagname is the name of the flag + flagname string + + // sourcePath contains either url or path to file with the url + sourcePath string +} + +type redactedURL struct { + *url.URL + redacted string +} + +// Get returns the current u address. +// +// It re-reads u value from the file:///path/to/file +// if they were passed to URL.Set. +func (u *URL) Get() string { + u.maybeRereadURL() + ru := *u.value.Load() + return ru.URL.String() +} + +// Get returns the current u redacted address. +// +// It re-reads u value from the file:///path/to/file +// if they were passed to URL.Set. +func (u *URL) String() string { + u.maybeRereadURL() + ru := *u.value.Load() + return ru.redacted +} + +func (u *URL) maybeRereadURL() { + if u.sourcePath == "" { + // Fast path - nothing to re-read + return + } + tsCurr := fasttime.UnixTimestamp() + tsNext := u.nextRefreshTimestamp.Load() + if tsCurr < tsNext { + // Fast path - nothing to re-read + return + } + + // Re-read value from s.sourcePath + u.nextRefreshTimestamp.Store(tsCurr + 2) + data, err := os.ReadFile(u.sourcePath) + if err != nil { + // cannot use lib/logger, since it can be uninitialized yet + log.Printf("flagutil: fall back to the previous url for -%s, since failed to re-read it from %q: cannot read %q: %s\n", u.flagname, u.sourcePath, u.sourcePath, err.Error()) + } else { + addr := strings.TrimRightFunc(string(data), unicode.IsSpace) + res, err := newRedactedURL(addr) + if err != nil { + log.Printf("flagutil: cannot parse %q: %s\n", u.flagname, err.Error()) + return + } + u.value.Store(&res) + } +} + +// Set implements flag.Value interface. +func (u *URL) Set(value string) error { + u.nextRefreshTimestamp.Store(0) + var s string + switch { + case strings.HasPrefix(value, "file://"): + u.sourcePath = strings.TrimPrefix(value, "file://") + data, err := os.ReadFile(u.sourcePath) + if err != nil { + // cannot use lib/logger, since it can be uninitialized yet + return fmt.Errorf("cannot read %q: %w", u.sourcePath, err) + } + s = strings.TrimRightFunc(string(data), unicode.IsSpace) + default: + u.sourcePath = "" + s = value + } + + res, err := newRedactedURL(s) + if err != nil { + return fmt.Errorf("cannot parse %q: %s", u.flagname, err) + } + u.value.Store(&res) + return nil +} + +var secretWordsRe = regexp.MustCompile("auth|pass|key|secret|token") + +func newRedactedURL(s string) (*redactedURL, error) { + u, err := url.Parse(s) + if err != nil { + return nil, fmt.Errorf("cannot parse URL: %s", err) + } + ru := &redactedURL{URL: u} + + // copy URL before mutating query params + u2 := *u + values := u2.Query() + for k, vs := range values { + if secretWordsRe.MatchString(k) { + for i := range vs { + vs[i] = "xxxxx" + } + } + } + u2.RawQuery = values.Encode() + if _, has := u2.User.Password(); has { + u2.User = url.UserPassword("xxxxx", "xxxxx") + } + ru.redacted = u2.String() + return ru, nil +} diff --git a/lib/flagutil/url_test.go b/lib/flagutil/url_test.go new file mode 100644 index 0000000000..4b36f54862 --- /dev/null +++ b/lib/flagutil/url_test.go @@ -0,0 +1,45 @@ +package flagutil + +import ( + "os" + "testing" +) + +func TestNewURL(t *testing.T) { + u := &URL{} + f := func(s, exp string) { + t.Helper() + if err := u.Set(s); err != nil { + t.Fatalf("failed to set %q value: %s", s, err) + } + if u.String() != exp { + t.Fatalf("expected to get %q; got %q instead", exp, u.String()) + } + } + + f("", "") + f("http://foo:8428", "http://foo:8428") + f("http://username:password@foo:8428", "http://xxxxx:xxxxx@foo:8428") + f("http://foo:8428?authToken=bar", "http://foo:8428?authToken=xxxxx") + f("http://username:password@foo:8428?authToken=bar", "http://xxxxx:xxxxx@foo:8428?authToken=xxxxx") + + file, err := os.CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(file.Name()) }() + + writeToFile(t, file.Name(), "http://foo:8428") + f("file://"+file.Name(), "http://foo:8428") + + writeToFile(t, file.Name(), "http://xxxxx:password@foo:8428?authToken=bar") + f("file://"+file.Name(), "http://xxxxx:xxxxx@foo:8428?authToken=xxxxx") +} + +func writeToFile(t *testing.T, file, b string) { + t.Helper() + err := os.WriteFile(file, []byte(b), 0644) + if err != nil { + t.Fatal(err) + } +} diff --git a/lib/snapshot/snapshot.go b/lib/snapshot/snapshot.go index 87bf8d09f6..c4324ea81d 100644 --- a/lib/snapshot/snapshot.go +++ b/lib/snapshot/snapshot.go @@ -51,13 +51,13 @@ func Create(createSnapshotURL string) (string, error) { return "", err } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body) + return "", fmt.Errorf("unexpected status code: %d; expecting %d; response body: %q", resp.StatusCode, http.StatusOK, body) } snap := snapshot{} err = json.Unmarshal(body, &snap) if err != nil { - return "", fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body) + return "", fmt.Errorf("cannot parse JSON response: %w; response body: %q", err, body) } if snap.Status == "ok" { @@ -67,7 +67,7 @@ func Create(createSnapshotURL string) (string, error) { if snap.Status == "error" { return "", errors.New(snap.Msg) } - return "", fmt.Errorf("Unkown status: %v", snap.Status) + return "", fmt.Errorf("unknown status: %v", snap.Status) } // Delete deletes a snapshot via the provided api endpoint @@ -95,13 +95,13 @@ func Delete(deleteSnapshotURL string, snapshotName string) error { return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body) + return fmt.Errorf("unexpected status code: %d; expecting %d; response body: %q", resp.StatusCode, http.StatusOK, body) } snap := snapshot{} err = json.Unmarshal(body, &snap) if err != nil { - return fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body) + return fmt.Errorf("cannot parse JSON response: %w; response body: %q", err, body) } if snap.Status == "ok" { @@ -111,5 +111,5 @@ func Delete(deleteSnapshotURL string, snapshotName string) error { if snap.Status == "error" { return errors.New(snap.Msg) } - return fmt.Errorf("Unkown status: %v", snap.Status) + return fmt.Errorf("unknown status: %v", snap.Status) } From 679844feafe0cc6c0187c0acb39ab5ad9f7d6eb5 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Wed, 24 Apr 2024 13:47:57 +0200 Subject: [PATCH 16/58] Revert "app/vmbackup: introduce new flag type URL (#6152)" This reverts commit 029060af60774bb8f3638d4ed09f88d29e7ca919. --- app/vmbackup/main.go | 35 +++++---- docs/CHANGELOG.md | 2 - lib/flagutil/url.go | 159 --------------------------------------- lib/flagutil/url_test.go | 45 ----------- lib/snapshot/snapshot.go | 12 +-- 5 files changed, 28 insertions(+), 225 deletions(-) delete mode 100644 lib/flagutil/url.go delete mode 100644 lib/flagutil/url_test.go diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go index 9f18cb41bc..3603f21050 100644 --- a/app/vmbackup/main.go +++ b/app/vmbackup/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -26,9 +27,9 @@ var ( httpListenAddr = flag.String("httpListenAddr", ":8420", "TCP address for exporting metrics at /metrics page") storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage") snapshotName = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set") - snapshotCreateURL = flagutil.NewURL("snapshot.createURL", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+ + snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+ "Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set") - snapshotDeleteURL = flagutil.NewURL("snapshot.deleteURL", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+ + snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+ "All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete") dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+ "Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup/dir\n"+ @@ -54,23 +55,31 @@ func main() { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055 deleteSnapshot := func() {} - if len(snapshotCreateURL.String()) > 0 { + if len(*snapshotCreateURL) > 0 { + // create net/url object + createURL, err := url.Parse(*snapshotCreateURL) + if err != nil { + logger.Fatalf("cannot parse snapshotCreateURL: %s", err) + } if len(*snapshotName) > 0 { logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case") } - - logger.Infof("Snapshot create url %s", snapshotCreateURL.String()) - if len(snapshotDeleteURL.String()) <= 0 { - err := flag.Set("snapshot.deleteURL", strings.Replace(snapshotCreateURL.Get(), "/create", "/delete", 1)) + logger.Infof("Snapshot create url %s", createURL.Redacted()) + if len(*snapshotDeleteURL) <= 0 { + err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1)) if err != nil { logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err) } } - - logger.Infof("Snapshot delete url %s", snapshotDeleteURL) - name, err := snapshot.Create(snapshotCreateURL.Get()) + deleteURL, err := url.Parse(*snapshotDeleteURL) if err != nil { - logger.Fatalf("cannot create snapshot via %q: %s", snapshotCreateURL, err) + logger.Fatalf("cannot parse snapshotDeleteURL: %s", err) + } + logger.Infof("Snapshot delete url %s", deleteURL.Redacted()) + + name, err := snapshot.Create(createURL.String()) + if err != nil { + logger.Fatalf("cannot create snapshot: %s", err) } err = flag.Set("snapshotName", name) if err != nil { @@ -78,9 +87,9 @@ func main() { } deleteSnapshot = func() { - err := snapshot.Delete(snapshotDeleteURL.Get(), name) + err := snapshot.Delete(deleteURL.String(), name) if err != nil { - logger.Fatalf("cannot delete snapshot via %q: %s", snapshotDeleteURL, err) + logger.Fatalf("cannot delete snapshot: %s", err) } } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6450cdd4ef..99f4b4b0cc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -41,8 +41,6 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): support regex matching when routing incoming requests based on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6070). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): optimize auto-suggestion performance for metric names when the database contains big number of unique time series. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): in the Select component, user-entered values are now preserved on blur if they match options in the list. -* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup): support loading values for flags `-snapshot.createURL` and `-snapshot.deleteURL` from files for security reasons. To load value from file use the following format `-snapshot.createURL=file:///abs/path/to/file` or `-snapshot.deleteURL=file://./relative/path/to/file`. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973). -* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup): hide sensitive information for flags `-snapshot.createURL` and `-snapshot.deleteURL` when printing logs. The change masks HTTP basic authentication user and password values, as well as GET params matching `auth`, `pass`, `key`, `secret`, `token` words. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973). Thanks to @wasim-nihal for [initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6060). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): avoid blocking `/api/v1/rules`, `/api/v1/alerts`, `/metrics` APIs when alerting rule uses template functions `query`, which could takes a while to execute. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079). diff --git a/lib/flagutil/url.go b/lib/flagutil/url.go deleted file mode 100644 index f8dcf8f65d..0000000000 --- a/lib/flagutil/url.go +++ /dev/null @@ -1,159 +0,0 @@ -package flagutil - -import ( - "flag" - "fmt" - "log" - "net/url" - "os" - "regexp" - "strings" - "sync/atomic" - "unicode" - - "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" -) - -// NewURL returns new `url` flag with the given name and description. -// -// The url value is redacted when calling URL.String() in the following way: -// 1. Basic Auth username and password are replaced with "xxxxx" -// 2. Values of GET params matching `secretWordsRe` expression are replaced with "xxxxx". -// -// Call URL.Get() for obtaining original URL address. -func NewURL(name, description string) *URL { - description += fmt.Sprintf("\nFlag value can be read from the given file when using -%s=file:///abs/path/to/file or -%s=file://./relative/path/to/file . ", name, name) - u := &URL{ - flagname: name, - } - ru := &redactedURL{ - URL: &url.URL{}, - redacted: "", - } - u.value.Store(&ru) - flag.Var(u, name, description) - return u -} - -// URL is a flag holding URL address -// -// If the flag value is file:///path/to/file, -// then its contents is automatically re-read from the given file on disk. -type URL struct { - nextRefreshTimestamp atomic.Uint64 - - value atomic.Pointer[*redactedURL] - - // flagname is the name of the flag - flagname string - - // sourcePath contains either url or path to file with the url - sourcePath string -} - -type redactedURL struct { - *url.URL - redacted string -} - -// Get returns the current u address. -// -// It re-reads u value from the file:///path/to/file -// if they were passed to URL.Set. -func (u *URL) Get() string { - u.maybeRereadURL() - ru := *u.value.Load() - return ru.URL.String() -} - -// Get returns the current u redacted address. -// -// It re-reads u value from the file:///path/to/file -// if they were passed to URL.Set. -func (u *URL) String() string { - u.maybeRereadURL() - ru := *u.value.Load() - return ru.redacted -} - -func (u *URL) maybeRereadURL() { - if u.sourcePath == "" { - // Fast path - nothing to re-read - return - } - tsCurr := fasttime.UnixTimestamp() - tsNext := u.nextRefreshTimestamp.Load() - if tsCurr < tsNext { - // Fast path - nothing to re-read - return - } - - // Re-read value from s.sourcePath - u.nextRefreshTimestamp.Store(tsCurr + 2) - data, err := os.ReadFile(u.sourcePath) - if err != nil { - // cannot use lib/logger, since it can be uninitialized yet - log.Printf("flagutil: fall back to the previous url for -%s, since failed to re-read it from %q: cannot read %q: %s\n", u.flagname, u.sourcePath, u.sourcePath, err.Error()) - } else { - addr := strings.TrimRightFunc(string(data), unicode.IsSpace) - res, err := newRedactedURL(addr) - if err != nil { - log.Printf("flagutil: cannot parse %q: %s\n", u.flagname, err.Error()) - return - } - u.value.Store(&res) - } -} - -// Set implements flag.Value interface. -func (u *URL) Set(value string) error { - u.nextRefreshTimestamp.Store(0) - var s string - switch { - case strings.HasPrefix(value, "file://"): - u.sourcePath = strings.TrimPrefix(value, "file://") - data, err := os.ReadFile(u.sourcePath) - if err != nil { - // cannot use lib/logger, since it can be uninitialized yet - return fmt.Errorf("cannot read %q: %w", u.sourcePath, err) - } - s = strings.TrimRightFunc(string(data), unicode.IsSpace) - default: - u.sourcePath = "" - s = value - } - - res, err := newRedactedURL(s) - if err != nil { - return fmt.Errorf("cannot parse %q: %s", u.flagname, err) - } - u.value.Store(&res) - return nil -} - -var secretWordsRe = regexp.MustCompile("auth|pass|key|secret|token") - -func newRedactedURL(s string) (*redactedURL, error) { - u, err := url.Parse(s) - if err != nil { - return nil, fmt.Errorf("cannot parse URL: %s", err) - } - ru := &redactedURL{URL: u} - - // copy URL before mutating query params - u2 := *u - values := u2.Query() - for k, vs := range values { - if secretWordsRe.MatchString(k) { - for i := range vs { - vs[i] = "xxxxx" - } - } - } - u2.RawQuery = values.Encode() - if _, has := u2.User.Password(); has { - u2.User = url.UserPassword("xxxxx", "xxxxx") - } - ru.redacted = u2.String() - return ru, nil -} diff --git a/lib/flagutil/url_test.go b/lib/flagutil/url_test.go deleted file mode 100644 index 4b36f54862..0000000000 --- a/lib/flagutil/url_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package flagutil - -import ( - "os" - "testing" -) - -func TestNewURL(t *testing.T) { - u := &URL{} - f := func(s, exp string) { - t.Helper() - if err := u.Set(s); err != nil { - t.Fatalf("failed to set %q value: %s", s, err) - } - if u.String() != exp { - t.Fatalf("expected to get %q; got %q instead", exp, u.String()) - } - } - - f("", "") - f("http://foo:8428", "http://foo:8428") - f("http://username:password@foo:8428", "http://xxxxx:xxxxx@foo:8428") - f("http://foo:8428?authToken=bar", "http://foo:8428?authToken=xxxxx") - f("http://username:password@foo:8428?authToken=bar", "http://xxxxx:xxxxx@foo:8428?authToken=xxxxx") - - file, err := os.CreateTemp("", "") - if err != nil { - t.Fatal(err) - } - defer func() { _ = os.Remove(file.Name()) }() - - writeToFile(t, file.Name(), "http://foo:8428") - f("file://"+file.Name(), "http://foo:8428") - - writeToFile(t, file.Name(), "http://xxxxx:password@foo:8428?authToken=bar") - f("file://"+file.Name(), "http://xxxxx:xxxxx@foo:8428?authToken=xxxxx") -} - -func writeToFile(t *testing.T, file, b string) { - t.Helper() - err := os.WriteFile(file, []byte(b), 0644) - if err != nil { - t.Fatal(err) - } -} diff --git a/lib/snapshot/snapshot.go b/lib/snapshot/snapshot.go index c4324ea81d..87bf8d09f6 100644 --- a/lib/snapshot/snapshot.go +++ b/lib/snapshot/snapshot.go @@ -51,13 +51,13 @@ func Create(createSnapshotURL string) (string, error) { return "", err } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code: %d; expecting %d; response body: %q", resp.StatusCode, http.StatusOK, body) + return "", fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body) } snap := snapshot{} err = json.Unmarshal(body, &snap) if err != nil { - return "", fmt.Errorf("cannot parse JSON response: %w; response body: %q", err, body) + return "", fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body) } if snap.Status == "ok" { @@ -67,7 +67,7 @@ func Create(createSnapshotURL string) (string, error) { if snap.Status == "error" { return "", errors.New(snap.Msg) } - return "", fmt.Errorf("unknown status: %v", snap.Status) + return "", fmt.Errorf("Unkown status: %v", snap.Status) } // Delete deletes a snapshot via the provided api endpoint @@ -95,13 +95,13 @@ func Delete(deleteSnapshotURL string, snapshotName string) error { return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d; expecting %d; response body: %q", resp.StatusCode, http.StatusOK, body) + return fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body) } snap := snapshot{} err = json.Unmarshal(body, &snap) if err != nil { - return fmt.Errorf("cannot parse JSON response: %w; response body: %q", err, body) + return fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body) } if snap.Status == "ok" { @@ -111,5 +111,5 @@ func Delete(deleteSnapshotURL string, snapshotName string) error { if snap.Status == "error" { return errors.New(snap.Msg) } - return fmt.Errorf("unknown status: %v", snap.Status) + return fmt.Errorf("Unkown status: %v", snap.Status) } From 2d9bbe1934e724fe3c49a349468cc03dc1bb568b Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Wed, 24 Apr 2024 17:03:21 +0200 Subject: [PATCH 17/58] docs/changelog: mention downsampling fixes for ENT version of VM Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 99f4b4b0cc..ffa33d40fc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -42,6 +42,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): optimize auto-suggestion performance for metric names when the database contains big number of unique time series. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): in the Select component, user-entered values are now preserved on blur if they match options in the list. +* BUGFIX: [downsampling](https://docs.victoriametrics.com/#downsampling): skip unnecessary index lookups if downsampling wasn't set for ENT versions of VictoriaMetrics. Before, users of VictoriaMetrics ENT could have experience elevated CPU usage even if no downsampling was configured. The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000). +* BUGFIX: [downsampling](https://docs.victoriametrics.com/#downsampling): properly populate downsampling metadata for data parts created by VictoriaMetrics ENT versions lower than v1.100.0. The bug could trigger the downsampling actions for parts that were downsampled already. This bug doesn't have any negative effect apart from spending extra CPU resources on the repeated downsampling. The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): avoid blocking `/api/v1/rules`, `/api/v1/alerts`, `/metrics` APIs when alerting rule uses template functions `query`, which could takes a while to execute. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix links with anchors in vmalert's UI. Starting from [v1.99.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.99.0) vmalert could ignore anchors pointing to specific rule groups if `search` param was present in URL. From 07496d7d922ed36e6f71e41ff129bfa48d8812b4 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Wed, 24 Apr 2024 19:29:32 +0300 Subject: [PATCH 18/58] deployment: update makefile package-* targets (#6172) Updated package targets in a same manner, how it's done for publish ones in https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7958f3886467aef1a386e3111136f1d3c59f7149 --- deployment/docker/Makefile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 00a73a48c5..9ae67e9aee 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -117,7 +117,7 @@ app-via-docker-linux-arm: CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-via-docker-goos-goarch app-via-docker-linux-arm64: - CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc \ + EXTRA_ENVS='CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc' \ CGO_ENABLED=1 GOOS=linux GOARCH=arm64 $(MAKE) app-via-docker-goos-goarch app-via-docker-linux-ppc64le: @@ -145,37 +145,28 @@ app-via-docker-windows-amd64: package-via-docker-goarch: APP_SUFFIX='-$(GOARCH)' \ - DOCKER_OPTS='--env CGO_ENABLED=$(CGO_ENABLED) --env GOOS=linux --env GOARCH=$(GOARCH)' \ + DOCKER_OPTS='--env CGO_ENABLED=$(CGO_ENABLED) --env GOOS=linux --env GOARCH=$(GOARCH) $(foreach v,$(EXTRA_ENVS),--env $(v))' \ $(MAKE) package-via-docker -package-via-docker-goarch-arm64: - APP_SUFFIX='-arm64' \ - DOCKER_OPTS='--env CGO_ENABLED=1 --env GOOS=linux --env GOARCH=arm64 --env CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc' \ - $(MAKE) package-via-docker - -package-via-docker-goarch-cgo: - CGO_ENABLED=1 $(MAKE) package-via-docker-goarch - -package-via-docker-goarch-nocgo: - CGO_ENABLED=0 $(MAKE) package-via-docker-goarch - package-via-docker-pure: APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) package-via-docker package-via-docker-amd64: - GOARCH=amd64 $(MAKE) package-via-docker-goarch-cgo + EXTRA_ENVS='CC=/opt/cross-builder/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc' \ + GOARCH=amd64 GOARCH=amd64 $(MAKE) package-via-docker-goarch package-via-docker-arm: - GOARCH=arm $(MAKE) package-via-docker-goarch-nocgo + GOARCH=arm CGO_ENABLED=0 $(MAKE) package-via-docker-goarch package-via-docker-arm64: - $(MAKE) package-via-docker-goarch-arm64 + EXTRA_ENVS='CC=/opt/cross-builder/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc' \ + CGO_ENABLED=1 GOARCH=arm64 $(MAKE) package-via-docker-goarch package-via-docker-ppc64le: - GOARCH=ppc64le $(MAKE) package-via-docker-goarch-nocgo + GOARCH=ppc64le CGO_ENABLED=0 $(MAKE) package-via-docker-goarch package-via-docker-386: - GOARCH=386 $(MAKE) package-via-docker-goarch-nocgo + GOARCH=386 CGO_ENABLED=0 $(MAKE) package-via-docker-goarch remove-docker-images: docker image ls --format '{{.ID}}' | xargs docker image rm -f From 6aaf1768f4b093eaa228771c8a368f8d677d33fb Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 25 Apr 2024 11:23:02 +0200 Subject: [PATCH 19/58] Revert "docs: removed code-style highlighting for commanad-line flags of VM components (#6147)" This reverts commit 9bedbcfa2f2a97875ec3425c0d73dd2002e27304. --- docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- docs/vmagent.md | 2 +- docs/vmalert.md | 2 +- docs/vmauth.md | 2 +- docs/vmbackup.md | 2 +- docs/vmbackupmanager.md | 2 +- docs/vmgateway.md | 2 +- docs/vmrestore.md | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index dcd35ae519..486003f451 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2679,7 +2679,7 @@ Files included in each folder: Pass `-help` to VictoriaMetrics in order to see the list of supported command-line flags with their description: -``` +```sh -bigMergeConcurrency int Deprecated: this flag does nothing -blockcache.missesBeforeCaching int diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index e3329834db..c98f0420b1 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2687,7 +2687,7 @@ Files included in each folder: Pass `-help` to VictoriaMetrics in order to see the list of supported command-line flags with their description: -``` +```sh -bigMergeConcurrency int Deprecated: this flag does nothing -blockcache.missesBeforeCaching int diff --git a/docs/vmagent.md b/docs/vmagent.md index 752efb903d..227c6e45d0 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1623,7 +1623,7 @@ It is safe sharing the collected profiles from security point of view, since the `vmagent` can be fine-tuned with various command-line flags. Run `./vmagent -help` in order to see the full list of these flags with their descriptions and default values: -``` +```sh ./vmagent -help vmagent collects metrics data via popular data ingestion protocols and routes them to VictoriaMetrics. diff --git a/docs/vmalert.md b/docs/vmalert.md index fbb78bd995..40dd5aa934 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -965,7 +965,7 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: -``` +```sh -clusterMode If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert/#multitenancy . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/enterprise/ -configCheckInterval duration diff --git a/docs/vmauth.md b/docs/vmauth.md index b69dd8d485..56d5367cc8 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -1121,7 +1121,7 @@ It is safe sharing the collected profiles from security point of view, since the Pass `-help` command-line arg to `vmauth` in order to see all the configuration options: -``` +```sh ./vmauth -help vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics. diff --git a/docs/vmbackup.md b/docs/vmbackup.md index 465e2f8a5e..628e364bf0 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -304,7 +304,7 @@ Refer to the respective documentation for your object storage provider for more Run `vmbackup -help` in order to see all the available options: -``` +```sh -concurrency int The number of concurrent workers. Higher concurrency may reduce backup duration (default 10) -configFilePath string diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index dcef2ab9c5..2bcd7dbdb6 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -415,7 +415,7 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: -``` +```text vmbackupmanager performs regular backups according to the provided configs. subcommands: diff --git a/docs/vmgateway.md b/docs/vmgateway.md index cc263e66a8..09445aee8b 100644 --- a/docs/vmgateway.md +++ b/docs/vmgateway.md @@ -273,7 +273,7 @@ Example usage for tokens issued by Google: Below is the list of configuration flags (it can be viewed by running `./vmgateway -help`): -``` +```sh -auth.httpHeader string HTTP header name to look for JWT authorization token (default "Authorization") -auth.jwksEndpoints array diff --git a/docs/vmrestore.md b/docs/vmrestore.md index f640539a92..a8dec2ce13 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -91,7 +91,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q * Run `vmrestore -help` in order to see all the available options: -``` +```sh -concurrency int The number of concurrent workers. Higher concurrency may reduce restore duration (default 10) -configFilePath string From 6193fa3dcfab29af6a2ac7dddebae8877288ebba Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Thu, 25 Apr 2024 12:48:49 +0200 Subject: [PATCH 20/58] vmui: trigger auto-suggestion at any cursor position (#6155) - Implemented auto-suggestion triggers for mid-string cursor positions in vmui. - Improved the suggestion list positioning to appear directly beneath the active text editing area. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5864 --- .../Configurators/QueryEditor/QueryEditor.tsx | 22 ++++- .../QueryEditor/QueryEditorAutocomplete.tsx | 91 ++++++++++++++----- .../Configurators/QueryEditor/style.scss | 9 ++ .../Main/Autocomplete/Autocomplete.tsx | 4 +- .../components/Main/TextField/TextField.tsx | 38 ++++++-- .../src/components/Main/TextField/style.scss | 2 +- .../vmui/src/hooks/useFetchQueryOptions.tsx | 5 +- docs/CHANGELOG.md | 1 + 8 files changed, 131 insertions(+), 41 deletions(-) diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx index a2a2c7a42d..2a42902641 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx @@ -9,6 +9,7 @@ import { partialWarning, seriesFetchedWarning } from "./warningText"; import { AutocompleteOptions } from "../../Main/Autocomplete/Autocomplete"; import useDeviceDetect from "../../../hooks/useDeviceDetect"; import { useQueryState } from "../../../state/query/QueryStateContext"; +import debounce from "lodash.debounce"; export interface QueryEditorProps { onChange: (query: string) => void; @@ -40,9 +41,12 @@ const QueryEditor: FC = ({ const { isMobile } = useDeviceDetect(); const [openAutocomplete, setOpenAutocomplete] = useState(false); - const [caretPosition, setCaretPosition] = useState([0, 0]); + const [caretPosition, setCaretPosition] = useState<[number, number]>([0, 0]); const autocompleteAnchorEl = useRef(null); + const [showAutocomplete, setShowAutocomplete] = useState(autocomplete); + const debouncedSetShowAutocomplete = useRef(debounce(setShowAutocomplete, 500)).current; + const warning = [ { show: stats?.seriesFetched === "0" && !stats.resultLength, @@ -58,8 +62,9 @@ const QueryEditor: FC = ({ label = `${label} (${stats.executionTimeMsec || 0}ms)`; } - const handleSelect = (val: string) => { + const handleSelect = (val: string, caretPosition: number) => { onChange(val); + setCaretPosition([caretPosition, caretPosition]); }; const handleKeyDown = (e: KeyboardEvent) => { @@ -100,14 +105,19 @@ const QueryEditor: FC = ({ setOpenAutocomplete(!!val.length); }; - const handleChangeCaret = (val: number[]) => { - setCaretPosition(val); + const handleChangeCaret = (val: [number, number]) => { + setCaretPosition(prev => prev[0] === val[0] && prev[1] === val[1] ? prev : val); }; useEffect(() => { setOpenAutocomplete(autocomplete); }, [autocompleteQuick]); + useEffect(() => { + setShowAutocomplete(false); + debouncedSetShowAutocomplete(true); + }, [caretPosition]); + return (
= ({ onChangeCaret={handleChangeCaret} disabled={disabled} inputmode={"search"} + caretPosition={caretPosition} /> - {autocomplete && ( + {showAutocomplete && autocomplete && ( diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx index edbd85d353..c19934895b 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx @@ -1,7 +1,6 @@ -import React, { FC, Ref, useState, useEffect, useMemo } from "preact/compat"; +import React, { FC, Ref, useState, useEffect, useMemo, useCallback } from "preact/compat"; import Autocomplete, { AutocompleteOptions } from "../../Main/Autocomplete/Autocomplete"; import { useFetchQueryOptions } from "../../../hooks/useFetchQueryOptions"; -import { getTextWidth } from "../../../utils/uplot"; import { escapeRegexp, hasUnclosedQuotes } from "../../../utils/regexp"; import useGetMetricsQL from "../../../hooks/useGetMetricsQL"; import { QueryContextType } from "../../../types"; @@ -10,8 +9,9 @@ import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete"; interface QueryEditorAutocompleteProps { value: string; anchorEl: Ref; - caretPosition: number[]; - onSelect: (val: string) => void; + caretPosition: [number, number]; // [start, end] + hasHelperText: boolean; + onSelect: (val: string, caretPosition: number) => void; onFoundOptions: (val: AutocompleteOptions[]) => void; } @@ -19,16 +19,24 @@ const QueryEditorAutocomplete: FC = ({ value, anchorEl, caretPosition, + hasHelperText, onSelect, onFoundOptions }) => { - const [leftOffset, setLeftOffset] = useState(0); + const [offsetPos, setOffsetPos] = useState({ top: 0, left: 0 }); const metricsqlFunctions = useGetMetricsQL(); + const values = useMemo(() => { + if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" }; + const beforeCursor = value.substring(0, caretPosition[0]); + const afterCursor = value.substring(caretPosition[1]); + return { beforeCursor, afterCursor }; + }, [value, caretPosition]); + const exprLastPart = useMemo(() => { - const parts = value.split("}"); + const parts = values.beforeCursor.split("}"); return parts[parts.length - 1]; - }, [value]); + }, [values]); const metric = useMemo(() => { const regexp = /\b[^{}(),\s]+(?={|$)/g; @@ -43,7 +51,7 @@ const QueryEditorAutocomplete: FC = ({ }, [exprLastPart]); const shouldSuppressAutoSuggestion = (value: string) => { - const pattern = /([(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right)\b)/; + const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right)\b)/; const parts = value.split(/\s+/); const partsCount = parts.length; const lastPart = parts[partsCount - 1]; @@ -55,7 +63,7 @@ const QueryEditorAutocomplete: FC = ({ }; const context = useMemo(() => { - if (!value || value.endsWith("}") || shouldSuppressAutoSuggestion(value)) { + if (!values.beforeCursor || values.beforeCursor.endsWith("}") || shouldSuppressAutoSuggestion(values.beforeCursor)) { return QueryContextType.empty; } @@ -63,19 +71,19 @@ const QueryEditorAutocomplete: FC = ({ const labelValueRegexp = new RegExp(`(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`, "g"); switch (true) { - case labelValueRegexp.test(value): + case labelValueRegexp.test(values.beforeCursor): return QueryContextType.labelValue; - case labelRegexp.test(value): + case labelRegexp.test(values.beforeCursor): return QueryContextType.label; default: return QueryContextType.metricsql; } - }, [value, metric, label]); + }, [values, metric, label]); const valueByContext = useMemo(() => { - const wordMatch = value.match(/([\w_\-.:/]+(?![},]))$/); + const wordMatch = values.beforeCursor.match(/([\w_\-.:/]+(?![},]))$/); return wordMatch ? wordMatch[0] : ""; - }, [value]); + }, [values.beforeCursor]); const { metrics, labels, labelValues, loading } = useFetchQueryOptions({ valueByContext, @@ -97,8 +105,10 @@ const QueryEditorAutocomplete: FC = ({ } }, [context, metrics, labels, labelValues]); - const handleSelect = (insert: string) => { + const handleSelect = useCallback((insert: string) => { // Find the start and end of valueByContext in the query string + const value = values.beforeCursor; + let valueAfterCursor = values.afterCursor; const startIndexOfValueByContext = value.lastIndexOf(valueByContext, caretPosition[0]); const endIndexOfValueByContext = startIndexOfValueByContext + valueByContext.length; @@ -110,26 +120,59 @@ const QueryEditorAutocomplete: FC = ({ if (context === QueryContextType.labelValue) { const quote = "\""; const needsQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext); + valueAfterCursor = valueAfterCursor.replace(/^[^\s"|},]*/, ""); insert = `${needsQuote ? quote : ""}${insert}`; } + if (context === QueryContextType.label) { + valueAfterCursor = valueAfterCursor.replace(/^[^\s=!,{}()"|+\-/*^]*/, ""); + } + + if (context === QueryContextType.metricsql) { + valueAfterCursor = valueAfterCursor.replace(/^[^\s[\]{}()"|+\-/*^]*/, ""); + } // Assemble the new value with the inserted text - const newVal = `${beforeValueByContext}${insert}${afterValueByContext}`; - onSelect(newVal); - }; + const newVal = `${beforeValueByContext}${insert}${afterValueByContext}${valueAfterCursor}`; + onSelect(newVal, beforeValueByContext.length + insert.length); + }, [values]); useEffect(() => { if (!anchorEl.current) { - setLeftOffset(0); + setOffsetPos({ top: 0, left: 0 }); return; } - const style = window.getComputedStyle(anchorEl.current); + const element = anchorEl.current.querySelector("textarea") || anchorEl.current; + const style = window.getComputedStyle(element); const fontSize = `${style.getPropertyValue("font-size")}`; const fontFamily = `${style.getPropertyValue("font-family")}`; - const offset = getTextWidth(value, `${fontSize} ${fontFamily}`); - setLeftOffset(offset); - }, [anchorEl, caretPosition]); + const lineHeight = parseInt(`${style.getPropertyValue("line-height")}`); + + const span = document.createElement("div"); + span.style.font = `${fontSize} ${fontFamily}`; + span.style.padding = style.getPropertyValue("padding"); + span.style.lineHeight = `${lineHeight}px`; + span.style.width = `${element.offsetWidth}px`; + span.style.maxWidth = `${element.offsetWidth}px`; + span.style.whiteSpace = style.getPropertyValue("white-space"); + span.style.overflowWrap = style.getPropertyValue("overflow-wrap"); + + const marker = document.createElement("span"); + span.appendChild(document.createTextNode(values.beforeCursor)); + span.appendChild(marker); + span.appendChild(document.createTextNode(values.afterCursor)); + document.body.appendChild(span); + + const spanRect = span.getBoundingClientRect(); + const markerRect = marker.getBoundingClientRect(); + + const leftOffset = markerRect.left - spanRect.left; + const topOffset = markerRect.bottom - spanRect.bottom - (hasHelperText ? lineHeight : 0); + setOffsetPos({ top: topOffset, left: leftOffset }); + + span.remove(); + marker.remove(); + }, [anchorEl, caretPosition, hasHelperText]); return ( <> @@ -140,7 +183,7 @@ const QueryEditorAutocomplete: FC = ({ options={options} anchor={anchorEl} minLength={0} - offset={{ top: 0, left: leftOffset }} + offset={offsetPos} onSelect={handleSelect} onFoundOptions={onFoundOptions} maxDisplayResults={{ diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss index db48a9cf13..b012403ae7 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss @@ -2,4 +2,13 @@ .vm-query-editor { position: relative; + + .marker-detection { + position: absolute; + top: 0; + left: 0; + pointer-events: none; + z-index: -9999; + visibility: hidden; + } } diff --git a/app/vmui/packages/vmui/src/components/Main/Autocomplete/Autocomplete.tsx b/app/vmui/packages/vmui/src/components/Main/Autocomplete/Autocomplete.tsx index ab83c25181..bbb15c76f1 100644 --- a/app/vmui/packages/vmui/src/components/Main/Autocomplete/Autocomplete.tsx +++ b/app/vmui/packages/vmui/src/components/Main/Autocomplete/Autocomplete.tsx @@ -120,7 +120,7 @@ const Autocomplete: FC = ({ const handleKeyDown = useCallback((e: KeyboardEvent) => { const { key, ctrlKey, metaKey, shiftKey } = e; const modifiers = ctrlKey || metaKey || shiftKey; - const hasOptions = foundOptions.length; + const hasOptions = foundOptions.length && !hideFoundedOptions; if (key === "ArrowUp" && !modifiers && hasOptions) { e.preventDefault(); @@ -148,7 +148,7 @@ const Autocomplete: FC = ({ if (key === "Escape") { handleCloseAutocomplete(); } - }, [focusOption, foundOptions, handleCloseAutocomplete, onSelect, selected]); + }, [focusOption, foundOptions, hideFoundedOptions, handleCloseAutocomplete, onSelect, selected]); useEffect(() => { setOpenAutocomplete(value.length >= minLength); diff --git a/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx b/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx index f95b4194e3..a2a938885d 100644 --- a/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx +++ b/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx @@ -1,6 +1,7 @@ import React, { FC, useEffect, + useState, useRef, useMemo, FormEvent, @@ -28,12 +29,13 @@ interface TextFieldProps { autofocus?: boolean helperText?: string inputmode?: "search" | "text" | "email" | "tel" | "url" | "none" | "numeric" | "decimal" + caretPosition?: [number, number] onChange?: (value: string) => void onEnter?: () => void onKeyDown?: (e: KeyboardEvent) => void onFocus?: () => void onBlur?: () => void - onChangeCaret?: (position: number[]) => void + onChangeCaret?: (position: [number, number]) => void } const TextField: FC = ({ @@ -49,6 +51,7 @@ const TextField: FC = ({ disabled = false, autofocus = false, inputmode = "text", + caretPosition, onChange, onEnter, onKeyDown, @@ -62,6 +65,7 @@ const TextField: FC = ({ const inputRef = useRef(null); const textareaRef = useRef(null); const fieldRef = useMemo(() => type === "textarea" ? textareaRef : inputRef, [type]); + const [selectionPos, setSelectionPos] = useState<[start: number, end: number]>([0, 0]); const inputClasses = classNames({ "vm-text-field__input": true, @@ -74,7 +78,7 @@ const TextField: FC = ({ const updateCaretPosition = (target: HTMLInputElement | HTMLTextAreaElement) => { const { selectionStart, selectionEnd } = target; - onChangeCaret && onChangeCaret([selectionStart || 0, selectionEnd || 0]); + setSelectionPos([selectionStart || 0, selectionEnd || 0]); }; const handleMouseUp = (e: MouseEvent) => { @@ -102,11 +106,6 @@ const TextField: FC = ({ updateCaretPosition(e.currentTarget); }; - useEffect(() => { - if (!autofocus || isMobile) return; - fieldRef?.current?.focus && fieldRef.current.focus(); - }, [fieldRef, autofocus]); - const handleFocus = () => { onFocus && onFocus(); }; @@ -115,6 +114,31 @@ const TextField: FC = ({ onBlur && onBlur(); }; + const setSelectionRange = (range: [number, number]) => { + try { + fieldRef.current && fieldRef.current.setSelectionRange(range[0], range[1]); + } catch (e) { + return e; + } + }; + + useEffect(() => { + if (!autofocus || isMobile) return; + fieldRef?.current?.focus && fieldRef.current.focus(); + }, [fieldRef, autofocus]); + + useEffect(() => { + onChangeCaret && onChangeCaret(selectionPos); + }, [selectionPos]); + + useEffect(() => { + setSelectionRange(selectionPos); + }, [value]); + + useEffect(() => { + caretPosition && setSelectionRange(caretPosition); + }, [caretPosition]); + return