From 62d19369a3821f990c1f253780e1f14aad3b3b80 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Tue, 13 Aug 2024 20:54:35 +0800 Subject: [PATCH] =?UTF-8?q?stream=20aggregation:=20do=20not=20allow=20to?= =?UTF-8?q?=20enable=20`-stream.keepInput`=20and=20`k=E2=80=A6=20(#6723)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …eep_metric_names` options in stream aggregation config together With aggregated data and raw data under the same metric, results would be confusing. --------- Signed-off-by: hagen1778 Co-authored-by: hagen1778 --- app/vmagent/remotewrite/streamaggr.go | 2 ++ docs/CHANGELOG.md | 3 ++- docs/stream-aggregation.md | 3 ++- lib/streamaggr/streamaggr.go | 9 +++++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/vmagent/remotewrite/streamaggr.go b/app/vmagent/remotewrite/streamaggr.go index f20df53f0..6275f95ee 100644 --- a/app/vmagent/remotewrite/streamaggr.go +++ b/app/vmagent/remotewrite/streamaggr.go @@ -202,6 +202,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) { DropInputLabels: *streamAggrGlobalDropInputLabels, IgnoreOldSamples: *streamAggrGlobalIgnoreOldSamples, IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals, + KeepInput: *streamAggrGlobalKeepInput, } sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global") @@ -230,6 +231,7 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag DropInputLabels: *streamAggrDropInputLabels, IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx), IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx), + KeepInput: streamAggrKeepInput.GetOptionalArg(idx), } sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5700f2bc2..57227ddb5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,11 +34,12 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): allow overriding the `sample_limit` option at [scrape_configs](https://docs.victoriametrics.com/sd_configs/#scrape_configs) when a label `__sample_limit__` is specified for target. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6665). Thanks to @zoglam for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6666). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): reduce memory usage when scraping targets with big response body. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6759). * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): use exponential backoff for retries when uploading or downloading data from S3. This should reduce the number of failed uploads and downloads when S3 is temporarily unavailable. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6732). +* FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): do not allow enabling `-stream.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config), as it may result in time series collision. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6784). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): respect HTTP headers defined in [notifier configuration file](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file) for each request to notifiers. Previously, this param was ignored by mistake. * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): correctly apply `-streamAggr.dropInputLabels` when global stream deduplication is enabled without `-streamAggr.config`. Previously, `-remoteWrite.streamAggr.dropInputLabels` was used instead. -* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): fix command-line flag `-remoteWrite.streamAggr.ignoreFirstIntervals` to accept multiple values and be applied per each corresponding `-remoteWrite.url`. Previously, this flag only could have been used globally for all URLs. +* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): fix command-line flag `-remoteWrite.streamAggr.ignoreFirstIntervals` to accept multiple values and be applied per each corresponding `-remoteWrite.url`. Previously, this flag only could have been used globally for all URLs. * BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): respect `-search.denyPartialResponse` cmd-line flag and `deny_partial_response` query GET param when serving requests via Graphite API. Before, partial responses were always denied. Thanks to @penguinlav for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6748). ## [v1.102.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.1) diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md index 5107fd4fd..f8a5cd8c1 100644 --- a/docs/stream-aggregation.md +++ b/docs/stream-aggregation.md @@ -1087,7 +1087,8 @@ specified individually per each `-remoteWrite.url`: outputs: [total] # keep_metric_names instructs keeping the original metric names for the aggregated samples. - # This option can be set only if outputs list contains only a single output. + # This option can't be enabled together with `-streamAggr.keepInput` or `-remoteWrite.streamAggr.keepInput`. + # This option can be set only if outputs list contains a single output. # By default, a special suffix is added to original metric names in the aggregated samples. # See https://docs.victoriametrics.com/stream-aggregation/#output-metric-names # diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index 6fc674e84..6188f8b80 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -135,6 +135,11 @@ type Options struct { // // This option can be overridden individually per each aggregation via ignore_first_intervals option. IgnoreFirstIntervals int + + // KeepInput enables keeping all the input samples after the aggregation. + // + // By default, aggregates samples are dropped, while the remaining samples are written to the corresponding -remoteWrite.url. + KeepInput bool } // Config is a configuration for a single stream aggregation. @@ -511,6 +516,10 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set, keepMetricNames = *v } if keepMetricNames { + if opts.KeepInput { + return nil, fmt.Errorf("`-streamAggr.keepInput` and `keep_metric_names` options can't be enabled in the same time," + + "as it may result in time series collision") + } if len(cfg.Outputs) != 1 { return nil, fmt.Errorf("`outputs` list must contain only a single entry if `keep_metric_names` is set; got %q; "+ "see https://docs.victoriametrics.com/stream-aggregation/#output-metric-names", cfg.Outputs)