From bdb743c88d737d18a5c7e6bec325deee0f35cd8a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Oct 2023 20:42:22 +0200 Subject: [PATCH] app/vmselect/promql: add drop_empty_series() function for dropping empty series before performing additional calculations This can be useful in the following queries: drop_empty_series(temperature <= 30) default 40 This query drops temperature series with all the values bigger than 30 on the selected time range, while replacing gaps in the remaining series with 40. The query without drop_empty_series: (temperature <= 30) default 40 would leave all the temperature series with all the values bigger than 30 on the selected time range, and replace all their values with 40. This is not what could be epxected in some cases like here - https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5071 --- app/vmselect/promql/exec_test.go | 42 +++++++++++++++++++ app/vmselect/promql/transform.go | 10 +++++ docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 11 +++++ go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/transform.go | 1 + vendor/modules.txt | 2 +- 8 files changed, 69 insertions(+), 4 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index d9d52fc46..fe4090318 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -2013,6 +2013,46 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`drop_empty_series()`, func(t *testing.T) { + t.Parallel() + q := `sort(drop_empty_series( + ( + alias(time(), "foo"), + alias(500 + time(), "bar"), + ) > 2000 + ) default 123)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 2100, 2300, 2500}, + Timestamps: timestampsExpected, + } + r.MetricName.MetricGroup = []byte("bar") + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run(`no drop_empty_series()`, func(t *testing.T) { + t.Parallel() + q := `sort(( + ( + alias(time(), "foo"), + alias(500 + time(), "bar"), + ) > 2000 + ) default 123)` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 123, 123, 123}, + Timestamps: timestampsExpected, + } + r1.MetricName.MetricGroup = []byte("foo") + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 2100, 2300, 2500}, + Timestamps: timestampsExpected, + } + r2.MetricName.MetricGroup = []byte("bar") + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`drop_common_labels(single_series)`, func(t *testing.T) { t.Parallel() q := `drop_common_labels(label_set(time(), "foo", "bar", "__name__", "xxx", "q", "we"))` @@ -8960,6 +9000,8 @@ func TestExecError(t *testing.T) { f(`delta_prometheus()`) f(`rollup_candlestick()`) f(`rollup()`) + f(`drop_empty_series()`) + f(`drop_common_labels()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 65a06971d..b7c94ea36 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -45,6 +45,7 @@ var transformFuncs = map[string]transformFunc{ "days_in_month": newTransformFuncDateTime(transformDaysInMonth), "deg": newTransformFuncOneArg(transformDeg), "drop_common_labels": transformDropCommonLabels, + "drop_empty_series": transformDropEmptySeries, "end": newTransformFuncZeroArgs(transformEnd), "exp": newTransformFuncOneArg(transformExp), "floor": newTransformFuncOneArg(transformFloor), @@ -1840,6 +1841,15 @@ func transformDropCommonLabels(tfa *transformFuncArg) ([]*timeseries, error) { return rvs, nil } +func transformDropEmptySeries(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) != 1 { + return nil, fmt.Errorf("unexpected number of args; got %d; want 1", len(args)) + } + rvs := removeEmptySeries(args[0]) + return rvs, nil +} + func transformLabelCopy(tfa *transformFuncArg) ([]*timeseries, error) { return transformLabelCopyExt(tfa, false) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c82622a18..44ffd5abe 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The sandbox cluster installation is running under the constant load generated by * SECURITY: upgrade Go builder from Go1.21.1 to Go1.21.3. See [the list of issues addressed in Go1.21.2](https://github.com/golang/go/issues?q=milestone%3AGo1.21.2+label%3ACherryPickApproved) and [the list of issues addressed in Go1.21.3](https://github.com/golang/go/issues?q=milestone%3AGo1.21.3+label%3ACherryPickApproved). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [drop_empty_series()](https://docs.victoriametrics.com/MetricsQL.html#drop_empty_series) function, which can be used for filtering out empty series before performing additional calculations as shown in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5071). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `eval_alignment` attribute for [Groups](https://docs.victoriametrics.com/vmalert.html#groups), it will align group query requests timestamp with interval like `datasource.queryTimeAlignment` did. This also means that `datasource.queryTimeAlignment` command-line flag becomes deprecated now and will have no effect if configured. If `datasource.queryTimeAlignment` was set to `false` before, then `eval_alignment` has to be set to `false` explicitly under group. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5049). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 4a6248458..462738dd6 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -1055,6 +1055,17 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke This function is supported by PromQL. See also [rad](#rad). +#### drop_empty_series + +`drop_empty_series(q) is a [transform function](#transform-functions), which drops empty series from `q`. + +This function can be used when `default` operator should be applied only to non-empty series. For example, +`drop_empty_series(temperature < 30) default 42` returns series, which have at least a single sample smaller than 30 on the selected time range, +while filling gaps in the returned series with 42. + +On the other hand `(temperature < 30) default 40` returns all the `temperature` series, even if they have no samples smaller than 30, +by replacing all the values bigger or equal to 30 with 40. + #### end `end()` is a [transform function](#transform-functions), which returns the unix timestamp in seconds for the last point. diff --git a/go.mod b/go.mod index bef3c9412..1ad509ca6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.66.1 + github.com/VictoriaMetrics/metricsql v0.67.0 github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 diff --git a/go.sum b/go.sum index fca23d048..631d229e7 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.66.1 h1:H+HfOHOznVNml0O3QkXarFxrlY+enrcWRUZ1xX73Kig= -github.com/VictoriaMetrics/metricsql v0.66.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.67.0 h1:IYbKA6rhd8UWmW1LbaKiEk1q5ixFQagu9jL6z3Mt03o= +github.com/VictoriaMetrics/metricsql v0.67.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index 22769604b..c281ba891 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -29,6 +29,7 @@ var transformFuncs = map[string]bool{ "days_in_month": true, "deg": true, "drop_common_labels": true, + "drop_empty_series": true, "end": true, "exp": true, "floor": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 5d11073e7..d6f336304 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.66.1 +# github.com/VictoriaMetrics/metricsql v0.67.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop