diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index d9d52fc46..fe4090318 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -2013,6 +2013,46 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`drop_empty_series()`, func(t *testing.T) { + t.Parallel() + q := `sort(drop_empty_series( + ( + alias(time(), "foo"), + alias(500 + time(), "bar"), + ) > 2000 + ) default 123)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 2100, 2300, 2500}, + Timestamps: timestampsExpected, + } + r.MetricName.MetricGroup = []byte("bar") + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run(`no drop_empty_series()`, func(t *testing.T) { + t.Parallel() + q := `sort(( + ( + alias(time(), "foo"), + alias(500 + time(), "bar"), + ) > 2000 + ) default 123)` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 123, 123, 123}, + Timestamps: timestampsExpected, + } + r1.MetricName.MetricGroup = []byte("foo") + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{123, 123, 123, 2100, 2300, 2500}, + Timestamps: timestampsExpected, + } + r2.MetricName.MetricGroup = []byte("bar") + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`drop_common_labels(single_series)`, func(t *testing.T) { t.Parallel() q := `drop_common_labels(label_set(time(), "foo", "bar", "__name__", "xxx", "q", "we"))` @@ -8960,6 +9000,8 @@ func TestExecError(t *testing.T) { f(`delta_prometheus()`) f(`rollup_candlestick()`) f(`rollup()`) + f(`drop_empty_series()`) + f(`drop_common_labels()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 65a06971d..b7c94ea36 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -45,6 +45,7 @@ var transformFuncs = map[string]transformFunc{ "days_in_month": newTransformFuncDateTime(transformDaysInMonth), "deg": newTransformFuncOneArg(transformDeg), "drop_common_labels": transformDropCommonLabels, + "drop_empty_series": transformDropEmptySeries, "end": newTransformFuncZeroArgs(transformEnd), "exp": newTransformFuncOneArg(transformExp), "floor": newTransformFuncOneArg(transformFloor), @@ -1840,6 +1841,15 @@ func transformDropCommonLabels(tfa *transformFuncArg) ([]*timeseries, error) { return rvs, nil } +func transformDropEmptySeries(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) != 1 { + return nil, fmt.Errorf("unexpected number of args; got %d; want 1", len(args)) + } + rvs := removeEmptySeries(args[0]) + return rvs, nil +} + func transformLabelCopy(tfa *transformFuncArg) ([]*timeseries, error) { return transformLabelCopyExt(tfa, false) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c82622a18..44ffd5abe 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The sandbox cluster installation is running under the constant load generated by * SECURITY: upgrade Go builder from Go1.21.1 to Go1.21.3. See [the list of issues addressed in Go1.21.2](https://github.com/golang/go/issues?q=milestone%3AGo1.21.2+label%3ACherryPickApproved) and [the list of issues addressed in Go1.21.3](https://github.com/golang/go/issues?q=milestone%3AGo1.21.3+label%3ACherryPickApproved). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [drop_empty_series()](https://docs.victoriametrics.com/MetricsQL.html#drop_empty_series) function, which can be used for filtering out empty series before performing additional calculations as shown in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5071). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `eval_alignment` attribute for [Groups](https://docs.victoriametrics.com/vmalert.html#groups), it will align group query requests timestamp with interval like `datasource.queryTimeAlignment` did. This also means that `datasource.queryTimeAlignment` command-line flag becomes deprecated now and will have no effect if configured. If `datasource.queryTimeAlignment` was set to `false` before, then `eval_alignment` has to be set to `false` explicitly under group. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5049). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 4a6248458..462738dd6 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -1055,6 +1055,17 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke This function is supported by PromQL. See also [rad](#rad). +#### drop_empty_series + +`drop_empty_series(q) is a [transform function](#transform-functions), which drops empty series from `q`. + +This function can be used when `default` operator should be applied only to non-empty series. For example, +`drop_empty_series(temperature < 30) default 42` returns series, which have at least a single sample smaller than 30 on the selected time range, +while filling gaps in the returned series with 42. + +On the other hand `(temperature < 30) default 40` returns all the `temperature` series, even if they have no samples smaller than 30, +by replacing all the values bigger or equal to 30 with 40. + #### end `end()` is a [transform function](#transform-functions), which returns the unix timestamp in seconds for the last point. diff --git a/go.mod b/go.mod index bef3c9412..1ad509ca6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.66.1 + github.com/VictoriaMetrics/metricsql v0.67.0 github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 diff --git a/go.sum b/go.sum index fca23d048..631d229e7 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.66.1 h1:H+HfOHOznVNml0O3QkXarFxrlY+enrcWRUZ1xX73Kig= -github.com/VictoriaMetrics/metricsql v0.66.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.67.0 h1:IYbKA6rhd8UWmW1LbaKiEk1q5ixFQagu9jL6z3Mt03o= +github.com/VictoriaMetrics/metricsql v0.67.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index 22769604b..c281ba891 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -29,6 +29,7 @@ var transformFuncs = map[string]bool{ "days_in_month": true, "deg": true, "drop_common_labels": true, + "drop_empty_series": true, "end": true, "exp": true, "floor": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 5d11073e7..d6f336304 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.66.1 +# github.com/VictoriaMetrics/metricsql v0.67.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop