From cc311e20fe8fc6b1f68473e83d647cd08c8cd20f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 12 May 2020 19:45:42 +0300 Subject: [PATCH] app/vmselect/promql: add `any(x) by (y)` aggregate function, which returns any time series from `q` for each group `y` --- app/vmselect/promql/aggr.go | 5 ++++ app/vmselect/promql/aggr_incremental.go | 27 +++++++++++++++++++ app/vmselect/promql/exec_test.go | 11 ++++++++ docs/MetricsQL.md | 4 ++- go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metricsql/aggr.go | 1 + vendor/modules.txt | 2 +- 8 files changed, 51 insertions(+), 5 deletions(-) diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index f73853a1d..8cf478dc4 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -43,6 +43,7 @@ var aggrFuncs = map[string]aggrFunc{ "bottomk_max": newAggrFuncRangeTopK(maxValue, true), "bottomk_avg": newAggrFuncRangeTopK(avgValue, true), "bottomk_median": newAggrFuncRangeTopK(medianValue, true), + "any": newAggrFunc(aggrFuncAny), } type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error) @@ -118,6 +119,10 @@ func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeserie return rvs, nil } +func aggrFuncAny(tss []*timeseries) []*timeseries { + return tss[:1] +} + func aggrFuncSum(tss []*timeseries) []*timeseries { if len(tss) == 1 { // Fast path - nothing to sum. diff --git a/app/vmselect/promql/aggr_incremental.go b/app/vmselect/promql/aggr_incremental.go index 3eb0a5e75..6ff37efea 100644 --- a/app/vmselect/promql/aggr_incremental.go +++ b/app/vmselect/promql/aggr_incremental.go @@ -48,6 +48,11 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{ mergeAggrFunc: mergeAggrGeomean, finalizeAggrFunc: finalizeAggrGeomean, }, + "any": { + updateAggrFunc: updateAggrAny, + mergeAggrFunc: mergeAggrAny, + finalizeAggrFunc: finalizeAggrCommon, + }, } type incrementalAggrFuncContext struct { @@ -458,3 +463,25 @@ func finalizeAggrGeomean(iac *incrementalAggrContext) { dstValues[i] = math.Pow(dstValues[i], 1/v) } } + +func updateAggrAny(iac *incrementalAggrContext, values []float64) { + dstValues := iac.ts.Values + dstCounts := iac.values + if dstCounts[0] > 0 { + return + } + dstCounts[0] = 1 + dstValues = append(dstValues[:0], values...) +} + +func mergeAggrAny(dst, src *incrementalAggrContext) { + srcValues := src.ts.Values + dstValues := dst.ts.Values + srcCounts := src.values + dstCounts := dst.values + if dstCounts[0] > 0 { + return + } + dstCounts[0] = srcCounts[0] + dstValues = append(dstValues[:0], srcValues...) +} diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 6960607a5..c7289e6a2 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -3783,6 +3783,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r1, r2} f(q, resultExpected) }) + t.Run(`any()`, func(t *testing.T) { + t.Parallel() + q := `any(label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{10, 10, 10, 10, 10, 10}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`topk(-1)`, func(t *testing.T) { t.Parallel() q := `sort(topk(-1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))` diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 048b7e50b..dea5008a8 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -22,7 +22,7 @@ Feel free [filing a feature request](https://github.com/VictoriaMetrics/Victoria This functionality can be tried at [an editable Grafana dashboard](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo). - [`WITH` templates](https://play.victoriametrics.com/promql/expand-with-exprs). This feature simplifies writing and managing complex queries. Go to [`WITH` templates playground](https://victoriametrics.com/promql/expand-with-exprs) and try it. -- Aggregate functions support optional `limit N` suffix in order to limit the number of output series. For example, `sum(x) by (y) limit 10` limits +- All the aggregate functions support optional `limit N` suffix in order to limit the number of output series. For example, `sum(x) by (y) limit 10` limits the number of output time series after the aggregation to 10. All the other time series are dropped. - Metric names and metric labels may contain escaped chars. For instance, `foo\-bar{baz\=aa="b"}` is valid expression. It returns time series with name `foo-bar` containing label `baz=aa` with value `b`. Additionally, `\xXX` escape sequence is supported, where `XX` is hexadecimal representation of escaped char. - `offset`, range duration and step value for range vector may refer to the current step aka `$__interval` value from Grafana. @@ -74,6 +74,8 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`. - `median(q)` - median aggregate. Shorthand to `quantile(0.5, q)`. - `limitk(k, q)` - limits the number of time series returned from `q` to `k`. +- `any(q) by (x)` - returns any time series from `q` for each group in `x`. Note that `any()` removes all the labels except of those listed in `by (x)`. + Use `limitk(1, q)` if you need retaining all the labels from `q`. - `keep_last_value(q)` - fills missing data (gaps) in `q` with the previous non-empty value. - `keep_next_value(q)` - fills missing data (gaps) in `q` with the next non-empty value. - `distinct_over_time(m[d])` - returns distinct number of values for `m` data points over `d` duration. diff --git a/go.mod b/go.mod index 59ffcdafb..b5bbfb5fd 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.1 github.com/VictoriaMetrics/metrics v1.11.2 - github.com/VictoriaMetrics/metricsql v0.2.0 + github.com/VictoriaMetrics/metricsql v0.2.1 github.com/aws/aws-sdk-go v1.30.25 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/protobuf v1.4.1 // indirect diff --git a/go.sum b/go.sum index 8e23a7f71..af2244af8 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.1 h1:I7YdbswTIW63WxoFoUOSNxeOEGB46rdKUL github.com/VictoriaMetrics/fasthttp v1.0.1/go.mod h1:BqgsieH90PR7x97c89j+eqZDloKkDhAEQTwhLw6jw/4= github.com/VictoriaMetrics/metrics v1.11.2 h1:t/ceLP6SvagUqypCKU7cI7+tQn54+TIV/tGoxihHvx8= github.com/VictoriaMetrics/metrics v1.11.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ= -github.com/VictoriaMetrics/metricsql v0.2.0 h1:tGWPFSCqDRKYLuc99lNmxa0md9uyWHVycbumGLAM4V0= -github.com/VictoriaMetrics/metricsql v0.2.0/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE= +github.com/VictoriaMetrics/metricsql v0.2.1 h1:OI/W2QCFiQiFULVN3ZiC/iCqZFt25rXp/O7P2NiAwYU= +github.com/VictoriaMetrics/metricsql v0.2.1/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aws/aws-sdk-go v1.30.25 h1:89NXJkfpjnMEnsxkP8MVX+LDsoiLCSqevraLb5y4Mjk= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go index 66a8f8b87..5cd88de20 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go @@ -33,6 +33,7 @@ var aggrFuncs = map[string]bool{ "bottomk_max": true, "bottomk_avg": true, "bottomk_median": true, + "any": true, } func isAggrFunc(s string) bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index 6fe84dea0..70f693a5a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -18,7 +18,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.11.2 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.2.0 +# github.com/VictoriaMetrics/metricsql v0.2.1 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.30.25