From da77f4deeb40c5526ced54dcdcb041bc0630ba9c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Oct 2023 21:50:09 +0200 Subject: [PATCH] app/vmselect/promql: add labels_equal(q, "label1", "label2", ...) function This function returns q series, which have identical values for the listed labels "label1", "label2", ... See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5148 --- app/vmselect/promql/exec_test.go | 45 +++++++++++++++++++ app/vmselect/promql/transform.go | 42 ++++++++++++++++- docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 21 ++++++--- go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/optimizer.go | 6 ++- .../VictoriaMetrics/metricsql/transform.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 109 insertions(+), 15 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index fe4090318..52e72f9a2 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -2013,6 +2013,50 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`labels_equal()`, func(t *testing.T) { + t.Parallel() + q := `sort(labels_equal(( + label_set(10, "instance", "qwe", "host", "rty"), + label_set(20, "instance", "qwe", "host", "qwe"), + label_set(30, "aaa", "bbb", "instance", "foo", "host", "foo"), + ), "instance", "host"))` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{20, 20, 20, 20, 20, 20}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("host"), + Value: []byte("qwe"), + }, + { + Key: []byte("instance"), + Value: []byte("qwe"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{30, 30, 30, 30, 30, 30}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("aaa"), + Value: []byte("bbb"), + }, + { + Key: []byte("host"), + Value: []byte("foo"), + }, + { + Key: []byte("instance"), + Value: []byte("foo"), + }, + } + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`drop_empty_series()`, func(t *testing.T) { t.Parallel() q := `sort(drop_empty_series( @@ -9002,6 +9046,7 @@ func TestExecError(t *testing.T) { f(`rollup()`) f(`drop_empty_series()`) f(`drop_common_labels()`) + f(`labels_equal()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index b7c94ea36..3ce79d016 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -75,6 +75,7 @@ var transformFuncs = map[string]transformFunc{ "label_uppercase": transformLabelUppercase, "label_value": transformLabelValue, "limit_offset": transformLimitOffset, + "labels_equal": transformLabelsEqual, "ln": newTransformFuncOneArg(transformLn), "log2": newTransformFuncOneArg(transformLog2), "log10": newTransformFuncOneArg(transformLog10), @@ -1843,8 +1844,8 @@ func transformDropCommonLabels(tfa *transformFuncArg) ([]*timeseries, error) { func transformDropEmptySeries(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args - if len(args) != 1 { - return nil, fmt.Errorf("unexpected number of args; got %d; want 1", len(args)) + if err := expectTransformArgsNum(args, 1); err != nil { + return nil, err } rvs := removeEmptySeries(args[0]) return rvs, nil @@ -2023,6 +2024,43 @@ func labelReplace(tss []*timeseries, srcLabel string, r *regexp.Regexp, dstLabel return tss, nil } +func transformLabelsEqual(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) < 3 { + return nil, fmt.Errorf("unexpected number of args; got %d; want at least 3", len(args)) + } + tss := args[0] + var labelNames []string + for i, ts := range args[1:] { + labelName, err := getString(ts, i+1) + if err != nil { + return nil, fmt.Errorf("cannot get label name: %w", err) + } + labelNames = append(labelNames, labelName) + } + rvs := tss[:0] + for _, ts := range tss { + if hasIdenticalLabelValues(&ts.MetricName, labelNames) { + rvs = append(rvs, ts) + } + } + return rvs, nil +} + +func hasIdenticalLabelValues(mn *storage.MetricName, labelNames []string) bool { + if len(labelNames) < 2 { + return true + } + labelValue := mn.GetTagValue(labelNames[0]) + for _, labelName := range labelNames[1:] { + b := mn.GetTagValue(labelName) + if string(labelValue) != string(b) { + return false + } + } + return true +} + func transformLabelValue(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 2); err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1b1729bff..dbf1eb3c1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,7 @@ The sandbox cluster installation is running under the constant load generated by * SECURITY: upgrade Go builder from Go1.21.1 to Go1.21.3. See [the list of issues addressed in Go1.21.2](https://github.com/golang/go/issues?q=milestone%3AGo1.21.2+label%3ACherryPickApproved) and [the list of issues addressed in Go1.21.3](https://github.com/golang/go/issues?q=milestone%3AGo1.21.3+label%3ACherryPickApproved). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [drop_empty_series()](https://docs.victoriametrics.com/MetricsQL.html#drop_empty_series) function, which can be used for filtering out empty series before performing additional calculations as shown in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5071). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [labels_equal()](https://docs.victoriametrics.com/MetricsQL.html#labels_equal) function, which can be used for searching series with identical values for the given labels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5148). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `eval_alignment` attribute for [Groups](https://docs.victoriametrics.com/vmalert.html#groups), it will align group query requests timestamp with interval like `datasource.queryTimeAlignment` did. This also means that `datasource.queryTimeAlignment` command-line flag becomes deprecated now and will have no effect if configured. If `datasource.queryTimeAlignment` was set to `false` before, then `eval_alignment` has to be set to `false` explicitly under group. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5049). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 28d11797c..ac16404e6 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -109,7 +109,7 @@ The list of MetricsQL features on top of PromQL: * [histogram_quantile](#histogram_quantile) accepts optional third arg - `boundsLabel`. In this case it returns `lower` and `upper` bounds for the estimated percentile. See [this issue for details](https://github.com/prometheus/prometheus/issues/5706). -* `default` binary operator. `q1 default q2` fills gaps in `q1` with the corresponding values from `q2`. +* `default` binary operator. `q1 default q2` fills gaps in `q1` with the corresponding values from `q2`. See also [drop_empty_series](#drop_empty_series). * `if` binary operator. `q1 if q2` removes values from `q1` for missing values from `q2`. * `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for existing values from `q2`. * `WITH` templates. This feature simplifies writing and managing complex queries. @@ -1602,7 +1602,7 @@ which maps `label` values from `src_*` to `dst*` for all the time series returne which drops time series from `q` with `label` not matching the given `regexp`. This function can be useful after [rollup](#rollup)-like functions, which may return multiple time series for every input series. -See also [label_mismatch](#label_mismatch). +See also [label_mismatch](#label_mismatch) and [labels_equal](#labels_equal). #### label_mismatch @@ -1610,7 +1610,7 @@ See also [label_mismatch](#label_mismatch). which drops time series from `q` with `label` matching the given `regexp`. This function can be useful after [rollup](#rollup)-like functions, which may return multiple time series for every input series. -See also [label_match](#label_match). +See also [label_match](#label_match) and [labels_equal](#labels_equal). #### label_move @@ -1653,23 +1653,30 @@ for the given `label` for every time series returned by `q`. For example, if `label_value(foo, "bar")` is applied to `foo{bar="1.234"}`, then it will return a time series `foo{bar="1.234"}` with `1.234` value. Function will return no data for non-numeric label values. +#### labels_equal + +`labels_equal(q, "label1", "label2", ...)` is [label manipulation function](#label-manipulation-functions), which returns `q` series with identical values for the listed labels +"label1", "label2", etc. + +See also [label_match](#label_match) and [label_mismatch](#label_mismatch). + #### sort_by_label -`sort_by_label(q, label1, ... labelN)` is [label manipulation function](#label-manipulation-functions), which sorts series in ascending order by the given set of labels. +`sort_by_label(q, "label1", ... "labelN")` is [label manipulation function](#label-manipulation-functions), which sorts series in ascending order by the given set of labels. For example, `sort_by_label(foo, "bar")` would sort `foo` series by values of the label `bar` in these series. See also [sort_by_label_desc](#sort_by_label_desc) and [sort_by_label_numeric](#sort_by_label_numeric). #### sort_by_label_desc -`sort_by_label_desc(q, label1, ... labelN)` is [label manipulation function](#label-manipulation-functions), which sorts series in descending order by the given set of labels. +`sort_by_label_desc(q, "label1", ... "labelN")` is [label manipulation function](#label-manipulation-functions), which sorts series in descending order by the given set of labels. For example, `sort_by_label(foo, "bar")` would sort `foo` series by values of the label `bar` in these series. See also [sort_by_label](#sort_by_label) and [sort_by_label_numeric_desc](#sort_by_label_numeric_desc). #### sort_by_label_numeric -`sort_by_label_numeric(q, label1, ... labelN)` is [label manipulation function](#label-manipulation-functions), which sorts series in ascending order by the given set of labels +`sort_by_label_numeric(q, "label1", ... "labelN")` is [label manipulation function](#label-manipulation-functions), which sorts series in ascending order by the given set of labels using [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html). For example, if `foo` series have `bar` label with values `1`, `101`, `15` and `2`, then `sort_by_label_numeric(foo, "bar")` would return series in the following order of `bar` label values: `1`, `2`, `15` and `101`. @@ -1678,7 +1685,7 @@ See also [sort_by_label_numeric_desc](#sort_by_label_numeric_desc) and [sort_by_ #### sort_by_label_numeric_desc -`sort_by_label_numeric_desc(q, label1, ... labelN)` is [label manipulation function](#label-manipulation-functions), which sorts series in descending order +`sort_by_label_numeric_desc(q, "label1", ... "labelN")` is [label manipulation function](#label-manipulation-functions), which sorts series in descending order by the given set of labels using [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html). For example, if `foo` series have `bar` label with values `1`, `101`, `15` and `2`, then `sort_by_label_numeric(foo, "bar")` would return series in the following order of `bar` label values: `101`, `15`, `2` and `1`. diff --git a/go.mod b/go.mod index 1ad509ca6..34414a059 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.67.0 + github.com/VictoriaMetrics/metricsql v0.68.0 github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 diff --git a/go.sum b/go.sum index 631d229e7..cd52e5753 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.67.0 h1:IYbKA6rhd8UWmW1LbaKiEk1q5ixFQagu9jL6z3Mt03o= -github.com/VictoriaMetrics/metricsql v0.67.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.68.0 h1:fAzYPjYkEipM/L/+WYbAK/gYuqt5rQHnb3cTY2cN628= +github.com/VictoriaMetrics/metricsql v0.68.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index 046a8f1b8..bd070be68 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -423,9 +423,11 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int { func isLabelManipulationFunc(funcName string) bool { switch strings.ToLower(funcName) { case "alias", "drop_common_labels", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", - "label_map", "label_match", "label_mismatch", "label_move", "label_replace", "label_set", "label_transform", - "label_uppercase", "label_value": + "label_map", "label_move", "label_replace", "label_set", "label_transform", "label_uppercase": return true + case "label_match", "label_mismatch", "label_value", "labels_equal": + // These functions aren't really label manipulation functions, since they do not change labels for the input series. + return false default: return false } diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index c281ba891..a7d9f6698 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -58,6 +58,7 @@ var transformFuncs = map[string]bool{ "label_transform": true, "label_uppercase": true, "label_value": true, + "labels_equal": true, "limit_offset": true, "ln": true, "log2": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index d6f336304..015c646bb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.67.0 +# github.com/VictoriaMetrics/metricsql v0.68.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop