From de892239a92fb02d599e18fd1e3ee56951a41087 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 21 Apr 2022 14:20:20 +0300 Subject: [PATCH] app/vmselect/promql: add `drop_common_labels()` function --- app/vmselect/promql/exec_test.go | 59 +++++++++++++++++++ app/vmselect/promql/transform.go | 38 ++++++++++++ docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 5 ++ go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/optimizer.go | 2 +- .../VictoriaMetrics/metricsql/transform.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 109 insertions(+), 5 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 1beb04039..25114e394 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -1832,6 +1832,65 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`drop_common_labels(single_series)`, func(t *testing.T) { + t.Parallel() + q := `drop_common_labels(label_set(time(), "foo", "bar", "__name__", "xxx", "q", "we"))` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1000, 1200, 1400, 1600, 1800, 2000}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run(`drop_common_labels(multi_series)`, func(t *testing.T) { + t.Parallel() + q := `drop_common_labels(( + label_set(time(), "foo", "bar", "__name__", "xxx", "q", "we"), + label_set(time()/10, "foo", "bar", "__name__", "yyy"), + ))` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1000, 1200, 1400, 1600, 1800, 2000}, + Timestamps: timestampsExpected, + } + r1.MetricName.MetricGroup = []byte("xxx") + r1.MetricName.Tags = []storage.Tag{{ + Key: []byte("q"), + Value: []byte("we"), + }} + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{100, 120, 140, 160, 180, 200}, + Timestamps: timestampsExpected, + } + r2.MetricName.MetricGroup = []byte("yyy") + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) + t.Run(`drop_common_labels(multi_args)`, func(t *testing.T) { + t.Parallel() + q := `drop_common_labels( + label_set(time(), "foo", "bar", "__name__", "xxx", "q", "we"), + label_set(time()/10, "foo", "bar", "__name__", "xxx"), + )` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{100, 120, 140, 160, 180, 200}, + Timestamps: timestampsExpected, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1000, 1200, 1400, 1600, 1800, 2000}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{{ + Key: []byte("q"), + Value: []byte("we"), + }} + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`label_keep(nolabels)`, func(t *testing.T) { t.Parallel() q := `label_keep(time(), "foo", "bar")` diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 8957554b4..a941e0a3f 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -42,6 +42,7 @@ var transformFuncs = map[string]transformFunc{ "day_of_week": newTransformFuncDateTime(transformDayOfWeek), "days_in_month": newTransformFuncDateTime(transformDaysInMonth), "deg": newTransformFuncOneArg(transformDeg), + "drop_common_labels": transformDropCommonLabels, "end": newTransformFuncZeroArgs(transformEnd), "exp": newTransformFuncOneArg(transformExp), "floor": newTransformFuncOneArg(transformFloor), @@ -1486,6 +1487,43 @@ func transformLabelMap(tfa *transformFuncArg) ([]*timeseries, error) { return rvs, nil } +func transformDropCommonLabels(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) < 1 { + return nil, fmt.Errorf(`not enough args; got %d; want at least %d`, len(args), 1) + } + rvs := args[0] + for _, tss := range args[1:] { + rvs = append(rvs, tss...) + } + m := make(map[string]map[string]int) + countLabel := func(name, value string) { + x := m[name] + if x == nil { + x = make(map[string]int) + m[name] = x + } + x[value]++ + } + for _, ts := range rvs { + countLabel("__name__", string(ts.MetricName.MetricGroup)) + for _, tag := range ts.MetricName.Tags { + countLabel(string(tag.Key), string(tag.Value)) + } + } + for labelName, x := range m { + for _, count := range x { + if count != len(rvs) { + continue + } + for _, ts := range rvs { + ts.MetricName.RemoveTag(labelName) + } + } + } + return rvs, nil +} + func transformLabelCopy(tfa *transformFuncArg) ([]*timeseries, error) { return transformLabelCopyExt(tfa, false) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5c4dc9e2d..2b9bade66 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: allow specifying TLS cipher suites for mTLS connections between cluster components via `-cluster.tlsCipherSuites` command-line flag. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): shown an empty graph on the selected time range when there is no data on it. Previously `No data to show` placeholder was shown instead of the graph in this case. This prevented from zooming and scrolling of such a graph. * FEATURE: expose `vm_indexdb_items_added_total` and `vm_indexdb_items_added_size_bytes_total` counters at `/metrics` page, which can be used for monitoring the rate for addition of new entries in `indexdb` (aka `inverted index`) alongside the total size in bytes for the added entries. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2471). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `drop_common_labels()` function, which drops common `label="name"` pairs from the passed time series. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#drop_common_labels). * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): return non-zero exit code on error. This allows handling `vmctl` errors in shell scripts. Previously `vmctl` was returning 0 exit code on error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2322). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly show `scrape_timeout` and `scrape_interval` options at `http://vmagent:8429/config` page. Previously these options weren't displayed even if they were set in `-promscrape.config`. diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 0b04dc67f..d06b2c784 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -713,6 +713,11 @@ See also [implicit query conversions](#implicit-query-conversions). `alias(q, "name")` sets the given `name` to all the time series returned by `q`. For example, `alias(up, "foobar")` would rename `up` series to `foobar` series. + +#### drop_common_labels + +`drop_common_labels(q1, ...., qN)` drops common `label="value"` paris among time series returned from `q1, ..., qN`. + #### label_copy `label_copy(q, "src_label1", "dst_label1", ..., "src_labelN", "dst_labelN")` copies label values from `src_label*` to `dst_label*` for all the time series returned by `q`. If `src_label` is empty, then the corresponding `dst_label` is left untouched. diff --git a/go.mod b/go.mod index f25ceb314..b6d71cc6a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.1 - github.com/VictoriaMetrics/metricsql v0.41.0 + github.com/VictoriaMetrics/metricsql v0.42.0 github.com/aws/aws-sdk-go v1.43.41 github.com/cespare/xxhash/v2 v2.1.2 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index 05029c516..57f327b4b 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.41.0 h1:fhWnSE9ZXVEbiXXGFY73YPLdovTaDRaDaFdxC3TTRZs= -github.com/VictoriaMetrics/metricsql v0.41.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.42.0 h1:E+NZWdpZHSLapQTuT9g+MB4vvD9JB6dSd/0L8QDkcQ4= +github.com/VictoriaMetrics/metricsql v0.42.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index 36d4553b4..03410f053 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -403,7 +403,7 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int { func isLabelManipulationFunc(funcName string) bool { switch strings.ToLower(funcName) { - case "alias", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", + case "alias", "drop_common_labels", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", "label_map", "label_match", "label_mismatch", "label_move", "label_replace", "label_set", "label_transform", "label_uppercase", "label_value": return true diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index b0bdc2527..865c68e87 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -28,6 +28,7 @@ var transformFuncs = map[string]bool{ "day_of_week": true, "days_in_month": true, "deg": true, + "drop_common_labels": true, "end": true, "exp": true, "floor": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c980dc68..ae65f369b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -27,7 +27,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.1 ## explicit; go 1.12 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.41.0 +# github.com/VictoriaMetrics/metricsql v0.42.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop