From d9c3a2b60516d86fb5a33b4bc7d316bd694c3342 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 21 Nov 2022 23:24:57 +0200 Subject: [PATCH] app/vmselect/promql: add `range_normalize(q1, ..., qN)` function for normalizing query results into [0..1] value range This may be useful for analyzing correlation between time series with different value ranges --- app/vmselect/promql/exec_test.go | 17 ++++++++++ app/vmselect/promql/transform.go | 34 +++++++++++++++++++ docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 5 +++ go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metricsql/optimizer.go | 2 +- .../VictoriaMetrics/metricsql/transform.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 63 insertions(+), 5 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 8930f6363..49eb8d540 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -6902,6 +6902,23 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`range_normalize(time(),alias(-time(),"negative"))`, func(t *testing.T) { + t.Parallel() + q := `range_normalize(time(),alias(-time(), "negative"))` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + Timestamps: timestampsExpected, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 0.8, 0.6, 0.4, 0.2, 0}, + Timestamps: timestampsExpected, + } + r2.MetricName.MetricGroup = []byte("negative") + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`range_first(time())`, func(t *testing.T) { t.Parallel() q := `range_first(time())` diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 3fa73ed6f..f4d151295 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -91,6 +91,7 @@ var transformFuncs = map[string]transformFunc{ "range_linear_regression": transformRangeLinearRegression, "range_max": newTransformFuncRange(runningMax), "range_min": newTransformFuncRange(runningMin), + "range_normalize": transformRangeNormalize, "range_quantile": transformRangeQuantile, "range_stddev": transformRangeStddev, "range_stdvar": transformRangeStdvar, @@ -142,6 +143,7 @@ var transformFuncsKeepMetricName = map[string]bool{ "range_linear_regression": true, "range_max": true, "range_min": true, + "range_normalize": true, "range_quantile": true, "range_stdvar": true, "range_sddev": true, @@ -1240,6 +1242,38 @@ func newTransformFuncRange(rf func(a, b float64, idx int) float64) transformFunc } } +func transformRangeNormalize(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + var rvs []*timeseries + for _, tss := range args { + for _, ts := range tss { + values := ts.Values + vMin := inf + vMax := -inf + for _, v := range values { + if math.IsNaN(v) { + continue + } + if v < vMin { + vMin = v + } + if v > vMax { + vMax = v + } + } + d := vMax - vMin + if math.IsInf(d, 0) { + continue + } + for i, v := range values { + values[i] = (v - vMin) / d + } + rvs = append(rvs, ts) + } + } + return rvs, nil +} + func transformRangeLinearRegression(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 1); err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fbd61aa35..558c36ed4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): add `-storageNode.filter` command-line flag for filtering the [discovered vmstorage nodes](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery) with arbitrary regular expressions. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3353). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow using numeric values with `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes inside MetricsQL queries. For example `8Ki` equals to `8*1024`, while `8.2M` equals to `8.2*1000*1000`. +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_normalize](https://docs.victoriametrics.com/MetricsQL.html#range_normalize) function for normalizing multiple time series into `[0...1]` value range. This function is useful for correlation analyzis of time series with distinct value ranges. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_linear_regression](https://docs.victoriametrics.com/MetricsQL.html#range_linear_regression) function for calculating [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression) over the input time series on the selected time range. This function is useful for predictions and capacity planning. For example, `range_linear_regression(process_resident_memory_bytes)` can predict future memory usage based on the past memory usage. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_stddev](https://docs.victoriametrics.com/MetricsQL.html#range_stddev) and [range_stdvar](https://docs.victoriametrics.com/MetricsQL.html#range_stdvar) functions. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): optimize `expr1 op expr2` query when `expr1` returns an empty result. In this case there is no sense in executing `expr2` for `op` not equal to `or`, since the end result will be empty according to [PromQL series matching rules](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349). Thanks to @jianglinjian for pointing to this case. diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 217c6f8da..e16949ec1 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -1221,6 +1221,11 @@ over the selected time range per each time series returned by `q`. This function `range_min(q)` is a [transform function](#transform-functions), which calculates the min value across points per each time series returned by `q`. +#### range_normalize + +`range_normalize(q1, ...)` is a [transform function](#transform-functions), which normalizes values for time series returned by `q1, ...` into `[0 ... 1]` range. +This function is useful for correlating time series with distinct value ranges. + #### range_quantile `range_quantile(phi, q)` is a [transform function](#transform-functions), which returns `phi`-quantile across points per each time series returned by `q`. diff --git a/go.mod b/go.mod index 0eeed2444..4ec30937b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.23.0 - github.com/VictoriaMetrics/metricsql v0.48.0 + github.com/VictoriaMetrics/metricsql v0.49.0 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.18.1 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.40 diff --git a/go.sum b/go.sum index 6f9c862bc..9ecd8cf86 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA= github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.48.0 h1:rq8ULfIDJ0QyDbyQWRuWrMTffEqL2sevU2Zs3Vx1pfw= -github.com/VictoriaMetrics/metricsql v0.48.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.49.0 h1:7R04eab3gU0PKu8Ksak7SJnORXm0K+hSGt2+t3XGyKg= +github.com/VictoriaMetrics/metricsql v0.49.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index e56659b86..3a432e63e 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -386,7 +386,7 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int { return -1 } switch funcName { - case "", "absent", "scalar", "union", "vector": + case "", "absent", "scalar", "union", "vector", "range_normalize": return -1 case "end", "now", "pi", "ru", "start", "step", "time": return -1 diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index 40d315832..5876c8290 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -76,6 +76,7 @@ var transformFuncs = map[string]bool{ "range_linear_regression": true, "range_max": true, "range_min": true, + "range_normalize": true, "range_quantile": true, "range_stddev": true, "range_stdvar": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index ad20c9371..c172a7d07 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -69,7 +69,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.23.0 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.48.0 +# github.com/VictoriaMetrics/metricsql v0.49.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop