From 98854e5f2b80dcc8e50bc16271ceef27ec24c284 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 24 Feb 2021 17:24:54 +0200 Subject: [PATCH] app/vmselect: add `sign(q)` and `clamp(q, min, max)` functions, which will be added in the upcoming Prometheus release See https://twitter.com/roidelapluie/status/1363428376162295811 The `last_over_time(m[d])` function already exists in MetricsQL. --- app/vmselect/promql/exec_test.go | 26 +++++++++- app/vmselect/promql/transform.go | 47 +++++++++++++++++++ docs/CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/transform.go | 2 + vendor/modules.txt | 2 +- 7 files changed, 79 insertions(+), 5 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index f12fbbda0..0bc3efc56 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -673,6 +673,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run("clamp(time(), 1400, 1800)", func(t *testing.T) { + t.Parallel() + q := `clamp(time(), 1400, 1800)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1400, 1400, 1400, 1600, 1800, 1800}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run("clamp_max(time(), 1400)", func(t *testing.T) { t.Parallel() q := `clamp_max(time(), 1400)` @@ -1716,6 +1727,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r1, r2} f(q, resultExpected) }) + t.Run(`sign(time()-1400)`, func(t *testing.T) { + t.Parallel() + q := `sign(time()-1400)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{-1, -1, 0, 1, 1, 1}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`round(time()/1e3)`, func(t *testing.T) { t.Parallel() q := `round(time()/1e3)` @@ -2763,7 +2785,7 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{} f(q, resultExpected) }) - t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) { + t.Run(`histogram_quantile(zero-value-inf-le)`, func(t *testing.T) { t.Parallel() q := `histogram_quantile(0.6, ( label_set(100, "le", "+Inf"), @@ -6307,6 +6329,7 @@ func TestExecError(t *testing.T) { f(`abs()`) f(`abs(1,2)`) f(`absent(1, 2)`) + f(`clamp()`) f(`clamp_max()`) f(`clamp_min(1,2,3)`) f(`hour(1,2)`) @@ -6323,6 +6346,7 @@ func TestExecError(t *testing.T) { f(`label_mismatch()`) f(`round()`) f(`round(1,2,3)`) + f(`sign()`) f(`scalar()`) f(`sort(1,2)`) f(`sort_desc()`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 20961e409..a893ce2e4 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -19,6 +19,7 @@ import ( var transformFuncsKeepMetricGroup = map[string]bool{ "ceil": true, + "clamp": true, "clamp_max": true, "clamp_min": true, "floor": true, @@ -44,6 +45,7 @@ var transformFuncs = map[string]transformFunc{ "abs": newTransformFuncOneArg(transformAbs), "absent": transformAbsent, "ceil": newTransformFuncOneArg(transformCeil), + "clamp": transformClamp, "clamp_max": transformClampMax, "clamp_min": transformClampMin, "day_of_month": newTransformFuncDateTime(transformDayOfMonth), @@ -61,6 +63,7 @@ var transformFuncs = map[string]transformFunc{ "minute": newTransformFuncDateTime(transformMinute), "month": newTransformFuncDateTime(transformMonth), "round": transformRound, + "sign": transformSign, "scalar": transformScalar, "sort": newTransformFuncSort(false), "sort_desc": newTransformFuncSort(true), @@ -215,6 +218,31 @@ func transformCeil(v float64) float64 { return math.Ceil(v) } +func transformClamp(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if err := expectTransformArgsNum(args, 3); err != nil { + return nil, err + } + mins, err := getScalar(args[1], 1) + if err != nil { + return nil, err + } + maxs, err := getScalar(args[2], 2) + if err != nil { + return nil, err + } + tf := func(values []float64) { + for i, v := range values { + if v > maxs[i] { + values[i] = maxs[i] + } else if v < mins[i] { + values[i] = mins[i] + } + } + } + return doTransformValues(args[0], tf, tfa.fe) +} + func transformClampMax(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 2); err != nil { @@ -1569,6 +1597,25 @@ func transformRound(tfa *transformFuncArg) ([]*timeseries, error) { return doTransformValues(args[0], tf, tfa.fe) } +func transformSign(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if err := expectTransformArgsNum(args, 1); err != nil { + return nil, err + } + tf := func(values []float64) { + for i, v := range values { + sign := float64(0) + if v < 0 { + sign = -1 + } else if v > 0 { + sign = 1 + } + values[i] = sign + } + } + return doTransformValues(args[0], tf, tfa.fe) +} + func transformScalar(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 1); err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 192a75ff5..c0c85e44a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,7 @@ # tip +* FEATURE: add `sign(q)` and `clamp(q, min, max)` functions, which are planned to be added in [the upcoming Prometheus release](https://twitter.com/roidelapluie/status/1363428376162295811) . The `last_over_time(m[d])` function is already supported in [MetricsQL](https://victoriametrics.github.io/MetricsQL.html). * FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details. * FEATURE: expose io-related metrics at `/metrics` page for every VictoriaMetrics component: * `process_io_read_bytes_total` - the number of bytes read via io syscalls such as read and pread diff --git a/go.mod b/go.mod index fe007c0dd..fa4503144 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.12 github.com/VictoriaMetrics/metrics v1.15.0 - github.com/VictoriaMetrics/metricsql v0.11.0 + github.com/VictoriaMetrics/metricsql v0.12.0 github.com/aws/aws-sdk-go v1.37.12 github.com/cespare/xxhash/v2 v2.1.1 github.com/cheggaaa/pb/v3 v3.0.6 diff --git a/go.sum b/go.sum index 343c504a0..8dfcad85f 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js= github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.11.0 h1:85zbY8NxWNALppctOAkWfLDC7dDFcvTEn5IHMKwOGag= -github.com/VictoriaMetrics/metricsql v0.11.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.12.0 h1:NMIu0MPBmGP34g4RUjI1U0xW5XYp7IVNXe9KtZI3PFQ= +github.com/VictoriaMetrics/metricsql v0.12.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index c1196badf..f8dd7cc84 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -10,6 +10,7 @@ var transformFuncs = map[string]bool{ "abs": true, "absent": true, "ceil": true, + "clamp": true, "clamp_max": true, "clamp_min": true, "day_of_month": true, @@ -28,6 +29,7 @@ var transformFuncs = map[string]bool{ "month": true, "round": true, "scalar": true, + "sign": true, "sort": true, "sort_desc": true, "sqrt": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index fa5b55f16..dcecec4b1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.15.0 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.11.0 +# github.com/VictoriaMetrics/metricsql v0.12.0 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/VividCortex/ewma v1.1.1