From fa03e0d21031c8de455eece2f4af792e28058239 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 22 Feb 2021 19:12:12 +0200 Subject: [PATCH] app/vmselect/promql: add `increase_pure()` function to MetricsQL Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962 --- app/vmselect/promql/exec_test.go | 11 +++++++++ app/vmselect/promql/rollup.go | 23 +++++++++++++++++++ app/vmselect/promql/rollup_test.go | 1 + docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 1 + go.mod | 2 +- go.sum | 4 ++-- .../VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 42 insertions(+), 4 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 6c3cd65df..8ed198353 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -5252,6 +5252,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`increase_pure(time())`, func(t *testing.T) { + t.Parallel() + q := `increase_pure(time())` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{200, 200, 200, 200, 200, 200}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`increase(time())`, func(t *testing.T) { t.Parallel() q := `increase(time())` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 3c62fbdc7..c7f973a82 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -53,6 +53,7 @@ var rollupFuncs = map[string]newRollupFunc{ "distinct_over_time": newRollupFuncOneArg(rollupDistinct), "increases_over_time": newRollupFuncOneArg(rollupIncreases), "decreases_over_time": newRollupFuncOneArg(rollupDecreases), + "increase_pure": newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets "integrate": newRollupFuncOneArg(rollupIntegrate), "ideriv": newRollupFuncOneArg(rollupIderiv), "lifetime": newRollupFuncOneArg(rollupLifetime), @@ -123,6 +124,7 @@ var rollupAggrFuncs = map[string]rollupFunc{ "distinct_over_time": rollupDistinct, "increases_over_time": rollupIncreases, "decreases_over_time": rollupDecreases, + "increase_pure": rollupIncreasePure, "integrate": rollupIntegrate, "ideriv": rollupIderiv, "lifetime": rollupLifetime, @@ -160,6 +162,7 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{ "distinct_over_time": true, "increases_over_time": true, "decreases_over_time": true, + "increase_pure": true, "integrate": true, "ascent_over_time": true, "descent_over_time": true, @@ -172,6 +175,7 @@ var rollupFuncsRemoveCounterResets = map[string]bool{ "rate": true, "rollup_rate": true, "rollup_increase": true, + "increase_pure": true, } var rollupFuncsKeepMetricGroup = map[string]bool{ @@ -1323,6 +1327,25 @@ func rollupStdvar(rfa *rollupFuncArg) float64 { return q / count } +func rollupIncreasePure(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + values := rfa.values + prevValue := rfa.prevValue + if math.IsNaN(prevValue) { + if len(values) == 0 { + return nan + } + // Assume the counter starts from 0. + prevValue = 0 + } + if len(values) == 0 { + // Assume the counter didsn't change since prevValue. + return 0 + } + return values[len(values)-1] - prevValue +} + func rollupDelta(rfa *rollupFuncArg) float64 { // There is no need in handling NaNs here, since they must be cleaned up // before calling rollup funcs. diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 2820b5b7f..4b5eb540f 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -476,6 +476,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) { f("ideriv", 0) f("decreases_over_time", 5) f("increases_over_time", 5) + f("increase_pure", 398) f("ascent_over_time", 142) f("descent_over_time", 231) f("zscore_over_time", -0.4254336383156416) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cb42435f6..a8ddf8c3e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -12,6 +12,7 @@ * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets. * FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases. +* FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details. * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065 diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 65d75612c..15bf2638c 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -72,6 +72,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query. - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`. - `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`. +- `increase_pure(m[d])` - for calculating increase of `m` over `d` without edge-case handling compared to `increase(m[d])`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details. - `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`. - `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range. - `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details. diff --git a/go.mod b/go.mod index a6fcfe489..fe007c0dd 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.12 github.com/VictoriaMetrics/metrics v1.15.0 - github.com/VictoriaMetrics/metricsql v0.10.1 + github.com/VictoriaMetrics/metricsql v0.11.0 github.com/aws/aws-sdk-go v1.37.12 github.com/cespare/xxhash/v2 v2.1.1 github.com/cheggaaa/pb/v3 v3.0.6 diff --git a/go.sum b/go.sum index 9de56d296..343c504a0 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js= github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.10.1 h1:wLl/YbMmBGFPyLKMfqNLC333iygibosSM5iSvlH2B4A= -github.com/VictoriaMetrics/metricsql v0.10.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.11.0 h1:85zbY8NxWNALppctOAkWfLDC7dDFcvTEn5IHMKwOGag= +github.com/VictoriaMetrics/metricsql v0.11.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index 80cfe58f9..085ee96c7 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -38,6 +38,7 @@ var rollupFuncs = map[string]bool{ "distinct_over_time": true, "increases_over_time": true, "decreases_over_time": true, + "increase_pure": true, "integrate": true, "ideriv": true, "lifetime": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 61b2281b4..fa5b55f16 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.15.0 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.10.1 +# github.com/VictoriaMetrics/metricsql v0.11.0 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/VividCortex/ewma v1.1.1