From 3e557c9861ea3f7ead0b60392f3ee521e5f16278 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 24 Jul 2020 01:13:05 +0300 Subject: [PATCH] app/vmselect/promql: add `rate_over_sum(m[d])` function to MetricsQL, which returns rate over sum of `m` values over `d` duration Something like `sum_over_time(m[d]) / d`, but more accurate. --- app/vmselect/promql/exec_test.go | 12 ++++++++ app/vmselect/promql/rollup.go | 28 +++++++++++++++++++ app/vmselect/promql/rollup_test.go | 15 ++++++++++ docs/MetricsQL.md | 3 +- go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metricsql/rollup.go | 2 ++ vendor/modules.txt | 2 +- 8 files changed, 63 insertions(+), 5 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index dc35c28e4e..4c2cd4138d 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -4537,6 +4537,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`rate_over_sum()`, func(t *testing.T) { + t.Parallel() + q := `rate_over_sum(round(time()/500)[100s:5s])` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0.4, 0.4, 0.6, 0.6, 0.71, 0.8}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`integrate(1)`, func(t *testing.T) { t.Parallel() q := `integrate(1)` @@ -5749,6 +5760,7 @@ func TestExecError(t *testing.T) { f(`outliersk()`) f(`outliersk(1)`) f(`mode_over_time()`) + f(`rate_over_sum()`) f(`mode()`) // Invalid argument type diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 188f81019a..f6973a70fc 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -82,6 +82,8 @@ var rollupFuncs = map[string]newRollupFunc{ // See https://en.wikipedia.org/wiki/Mode_(statistics) "mode_over_time": newRollupFuncOneArg(rollupModeOverTime), + + "rate_over_sum": newRollupFuncOneArg(rollupRateOverSum), } // rollupAggrFuncs are functions that can be passed to `aggr_over_time()` @@ -125,6 +127,7 @@ var rollupAggrFuncs = map[string]rollupFunc{ "descent_over_time": rollupDescentOverTime, "timestamp": rollupTimestamp, "mode_over_time": rollupModeOverTime, + "rate_over_sum": rollupRateOverSum, } var rollupFuncsCannotAdjustWindow = map[string]bool{ @@ -1083,6 +1086,31 @@ func rollupSum(rfa *rollupFuncArg) float64 { return sum } +func rollupRateOverSum(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + values := rfa.values + timestamps := rfa.timestamps + prevTimestamp := rfa.prevTimestamp + if math.IsNaN(rfa.prevValue) { + if len(values) == 0 { + return nan + } + prevTimestamp = timestamps[1] + values = values[1:] + timestamps = timestamps[1:] + } + if len(values) == 0 { + return nan + } + sum := float64(0) + for _, v := range values { + sum += v + } + dt := timestamps[len(timestamps)-1] - prevTimestamp + return sum / (float64(dt) / 1e3) +} + func rollupRange(rfa *rollupFuncArg) float64 { max := rollupMax(rfa) min := rollupMin(rfa) diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 65aeee9d13..b543c42804 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -393,6 +393,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) { f("descent_over_time", 231) f("timestamp", 0.13) f("mode_over_time", 34) + f("rate_over_sum", 3843.478260869565) } func TestRollupNewRollupFuncError(t *testing.T) { @@ -967,6 +968,20 @@ func TestRollupFuncsNoWindow(t *testing.T) { timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) + t.Run("rate_over_sum", func(t *testing.T) { + rc := rollupConfig{ + Func: rollupRateOverSum, + Start: 0, + End: 160, + Step: 40, + Window: 80, + } + rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) + values := rc.Do(nil, testValues, testTimestamps) + valuesExpected := []float64{nan, 4238.095238095238, 3738.461538461538, 4059.523809523809, 6200} + timestampsExpected := []int64{0, 40, 80, 120, 160} + testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) + }) } func TestRollupBigNumberOfValues(t *testing.T) { diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 62080faf24..3342cc6b4d 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -124,4 +124,5 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `ascent_over_time(m[d])` - returns the sum of positive deltas between adjancent data points in `m` over `d`. Useful for tracking height gains in GPS track. - `descent_over_time(m[d])` - returns the absolute sum of negative deltas between adjancent data points in `m` over `d`. Useful for tracking height loss in GPS track. - `mode_over_time(m[d])` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for `m` values over `d`. It is expected that `m` values are discrete. -- `mode(q) by (x)` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for each point in `q` grouped by `x`. +- `mode(q) by (x)` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for each point in `q` grouped by `x`. It is expected that `q` points are discrete. +- `rate_over_sum(m[d])` - returns rate over the sum of `m` values over `d` duration. diff --git a/go.mod b/go.mod index 5bb4b5ff58..20afb43f75 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.1 github.com/VictoriaMetrics/metrics v1.12.0 - github.com/VictoriaMetrics/metricsql v0.2.7 + github.com/VictoriaMetrics/metricsql v0.2.8 github.com/aws/aws-sdk-go v1.33.9 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.1 diff --git a/go.sum b/go.sum index aa66f72376..f18b0b025f 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/VictoriaMetrics/metrics v1.11.2 h1:t/ceLP6SvagUqypCKU7cI7+tQn54+TIV/t github.com/VictoriaMetrics/metrics v1.11.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ= github.com/VictoriaMetrics/metrics v1.12.0 h1:BudxtRYSA6j8H9mzjhXNEIsCPIEUPCb76QwFEptQzvQ= github.com/VictoriaMetrics/metrics v1.12.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.2.7 h1:4FXyJJjXTbAPVAakEEwaFSD0YOEPKEjdZKNQrWN76Ts= -github.com/VictoriaMetrics/metricsql v0.2.7/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE= +github.com/VictoriaMetrics/metricsql v0.2.8 h1:RET+5ZKSHFpcm7RNEEHFMiSNYtd6GlGKyNn/ZO53zhA= +github.com/VictoriaMetrics/metricsql v0.2.8/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aws/aws-sdk-go v1.33.9 h1:nkC8YxL1nxwshIoO3UM2486Ph+zs7IZWjhRHjmXeCPw= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index 8889965896..3132925290 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -66,6 +66,8 @@ var rollupFuncs = map[string]bool{ // See https://en.wikipedia.org/wiki/Mode_(statistics) "mode_over_time": true, + + "rate_over_sum": true, } // IsRollupFunc returns whether funcName is known rollup function. diff --git a/vendor/modules.txt b/vendor/modules.txt index f8a6a0e7c8..1b89543e81 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.12.0 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.2.7 +# github.com/VictoriaMetrics/metricsql v0.2.8 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.33.9