app/vmselect/promql: add rate_over_sum(m[d]) function to MetricsQL, which returns rate over sum of m values over d duration

Something like `sum_over_time(m[d]) / d`, but more accurate.

app/vmselect/promql/exec_test.go

@@ -4537,6 +4537,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`rate_over_sum()`, func(t *testing.T) {
+		t.Parallel()
+		q := `rate_over_sum(round(time()/500)[100s:5s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.4, 0.4, 0.6, 0.6, 0.71, 0.8},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`integrate(1)`, func(t *testing.T) {
 		t.Parallel()
 		q := `integrate(1)`
@@ -5749,6 +5760,7 @@ func TestExecError(t *testing.T) {
 	f(`outliersk()`)
 	f(`outliersk(1)`)
 	f(`mode_over_time()`)
+	f(`rate_over_sum()`)
 	f(`mode()`)
 
 	// Invalid argument type

app/vmselect/promql/rollup.go

@@ -82,6 +82,8 @@ var rollupFuncs = map[string]newRollupFunc{
 
 	// See https://en.wikipedia.org/wiki/Mode_(statistics)
 	"mode_over_time": newRollupFuncOneArg(rollupModeOverTime),
+
+	"rate_over_sum": newRollupFuncOneArg(rollupRateOverSum),
 }
 
 // rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
@@ -125,6 +127,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"descent_over_time":   rollupDescentOverTime,
 	"timestamp":           rollupTimestamp,
 	"mode_over_time":      rollupModeOverTime,
+	"rate_over_sum":       rollupRateOverSum,
 }
 
 var rollupFuncsCannotAdjustWindow = map[string]bool{
@@ -1083,6 +1086,31 @@ func rollupSum(rfa *rollupFuncArg) float64 {
 	return sum
 }
 
+func rollupRateOverSum(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	timestamps := rfa.timestamps
+	prevTimestamp := rfa.prevTimestamp
+	if math.IsNaN(rfa.prevValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		prevTimestamp = timestamps[1]
+		values = values[1:]
+		timestamps = timestamps[1:]
+	}
+	if len(values) == 0 {
+		return nan
+	}
+	sum := float64(0)
+	for _, v := range values {
+		sum += v
+	}
+	dt := timestamps[len(timestamps)-1] - prevTimestamp
+	return sum / (float64(dt) / 1e3)
+}
+
 func rollupRange(rfa *rollupFuncArg) float64 {
 	max := rollupMax(rfa)
 	min := rollupMin(rfa)

app/vmselect/promql/rollup_test.go

@@ -393,6 +393,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("descent_over_time", 231)
 	f("timestamp", 0.13)
 	f("mode_over_time", 34)
+	f("rate_over_sum", 3843.478260869565)
 }
 
 func TestRollupNewRollupFuncError(t *testing.T) {
@@ -967,6 +968,20 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("rate_over_sum", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:   rollupRateOverSum,
+			Start:  0,
+			End:    160,
+			Step:   40,
+			Window: 80,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{nan, 4238.095238095238, 3738.461538461538, 4059.523809523809, 6200}
+		timestampsExpected := []int64{0, 40, 80, 120, 160}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
 }
 
 func TestRollupBigNumberOfValues(t *testing.T) {

docs/MetricsQL.md

@@ -124,4 +124,5 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `ascent_over_time(m[d])` - returns the sum of positive deltas between adjancent data points in `m` over `d`. Useful for tracking height gains in GPS track.
 - `descent_over_time(m[d])` - returns the absolute sum of negative deltas between adjancent data points in `m` over `d`. Useful for tracking height loss in GPS track.
 - `mode_over_time(m[d])` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for `m` values over `d`. It is expected that `m` values are discrete.
-- `mode(q) by (x)` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for each point in `q` grouped by `x`.
+- `mode(q) by (x)` - returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) for each point in `q` grouped by `x`. It is expected that `q` points are discrete.
+- `rate_over_sum(m[d])` - returns rate over the sum of `m` values over `d` duration.

go.mod

@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.1
 	github.com/VictoriaMetrics/metrics v1.12.0
-	github.com/VictoriaMetrics/metricsql v0.2.7
+	github.com/VictoriaMetrics/metricsql v0.2.8
 	github.com/aws/aws-sdk-go v1.33.9
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/golang/snappy v0.0.1

go.sum

@@ -53,8 +53,8 @@ github.com/VictoriaMetrics/metrics v1.11.2 h1:t/ceLP6SvagUqypCKU7cI7+tQn54+TIV/t
 github.com/VictoriaMetrics/metrics v1.11.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
 github.com/VictoriaMetrics/metrics v1.12.0 h1:BudxtRYSA6j8H9mzjhXNEIsCPIEUPCb76QwFEptQzvQ=
 github.com/VictoriaMetrics/metrics v1.12.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.2.7 h1:4FXyJJjXTbAPVAakEEwaFSD0YOEPKEjdZKNQrWN76Ts=
+github.com/VictoriaMetrics/metricsql v0.2.8 h1:RET+5ZKSHFpcm7RNEEHFMiSNYtd6GlGKyNn/ZO53zhA=
-github.com/VictoriaMetrics/metricsql v0.2.7/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE=
+github.com/VictoriaMetrics/metricsql v0.2.8/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/aws/aws-sdk-go v1.33.9 h1:nkC8YxL1nxwshIoO3UM2486Ph+zs7IZWjhRHjmXeCPw=

vendor/github.com/VictoriaMetrics/metricsql/rollup.go

@@ -66,6 +66,8 @@ var rollupFuncs = map[string]bool{
 
 	// See https://en.wikipedia.org/wiki/Mode_(statistics)
 	"mode_over_time": true,
+
+	"rate_over_sum": true,
 }
 
 // IsRollupFunc returns whether funcName is known rollup function.

vendor/modules.txt

@@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.12.0
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.2.7
+# github.com/VictoriaMetrics/metricsql v0.2.8
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/aws/aws-sdk-go v1.33.9