app/vmselect/promql: add increase_pure() function to MetricsQL

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962

app/vmselect/promql/exec_test.go

@@ -5252,6 +5252,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`increase_pure(time())`, func(t *testing.T) {
+		t.Parallel()
+		q := `increase_pure(time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increase(time())`, func(t *testing.T) {
 		t.Parallel()
 		q := `increase(time())`

app/vmselect/promql/rollup.go

@@ -53,6 +53,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"distinct_over_time":    newRollupFuncOneArg(rollupDistinct),
 	"increases_over_time":   newRollupFuncOneArg(rollupIncreases),
 	"decreases_over_time":   newRollupFuncOneArg(rollupDecreases),
+	"increase_pure":         newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
 	"integrate":             newRollupFuncOneArg(rollupIntegrate),
 	"ideriv":                newRollupFuncOneArg(rollupIderiv),
 	"lifetime":              newRollupFuncOneArg(rollupLifetime),
@@ -123,6 +124,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"distinct_over_time":  rollupDistinct,
 	"increases_over_time": rollupIncreases,
 	"decreases_over_time": rollupDecreases,
+	"increase_pure":       rollupIncreasePure,
 	"integrate":           rollupIntegrate,
 	"ideriv":              rollupIderiv,
 	"lifetime":            rollupLifetime,
@@ -160,6 +162,7 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{
 	"distinct_over_time":  true,
 	"increases_over_time": true,
 	"decreases_over_time": true,
+	"increase_pure":       true,
 	"integrate":           true,
 	"ascent_over_time":    true,
 	"descent_over_time":   true,
@@ -172,6 +175,7 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 	"rate":            true,
 	"rollup_rate":     true,
 	"rollup_increase": true,
+	"increase_pure":   true,
 }
 
 var rollupFuncsKeepMetricGroup = map[string]bool{
@@ -1323,6 +1327,25 @@ func rollupStdvar(rfa *rollupFuncArg) float64 {
 	return q / count
 }
 
+func rollupIncreasePure(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	prevValue := rfa.prevValue
+	if math.IsNaN(prevValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		// Assume the counter starts from 0.
+		prevValue = 0
+	}
+	if len(values) == 0 {
+		// Assume the counter didsn't change since prevValue.
+		return 0
+	}
+	return values[len(values)-1] - prevValue
+}
+
 func rollupDelta(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.

app/vmselect/promql/rollup_test.go

@@ -476,6 +476,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("ideriv", 0)
 	f("decreases_over_time", 5)
 	f("increases_over_time", 5)
+	f("increase_pure", 398)
 	f("ascent_over_time", 142)
 	f("descent_over_time", 231)
 	f("zscore_over_time", -0.4254336383156416)

docs/CHANGELOG.md

@@ -12,6 +12,7 @@
   * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
 * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
 * FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
+* FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
 
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065

docs/MetricsQL.md

@@ -72,6 +72,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
 - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
 - `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`.
+- `increase_pure(m[d])` - for calculating increase of `m` over `d` without edge-case handling compared to `increase(m[d])`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
 - `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`.
 - `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range.
 - `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details.

go.mod

@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.12
 	github.com/VictoriaMetrics/metrics v1.15.0
-	github.com/VictoriaMetrics/metricsql v0.10.1
+	github.com/VictoriaMetrics/metricsql v0.11.0
 	github.com/aws/aws-sdk-go v1.37.12
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/cheggaaa/pb/v3 v3.0.6

go.sum

@@ -87,8 +87,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js=
 github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.10.1 h1:wLl/YbMmBGFPyLKMfqNLC333iygibosSM5iSvlH2B4A=
-github.com/VictoriaMetrics/metricsql v0.10.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.11.0 h1:85zbY8NxWNALppctOAkWfLDC7dDFcvTEn5IHMKwOGag=
+github.com/VictoriaMetrics/metricsql v0.11.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=

vendor/github.com/VictoriaMetrics/metricsql/rollup.go

@@ -38,6 +38,7 @@ var rollupFuncs = map[string]bool{
 	"distinct_over_time":    true,
 	"increases_over_time":   true,
 	"decreases_over_time":   true,
+	"increase_pure":         true,
 	"integrate":             true,
 	"ideriv":                true,
 	"lifetime":              true,

vendor/modules.txt

@@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.15.0
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.10.1
+# github.com/VictoriaMetrics/metricsql v0.11.0
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/VividCortex/ewma v1.1.1