From e92fde794524e87943bda676ff9a5dad1ce9bb32 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 3 Aug 2021 12:21:05 +0300 Subject: [PATCH] app/vmselect/promql: add `present_over_time(m[d])` function, which will be available starting from Prometheus 2.29.0 See https://github.com/prometheus/prometheus/releases/tag/v2.29.0-rc.0 and https://github.com/prometheus/prometheus/pull/9097 --- app/vmselect/promql/exec_test.go | 33 +++++++++++++ app/vmselect/promql/rollup.go | 46 ++++++++++++------- docs/CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 7 files changed, 68 insertions(+), 21 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 2e6efc982..9e7830381 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -698,6 +698,39 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{} f(q, resultExpected) }) + t.Run("present_over_time(time())", func(t *testing.T) { + t.Parallel() + q := `present_over_time(time())` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, 1, 1, 1}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("present_over_time(time()[100:300])", func(t *testing.T) { + t.Parallel() + q := `present_over_time(time()[100:300])` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{nan, 1, nan, nan, 1, nan}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("present_over_time(time()<10m)", func(t *testing.T) { + t.Parallel() + q := `present_over_time(time()<1600)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, nan, nan, nan}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run("absent(123)", func(t *testing.T) { t.Parallel() q := `absent(123)` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 377b7cc7a..97c353fcb 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -42,6 +42,7 @@ var rollupFuncs = map[string]newRollupFunc{ "stddev_over_time": newRollupFuncOneArg(rollupStddev), "stdvar_over_time": newRollupFuncOneArg(rollupStdvar), "absent_over_time": newRollupFuncOneArg(rollupAbsent), + "present_over_time": newRollupFuncOneArg(rollupPresent), // Additional rollup funcs. "default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func @@ -97,23 +98,24 @@ var rollupFuncs = map[string]newRollupFunc{ // rollupAggrFuncs are functions that can be passed to `aggr_over_time()` var rollupAggrFuncs = map[string]rollupFunc{ // Standard rollup funcs from PromQL. - "changes": rollupChanges, - "delta": rollupDelta, - "deriv": rollupDerivSlow, - "deriv_fast": rollupDerivFast, - "idelta": rollupIdelta, - "increase": rollupDelta, // + rollupFuncsRemoveCounterResets - "irate": rollupIderiv, // + rollupFuncsRemoveCounterResets - "rate": rollupDerivFast, // + rollupFuncsRemoveCounterResets - "resets": rollupResets, - "avg_over_time": rollupAvg, - "min_over_time": rollupMin, - "max_over_time": rollupMax, - "sum_over_time": rollupSum, - "count_over_time": rollupCount, - "stddev_over_time": rollupStddev, - "stdvar_over_time": rollupStdvar, - "absent_over_time": rollupAbsent, + "changes": rollupChanges, + "delta": rollupDelta, + "deriv": rollupDerivSlow, + "deriv_fast": rollupDerivFast, + "idelta": rollupIdelta, + "increase": rollupDelta, // + rollupFuncsRemoveCounterResets + "irate": rollupIderiv, // + rollupFuncsRemoveCounterResets + "rate": rollupDerivFast, // + rollupFuncsRemoveCounterResets + "resets": rollupResets, + "avg_over_time": rollupAvg, + "min_over_time": rollupMin, + "max_over_time": rollupMax, + "sum_over_time": rollupSum, + "count_over_time": rollupCount, + "stddev_over_time": rollupStddev, + "stdvar_over_time": rollupStdvar, + "absent_over_time": rollupAbsent, + "present_over_time": rollupPresent, // Additional rollup funcs. "range_over_time": rollupRange, @@ -157,6 +159,7 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{ "stddev_over_time": true, "stdvar_over_time": true, "absent_over_time": true, + "present_over_time": true, "sum2_over_time": true, "geomean_over_time": true, "distinct_over_time": true, @@ -1284,6 +1287,15 @@ func rollupAbsent(rfa *rollupFuncArg) float64 { return nan } +func rollupPresent(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + if len(rfa.values) > 0 { + return 1 + } + return nan +} + func rollupCount(rfa *rollupFuncArg) float64 { // There is no need in handling NaNs here, since they must be cleaned up // before calling rollup funcs. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bbb91ef25..afecb124c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,7 @@ sort: 15 ## tip +* FEATURE: add `present_over_time(m[d])` function, which returns 1 if `m` has a least a single sample over the previous duration `d`. This function has been added also to [Prometheus 2.29](https://github.com/prometheus/prometheus/releases/tag/v2.29.0-rc.0). * FEATURE: add `-search.maxSamplesPerSeries` command-line flag for limiting the number of raw samples a single query can process per each time series. This option can protect from out of memory errors when a query processes tens of millions of raw samples per series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1067). * FEATURE: add `-search.maxSamplesPerQuery` command-line flag for limiting the number of raw samples a single query can process across all the time series. This option can protect from heavy queries, which select too big number of raw samples. Thanks to @jiangxinlingdu for [the initial pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1478). * FEATURE: improve performance for queries that process big number of time series and/or samples on systems with big number of CPU cores. diff --git a/go.mod b/go.mod index d7d256202..25e5c886a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.16 github.com/VictoriaMetrics/metrics v1.17.3 - github.com/VictoriaMetrics/metricsql v0.16.0 + github.com/VictoriaMetrics/metricsql v0.17.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.40.7 github.com/cespare/xxhash/v2 v2.1.1 diff --git a/go.sum b/go.sum index 7f4fdf305..29885c26e 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4= github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.16.0 h1:YzrMnGUs6Y6f5LxsH8eSAoik98aEzlc1TiYgOONgr3Q= -github.com/VictoriaMetrics/metricsql v0.16.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.17.0 h1:OdOkx5GK2p1+EnHn/+zpLWW6ze96L1AmMV5X1HOzyfY= +github.com/VictoriaMetrics/metricsql v0.17.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index 085ee96c7..4db4770f4 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -27,6 +27,7 @@ var rollupFuncs = map[string]bool{ "stddev_over_time": true, "stdvar_over_time": true, "absent_over_time": true, + "present_over_time": true, // Additional rollup funcs. "default_rollup": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 554aa9653..2b75612d2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.17.3 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.16.0 +# github.com/VictoriaMetrics/metricsql v0.17.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop