From e2e8ef86d9125a5406b7310e2ff25e5ec65f95be Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 5 Dec 2020 12:30:46 +0200 Subject: [PATCH] app/vmselect/promql: add `count_eq_over_time(m[d], N)` and `count_ne_over_time(m[d], N)` for calculating the number of samples in `m` over `d` that are equal / not equal to `N` --- app/vmselect/promql/exec_test.go | 24 ++++++++++++ app/vmselect/promql/rollup.go | 30 +++++++++++++++ app/vmselect/promql/rollup_test.go | 38 +++++++++++++++++++ docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 2 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/rollup.go | 2 + vendor/modules.txt | 2 +- 9 files changed, 101 insertions(+), 4 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 7978d9d72..0090e00df 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -4135,6 +4135,28 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`count_eq_over_time`, func(t *testing.T) { + t.Parallel() + q := `count_eq_over_time(round(5*rand(0))[200s:10s], 1)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{2, 4, 5, 2, 6, 6}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run(`count_ne_over_time`, func(t *testing.T) { + t.Parallel() + q := `count_ne_over_time(round(5*rand(0))[200s:10s], 1)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{18, 16, 15, 18, 14, 14}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`increases_over_time`, func(t *testing.T) { t.Parallel() q := `increases_over_time(rand(0)[200s:10s])` @@ -6237,6 +6259,8 @@ func TestExecError(t *testing.T) { f(`share_gt_over_time()`) f(`count_le_over_time()`) f(`count_gt_over_time()`) + f(`count_eq_over_time()`) + f(`count_ne_over_time()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 2685f72ca..5245c63c3 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -64,6 +64,8 @@ var rollupFuncs = map[string]newRollupFunc{ "share_gt_over_time": newRollupShareGT, "count_le_over_time": newRollupCountLE, "count_gt_over_time": newRollupCountGT, + "count_eq_over_time": newRollupCountEQ, + "count_ne_over_time": newRollupCountNE, "histogram_over_time": newRollupFuncOneArg(rollupHistogram), "rollup": newRollupFuncOneArg(rollupFake), "rollup_rate": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets @@ -895,6 +897,26 @@ func countFilterGT(values []float64, gt float64) int { return n } +func countFilterEQ(values []float64, eq float64) int { + n := 0 + for _, v := range values { + if v == eq { + n++ + } + } + return n +} + +func countFilterNE(values []float64, ne float64) int { + n := 0 + for _, v := range values { + if v != ne { + n++ + } + } + return n +} + func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) { rf, err := newRollupCountFilter(args, countFilter) if err != nil { @@ -914,6 +936,14 @@ func newRollupCountGT(args []interface{}) (rollupFunc, error) { return newRollupCountFilter(args, countFilterGT) } +func newRollupCountEQ(args []interface{}) (rollupFunc, error) { + return newRollupCountFilter(args, countFilterEQ) +} + +func newRollupCountNE(args []interface{}) (rollupFunc, error) { + return newRollupCountFilter(args, countFilterNE) +} + func newRollupCountFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) { if err := expectRollupArgsNum(args, 2); err != nil { return nil, err diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index dff89e2c8..ae3268c7f 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -285,6 +285,44 @@ func TestRollupCountGTOverTime(t *testing.T) { f(1000, 0) } +func TestRollupCountEQOverTime(t *testing.T) { + f := func(eq, vExpected float64) { + t.Helper() + eqs := []*timeseries{{ + Values: []float64{eq}, + Timestamps: []int64{123}, + }} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}, eqs} + testRollupFunc(t, "count_eq_over_time", args, &me, vExpected) + } + + f(-123, 0) + f(0, 0) + f(34, 4) + f(123, 1) + f(12, 1) +} + +func TestRollupCountNEOverTime(t *testing.T) { + f := func(ne, vExpected float64) { + t.Helper() + nes := []*timeseries{{ + Values: []float64{ne}, + Timestamps: []int64{123}, + }} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}, nes} + testRollupFunc(t, "count_ne_over_time", args, &me, vExpected) + } + + f(-123, 12) + f(0, 12) + f(34, 8) + f(123, 11) + f(12, 11) +} + func TestRollupQuantileOverTime(t *testing.T) { f := func(phi, vExpected float64) { t.Helper() diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3be27d898..b9550437a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,7 @@ * FEATURE: optimize Consul service discovery speed when discovering big number of services. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574 * FEATURE: add `label_uppercase(q, label1, ... labelN)` and `label_lowercase(q, label1, ... labelN)` function to [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) for uppercasing and lowercasing values for the given labels. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/936 +* FEATURE: add `count_eq_over_time(m[d], N)` and `count_ne_over_time(m[d], N)` for counting the number of samples for `m` over `d` that (equal / not equal) to `N`. * FEATURE: do not print usage info for all the command-line flags when incorrect command-line flag is passed. Previously it could be hard reading the error message about incorrect command-line flag because of too big usage info for all the flags. diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 9fdd5d983..8461ac2f8 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -123,6 +123,8 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g Example: `share_gt_over_time(up[24h], 0)` - returns service availability for the last 24 hours. - `count_le_over_time(m[d], le)` - returns the number of raw samples for `m` over `d`, which don't exceed `le`. - `count_gt_over_time(m[d], gt)` - returns the number of raw samples for `m` over `d`, which are bigger than `gt`. +- `count_eq_over_time(m[d], N)` - returns the number of raw samples for `m` over `d` with values equal to `N`. +- `count_ne_over_time(m[d], N)` - returns the number of raw samples for `m` over `d` with values not equal to `N`. - `tmin_over_time(m[d])` - returns timestamp for the minimum value for `m` over `d` time range. - `tmax_over_time(m[d])` - returns timestamp for the maximum value for `m` over `d` time range. - `aggr_over_time(("aggr_func1", "aggr_func2", ...), m[d])` - simultaneously calculates all the listed `aggr_func*` for `m` over `d` time range. diff --git a/go.mod b/go.mod index 34451eb8d..eaac1e3c5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.9 github.com/VictoriaMetrics/metrics v1.12.3 - github.com/VictoriaMetrics/metricsql v0.8.0 + github.com/VictoriaMetrics/metricsql v0.9.0 github.com/aws/aws-sdk-go v1.36.0 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 diff --git a/go.sum b/go.sum index 219c369bd..36ee06271 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.9/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtN github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.8.0 h1:fUZWNjXYJhiCKoEXr86pM4RaAxfA3SqkAhJLe1gr/mo= -github.com/VictoriaMetrics/metricsql v0.8.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.9.0 h1:mO4YmVRVHQmipTHcSMlCJ7Rctsol7vlu1l2ifh+ibqI= +github.com/VictoriaMetrics/metricsql v0.9.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index 283f08c3e..33b0827f3 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -49,6 +49,8 @@ var rollupFuncs = map[string]bool{ "share_gt_over_time": true, "count_le_over_time": true, "count_gt_over_time": true, + "count_eq_over_time": true, + "count_ne_over_time": true, "histogram_over_time": true, "rollup": true, "rollup_rate": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index e8753f365..6b03b84f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.8.0 +# github.com/VictoriaMetrics/metricsql v0.9.0 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.36.0