From b11bdc46bebcd11098c5e0414f7dacab6fdc7945 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 11 Feb 2023 01:06:16 -0800 Subject: [PATCH] app/vmselect/promql: add `mad_over_time(m[d])` function See https://github.com/prometheus/prometheus/issues/5514 --- app/vmselect/promql/exec_test.go | 11 ++++++++++ app/vmselect/promql/rollup.go | 20 +++++++++++++++++++ app/vmselect/promql/rollup_test.go | 1 + docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- .../VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 45 insertions(+), 4 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index fca14d550..15317a698 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -3630,6 +3630,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`mad_over_time()`, func(t *testing.T) { + t.Parallel() + q := `round(mad_over_time(rand(0)[200s:5s]), 0.001)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0.243, 0.274, 0.256, 0.185, 0.266, 0.256}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`stddev_over_time()`, func(t *testing.T) { t.Parallel() q := `round(stddev_over_time(rand(0)[200s:5s]), 0.001)` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 8acb8aa09..993c24761 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -56,6 +56,7 @@ var rollupFuncs = map[string]newRollupFunc{ "lag": newRollupFuncOneArg(rollupLag), "last_over_time": newRollupFuncOneArg(rollupLast), "lifetime": newRollupFuncOneArg(rollupLifetime), + "mad_over_time": newRollupFuncOneArg(rollupMAD), "max_over_time": newRollupFuncOneArg(rollupMax), "min_over_time": newRollupFuncOneArg(rollupMin), "mode_over_time": newRollupFuncOneArg(rollupModeOverTime), @@ -121,6 +122,7 @@ var rollupAggrFuncs = map[string]rollupFunc{ "lag": rollupLag, "last_over_time": rollupLast, "lifetime": rollupLifetime, + "mad_over_time": rollupMAD, "max_over_time": rollupMax, "min_over_time": rollupMin, "mode_over_time": rollupModeOverTime, @@ -1213,6 +1215,24 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) { return rf, nil } +func rollupMAD(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + + // See https://en.wikipedia.org/wiki/Median_absolute_deviation + values := rfa.values + median := quantile(0.5, values) + a := getFloat64s() + ds := a.A[:0] + for _, v := range values { + ds = append(ds, math.Abs(v-median)) + } + mad := quantile(0.5, ds) + a.A = ds + putFloat64s(a) + return mad +} + func rollupHistogram(rfa *rollupFuncArg) float64 { values := rfa.values tsm := rfa.tsm diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index e7b48b887..25caa7c32 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -499,6 +499,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) { f("resets", 5) f("range_over_time", 111) f("avg_over_time", 47.083333333333336) + f("mad_over_time", 10) f("min_over_time", 12) f("max_over_time", 123) f("tmin_over_time", 0.08) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 61cf17cf0..9b08c5b55 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -21,6 +21,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time. * FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `mad_over_time(m[d])` function for calculating the [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) over raw samples on the lookbehind window `d`. See [this feature request](https://github.com/prometheus/prometheus/issues/5514). ## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1) diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index b302b63b1..caf714274 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -503,6 +503,13 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k See also [duration_over_time](#duration_over_time) and [lag](#lag). +#### mad_over_time + +`mad_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) +over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). + +See also [mad](#mad). + #### max_over_time `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over raw samples diff --git a/go.mod b/go.mod index 42035cb83..6f8536613 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.23.1 - github.com/VictoriaMetrics/metricsql v0.51.2 + github.com/VictoriaMetrics/metricsql v0.53.0 github.com/aws/aws-sdk-go-v2 v1.17.4 github.com/aws/aws-sdk-go-v2/config v1.18.12 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 diff --git a/go.sum b/go.sum index bee6cf9b2..8d12331af 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI= -github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.53.0 h1:R//oEGo+G0DtmNxF111ClM2e2pjC4sG14geyZzXfbjU= +github.com/VictoriaMetrics/metricsql v0.53.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index eb73d2f84..b04950895 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -42,6 +42,7 @@ var rollupFuncs = map[string]bool{ "lag": true, "last_over_time": true, "lifetime": true, + "mad_over_time": true, "max_over_time": true, "min_over_time": true, "mode_over_time": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 3cb30be1f..b2db10883 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -71,7 +71,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.23.1 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.51.2 +# github.com/VictoriaMetrics/metricsql v0.53.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop