From ed4492ddd5bc7e5a2f0ba5d6b050dd75e43a7533 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin <valyala@victoriametrics.com> Date: Sat, 11 Feb 2023 00:37:14 -0800 Subject: [PATCH 1/2] all: update alpine base docker image from 1.17.1 to 1.17.2 See https://alpinelinux.org/posts/Alpine-3.17.2-released.html --- app/vmui/Dockerfile-web | 2 +- deployment/docker/Makefile | 4 ++-- docs/CHANGELOG.md | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web index c702c718c5..7a85f9780b 100644 --- a/app/vmui/Dockerfile-web +++ b/app/vmui/Dockerfile-web @@ -6,7 +6,7 @@ COPY web/ /build/ RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/ -FROM alpine:3.17.1 +FROM alpine:3.17.2 USER root COPY --from=build-web-stage /build/web-amd64 /app/web diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index c4a288a4f9..577de6dda7 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -2,8 +2,8 @@ DOCKER_NAMESPACE := victoriametrics -ROOT_IMAGE ?= alpine:3.17.1 -CERTS_IMAGE := alpine:3.17.1 +ROOT_IMAGE ?= alpine:3.17.2 +CERTS_IMAGE := alpine:3.17.2 GO_BUILDER_IMAGE := golang:1.20.0-alpine BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 23ea1ce7bf..61cf17cf0d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* SECURITY: upgrade base docker image (alpine) from 3.17.1 to 3.17.2. See [alpine 3.17.2 release notes](https://alpinelinux.org/posts/Alpine-3.17.2-released.html). + * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add the ability to limit the number of concurrent requests on a per-user basis via `-maxConcurrentPerUserRequests` command-line flag and via `max_concurrent_requests` config option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346) and [these docs](https://docs.victoriametrics.com/vmauth.html#concurrency-limiting). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time. From b11bdc46bebcd11098c5e0414f7dacab6fdc7945 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin <valyala@victoriametrics.com> Date: Sat, 11 Feb 2023 01:06:16 -0800 Subject: [PATCH 2/2] app/vmselect/promql: add `mad_over_time(m[d])` function See https://github.com/prometheus/prometheus/issues/5514 --- app/vmselect/promql/exec_test.go | 11 ++++++++++ app/vmselect/promql/rollup.go | 20 +++++++++++++++++++ app/vmselect/promql/rollup_test.go | 1 + docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- .../VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 9 files changed, 45 insertions(+), 4 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index fca14d550f..15317a698b 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -3630,6 +3630,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`mad_over_time()`, func(t *testing.T) { + t.Parallel() + q := `round(mad_over_time(rand(0)[200s:5s]), 0.001)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0.243, 0.274, 0.256, 0.185, 0.266, 0.256}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`stddev_over_time()`, func(t *testing.T) { t.Parallel() q := `round(stddev_over_time(rand(0)[200s:5s]), 0.001)` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 8acb8aa090..993c247614 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -56,6 +56,7 @@ var rollupFuncs = map[string]newRollupFunc{ "lag": newRollupFuncOneArg(rollupLag), "last_over_time": newRollupFuncOneArg(rollupLast), "lifetime": newRollupFuncOneArg(rollupLifetime), + "mad_over_time": newRollupFuncOneArg(rollupMAD), "max_over_time": newRollupFuncOneArg(rollupMax), "min_over_time": newRollupFuncOneArg(rollupMin), "mode_over_time": newRollupFuncOneArg(rollupModeOverTime), @@ -121,6 +122,7 @@ var rollupAggrFuncs = map[string]rollupFunc{ "lag": rollupLag, "last_over_time": rollupLast, "lifetime": rollupLifetime, + "mad_over_time": rollupMAD, "max_over_time": rollupMax, "min_over_time": rollupMin, "mode_over_time": rollupModeOverTime, @@ -1213,6 +1215,24 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) { return rf, nil } +func rollupMAD(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + + // See https://en.wikipedia.org/wiki/Median_absolute_deviation + values := rfa.values + median := quantile(0.5, values) + a := getFloat64s() + ds := a.A[:0] + for _, v := range values { + ds = append(ds, math.Abs(v-median)) + } + mad := quantile(0.5, ds) + a.A = ds + putFloat64s(a) + return mad +} + func rollupHistogram(rfa *rollupFuncArg) float64 { values := rfa.values tsm := rfa.tsm diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index e7b48b887e..25caa7c328 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -499,6 +499,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) { f("resets", 5) f("range_over_time", 111) f("avg_over_time", 47.083333333333336) + f("mad_over_time", 10) f("min_over_time", 12) f("max_over_time", 123) f("tmin_over_time", 0.08) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 61cf17cf0d..9b08c5b555 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -21,6 +21,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time. * FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `mad_over_time(m[d])` function for calculating the [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) over raw samples on the lookbehind window `d`. See [this feature request](https://github.com/prometheus/prometheus/issues/5514). ## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1) diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index b302b63b19..caf7142741 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -503,6 +503,13 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k See also [duration_over_time](#duration_over_time) and [lag](#lag). +#### mad_over_time + +`mad_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) +over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). + +See also [mad](#mad). + #### max_over_time `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over raw samples diff --git a/go.mod b/go.mod index 42035cb832..6f8536613b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.23.1 - github.com/VictoriaMetrics/metricsql v0.51.2 + github.com/VictoriaMetrics/metricsql v0.53.0 github.com/aws/aws-sdk-go-v2 v1.17.4 github.com/aws/aws-sdk-go-v2/config v1.18.12 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 diff --git a/go.sum b/go.sum index bee6cf9b25..8d12331afd 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI= -github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.53.0 h1:R//oEGo+G0DtmNxF111ClM2e2pjC4sG14geyZzXfbjU= +github.com/VictoriaMetrics/metricsql v0.53.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index eb73d2f843..b049508955 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -42,6 +42,7 @@ var rollupFuncs = map[string]bool{ "lag": true, "last_over_time": true, "lifetime": true, + "mad_over_time": true, "max_over_time": true, "min_over_time": true, "mode_over_time": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 3cb30be1f5..b2db10883c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -71,7 +71,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.23.1 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.51.2 +# github.com/VictoriaMetrics/metricsql v0.53.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop