From ed4492ddd5bc7e5a2f0ba5d6b050dd75e43a7533 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 11 Feb 2023 00:37:14 -0800
Subject: [PATCH 1/2] all: update alpine base docker image from 1.17.1 to
1.17.2
See https://alpinelinux.org/posts/Alpine-3.17.2-released.html
---
app/vmui/Dockerfile-web | 2 +-
deployment/docker/Makefile | 4 ++--
docs/CHANGELOG.md | 2 ++
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web
index c702c718c5..7a85f9780b 100644
--- a/app/vmui/Dockerfile-web
+++ b/app/vmui/Dockerfile-web
@@ -6,7 +6,7 @@ COPY web/ /build/
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
-FROM alpine:3.17.1
+FROM alpine:3.17.2
USER root
COPY --from=build-web-stage /build/web-amd64 /app/web
diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index c4a288a4f9..577de6dda7 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -2,8 +2,8 @@
DOCKER_NAMESPACE := victoriametrics
-ROOT_IMAGE ?= alpine:3.17.1
-CERTS_IMAGE := alpine:3.17.1
+ROOT_IMAGE ?= alpine:3.17.2
+CERTS_IMAGE := alpine:3.17.2
GO_BUILDER_IMAGE := golang:1.20.0-alpine
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 23ea1ce7bf..61cf17cf0d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
+* SECURITY: upgrade base docker image (alpine) from 3.17.1 to 3.17.2. See [alpine 3.17.2 release notes](https://alpinelinux.org/posts/Alpine-3.17.2-released.html).
+
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add the ability to limit the number of concurrent requests on a per-user basis via `-maxConcurrentPerUserRequests` command-line flag and via `max_concurrent_requests` config option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346) and [these docs](https://docs.victoriametrics.com/vmauth.html#concurrency-limiting).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends.
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time.
From b11bdc46bebcd11098c5e0414f7dacab6fdc7945 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 11 Feb 2023 01:06:16 -0800
Subject: [PATCH 2/2] app/vmselect/promql: add `mad_over_time(m[d])` function
See https://github.com/prometheus/prometheus/issues/5514
---
app/vmselect/promql/exec_test.go | 11 ++++++++++
app/vmselect/promql/rollup.go | 20 +++++++++++++++++++
app/vmselect/promql/rollup_test.go | 1 +
docs/CHANGELOG.md | 1 +
docs/MetricsQL.md | 7 +++++++
go.mod | 2 +-
go.sum | 4 ++--
.../VictoriaMetrics/metricsql/rollup.go | 1 +
vendor/modules.txt | 2 +-
9 files changed, 45 insertions(+), 4 deletions(-)
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index fca14d550f..15317a698b 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -3630,6 +3630,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
+ t.Run(`mad_over_time()`, func(t *testing.T) {
+ t.Parallel()
+ q := `round(mad_over_time(rand(0)[200s:5s]), 0.001)`
+ r := netstorage.Result{
+ MetricName: metricNameExpected,
+ Values: []float64{0.243, 0.274, 0.256, 0.185, 0.266, 0.256},
+ Timestamps: timestampsExpected,
+ }
+ resultExpected := []netstorage.Result{r}
+ f(q, resultExpected)
+ })
t.Run(`stddev_over_time()`, func(t *testing.T) {
t.Parallel()
q := `round(stddev_over_time(rand(0)[200s:5s]), 0.001)`
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 8acb8aa090..993c247614 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -56,6 +56,7 @@ var rollupFuncs = map[string]newRollupFunc{
"lag": newRollupFuncOneArg(rollupLag),
"last_over_time": newRollupFuncOneArg(rollupLast),
"lifetime": newRollupFuncOneArg(rollupLifetime),
+ "mad_over_time": newRollupFuncOneArg(rollupMAD),
"max_over_time": newRollupFuncOneArg(rollupMax),
"min_over_time": newRollupFuncOneArg(rollupMin),
"mode_over_time": newRollupFuncOneArg(rollupModeOverTime),
@@ -121,6 +122,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
"lag": rollupLag,
"last_over_time": rollupLast,
"lifetime": rollupLifetime,
+ "mad_over_time": rollupMAD,
"max_over_time": rollupMax,
"min_over_time": rollupMin,
"mode_over_time": rollupModeOverTime,
@@ -1213,6 +1215,24 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) {
return rf, nil
}
+func rollupMAD(rfa *rollupFuncArg) float64 {
+ // There is no need in handling NaNs here, since they must be cleaned up
+ // before calling rollup funcs.
+
+ // See https://en.wikipedia.org/wiki/Median_absolute_deviation
+ values := rfa.values
+ median := quantile(0.5, values)
+ a := getFloat64s()
+ ds := a.A[:0]
+ for _, v := range values {
+ ds = append(ds, math.Abs(v-median))
+ }
+ mad := quantile(0.5, ds)
+ a.A = ds
+ putFloat64s(a)
+ return mad
+}
+
func rollupHistogram(rfa *rollupFuncArg) float64 {
values := rfa.values
tsm := rfa.tsm
diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go
index e7b48b887e..25caa7c328 100644
--- a/app/vmselect/promql/rollup_test.go
+++ b/app/vmselect/promql/rollup_test.go
@@ -499,6 +499,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
f("resets", 5)
f("range_over_time", 111)
f("avg_over_time", 47.083333333333336)
+ f("mad_over_time", 10)
f("min_over_time", 12)
f("max_over_time", 123)
f("tmin_over_time", 0.08)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 61cf17cf0d..9b08c5b555 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -21,6 +21,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends.
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time.
* FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage).
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `mad_over_time(m[d])` function for calculating the [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) over raw samples on the lookbehind window `d`. See [this feature request](https://github.com/prometheus/prometheus/issues/5514).
## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1)
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index b302b63b19..caf7142741 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -503,6 +503,13 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
See also [duration_over_time](#duration_over_time) and [lag](#lag).
+#### mad_over_time
+
+`mad_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation)
+over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+See also [mad](#mad).
+
#### max_over_time
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over raw samples
diff --git a/go.mod b/go.mod
index 42035cb832..6f8536613b 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.23.1
- github.com/VictoriaMetrics/metricsql v0.51.2
+ github.com/VictoriaMetrics/metricsql v0.53.0
github.com/aws/aws-sdk-go-v2 v1.17.4
github.com/aws/aws-sdk-go-v2/config v1.18.12
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51
diff --git a/go.sum b/go.sum
index bee6cf9b25..8d12331afd 100644
--- a/go.sum
+++ b/go.sum
@@ -69,8 +69,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0=
github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
-github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI=
-github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
+github.com/VictoriaMetrics/metricsql v0.53.0 h1:R//oEGo+G0DtmNxF111ClM2e2pjC4sG14geyZzXfbjU=
+github.com/VictoriaMetrics/metricsql v0.53.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
index eb73d2f843..b049508955 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
@@ -42,6 +42,7 @@ var rollupFuncs = map[string]bool{
"lag": true,
"last_over_time": true,
"lifetime": true,
+ "mad_over_time": true,
"max_over_time": true,
"min_over_time": true,
"mode_over_time": true,
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 3cb30be1f5..b2db10883c 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -71,7 +71,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
# github.com/VictoriaMetrics/metrics v1.23.1
## explicit; go 1.15
github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.51.2
+# github.com/VictoriaMetrics/metricsql v0.53.0
## explicit; go 1.13
github.com/VictoriaMetrics/metricsql
github.com/VictoriaMetrics/metricsql/binaryop