From d856b598a89dc8bc27a3e86b9fbb4d724475bcef Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 12 May 2024 10:10:46 +0200 Subject: [PATCH] app/vmselect/promql: properly estimate the needed amounts of memory for executing aggregate function over rollup function in incremental mode Incremental aggregation processes only GOMAXPROCS time series at a time, so its' memory usage doesn't depend on the number of input time series. The issue has been introduced in 5138eaeea0791caa34bcfab410e0ca9cd253cd8f Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3203 --- app/vmselect/promql/eval.go | 2 +- docs/CHANGELOG.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index bcecda235e..40d866577d 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -1126,7 +1126,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa } } rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs))) - rollupMemorySize = sumNoOverflow(mulNoOverflow(int64(rssLen), 1000), mulNoOverflow(rollupPoints, 16)) + rollupMemorySize = sumNoOverflow(mulNoOverflow(int64(timeseriesLen), 1000), mulNoOverflow(rollupPoints, 16)) if maxMemory := int64(logQueryMemoryUsage.N); maxMemory > 0 && rollupMemorySize > maxMemory { requestURI := ec.GetRequestURI() logger.Warnf("remoteAddr=%s, requestURI=%s: the %s requires %d bytes of memory for processing; "+ diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d6d27f1887..85bc5f9ac0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components * SECURITY: upgrade Go builder from Go1.22.2 to Go1.22.3. See [the list of issues addressed in Go1.22.3](https://github.com/golang/go/issues?q=milestone%3AGo1.22.3+label%3ACherryPickApproved). +* BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/changelog_2022/#v1830). + ## [v1.93.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.14) Released at 2024-04-19