From 024e2f18da05b64781a6a34497e12634e7f91185 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 2 Sep 2022 19:46:25 +0300
Subject: [PATCH] app/vmselect/promql: evaluate `union()` args in parallel in
order to increase query performance
Note that the parallel execution of `union()` args may take more memory and CPU time
than the sequential execution if args contain heavy queries, which may load all the available CPU,
disk and memory resources and vmselect and vmstorage levels.
---
app/vmselect/promql/eval.go | 43 ++++++++++++++++++++++++++++++++++---
docs/CHANGELOG.md | 2 ++
2 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go
index 6af5859069..423403774b 100644
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -318,7 +318,14 @@ func evalTransformFunc(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.Fun
Err: fmt.Errorf(`unknown func %q`, fe.Name),
}
}
- args, err := evalExprs(qt, ec, fe.Args)
+ var args [][]*timeseries
+ var err error
+ switch fe.Name {
+ case "", "union":
+ args, err = evalExprsInParallel(qt, ec, fe.Args)
+ default:
+ args, err = evalExprsSequentially(qt, ec, fe.Args)
+ }
if err != nil {
return nil, err
}
@@ -354,7 +361,7 @@ func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFunc
return evalRollupFunc(qt, ec, fe.Name, rf, ae, re, iafc)
}
}
- args, err := evalExprs(qt, ec, ae.Args)
+ args, err := evalExprsInParallel(qt, ec, ae.Args)
if err != nil {
return nil, err
}
@@ -633,7 +640,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
return nil, nil
}
-func evalExprs(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
+func evalExprsSequentially(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
var rvs [][]*timeseries
for _, e := range es {
rv, err := evalExpr(qt, ec, e)
@@ -645,6 +652,36 @@ func evalExprs(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][
return rvs, nil
}
+func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
+ if len(es) < 2 {
+ return evalExprsSequentially(qt, ec, es)
+ }
+ rvs := make([][]*timeseries, len(es))
+ errs := make([]error, len(es))
+ var wg sync.WaitGroup
+ for i, e := range es {
+ qt.Printf("eval function args in parallel")
+ wg.Add(1)
+ qtChild := qt.NewChild("eval arg %d", i)
+ go func(e metricsql.Expr, i int) {
+ defer func() {
+ qtChild.Done()
+ wg.Done()
+ }()
+ rv, err := evalExpr(qtChild, ec, e)
+ rvs[i] = rv
+ errs[i] = err
+ }(e, i)
+ }
+ wg.Wait()
+ for _, err := range errs {
+ if err != nil {
+ return nil, err
+ }
+ }
+ return rvs, nil
+}
+
func evalRollupFuncArgs(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
var re *metricsql.RollupExpr
rollupArgIdx := metricsql.GetRollupArgIdx(fe)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index eb65f38d39..914c3dffbc 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): evaluate `q1`, ..., `qN` in parallel when calculating `union(q1, .., qN)`. Previously [union](https://docs.victoriametrics.com/MetricsQL.html#union) args were evaluated sequentially. This could result in lower than expected performance.
+
## [v1.81.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.0)
Released at 31-08-2022