github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 15:16:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmselect/promql: evaluate union() args in parallel in order to increase query performance

Browse source 
Note that the parallel execution of `union()` args may take more memory and CPU time
than the sequential execution if args contain heavy queries, which may load all the available CPU,
disk and memory resources and vmselect and vmstorage levels.
This commit is contained in:
Aliaksandr Valialkin 2022-09-02 19:46:25 +03:00
parent f9d4ade35a
commit 4076277cf0
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
2 changed files with 42 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
app/vmselect/promql/eval.go
View file

@ -301,7 +301,14 @@ func evalTransformFunc(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.Fun
Err: fmt.Errorf(`unknown func %q`, fe.Name), Err: fmt.Errorf(`unknown func %q`, fe.Name),
} }
} }
args, err := evalExprs(qt, ec, fe.Args) var args [][]*timeseries
var err error
switch fe.Name {
case "", "union":
args, err = evalExprsInParallel(qt, ec, fe.Args)
default:
args, err = evalExprsSequentially(qt, ec, fe.Args)
}
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -337,7 +344,7 @@ func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFunc
return evalRollupFunc(qt, ec, fe.Name, rf, ae, re, iafc) return evalRollupFunc(qt, ec, fe.Name, rf, ae, re, iafc)
} }
} }
args, err := evalExprs(qt, ec, ae.Args) args, err := evalExprsInParallel(qt, ec, ae.Args)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -616,7 +623,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
return nil, nil return nil, nil
} }
func evalExprs(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) { func evalExprsSequentially(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
var rvs [][]*timeseries var rvs [][]*timeseries
for _, e := range es { for _, e := range es {
rv, err := evalExpr(qt, ec, e) rv, err := evalExpr(qt, ec, e)
@ -628,6 +635,36 @@ func evalExprs(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][
return rvs, nil return rvs, nil
} }
func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
if len(es) < 2 {
return evalExprsSequentially(qt, ec, es)
}
rvs := make([][]*timeseries, len(es))
errs := make([]error, len(es))
var wg sync.WaitGroup
for i, e := range es {
qt.Printf("eval function args in parallel")
wg.Add(1)
qtChild := qt.NewChild("eval arg %d", i)
go func(e metricsql.Expr, i int) {
defer func() {
qtChild.Done()
wg.Done()
}()
rv, err := evalExpr(qtChild, ec, e)
rvs[i] = rv
errs[i] = err
}(e, i)
}
wg.Wait()
for _, err := range errs {
if err != nil {
return nil, err
}
}
return rvs, nil
}
func evalRollupFuncArgs(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) { func evalRollupFuncArgs(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
var re *metricsql.RollupExpr var re *metricsql.RollupExpr
rollupArgIdx := metricsql.GetRollupArgIdx(fe) rollupArgIdx := metricsql.GetRollupArgIdx(fe)

2
docs/CHANGELOG.md
View file

@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip ## tip
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): evaluate `q1`, ..., `qN` in parallel when calculating `union(q1, .., qN)`. Previously [union](https://docs.victoriametrics.com/MetricsQL.html#union) args were evaluated sequentially. This could result in lower than expected performance.
## [v1.81.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.0) ## [v1.81.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.0)
Released at 31-08-2022 Released at 31-08-2022