From 4ed7548cedecee2c02a6e5607eb3273278b285dd Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin <valyala@gmail.com> Date: Fri, 17 Sep 2021 12:33:40 +0300 Subject: [PATCH] app/vmselect/promql: optimize quantiles() calculation Calculate quantiles in one go instead of calculating each quantile individually --- app/vmselect/promql/aggr.go | 48 ++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index 744ffbf38e..fdcd0d1361 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -966,30 +966,46 @@ func aggrFuncQuantiles(afa *aggrFuncArg) ([]*timeseries, error) { return nil, fmt.Errorf("cannot obtain dstLabel: %w", err) } phiArgs := args[1 : len(args)-1] - argOrig := args[len(args)-1] - var rvs []*timeseries + phis := make([]float64, len(phiArgs)) for i, phiArg := range phiArgs { - phis, err := getScalar(phiArg, i+1) + phisLocal, err := getScalar(phiArg, i+1) if err != nil { return nil, err } if len(phis) == 0 { logger.Panicf("BUG: expecting at least a single sample") } - phi := phis[0] - afe := newAggrQuantileFunc(phis) - arg := copyTimeseries(argOrig) - tss, err := aggrFuncExt(afe, arg, &afa.ae.Modifier, afa.ae.Limit, false) - if err != nil { - return nil, fmt.Errorf("cannot calculate quantile %g: %w", phi, err) - } - for _, ts := range tss { - ts.MetricName.RemoveTag(dstLabel) - ts.MetricName.AddTag(dstLabel, fmt.Sprintf("%g", phi)) - } - rvs = append(rvs, tss...) + phis[i] = phisLocal[0] } - return rvs, nil + argOrig := args[len(args)-1] + afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries { + tssDst := make([]*timeseries, len(phiArgs)) + for j := range tssDst { + ts := ×eries{} + ts.CopyFromShallowTimestamps(tss[0]) + ts.MetricName.RemoveTag(dstLabel) + ts.MetricName.AddTag(dstLabel, fmt.Sprintf("%g", phis[j])) + tssDst[j] = ts + } + h := histogram.GetFast() + defer histogram.PutFast(h) + var qs []float64 + for n := range tss[0].Values { + h.Reset() + for j := range tss { + v := tss[j].Values[n] + if !math.IsNaN(v) { + h.Update(v) + } + } + qs = h.Quantiles(qs[:0], phis) + for j := range tssDst { + tssDst[j].Values[n] = qs[j] + } + } + return tssDst + } + return aggrFuncExt(afe, argOrig, &afa.ae.Modifier, afa.ae.Limit, false) } func aggrFuncQuantile(afa *aggrFuncArg) ([]*timeseries, error) {