From e9f2e2cbc971fd0c4ef9cd02ce69c9336e8a60eb Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 7 Oct 2020 21:15:06 +0300 Subject: [PATCH] app/vmselect/promql: add missing label filters to binary operands before query execution This implements the optimization described at https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization See also https://github.com/cortexproject/cortex/issues/3253 --- app/vmselect/promql/exec.go | 1 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/optimizer.go | 83 +++++++++++++++++++ vendor/modules.txt | 2 +- 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 vendor/github.com/VictoriaMetrics/metricsql/optimizer.go diff --git a/app/vmselect/promql/exec.go b/app/vmselect/promql/exec.go index 6d92314c4..789f74ab6 100644 --- a/app/vmselect/promql/exec.go +++ b/app/vmselect/promql/exec.go @@ -175,6 +175,7 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) { if pcv == nil { e, err := metricsql.Parse(q) if err == nil { + e = metricsql.Optimize(e) e = adjustCmpOps(e) } pcv = &parseCacheValue{ diff --git a/go.mod b/go.mod index 10559cb36..6a735452a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.7 github.com/VictoriaMetrics/metrics v1.12.3 - github.com/VictoriaMetrics/metricsql v0.6.0 + github.com/VictoriaMetrics/metricsql v0.7.0 github.com/aws/aws-sdk-go v1.35.3 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 diff --git a/go.sum b/go.sum index b14e90695..ee31b099a 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/VictoriaMetrics/metrics v1.12.2 h1:SG8iAmqavDNuh7GIdHPoGHUhDL23KeKfvS github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.6.0 h1:JnHUmifuA3fdy1GQrmkZJFO+CwFrhLxKwzMv89wNgJ4= -github.com/VictoriaMetrics/metricsql v0.6.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.7.0 h1:YR/OvbsCH0dwUuc3r5GayTcuTdgWHJZo+4bqbaGl7WM= +github.com/VictoriaMetrics/metricsql v0.7.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go new file mode 100644 index 000000000..9c260508a --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -0,0 +1,83 @@ +package metricsql + +import ( + "sort" +) + +// Optimize optimizes e in order to improve its performance. +func Optimize(e Expr) Expr { + switch t := e.(type) { + case *BinaryOpExpr: + // Convert `foo{filters1} op bar{filters2}` to `foo{filters1, filters2} op bar{filters1, filters2}`. + // This should reduce the number of operations + // See https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization + // for details. + switch t.Op { + case "+", "-", "*", "/", "%", "^", + "==", "!=", ">", "<", ">=", "<=", + "if", "ifnot", "default": + // The optimization can be applied only to these operations. + default: + return optimizeBinaryOpArgs(t) + } + if t.JoinModifier.Op != "" { + return optimizeBinaryOpArgs(t) + } + if t.GroupModifier.Op != "" { + return optimizeBinaryOpArgs(t) + } + meLeft, ok := t.Left.(*MetricExpr) + if !ok || !meLeft.hasNonEmptyMetricGroup() { + return optimizeBinaryOpArgs(t) + } + meRight, ok := t.Right.(*MetricExpr) + if !ok || !meRight.hasNonEmptyMetricGroup() { + return optimizeBinaryOpArgs(t) + } + lfs := intersectLabelFilters(meLeft.LabelFilters[1:], meRight.LabelFilters[1:]) + meLeft.LabelFilters = append(meLeft.LabelFilters[:1], lfs...) + meRight.LabelFilters = append(meRight.LabelFilters[:1], lfs...) + return t + case *FuncExpr: + for i := range t.Args { + t.Args[i] = Optimize(t.Args[i]) + } + return t + case *AggrFuncExpr: + for i := range t.Args { + t.Args[i] = Optimize(t.Args[i]) + } + return t + default: + return e + } +} + +func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr { + be.Left = Optimize(be.Left) + be.Right = Optimize(be.Right) + return be +} + +func intersectLabelFilters(a, b []LabelFilter) []LabelFilter { + m := make(map[string]LabelFilter, len(a)+len(b)) + var buf []byte + for _, lf := range a { + buf = lf.AppendString(buf[:0]) + m[string(buf)] = lf + } + for _, lf := range b { + buf = lf.AppendString(buf[:0]) + m[string(buf)] = lf + } + ss := make([]string, 0, len(m)) + for s := range m { + ss = append(ss, s) + } + sort.Strings(ss) + lfs := make([]LabelFilter, 0, len(ss)) + for _, s := range ss { + lfs = append(lfs, m[s]) + } + return lfs +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d621ffd8f..6b35c1fdd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.6.0 +# github.com/VictoriaMetrics/metricsql v0.7.0 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.35.3