From 96cdfcba501fed1c3df7ff12c28e8b5015b81e31 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 16 Oct 2020 12:53:34 +0300 Subject: [PATCH] vendor: update github.com/VictoriaMetrics/metricsql from v0.7.1 to v0.7.2 The new release of github.com/VictoriaMetrics/metricsql adds more optimizations for `foo{filters1} op bar{filters2}`: * rollup_func(foo[d]) op bar{filters} * transform_func(foo) op bar{filters} * num_or_scalar op bar op baz{filters} --- go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/optimizer.go | 110 +++++++++++++++--- vendor/modules.txt | 2 +- 4 files changed, 98 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 7a6850dcb..5e0f6bf26 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.7 github.com/VictoriaMetrics/metrics v1.12.3 - github.com/VictoriaMetrics/metricsql v0.7.1 + github.com/VictoriaMetrics/metricsql v0.7.2 github.com/aws/aws-sdk-go v1.35.5 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 diff --git a/go.sum b/go.sum index 5c18acacf..3c9c0b92a 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/VictoriaMetrics/metrics v1.12.2 h1:SG8iAmqavDNuh7GIdHPoGHUhDL23KeKfvS github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.7.1 h1:2V7EbbfKkU2pDzs+D/S0IKYvNSQzDYBlbS8afXD7ntE= -github.com/VictoriaMetrics/metricsql v0.7.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.7.2 h1:ZdFPiA9Etrf3dow43IcPvLjPi5BYWIYj194wPKIhKfs= +github.com/VictoriaMetrics/metricsql v0.7.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index 9424638da..b337b22dd 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -2,9 +2,16 @@ package metricsql import ( "sort" + "strings" ) // Optimize optimizes e in order to improve its performance. +// +// It performs the following optimizations: +// +// - Adds missing filters to `foo{filters1} op bar{filters2}` +// according to https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization +// I.e. such query is converted to `foo{filters1, filters2} op bar{filters1, filters2}` func Optimize(e Expr) Expr { switch t := e.(type) { case *BinaryOpExpr: @@ -12,26 +19,15 @@ func Optimize(e Expr) Expr { // This should reduce the number of operations // See https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization // for details. - switch t.Op { - case "+", "-", "*", "/", "%", "^", - "==", "!=", ">", "<", ">=", "<=", - "and", "if", "ifnot", "default": - // The optimization can be applied only to these operations. - default: + if !canOptimizeBinaryOp(t) { return optimizeBinaryOpArgs(t) } - if t.JoinModifier.Op != "" { + meLeft := getMetricExprForOptimization(t.Left) + if meLeft == nil || !meLeft.hasNonEmptyMetricGroup() { return optimizeBinaryOpArgs(t) } - if t.GroupModifier.Op != "" { - return optimizeBinaryOpArgs(t) - } - meLeft, ok := t.Left.(*MetricExpr) - if !ok || !meLeft.hasNonEmptyMetricGroup() { - return optimizeBinaryOpArgs(t) - } - meRight, ok := t.Right.(*MetricExpr) - if !ok || !meRight.hasNonEmptyMetricGroup() { + meRight := getMetricExprForOptimization(t.Right) + if meRight == nil || !meRight.hasNonEmptyMetricGroup() { return optimizeBinaryOpArgs(t) } lfs := intersectLabelFilters(meLeft.LabelFilters[1:], meRight.LabelFilters[1:]) @@ -53,12 +49,94 @@ func Optimize(e Expr) Expr { } } +func canOptimizeBinaryOp(be *BinaryOpExpr) bool { + if be.JoinModifier.Op != "" || be.GroupModifier.Op != "" { + return false + } + switch be.Op { + case "+", "-", "*", "/", "%", "^", + "==", "!=", ">", "<", ">=", "<=", + "and", "if", "ifnot", "default": + return true + default: + return false + } +} + func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr { be.Left = Optimize(be.Left) be.Right = Optimize(be.Right) return be } +func getMetricExprForOptimization(e Expr) *MetricExpr { + me, ok := e.(*MetricExpr) + if ok { + // Ordinary metric expression, i.e. `foo{bar="baz"}` + return me + } + be, ok := e.(*BinaryOpExpr) + if ok { + if !canOptimizeBinaryOp(be) { + return nil + } + if me, ok := be.Left.(*MetricExpr); ok && isNumberOrScalar(be.Right) { + // foo{bar="baz"} * num_or_scalar + return me + } + if me, ok := be.Right.(*MetricExpr); ok && isNumberOrScalar(be.Left) { + // num_or_scalar * foo{bar="baz"} + return me + } + return nil + } + fe, ok := e.(*FuncExpr) + if !ok { + return nil + } + if IsRollupFunc(fe.Name) { + for _, arg := range fe.Args { + re, ok := arg.(*RollupExpr) + if !ok { + continue + } + if me, ok := re.Expr.(*MetricExpr); ok { + // rollup_func(foo{bar="baz"}[d]) + return me + } + } + return nil + } + if IsTransformFunc(fe.Name) { + switch strings.ToLower(fe.Name) { + case "absent", "histogram_quantile", "label_join", "label_replace", "scalar", "vector", + "label_set", "label_map", "label_del", "label_keep", "label_copy", + "label_move", "label_transform", "label_value", "label_match", "label_mismatch", + "prometheus_buckets", "buckets_limit", "histogram_share", "union", "": + // metric expressions for these functions cannot be optimized. + return nil + } + for _, arg := range fe.Args { + if me, ok := arg.(*MetricExpr); ok { + // transform_func(foo{bar="baz"}) + return me + } + } + return nil + } + return nil +} + +func isNumberOrScalar(e Expr) bool { + if _, ok := e.(*NumberExpr); ok { + return true + } + if fe, ok := e.(*FuncExpr); ok && strings.ToLower(fe.Name) == "scalar" { + return true + } + return false +} + func intersectLabelFilters(a, b []LabelFilter) []LabelFilter { m := make(map[string]LabelFilter, len(a)+len(b)) var buf []byte diff --git a/vendor/modules.txt b/vendor/modules.txt index eb116ad71..af622f1b4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.7.1 +# github.com/VictoriaMetrics/metricsql v0.7.2 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.35.5