From e02e0508dafd8054e5149fa9c85b20816f085e80 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 27 Jan 2022 18:59:25 +0200 Subject: [PATCH] vendor: update github.com/VictoriaMetrics/metricsql from v0.37.0 to v0.38.0 This adds more optimization cases for https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization For example: * Multi-level transform functions. For example, abs(round(foo{a="b"})) + bar{x="y"} is now optimized to abs(round(foo{a="b",x="y"})) + bar{a="b",x="y"} * Binary operations with `on()`, `without()`, `group_left()` and `group_right()` modifiers. For example, foo{a="b"} on (a) + bar is now optimized to foo{a="b"} on (a) + bar{a="b"} * Multi-level binary operations. For example, foo{a="b"} + bar{x="y"} + baz{z="q"} is now optimized to foo{a="b",x="y",z="q"} + bar{a="b",x="y",z="q"} + baz{a="b",x="y",z="q"} * Aggregate functions. For example, sum(foo{a="b"}) by (c) + bar{c="d"} is now optimized to sum(foo{a="b",c="d"}) by (c) + bar{c="d"} --- docs/CHANGELOG.md | 6 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/optimizer.go | 374 ++++++++++++------ .../VictoriaMetrics/metricsql/parser.go | 7 +- .../VictoriaMetrics/metricsql/rollup.go | 3 +- vendor/modules.txt | 2 +- 7 files changed, 271 insertions(+), 127 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 74e588452..774fe383a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,12 @@ sort: 15 ## tip +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): cover more cases with the [label filters' propagation optimization](https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization). This should improve the average performance for practical queries. The following cases are additionally covered: + * Multi-level [transform functions](https://docs.victoriametrics.com/MetricsQL.html#transform-functions). For example, `abs(round(foo{a="b"})) + bar{x="y"}` is now optimized to `abs(round(foo{a="b",x="y"})) + bar{a="b",x="y"}` + * Binary operations with `on()`, `without()`, `group_left()` and `group_right()` modifiers. For example, `foo{a="b"} on (a) + bar` is now optimized to `foo{a="b"} on (a) + bar{a="b"}` + * Multi-level binary operations. For example, `foo{a="b"} + bar{x="y"} + baz{z="q"}` is now optimized to `foo{a="b",x="y",z="q"} + bar{a="b",x="y",z="q"} + baz{a="b",x="y",z="q"}` + * Aggregate functions. For example, `sum(foo{a="b"}) by (c) + bar{c="d"}` is now optimized to `sum(foo{a="b",c="d"}) by (c) + bar{c="d"}` + * BUGFIX: return proper results from `highestMax()` function at [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage). Previously it was incorrectly returning timeseries with min peaks instead of max peaks. * BUGFIX: properly limit indexdb cache sizes. Previously they could exceed values set via `-memory.allowedPercent` and/or `-memory.allowedBytes` when `indexdb` contained many data parts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a bug, which could break time range picker when editing `From` or `To` input fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2080). diff --git a/go.mod b/go.mod index 2dab4494f..b4f7bba58 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.1 - github.com/VictoriaMetrics/metricsql v0.37.0 + github.com/VictoriaMetrics/metricsql v0.38.0 github.com/aws/aws-sdk-go v1.42.42 github.com/cespare/xxhash/v2 v2.1.2 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index a1bfcad30..f8a685fa1 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.37.0 h1:zFKC+XJpEhp0TtTa6pD0pnyg9sDLH4U5nCeDUT8eUAw= -github.com/VictoriaMetrics/metricsql v0.37.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.38.0 h1:YBAzxKyr2QLFXYap8Nd0bxIr0e8mE/aUIyBYgDFMpK4= +github.com/VictoriaMetrics/metricsql v0.38.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index a644d63f2..4d76652b4 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -14,45 +14,194 @@ import ( // I.e. such query is converted to `foo{filters1, filters2} op bar{filters1, filters2}` func Optimize(e Expr) Expr { switch t := e.(type) { - case *BinaryOpExpr: - // Convert `foo{filters1} op bar{filters2}` to `foo{filters1, filters2} op bar{filters1, filters2}`. - // This should reduce the number of operations - // See https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization - // for details. - if !canOptimizeBinaryOp(t) { - return optimizeBinaryOpArgs(t) - } - meLeft := getMetricExprForOptimization(t.Left) - if meLeft == nil || !meLeft.hasNonEmptyMetricGroup() { - return optimizeBinaryOpArgs(t) - } - meRight := getMetricExprForOptimization(t.Right) - if meRight == nil || !meRight.hasNonEmptyMetricGroup() { - return optimizeBinaryOpArgs(t) - } - lfs := intersectLabelFilters(meLeft.LabelFilters[1:], meRight.LabelFilters[1:]) - meLeft.LabelFilters = append(meLeft.LabelFilters[:1], lfs...) - meRight.LabelFilters = append(meRight.LabelFilters[:1], lfs...) - return t + case *RollupExpr: + t.Expr = Optimize(t.Expr) + t.At = Optimize(t.At) case *FuncExpr: - for i := range t.Args { - t.Args[i] = Optimize(t.Args[i]) - } - return t + optimizeFuncArgs(t.Args) case *AggrFuncExpr: - for i := range t.Args { - t.Args[i] = Optimize(t.Args[i]) - } - return t - default: - return e + optimizeFuncArgs(t.Args) + case *BinaryOpExpr: + t.Left = Optimize(t.Left) + t.Right = Optimize(t.Right) + lfs := getCommonLabelFilters(t) + pushdownLabelFilters(t, lfs) + } + return e +} + +func optimizeFuncArgs(args []Expr) { + for i := range args { + args[i] = Optimize(args[i]) } } -func canOptimizeBinaryOp(be *BinaryOpExpr) bool { - if be.JoinModifier.Op != "" || be.GroupModifier.Op != "" { - return false +func getCommonLabelFilters(e Expr) []LabelFilter { + switch t := e.(type) { + case *MetricExpr: + return getLabelFiltersWithoutMetricName(t.LabelFilters) + case *RollupExpr: + return getCommonLabelFilters(t.Expr) + case *FuncExpr: + arg := getFuncArgForOptimization(t.Name, t.Args) + if arg == nil { + return nil + } + return getCommonLabelFilters(arg) + case *AggrFuncExpr: + arg := getFuncArgForOptimization(t.Name, t.Args) + if arg == nil { + return nil + } + lfs := getCommonLabelFilters(arg) + return filterLabelFiltersByAggrModifier(lfs, t) + case *BinaryOpExpr: + if !canOptimizeBinaryOp(t) { + return nil + } + lfsLeft := getCommonLabelFilters(t.Left) + lfsRight := getCommonLabelFilters(t.Right) + lfs := unionLabelFilters(lfsLeft, lfsRight) + return filterLabelFiltersByGroupModifier(lfs, t) + default: + return nil } +} + +func filterLabelFiltersByAggrModifier(lfs []LabelFilter, afe *AggrFuncExpr) []LabelFilter { + switch strings.ToLower(afe.Modifier.Op) { + case "by": + return filterLabelFiltersOn(lfs, afe.Modifier.Args) + case "without": + return filterLabelFiltersIgnoring(lfs, afe.Modifier.Args) + default: + return nil + } +} + +func filterLabelFiltersByGroupModifier(lfs []LabelFilter, be *BinaryOpExpr) []LabelFilter { + switch strings.ToLower(be.GroupModifier.Op) { + case "on": + return filterLabelFiltersOn(lfs, be.GroupModifier.Args) + case "ignoring": + return filterLabelFiltersIgnoring(lfs, be.GroupModifier.Args) + default: + return lfs + } +} + +func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter { + lfsNew := make([]LabelFilter, 0, len(lfs)) + for _, lf := range lfs { + if lf.Label != "__name__" { + lfsNew = append(lfsNew, lf) + } + } + return lfsNew +} + +func pushdownLabelFilters(e Expr, lfs []LabelFilter) { + if len(lfs) == 0 { + return + } + switch t := e.(type) { + case *MetricExpr: + t.LabelFilters = unionLabelFilters(t.LabelFilters, lfs) + sortLabelFilters(t.LabelFilters) + case *RollupExpr: + pushdownLabelFilters(t.Expr, lfs) + case *FuncExpr: + arg := getFuncArgForOptimization(t.Name, t.Args) + if arg != nil { + pushdownLabelFilters(arg, lfs) + } + case *AggrFuncExpr: + lfs = filterLabelFiltersByAggrModifier(lfs, t) + arg := getFuncArgForOptimization(t.Name, t.Args) + if arg != nil { + pushdownLabelFilters(arg, lfs) + } + case *BinaryOpExpr: + if canOptimizeBinaryOp(t) { + lfs = filterLabelFiltersByGroupModifier(lfs, t) + pushdownLabelFilters(t.Left, lfs) + pushdownLabelFilters(t.Right, lfs) + } + } +} + +func unionLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { + if len(lfsA) == 0 { + return lfsB + } + if len(lfsB) == 0 { + return lfsA + } + m := make(map[string]struct{}, len(lfsA)) + var b []byte + for _, lf := range lfsA { + b = lf.AppendString(b[:0]) + m[string(b)] = struct{}{} + } + lfs := append([]LabelFilter{}, lfsA...) + for _, lf := range lfsB { + b = lf.AppendString(b[:0]) + if _, ok := m[string(b)]; !ok { + lfs = append(lfs, lf) + } + } + return lfs +} + +func sortLabelFilters(lfs []LabelFilter) { + // Make sure the first label filter is __name__ (if any) + if len(lfs) > 0 && lfs[0].isMetricNameFilter() { + lfs = lfs[1:] + } + sort.Slice(lfs, func(i, j int) bool { + a, b := lfs[i], lfs[j] + if a.Label != b.Label { + return a.Label < b.Label + } + return a.Value < b.Value + }) +} + +func filterLabelFiltersOn(lfs []LabelFilter, args []string) []LabelFilter { + if len(args) == 0 { + return nil + } + m := make(map[string]struct{}, len(args)) + for _, arg := range args { + m[arg] = struct{}{} + } + var lfsNew []LabelFilter + for _, lf := range lfs { + if _, ok := m[lf.Label]; ok { + lfsNew = append(lfsNew, lf) + } + } + return lfsNew +} + +func filterLabelFiltersIgnoring(lfs []LabelFilter, args []string) []LabelFilter { + if len(args) == 0 { + return lfs + } + m := make(map[string]struct{}, len(args)) + for _, arg := range args { + m[arg] = struct{}{} + } + var lfsNew []LabelFilter + for _, lf := range lfs { + if _, ok := m[lf.Label]; !ok { + lfsNew = append(lfsNew, lf) + } + } + return lfsNew +} + +func canOptimizeBinaryOp(be *BinaryOpExpr) bool { switch be.Op { case "+", "-", "*", "/", "%", "^", "==", "!=", ">", "<", ">=", "<=", @@ -63,99 +212,86 @@ func canOptimizeBinaryOp(be *BinaryOpExpr) bool { } } -func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr { - be.Left = Optimize(be.Left) - be.Right = Optimize(be.Right) - return be +func getFuncArgForOptimization(funcName string, args []Expr) Expr { + idx := getFuncArgIdxForOptimization(funcName, args) + if idx < 0 || idx >= len(args) { + return nil + } + return args[idx] } -func getMetricExprForOptimization(e Expr) *MetricExpr { - re, ok := e.(*RollupExpr) - if ok { - // Try optimizing the inner expression in RollupExpr. - return getMetricExprForOptimization(re.Expr) +func getFuncArgIdxForOptimization(funcName string, args []Expr) int { + funcName = strings.ToLower(funcName) + if IsRollupFunc(funcName) { + return getRollupArgIdxForOptimization(funcName, args) } - me, ok := e.(*MetricExpr) - if ok { - // Ordinary metric expression, i.e. `foo{bar="baz"}` - return me + if IsTransformFunc(funcName) { + return getTransformArgIdxForOptimization(funcName, args) } - be, ok := e.(*BinaryOpExpr) - if ok { - if !canOptimizeBinaryOp(be) { - return nil - } - if me, ok := be.Left.(*MetricExpr); ok && isNumberOrScalar(be.Right) { - // foo{bar="baz"} * num_or_scalar - return me - } - if me, ok := be.Right.(*MetricExpr); ok && isNumberOrScalar(be.Left) { - // num_or_scalar * foo{bar="baz"} - return me - } - return nil + if isAggrFunc(funcName) { + return getAggrArgIdxForOptimization(funcName, args) } - fe, ok := e.(*FuncExpr) - if !ok { - return nil - } - if IsRollupFunc(fe.Name) { - argIdx := GetRollupArgIdx(fe) - if argIdx >= len(fe.Args) { - return nil - } - arg := fe.Args[argIdx] - return getMetricExprForOptimization(arg) - } - if IsTransformFunc(fe.Name) { - switch strings.ToLower(fe.Name) { - case "absent", "histogram_quantile", "label_join", "label_replace", "scalar", "vector", - "label_set", "label_map", "label_uppercase", "label_lowercase", "label_del", "label_keep", "label_copy", - "label_move", "label_transform", "label_value", "label_match", "label_mismatch", "label_graphite_group", - "prometheus_buckets", "buckets_limit", "histogram_share", "histogram_avg", "histogram_stdvar", "histogram_stddev", "union", "": - // metric expressions for these functions cannot be optimized. - return nil - } - for _, arg := range fe.Args { - if me, ok := arg.(*MetricExpr); ok { - // transform_func(foo{bar="baz"}) - return me - } - } - return nil - } - return nil + return -1 } -func isNumberOrScalar(e Expr) bool { - if _, ok := e.(*NumberExpr); ok { +func getAggrArgIdxForOptimization(funcName string, args []Expr) int { + switch strings.ToLower(funcName) { + case "bottomk", "bottomk_avg", "bottomk_max", "bottomk_median", "bottomk_last", "bottomk_min", + "limitk", "outliers_mad", "outliersk", "quantile", + "topk", "topk_avg", "topk_max", "topk_median", "topk_last", "topk_min": + return 1 + case "count_values": + return -1 + case "quantiles": + return len(args) - 1 + default: + return 0 + } +} + +func getRollupArgIdxForOptimization(funcName string, args []Expr) int { + // This must be kept in sync with GetRollupArgIdx() + switch strings.ToLower(funcName) { + case "absent_over_time": + return -1 + case "quantile_over_time", "aggr_over_time", + "hoeffding_bound_lower", "hoeffding_bound_upper": + return 1 + case "quantiles_over_time": + return len(args) - 1 + default: + return 0 + } +} + +func getTransformArgIdxForOptimization(funcName string, args []Expr) int { + funcName = strings.ToLower(funcName) + if isLabelManipulationFunc(funcName) { + return -1 + } + switch funcName { + case "", "absent", "scalar", "union": + return -1 + case "end", "now", "pi", "ru", "start", "step", "time": + return -1 + case "limit_offset": + return 2 + case "buckets_limit", "histogram_quantile", "histogram_share", "range_quantile": + return 1 + case "histogram_quantiles": + return len(args) - 1 + default: + return 0 + } +} + +func isLabelManipulationFunc(funcName string) bool { + switch strings.ToLower(funcName) { + case "alias", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", + "label_map", "label_match", "label_mismatch", "label_move", "label_replace", "label_set", "label_transform", + "label_uppercase", "label_value": return true + default: + return false } - if fe, ok := e.(*FuncExpr); ok && strings.ToLower(fe.Name) == "scalar" { - return true - } - return false -} - -func intersectLabelFilters(a, b []LabelFilter) []LabelFilter { - m := make(map[string]LabelFilter, len(a)+len(b)) - var buf []byte - for _, lf := range a { - buf = lf.AppendString(buf[:0]) - m[string(buf)] = lf - } - for _, lf := range b { - buf = lf.AppendString(buf[:0]) - m[string(buf)] = lf - } - ss := make([]string, 0, len(m)) - for s := range m { - ss = append(ss, s) - } - sort.Strings(ss) - lfs := make([]LabelFilter, 0, len(ss)) - for _, s := range ss { - lfs = append(lfs, m[s]) - } - return lfs } diff --git a/vendor/github.com/VictoriaMetrics/metricsql/parser.go b/vendor/github.com/VictoriaMetrics/metricsql/parser.go index 1a39ec2b5..403c40d09 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/parser.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/parser.go @@ -1836,7 +1836,7 @@ func (lf *LabelFilter) AppendString(dst []byte) []byte { // MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. type MetricExpr struct { // LabelFilters contains a list of label filters from curly braces. - // Metric name if present must be the first. + // Filter or metric name must be the first if present. LabelFilters []LabelFilter // labelFilters must be expanded to LabelFilters by expandWithExpr. @@ -1884,6 +1884,9 @@ func (me *MetricExpr) hasNonEmptyMetricGroup() bool { if len(me.LabelFilters) == 0 { return false } - lf := &me.LabelFilters[0] + return me.LabelFilters[0].isMetricNameFilter() +} + +func (lf *LabelFilter) isMetricNameFilter() bool { return lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp } diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index c121f9dc5..7fbd5eb8c 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -90,8 +90,7 @@ func IsRollupFunc(funcName string) bool { // // -1 is returned if fe isn't a rollup function. func GetRollupArgIdx(fe *FuncExpr) int { - funcName := fe.Name - funcName = strings.ToLower(funcName) + funcName := strings.ToLower(fe.Name) if !rollupFuncs[funcName] { return -1 } diff --git a/vendor/modules.txt b/vendor/modules.txt index b4c31e9a9..4fbb8a013 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -26,7 +26,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.1 ## explicit; go 1.12 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.37.0 +# github.com/VictoriaMetrics/metricsql v0.38.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop