vendor: update github.com/VictoriaMetrics/metricsql from v0.37.0 to v0.38.0

This adds more optimization cases for https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization

For example:

* Multi-level transform functions. For example, abs(round(foo{a="b"})) + bar{x="y"}
  is now optimized to abs(round(foo{a="b",x="y"})) + bar{a="b",x="y"}
* Binary operations with `on()`, `without()`, `group_left()` and `group_right()` modifiers.
  For example, foo{a="b"} on (a) + bar is now optimized to foo{a="b"} on (a) + bar{a="b"}
* Multi-level binary operations. For example, foo{a="b"} + bar{x="y"} + baz{z="q"}
  is now optimized to foo{a="b",x="y",z="q"} + bar{a="b",x="y",z="q"} + baz{a="b",x="y",z="q"}
* Aggregate functions. For example, sum(foo{a="b"}) by (c) + bar{c="d"}
  is now optimized to sum(foo{a="b",c="d"}) by (c) + bar{c="d"}

docs/CHANGELOG.md

@@ -6,6 +6,12 @@ sort: 15
 
 ## tip
 
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): cover more cases with the [label filters' propagation optimization](https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization). This should improve the average performance for practical queries. The following cases are additionally covered:
+  * Multi-level [transform functions](https://docs.victoriametrics.com/MetricsQL.html#transform-functions). For example, `abs(round(foo{a="b"})) + bar{x="y"}` is now optimized to `abs(round(foo{a="b",x="y"})) + bar{a="b",x="y"}`
+  * Binary operations with `on()`, `without()`, `group_left()` and `group_right()` modifiers. For example, `foo{a="b"} on (a) + bar` is now optimized to `foo{a="b"} on (a) + bar{a="b"}`
+  * Multi-level binary operations. For example, `foo{a="b"} + bar{x="y"} + baz{z="q"}` is now optimized to `foo{a="b",x="y",z="q"} + bar{a="b",x="y",z="q"} + baz{a="b",x="y",z="q"}`
+  * Aggregate functions. For example, `sum(foo{a="b"}) by (c) + bar{c="d"}` is now optimized to `sum(foo{a="b",c="d"}) by (c) + bar{c="d"}`
+
 * BUGFIX: return proper results from `highestMax()` function at [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage). Previously it was incorrectly returning timeseries with min peaks instead of max peaks.
 * BUGFIX: properly limit indexdb cache sizes. Previously they could exceed values set via `-memory.allowedPercent` and/or `-memory.allowedBytes` when `indexdb` contained many data parts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a bug, which could break time range picker when editing `From` or `To` input fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2080).

go.mod

@@ -10,7 +10,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.1.0
 	github.com/VictoriaMetrics/metrics v1.18.1
-	github.com/VictoriaMetrics/metricsql v0.37.0
+	github.com/VictoriaMetrics/metricsql v0.38.0
 	github.com/aws/aws-sdk-go v1.42.42
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb/v3 v3.0.8

go.sum

@@ -115,8 +115,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a
 github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ=
 github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
 github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
-github.com/VictoriaMetrics/metricsql v0.37.0 h1:zFKC+XJpEhp0TtTa6pD0pnyg9sDLH4U5nCeDUT8eUAw=
-github.com/VictoriaMetrics/metricsql v0.37.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
+github.com/VictoriaMetrics/metricsql v0.38.0 h1:YBAzxKyr2QLFXYap8Nd0bxIr0e8mE/aUIyBYgDFMpK4=
+github.com/VictoriaMetrics/metricsql v0.38.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=

vendor/github.com/VictoriaMetrics/metricsql/optimizer.go

@@ -14,45 +14,194 @@ import (
 //   I.e. such query is converted to `foo{filters1, filters2} op bar{filters1, filters2}`
 func Optimize(e Expr) Expr {
 	switch t := e.(type) {
-	case *BinaryOpExpr:
-		// Convert `foo{filters1} op bar{filters2}` to `foo{filters1, filters2} op bar{filters1, filters2}`.
-		// This should reduce the number of operations
-		// See https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization
-		// for details.
-		if !canOptimizeBinaryOp(t) {
-			return optimizeBinaryOpArgs(t)
-		}
-		meLeft := getMetricExprForOptimization(t.Left)
-		if meLeft == nil || !meLeft.hasNonEmptyMetricGroup() {
-			return optimizeBinaryOpArgs(t)
-		}
-		meRight := getMetricExprForOptimization(t.Right)
-		if meRight == nil || !meRight.hasNonEmptyMetricGroup() {
-			return optimizeBinaryOpArgs(t)
-		}
-		lfs := intersectLabelFilters(meLeft.LabelFilters[1:], meRight.LabelFilters[1:])
-		meLeft.LabelFilters = append(meLeft.LabelFilters[:1], lfs...)
-		meRight.LabelFilters = append(meRight.LabelFilters[:1], lfs...)
-		return t
+	case *RollupExpr:
+		t.Expr = Optimize(t.Expr)
+		t.At = Optimize(t.At)
 	case *FuncExpr:
-		for i := range t.Args {
-			t.Args[i] = Optimize(t.Args[i])
-		}
-		return t
+		optimizeFuncArgs(t.Args)
 	case *AggrFuncExpr:
-		for i := range t.Args {
-			t.Args[i] = Optimize(t.Args[i])
+		optimizeFuncArgs(t.Args)
+	case *BinaryOpExpr:
+		t.Left = Optimize(t.Left)
+		t.Right = Optimize(t.Right)
+		lfs := getCommonLabelFilters(t)
+		pushdownLabelFilters(t, lfs)
 	}
-		return t
-	default:
 	return e
 }
+
+func optimizeFuncArgs(args []Expr) {
+	for i := range args {
+		args[i] = Optimize(args[i])
+	}
 }
 
-func canOptimizeBinaryOp(be *BinaryOpExpr) bool {
-	if be.JoinModifier.Op != "" || be.GroupModifier.Op != "" {
-		return false
+func getCommonLabelFilters(e Expr) []LabelFilter {
+	switch t := e.(type) {
+	case *MetricExpr:
+		return getLabelFiltersWithoutMetricName(t.LabelFilters)
+	case *RollupExpr:
+		return getCommonLabelFilters(t.Expr)
+	case *FuncExpr:
+		arg := getFuncArgForOptimization(t.Name, t.Args)
+		if arg == nil {
+			return nil
 		}
+		return getCommonLabelFilters(arg)
+	case *AggrFuncExpr:
+		arg := getFuncArgForOptimization(t.Name, t.Args)
+		if arg == nil {
+			return nil
+		}
+		lfs := getCommonLabelFilters(arg)
+		return filterLabelFiltersByAggrModifier(lfs, t)
+	case *BinaryOpExpr:
+		if !canOptimizeBinaryOp(t) {
+			return nil
+		}
+		lfsLeft := getCommonLabelFilters(t.Left)
+		lfsRight := getCommonLabelFilters(t.Right)
+		lfs := unionLabelFilters(lfsLeft, lfsRight)
+		return filterLabelFiltersByGroupModifier(lfs, t)
+	default:
+		return nil
+	}
+}
+
+func filterLabelFiltersByAggrModifier(lfs []LabelFilter, afe *AggrFuncExpr) []LabelFilter {
+	switch strings.ToLower(afe.Modifier.Op) {
+	case "by":
+		return filterLabelFiltersOn(lfs, afe.Modifier.Args)
+	case "without":
+		return filterLabelFiltersIgnoring(lfs, afe.Modifier.Args)
+	default:
+		return nil
+	}
+}
+
+func filterLabelFiltersByGroupModifier(lfs []LabelFilter, be *BinaryOpExpr) []LabelFilter {
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		return filterLabelFiltersOn(lfs, be.GroupModifier.Args)
+	case "ignoring":
+		return filterLabelFiltersIgnoring(lfs, be.GroupModifier.Args)
+	default:
+		return lfs
+	}
+}
+
+func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter {
+	lfsNew := make([]LabelFilter, 0, len(lfs))
+	for _, lf := range lfs {
+		if lf.Label != "__name__" {
+			lfsNew = append(lfsNew, lf)
+		}
+	}
+	return lfsNew
+}
+
+func pushdownLabelFilters(e Expr, lfs []LabelFilter) {
+	if len(lfs) == 0 {
+		return
+	}
+	switch t := e.(type) {
+	case *MetricExpr:
+		t.LabelFilters = unionLabelFilters(t.LabelFilters, lfs)
+		sortLabelFilters(t.LabelFilters)
+	case *RollupExpr:
+		pushdownLabelFilters(t.Expr, lfs)
+	case *FuncExpr:
+		arg := getFuncArgForOptimization(t.Name, t.Args)
+		if arg != nil {
+			pushdownLabelFilters(arg, lfs)
+		}
+	case *AggrFuncExpr:
+		lfs = filterLabelFiltersByAggrModifier(lfs, t)
+		arg := getFuncArgForOptimization(t.Name, t.Args)
+		if arg != nil {
+			pushdownLabelFilters(arg, lfs)
+		}
+	case *BinaryOpExpr:
+		if canOptimizeBinaryOp(t) {
+			lfs = filterLabelFiltersByGroupModifier(lfs, t)
+			pushdownLabelFilters(t.Left, lfs)
+			pushdownLabelFilters(t.Right, lfs)
+		}
+	}
+}
+
+func unionLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter {
+	if len(lfsA) == 0 {
+		return lfsB
+	}
+	if len(lfsB) == 0 {
+		return lfsA
+	}
+	m := make(map[string]struct{}, len(lfsA))
+	var b []byte
+	for _, lf := range lfsA {
+		b = lf.AppendString(b[:0])
+		m[string(b)] = struct{}{}
+	}
+	lfs := append([]LabelFilter{}, lfsA...)
+	for _, lf := range lfsB {
+		b = lf.AppendString(b[:0])
+		if _, ok := m[string(b)]; !ok {
+			lfs = append(lfs, lf)
+		}
+	}
+	return lfs
+}
+
+func sortLabelFilters(lfs []LabelFilter) {
+	// Make sure the first label filter is __name__ (if any)
+	if len(lfs) > 0 && lfs[0].isMetricNameFilter() {
+		lfs = lfs[1:]
+	}
+	sort.Slice(lfs, func(i, j int) bool {
+		a, b := lfs[i], lfs[j]
+		if a.Label != b.Label {
+			return a.Label < b.Label
+		}
+		return a.Value < b.Value
+	})
+}
+
+func filterLabelFiltersOn(lfs []LabelFilter, args []string) []LabelFilter {
+	if len(args) == 0 {
+		return nil
+	}
+	m := make(map[string]struct{}, len(args))
+	for _, arg := range args {
+		m[arg] = struct{}{}
+	}
+	var lfsNew []LabelFilter
+	for _, lf := range lfs {
+		if _, ok := m[lf.Label]; ok {
+			lfsNew = append(lfsNew, lf)
+		}
+	}
+	return lfsNew
+}
+
+func filterLabelFiltersIgnoring(lfs []LabelFilter, args []string) []LabelFilter {
+	if len(args) == 0 {
+		return lfs
+	}
+	m := make(map[string]struct{}, len(args))
+	for _, arg := range args {
+		m[arg] = struct{}{}
+	}
+	var lfsNew []LabelFilter
+	for _, lf := range lfs {
+		if _, ok := m[lf.Label]; !ok {
+			lfsNew = append(lfsNew, lf)
+		}
+	}
+	return lfsNew
+}
+
+func canOptimizeBinaryOp(be *BinaryOpExpr) bool {
 	switch be.Op {
 	case "+", "-", "*", "/", "%", "^",
 		"==", "!=", ">", "<", ">=", "<=",
@@ -63,99 +212,86 @@ func canOptimizeBinaryOp(be *BinaryOpExpr) bool {
 	}
 }
 
-func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr {
-	be.Left = Optimize(be.Left)
-	be.Right = Optimize(be.Right)
-	return be
+func getFuncArgForOptimization(funcName string, args []Expr) Expr {
+	idx := getFuncArgIdxForOptimization(funcName, args)
+	if idx < 0 || idx >= len(args) {
+		return nil
+	}
+	return args[idx]
 }
 
-func getMetricExprForOptimization(e Expr) *MetricExpr {
-	re, ok := e.(*RollupExpr)
-	if ok {
-		// Try optimizing the inner expression in RollupExpr.
-		return getMetricExprForOptimization(re.Expr)
+func getFuncArgIdxForOptimization(funcName string, args []Expr) int {
+	funcName = strings.ToLower(funcName)
+	if IsRollupFunc(funcName) {
+		return getRollupArgIdxForOptimization(funcName, args)
 	}
-	me, ok := e.(*MetricExpr)
-	if ok {
-		// Ordinary metric expression, i.e. `foo{bar="baz"}`
-		return me
+	if IsTransformFunc(funcName) {
+		return getTransformArgIdxForOptimization(funcName, args)
 	}
-	be, ok := e.(*BinaryOpExpr)
-	if ok {
-		if !canOptimizeBinaryOp(be) {
-			return nil
+	if isAggrFunc(funcName) {
+		return getAggrArgIdxForOptimization(funcName, args)
 	}
-		if me, ok := be.Left.(*MetricExpr); ok && isNumberOrScalar(be.Right) {
-			// foo{bar="baz"} * num_or_scalar
-			return me
-		}
-		if me, ok := be.Right.(*MetricExpr); ok && isNumberOrScalar(be.Left) {
-			// num_or_scalar * foo{bar="baz"}
-			return me
-		}
-		return nil
-	}
-	fe, ok := e.(*FuncExpr)
-	if !ok {
-		return nil
-	}
-	if IsRollupFunc(fe.Name) {
-		argIdx := GetRollupArgIdx(fe)
-		if argIdx >= len(fe.Args) {
-			return nil
-		}
-		arg := fe.Args[argIdx]
-		return getMetricExprForOptimization(arg)
-	}
-	if IsTransformFunc(fe.Name) {
-		switch strings.ToLower(fe.Name) {
-		case "absent", "histogram_quantile", "label_join", "label_replace", "scalar", "vector",
-			"label_set", "label_map", "label_uppercase", "label_lowercase", "label_del", "label_keep", "label_copy",
-			"label_move", "label_transform", "label_value", "label_match", "label_mismatch", "label_graphite_group",
-			"prometheus_buckets", "buckets_limit", "histogram_share", "histogram_avg", "histogram_stdvar", "histogram_stddev", "union", "":
-			// metric expressions for these functions cannot be optimized.
-			return nil
-		}
-		for _, arg := range fe.Args {
-			if me, ok := arg.(*MetricExpr); ok {
-				// transform_func(foo{bar="baz"})
-				return me
-			}
-		}
-		return nil
-	}
-	return nil
+	return -1
 }
 
-func isNumberOrScalar(e Expr) bool {
-	if _, ok := e.(*NumberExpr); ok {
-		return true
+func getAggrArgIdxForOptimization(funcName string, args []Expr) int {
+	switch strings.ToLower(funcName) {
+	case "bottomk", "bottomk_avg", "bottomk_max", "bottomk_median", "bottomk_last", "bottomk_min",
+		"limitk", "outliers_mad", "outliersk", "quantile",
+		"topk", "topk_avg", "topk_max", "topk_median", "topk_last", "topk_min":
+		return 1
+	case "count_values":
+		return -1
+	case "quantiles":
+		return len(args) - 1
+	default:
+		return 0
 	}
-	if fe, ok := e.(*FuncExpr); ok && strings.ToLower(fe.Name) == "scalar" {
-		return true
 }
+
+func getRollupArgIdxForOptimization(funcName string, args []Expr) int {
+	// This must be kept in sync with GetRollupArgIdx()
+	switch strings.ToLower(funcName) {
+	case "absent_over_time":
+		return -1
+	case "quantile_over_time", "aggr_over_time",
+		"hoeffding_bound_lower", "hoeffding_bound_upper":
+		return 1
+	case "quantiles_over_time":
+		return len(args) - 1
+	default:
+		return 0
+	}
+}
+
+func getTransformArgIdxForOptimization(funcName string, args []Expr) int {
+	funcName = strings.ToLower(funcName)
+	if isLabelManipulationFunc(funcName) {
+		return -1
+	}
+	switch funcName {
+	case "", "absent", "scalar", "union":
+		return -1
+	case "end", "now", "pi", "ru", "start", "step", "time":
+		return -1
+	case "limit_offset":
+		return 2
+	case "buckets_limit", "histogram_quantile", "histogram_share", "range_quantile":
+		return 1
+	case "histogram_quantiles":
+		return len(args) - 1
+	default:
+		return 0
+	}
+}
+
+func isLabelManipulationFunc(funcName string) bool {
+	switch strings.ToLower(funcName) {
+	case "alias", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase",
+		"label_map", "label_match", "label_mismatch", "label_move", "label_replace", "label_set", "label_transform",
+		"label_uppercase", "label_value":
+		return true
+	default:
 		return false
 	}
-
-func intersectLabelFilters(a, b []LabelFilter) []LabelFilter {
-	m := make(map[string]LabelFilter, len(a)+len(b))
-	var buf []byte
-	for _, lf := range a {
-		buf = lf.AppendString(buf[:0])
-		m[string(buf)] = lf
-	}
-	for _, lf := range b {
-		buf = lf.AppendString(buf[:0])
-		m[string(buf)] = lf
-	}
-	ss := make([]string, 0, len(m))
-	for s := range m {
-		ss = append(ss, s)
-	}
-	sort.Strings(ss)
-	lfs := make([]LabelFilter, 0, len(ss))
-	for _, s := range ss {
-		lfs = append(lfs, m[s])
-	}
-	return lfs
 }

vendor/github.com/VictoriaMetrics/metricsql/parser.go

@@ -1836,7 +1836,7 @@ func (lf *LabelFilter) AppendString(dst []byte) []byte {
 // MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`.
 type MetricExpr struct {
 	// LabelFilters contains a list of label filters from curly braces.
-	// Metric name if present must be the first.
+	// Filter or metric name must be the first if present.
 	LabelFilters []LabelFilter
 
 	// labelFilters must be expanded to LabelFilters by expandWithExpr.
@@ -1884,6 +1884,9 @@ func (me *MetricExpr) hasNonEmptyMetricGroup() bool {
 	if len(me.LabelFilters) == 0 {
 		return false
 	}
-	lf := &me.LabelFilters[0]
+	return me.LabelFilters[0].isMetricNameFilter()
+}
+
+func (lf *LabelFilter) isMetricNameFilter() bool {
 	return lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp
 }

vendor/github.com/VictoriaMetrics/metricsql/rollup.go

@@ -90,8 +90,7 @@ func IsRollupFunc(funcName string) bool {
 //
 // -1 is returned if fe isn't a rollup function.
 func GetRollupArgIdx(fe *FuncExpr) int {
-	funcName := fe.Name
-	funcName = strings.ToLower(funcName)
+	funcName := strings.ToLower(fe.Name)
 	if !rollupFuncs[funcName] {
 		return -1
 	}

vendor/modules.txt

@@ -26,7 +26,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.18.1
 ## explicit; go 1.12
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.37.0
+# github.com/VictoriaMetrics/metricsql v0.38.0
 ## explicit; go 1.13
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop