diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c6535c5ac..b9e66ea7c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.tlsHandshakeTimeout` command-line flag for tuning the timeout needed for establishing TLS connections to `-remoteWrite.url`. Setting bigger tls handshake timeouts should reduce the probability of `http: TLS handshake error from ...: EOF` errors at the remote storage side under high load. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1699). * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add `-disableReroutingOnUnavailable` command-line flag to `vminsert`, which can be used for reducing resource usage spikes at `vmstorage` nodes during rolling restart. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#improving-re-routing-performance-during-restart). Thanks to @Muxa1L for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5713). * FEATURE: add `-search.resetRollupResultCacheOnStartup` command-line flag for resetting [query cache](https://docs.victoriametrics.com/#rollup-result-cache) on startup. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/834). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): propagate label filters across [label_set](https://docs.victoriametrics.com/MetricsQL.html#label_set) and [alias](https://docs.victoriametrics.com/MetricsQL.html#alias) functions. For example, `label_set(q1, "a", "b") + q2{c="d"}` is automatically transformed to `label_set(q1{c="d"}, "a", "b") + q2{a="b",c="d"}` now. This should improve performance for such queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1827#issuecomment-1654095358). * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Targets scraped/s` stat panel showing the number of targets scraped by the vmagent per-second. * FEATURE: [dashboards/all](https://grafana.com/orgs/victoriametrics): add new panel `CPU spent on GC`. It should help identifying cases when too much CPU is spent on garbage collection, and advice users on how this can be addressed. * FEATURE: [vmalert](https://docs.victoriametrics.com/#vmalert): support [filtering](https://prometheus.io/docs/prometheus/2.49/querying/api/#rules) for `/api/v1/rules` API. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5749) by @victoramsantos. diff --git a/go.mod b/go.mod index dc25e98b5..0887126cb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/VictoriaMetrics/easyproto v0.1.4 github.com/VictoriaMetrics/fastcache v1.12.2 github.com/VictoriaMetrics/metrics v1.31.0 - github.com/VictoriaMetrics/metricsql v0.70.1 + github.com/VictoriaMetrics/metricsql v0.71.0 github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/config v1.26.6 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 diff --git a/go.sum b/go.sum index 628611c7a..fed27175d 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= github.com/VictoriaMetrics/metrics v1.31.0 h1:X6+nBvAP0UB+GjR0Ht9hhQ3pjL1AN4b8dt9zFfzTsUo= github.com/VictoriaMetrics/metrics v1.31.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= -github.com/VictoriaMetrics/metricsql v0.70.1 h1:9WneeSk9HAGf9E8qZmuPnE3KaflBpVkrLjpPtbAlzoU= -github.com/VictoriaMetrics/metricsql v0.70.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.71.0 h1:i6uJPTPY2CTGvuWlxD0+l9HrhPHFbRPqvEopp6YzCSI= +github.com/VictoriaMetrics/metricsql v0.71.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index f1e04ab5b..507e93160 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -82,6 +82,9 @@ func getCommonLabelFilters(e Expr) []LabelFilter { case *RollupExpr: return getCommonLabelFilters(t.Expr) case *FuncExpr: + if strings.ToLower(t.Name) == "label_set" { + return getCommonLabelFiltersForLabelSet(t.Args) + } arg := getFuncArgForOptimization(t.Name, t.Args) if arg == nil { return nil @@ -161,6 +164,46 @@ func getCommonLabelFilters(e Expr) []LabelFilter { } } +func getCommonLabelFiltersForLabelSet(args []Expr) []LabelFilter { + if len(args) == 0 { + return nil + } + lfs := getCommonLabelFilters(args[0]) + args = args[1:] + for i := 0; i < len(args); i += 2 { + labelName := args[i] + if i+1 >= len(args) { + return nil + } + labelValue := args[i+1] + + seLabelName, ok := labelName.(*StringExpr) + if !ok { + return nil + } + seLabelValue, ok := labelValue.(*StringExpr) + if !ok { + return nil + } + + if seLabelName.S == "__name__" { + continue + } + + lfsDst := lfs[:0] + for _, lf := range lfs { + if lf.Label != seLabelName.S { + lfsDst = append(lfsDst, lf) + } + } + lfs = append(lfsDst, LabelFilter{ + Label: seLabelName.S, + Value: seLabelValue.S, + }) + } + return lfs +} + func trimFiltersByAggrModifier(lfs []LabelFilter, afe *AggrFuncExpr) []LabelFilter { switch strings.ToLower(afe.Modifier.Op) { case "by": @@ -245,9 +288,15 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) { case *RollupExpr: pushdownBinaryOpFiltersInplace(t.Expr, lfs) case *FuncExpr: - arg := getFuncArgForOptimization(t.Name, t.Args) - if arg != nil { + if strings.ToLower(t.Name) == "label_set" && len(t.Args) > 0 { + arg := t.Args[0] + lfs = getPushdownLabelFiltersForLabelSet(t.Args[1:], lfs) pushdownBinaryOpFiltersInplace(arg, lfs) + } else { + arg := getFuncArgForOptimization(t.Name, t.Args) + if arg != nil { + pushdownBinaryOpFiltersInplace(arg, lfs) + } } case *AggrFuncExpr: lfs = trimFiltersByAggrModifier(lfs, t) @@ -269,6 +318,26 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) { } } +func getPushdownLabelFiltersForLabelSet(args []Expr, lfs []LabelFilter) []LabelFilter { + m := make(map[string]struct{}) + for i := 0; i < len(args); i += 2 { + labelName := args[i] + seLabelName, ok := labelName.(*StringExpr) + if !ok { + return nil + } + m[seLabelName.S] = struct{}{} + } + + var lfsDst []LabelFilter + for _, lf := range lfs { + if _, ok := m[lf.Label]; !ok { + lfsDst = append(lfsDst, lf) + } + } + return lfsDst +} + func intersectLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { if len(lfsA) == 0 || len(lfsB) == 0 { return nil @@ -428,11 +497,7 @@ func getRollupArgIdxForOptimization(funcName string, args []Expr) int { } func getTransformArgIdxForOptimization(funcName string, args []Expr) int { - funcName = strings.ToLower(funcName) - if isLabelManipulationFunc(funcName) { - return -1 - } - switch funcName { + switch strings.ToLower(funcName) { case "", "absent", "scalar", "union", "vector", "range_normalize": return -1 case "end", "now", "pi", "ru", "start", "step", "time": @@ -444,20 +509,10 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int { return 1 case "histogram_quantiles": return len(args) - 1 + case "drop_common_labels", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", + "label_map", "label_move", "label_replace", "label_set", "label_transform", "label_uppercase": + return -1 default: return 0 } } - -func isLabelManipulationFunc(funcName string) bool { - switch strings.ToLower(funcName) { - case "alias", "drop_common_labels", "label_copy", "label_del", "label_graphite_group", "label_join", "label_keep", "label_lowercase", - "label_map", "label_move", "label_replace", "label_set", "label_transform", "label_uppercase": - return true - case "label_match", "label_mismatch", "label_value", "labels_equal": - // These functions aren't really label manipulation functions, since they do not change labels for the input series. - return false - default: - return false - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index dfe79cd47..05a750d03 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -102,7 +102,7 @@ github.com/VictoriaMetrics/fastcache # github.com/VictoriaMetrics/metrics v1.31.0 ## explicit; go 1.17 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.70.1 +# github.com/VictoriaMetrics/metricsql v0.71.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop