diff --git a/go.mod b/go.mod index 68f0c3d64..3c2e48268 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.1 - github.com/VictoriaMetrics/metricsql v0.39.0 + github.com/VictoriaMetrics/metricsql v0.40.0 github.com/aws/aws-sdk-go v1.42.44 github.com/cespare/xxhash/v2 v2.1.2 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index ea6f537a7..77165a794 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.39.0 h1:tm1hneyxVhm1oeJ/1T4007Y5Bn+LKN+Aw3l6XGwvgRM= -github.com/VictoriaMetrics/metricsql v0.39.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.40.0 h1:QDzuhzsP2cZJyrijIptDJ6gnxd3qWGzQi4Fhj8mOLHo= +github.com/VictoriaMetrics/metricsql v0.40.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index 88a54876e..36d4553b4 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -95,13 +95,58 @@ func getCommonLabelFilters(e Expr) []LabelFilter { lfs := getCommonLabelFilters(arg) return trimFiltersByAggrModifier(lfs, t) case *BinaryOpExpr: - if !canOptimizeBinaryOp(t) { - return nil - } lfsLeft := getCommonLabelFilters(t.Left) lfsRight := getCommonLabelFilters(t.Right) - lfs := unionLabelFilters(lfsLeft, lfsRight) - return TrimFiltersByGroupModifier(lfs, t) + var lfs []LabelFilter + switch strings.ToLower(t.Op) { + case "or": + // {fCommon, f1} or {fCommon, f2} -> {fCommon} + // {fCommon, f1} or on() {fCommon, f2} -> {} + // {fCommon, f1} or on(fCommon) {fCommon, f2} -> {fCommon} + // {fCommon, f1} or on(f1) {fCommon, f2} -> {} + // {fCommon, f1} or on(f2) {fCommon, f2} -> {} + // {fCommon, f1} or on(f3) {fCommon, f2} -> {} + lfs = intersectLabelFilters(lfsLeft, lfsRight) + return TrimFiltersByGroupModifier(lfs, t) + case "unless": + // {f1} unless {f2} -> {f1} + // {f1} unless on() {f2} -> {} + // {f1} unless on(f1) {f2} -> {f1} + // {f1} unless on(f2) {f2} -> {} + // {f1} unless on(f1, f2) {f2} -> {f1} + // {f1} unless on(f3) {f2} -> {} + return TrimFiltersByGroupModifier(lfsLeft, t) + default: + switch strings.ToLower(t.JoinModifier.Op) { + case "group_left": + // {f1} * group_left() {f2} -> {f1, f2} + // {f1} * on() group_left() {f2} -> {f1} + // {f1} * on(f1) group_left() {f2} -> {f1} + // {f1} * on(f2) group_left() {f2} -> {f1, f2} + // {f1} * on(f1, f2) group_left() {f2} -> {f1, f2} + // {f1} * on(f3) group_left() {f2} -> {f1} + lfsRight = TrimFiltersByGroupModifier(lfsRight, t) + return unionLabelFilters(lfsLeft, lfsRight) + case "group_right": + // {f1} * group_right() {f2} -> {f1, f2} + // {f1} * on() group_right() {f2} -> {f2} + // {f1} * on(f1) group_right() {f2} -> {f1, f2} + // {f1} * on(f2) group_right() {f2} -> {f2} + // {f1} * on(f1, f2) group_right() {f2} -> {f1, f2} + // {f1} * on(f3) group_right() {f2} -> {f2} + lfsLeft = TrimFiltersByGroupModifier(lfsLeft, t) + return unionLabelFilters(lfsLeft, lfsRight) + default: + // {f1} * {f2} -> {f1, f2} + // {f1} * on() {f2} -> {} + // {f1} * on(f1) {f2} -> {f1} + // {f1} * on(f2) {f2} -> {f2} + // {f1} * on(f1, f2) {f2} -> {f2} + // {f1} * on(f3} {f2} -> {} + lfs = unionLabelFilters(lfsLeft, lfsRight) + return TrimFiltersByGroupModifier(lfs, t) + } + } default: return nil } @@ -184,12 +229,26 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) { pushdownBinaryOpFiltersInplace(arg, lfs) } case *BinaryOpExpr: - if canOptimizeBinaryOp(t) { - lfs = TrimFiltersByGroupModifier(lfs, t) - pushdownBinaryOpFiltersInplace(t.Left, lfs) - pushdownBinaryOpFiltersInplace(t.Right, lfs) + lfs = TrimFiltersByGroupModifier(lfs, t) + pushdownBinaryOpFiltersInplace(t.Left, lfs) + pushdownBinaryOpFiltersInplace(t.Right, lfs) + } +} + +func intersectLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { + if len(lfsA) == 0 || len(lfsB) == 0 { + return nil + } + m := getLabelFiltersMap(lfsA) + var b []byte + var lfs []LabelFilter + for _, lf := range lfsB { + b = lf.AppendString(b[:0]) + if _, ok := m[string(b)]; ok { + lfs = append(lfs, lf) } } + return lfs } func unionLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { @@ -199,12 +258,8 @@ func unionLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { if len(lfsB) == 0 { return lfsA } - m := make(map[string]struct{}, len(lfsA)) + m := getLabelFiltersMap(lfsA) var b []byte - for _, lf := range lfsA { - b = lf.AppendString(b[:0]) - m[string(b)] = struct{}{} - } lfs := append([]LabelFilter{}, lfsA...) for _, lf := range lfsB { b = lf.AppendString(b[:0]) @@ -215,6 +270,16 @@ func unionLabelFilters(lfsA, lfsB []LabelFilter) []LabelFilter { return lfs } +func getLabelFiltersMap(lfs []LabelFilter) map[string]struct{} { + m := make(map[string]struct{}, len(lfs)) + var b []byte + for _, lf := range lfs { + b = lf.AppendString(b[:0]) + m[string(b)] = struct{}{} + } + return m +} + func sortLabelFilters(lfs []LabelFilter) { // Make sure the first label filter is __name__ (if any) if len(lfs) > 0 && lfs[0].isMetricNameFilter() { @@ -263,17 +328,6 @@ func filterLabelFiltersIgnoring(lfs []LabelFilter, args []string) []LabelFilter return lfsNew } -func canOptimizeBinaryOp(be *BinaryOpExpr) bool { - switch be.Op { - case "+", "-", "*", "/", "%", "^", - "==", "!=", ">", "<", ">=", "<=", - "and", "if", "ifnot", "default": - return true - default: - return false - } -} - func getFuncArgForOptimization(funcName string, args []Expr) Expr { idx := getFuncArgIdxForOptimization(funcName, args) if idx < 0 || idx >= len(args) { @@ -332,7 +386,7 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int { return -1 } switch funcName { - case "", "absent", "scalar", "union": + case "", "absent", "scalar", "union", "vector": return -1 case "end", "now", "pi", "ru", "start", "step", "time": return -1 diff --git a/vendor/modules.txt b/vendor/modules.txt index 00c0597cd..c9e4cf0a8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -26,7 +26,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.1 ## explicit; go 1.12 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.39.0 +# github.com/VictoriaMetrics/metricsql v0.40.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop