diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 35797ba1a..4260359ad 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -15,6 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" "github.com/valyala/quicktemplate" @@ -634,14 +635,14 @@ func parseDuration(s string, step int64) (int64, error) { if len(s) == 0 { return 0, nil } - return promql.DurationValue(s, step) + return metricsql.DurationValue(s, step) } func parsePositiveDuration(s string, step int64) (int64, error) { if len(s) == 0 { return 0, nil } - return promql.PositiveDurationValue(s, step) + return metricsql.PositiveDurationValue(s, step) } // QueryRangeHandler processes /api/v1/query_range request. diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index ba8d7013e..684559049 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" "github.com/valyala/histogram" @@ -49,7 +49,7 @@ type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error) type aggrFuncArg struct { args [][]*timeseries - ae *promql.AggrFuncExpr + ae *metricsql.AggrFuncExpr ec *EvalConfig } @@ -68,7 +68,7 @@ func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc { } } -func removeGroupTags(metricName *storage.MetricName, modifier *promql.ModifierExpr) { +func removeGroupTags(metricName *storage.MetricName, modifier *metricsql.ModifierExpr) { groupOp := strings.ToLower(modifier.Op) switch groupOp { case "", "by": @@ -80,7 +80,7 @@ func removeGroupTags(metricName *storage.MetricName, modifier *promql.ModifierEx } } -func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *promql.ModifierExpr, keepOriginal bool) ([]*timeseries, error) { +func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *metricsql.ModifierExpr, keepOriginal bool) ([]*timeseries, error) { arg := copyTimeseriesMetricNames(argOrig) // Perform grouping. diff --git a/app/vmselect/promql/aggr_incremental.go b/app/vmselect/promql/aggr_incremental.go index c980816b6..985f35f56 100644 --- a/app/vmselect/promql/aggr_incremental.go +++ b/app/vmselect/promql/aggr_incremental.go @@ -5,11 +5,11 @@ import ( "strings" "sync" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" ) // callbacks for optimized incremental calculations for aggregate functions -// over rollups over metricExpr. +// over rollups over metricsql.MetricExpr. // // These calculations save RAM for aggregates over big number of time series. var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{ @@ -51,7 +51,7 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{ } type incrementalAggrFuncContext struct { - ae *promql.AggrFuncExpr + ae *metricsql.AggrFuncExpr mLock sync.Mutex m map[uint]map[string]*incrementalAggrContext @@ -59,7 +59,7 @@ type incrementalAggrFuncContext struct { callbacks *incrementalAggrFuncCallbacks } -func newIncrementalAggrFuncContext(ae *promql.AggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext { +func newIncrementalAggrFuncContext(ae *metricsql.AggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext { return &incrementalAggrFuncContext{ ae: ae, m: make(map[uint]map[string]*incrementalAggrContext), diff --git a/app/vmselect/promql/aggr_incremental_test.go b/app/vmselect/promql/aggr_incremental_test.go index 2030f171a..e31c9bee9 100644 --- a/app/vmselect/promql/aggr_incremental_test.go +++ b/app/vmselect/promql/aggr_incremental_test.go @@ -8,7 +8,7 @@ import ( "sync" "testing" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" ) func TestIncrementalAggr(t *testing.T) { @@ -44,7 +44,7 @@ func TestIncrementalAggr(t *testing.T) { f := func(name string, valuesExpected []float64) { t.Helper() callbacks := getIncrementalAggrFuncCallbacks(name) - ae := &promql.AggrFuncExpr{ + ae := &metricsql.AggrFuncExpr{ Name: name, } tssExpected := []*timeseries{{ diff --git a/app/vmselect/promql/alias.go b/app/vmselect/promql/alias.go deleted file mode 100644 index f9c2031ed..000000000 --- a/app/vmselect/promql/alias.go +++ /dev/null @@ -1,9 +0,0 @@ -package promql - -import ( - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" -) - -// DurationValue returns the duration in milliseconds for the given s -// and the given step. -var DurationValue = promql.DurationValue diff --git a/app/vmselect/promql/binary_op.go b/app/vmselect/promql/binary_op.go index 34ee59adc..fe4c39fab 100644 --- a/app/vmselect/promql/binary_op.go +++ b/app/vmselect/promql/binary_op.go @@ -6,25 +6,26 @@ import ( "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql/binaryop" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) var binaryOpFuncs = map[string]binaryOpFunc{ - "+": newBinaryOpArithFunc(binaryOpPlus), - "-": newBinaryOpArithFunc(binaryOpMinus), - "*": newBinaryOpArithFunc(binaryOpMul), - "/": newBinaryOpArithFunc(binaryOpDiv), - "%": newBinaryOpArithFunc(binaryOpMod), - "^": newBinaryOpArithFunc(binaryOpPow), + "+": newBinaryOpArithFunc(binaryop.Plus), + "-": newBinaryOpArithFunc(binaryop.Minus), + "*": newBinaryOpArithFunc(binaryop.Mul), + "/": newBinaryOpArithFunc(binaryop.Div), + "%": newBinaryOpArithFunc(binaryop.Mod), + "^": newBinaryOpArithFunc(binaryop.Pow), // cmp ops - "==": newBinaryOpCmpFunc(binaryOpEq), - "!=": newBinaryOpCmpFunc(binaryOpNeq), - ">": newBinaryOpCmpFunc(binaryOpGt), - "<": newBinaryOpCmpFunc(binaryOpLt), - ">=": newBinaryOpCmpFunc(binaryOpGte), - "<=": newBinaryOpCmpFunc(binaryOpLte), + "==": newBinaryOpCmpFunc(binaryop.Eq), + "!=": newBinaryOpCmpFunc(binaryop.Neq), + ">": newBinaryOpCmpFunc(binaryop.Gt), + "<": newBinaryOpCmpFunc(binaryop.Lt), + ">=": newBinaryOpCmpFunc(binaryop.Gte), + "<=": newBinaryOpCmpFunc(binaryop.Lte), // logical set ops "and": binaryOpAnd, @@ -32,9 +33,9 @@ var binaryOpFuncs = map[string]binaryOpFunc{ "unless": binaryOpUnless, // New op - "if": newBinaryOpArithFunc(binaryOpIf), - "ifnot": newBinaryOpArithFunc(binaryOpIfnot), - "default": newBinaryOpArithFunc(binaryOpDefault), + "if": newBinaryOpArithFunc(binaryop.If), + "ifnot": newBinaryOpArithFunc(binaryop.Ifnot), + "default": newBinaryOpArithFunc(binaryop.Default), } func getBinaryOpFunc(op string) binaryOpFunc { @@ -42,80 +43,8 @@ func getBinaryOpFunc(op string) binaryOpFunc { return binaryOpFuncs[op] } -func isBinaryOpCmp(op string) bool { - switch op { - case "==", "!=", ">", "<", ">=", "<=": - return true - default: - return false - } -} - -func binaryOpConstants(op string, left, right float64, isBool bool) float64 { - if isBinaryOpCmp(op) { - evalCmp := func(cf func(left, right float64) bool) float64 { - if isBool { - if cf(left, right) { - return 1 - } - return 0 - } - if cf(left, right) { - return left - } - return nan - } - switch op { - case "==": - left = evalCmp(binaryOpEq) - case "!=": - left = evalCmp(binaryOpNeq) - case ">": - left = evalCmp(binaryOpGt) - case "<": - left = evalCmp(binaryOpLt) - case ">=": - left = evalCmp(binaryOpGte) - case "<=": - left = evalCmp(binaryOpLte) - default: - logger.Panicf("BUG: unexpected comparison binaryOp: %q", op) - } - } else { - switch op { - case "+": - left = binaryOpPlus(left, right) - case "-": - left = binaryOpMinus(left, right) - case "*": - left = binaryOpMul(left, right) - case "/": - left = binaryOpDiv(left, right) - case "%": - left = binaryOpMod(left, right) - case "^": - left = binaryOpPow(left, right) - case "and": - // Nothing to do - case "or": - // Nothing to do - case "unless": - left = nan - case "default": - left = binaryOpDefault(left, right) - case "if": - left = binaryOpIf(left, right) - case "ifnot": - left = binaryOpIfnot(left, right) - default: - logger.Panicf("BUG: unexpected non-comparison binaryOp: %q", op) - } - } - return left -} - type binaryOpFuncArg struct { - be *promql.BinaryOpExpr + be *metricsql.BinaryOpExpr left []*timeseries right []*timeseries } @@ -175,7 +104,7 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp } } -func adjustBinaryOpTags(be *promql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) { +func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) { if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 { if isScalar(left) { // Fast path: `scalar op vector` @@ -256,7 +185,7 @@ func adjustBinaryOpTags(be *promql.BinaryOpExpr, left, right []*timeseries) ([]* return rvsLeft, rvsRight, dst, nil } -func ensureSingleTimeseries(side string, be *promql.BinaryOpExpr, tss []*timeseries) error { +func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error { if len(tss) == 0 { logger.Panicf("BUG: tss must contain at least one value") } @@ -270,7 +199,7 @@ func ensureSingleTimeseries(side string, be *promql.BinaryOpExpr, tss []*timeser return nil } -func groupJoin(singleTimeseriesSide string, be *promql.BinaryOpExpr, rvsLeft, rvsRight, tssLeft, tssRight []*timeseries) ([]*timeseries, []*timeseries, error) { +func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft, rvsRight, tssLeft, tssRight []*timeseries) ([]*timeseries, []*timeseries, error) { joinTags := be.JoinModifier.Args var m map[string]*timeseries for _, tsLeft := range tssLeft { @@ -340,8 +269,8 @@ func mergeNonOverlappingTimeseries(dst, src *timeseries) bool { return true } -func resetMetricGroupIfRequired(be *promql.BinaryOpExpr, ts *timeseries) { - if isBinaryOpCmp(be.Op) && !be.Bool { +func resetMetricGroupIfRequired(be *metricsql.BinaryOpExpr, ts *timeseries) { + if metricsql.IsBinaryOpCmp(be.Op) && !be.Bool { // Do not reset MetricGroup for non-boolean `compare` binary ops like Prometheus does. return } @@ -353,90 +282,6 @@ func resetMetricGroupIfRequired(be *promql.BinaryOpExpr, ts *timeseries) { ts.MetricName.ResetMetricGroup() } -func binaryOpPlus(left, right float64) float64 { - return left + right -} - -func binaryOpMinus(left, right float64) float64 { - return left - right -} - -func binaryOpMul(left, right float64) float64 { - return left * right -} - -func binaryOpDiv(left, right float64) float64 { - return left / right -} - -func binaryOpMod(left, right float64) float64 { - return math.Mod(left, right) -} - -func binaryOpPow(left, right float64) float64 { - return math.Pow(left, right) -} - -func binaryOpDefault(left, right float64) float64 { - if math.IsNaN(left) { - return right - } - return left -} - -func binaryOpIf(left, right float64) float64 { - if math.IsNaN(right) { - return nan - } - return left -} - -func binaryOpIfnot(left, right float64) float64 { - if math.IsNaN(right) { - return left - } - return nan -} - -func binaryOpEq(left, right float64) bool { - // Special handling for nan == nan. - // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 . - if math.IsNaN(left) { - return math.IsNaN(right) - } - - return left == right -} - -func binaryOpNeq(left, right float64) bool { - // Special handling for comparison with nan. - // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 . - if math.IsNaN(left) { - return !math.IsNaN(right) - } - if math.IsNaN(right) { - return true - } - - return left != right -} - -func binaryOpGt(left, right float64) bool { - return left > right -} - -func binaryOpLt(left, right float64) bool { - return left < right -} - -func binaryOpGte(left, right float64) bool { - return left >= right -} - -func binaryOpLte(left, right float64) bool { - return left <= right -} - func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) { mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right) var rvs []*timeseries @@ -473,7 +318,7 @@ func binaryOpUnless(bfa *binaryOpFuncArg) ([]*timeseries, error) { return rvs, nil } -func createTimeseriesMapByTagSet(be *promql.BinaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) { +func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) { groupTags := be.GroupModifier.Args groupOp := strings.ToLower(be.GroupModifier.Op) if len(groupOp) == 0 { diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index 3f452eef0..07bd6d372 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -6,13 +6,12 @@ import ( "math" "runtime" "sync" - "unsafe" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" ) @@ -154,9 +153,9 @@ func getTimestamps(start, end, step int64) []int64 { return timestamps } -func evalExpr(ec *EvalConfig, e promql.Expr) ([]*timeseries, error) { - if me, ok := e.(*promql.MetricExpr); ok { - re := &promql.RollupExpr{ +func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) { + if me, ok := e.(*metricsql.MetricExpr); ok { + re := &metricsql.RollupExpr{ Expr: me, } rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, re, nil) @@ -165,14 +164,14 @@ func evalExpr(ec *EvalConfig, e promql.Expr) ([]*timeseries, error) { } return rv, nil } - if re, ok := e.(*promql.RollupExpr); ok { + if re, ok := e.(*metricsql.RollupExpr); ok { rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, re, nil) if err != nil { return nil, fmt.Errorf(`cannot evaluate %q: %s`, re.AppendString(nil), err) } return rv, nil } - if fe, ok := e.(*promql.FuncExpr); ok { + if fe, ok := e.(*metricsql.FuncExpr); ok { nrf := getRollupFunc(fe.Name) if nrf == nil { args, err := evalExprs(ec, fe.Args) @@ -208,11 +207,11 @@ func evalExpr(ec *EvalConfig, e promql.Expr) ([]*timeseries, error) { } return rv, nil } - if ae, ok := e.(*promql.AggrFuncExpr); ok { + if ae, ok := e.(*metricsql.AggrFuncExpr); ok { if callbacks := getIncrementalAggrFuncCallbacks(ae.Name); callbacks != nil { fe, nrf := tryGetArgRollupFuncWithMetricExpr(ae) if fe != nil { - // There is an optimized path for calculating aggrFuncExpr over rollupFunc over metricExpr. + // There is an optimized path for calculating metricsql.AggrFuncExpr over rollupFunc over metricsql.MetricExpr. // The optimized path saves RAM for aggregates over big number of time series. args, re, err := evalRollupFuncArgs(ec, fe) if err != nil { @@ -245,7 +244,7 @@ func evalExpr(ec *EvalConfig, e promql.Expr) ([]*timeseries, error) { } return rv, nil } - if be, ok := e.(*promql.BinaryOpExpr); ok { + if be, ok := e.(*metricsql.BinaryOpExpr); ok { left, err := evalExpr(ec, be.Left) if err != nil { return nil, err @@ -269,18 +268,18 @@ func evalExpr(ec *EvalConfig, e promql.Expr) ([]*timeseries, error) { } return rv, nil } - if ne, ok := e.(*promql.NumberExpr); ok { + if ne, ok := e.(*metricsql.NumberExpr); ok { rv := evalNumber(ec, ne.N) return rv, nil } - if se, ok := e.(*promql.StringExpr); ok { + if se, ok := e.(*metricsql.StringExpr); ok { rv := evalString(ec, se.S) return rv, nil } return nil, fmt.Errorf("unexpected expression %q", e.AppendString(nil)) } -func tryGetArgRollupFuncWithMetricExpr(ae *promql.AggrFuncExpr) (*promql.FuncExpr, newRollupFunc) { +func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.FuncExpr, newRollupFunc) { if len(ae.Args) != 1 { return nil, nil } @@ -291,31 +290,31 @@ func tryGetArgRollupFuncWithMetricExpr(ae *promql.AggrFuncExpr) (*promql.FuncExp // - rollupFunc(metricExpr) // - rollupFunc(metricExpr[d]) - if me, ok := e.(*promql.MetricExpr); ok { + if me, ok := e.(*metricsql.MetricExpr); ok { // e = metricExpr if me.IsEmpty() { return nil, nil } - fe := &promql.FuncExpr{ + fe := &metricsql.FuncExpr{ Name: "default_rollup", - Args: []promql.Expr{me}, + Args: []metricsql.Expr{me}, } nrf := getRollupFunc(fe.Name) return fe, nrf } - if re, ok := e.(*promql.RollupExpr); ok { - if me, ok := re.Expr.(*promql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() { + if re, ok := e.(*metricsql.RollupExpr); ok { + if me, ok := re.Expr.(*metricsql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() { return nil, nil } // e = metricExpr[d] - fe := &promql.FuncExpr{ + fe := &metricsql.FuncExpr{ Name: "default_rollup", - Args: []promql.Expr{re}, + Args: []metricsql.Expr{re}, } nrf := getRollupFunc(fe.Name) return fe, nrf } - fe, ok := e.(*promql.FuncExpr) + fe, ok := e.(*metricsql.FuncExpr) if !ok { return nil, nil } @@ -325,18 +324,18 @@ func tryGetArgRollupFuncWithMetricExpr(ae *promql.AggrFuncExpr) (*promql.FuncExp } rollupArgIdx := getRollupArgIdx(fe.Name) arg := fe.Args[rollupArgIdx] - if me, ok := arg.(*promql.MetricExpr); ok { + if me, ok := arg.(*metricsql.MetricExpr); ok { if me.IsEmpty() { return nil, nil } // e = rollupFunc(metricExpr) - return &promql.FuncExpr{ + return &metricsql.FuncExpr{ Name: fe.Name, - Args: []promql.Expr{me}, + Args: []metricsql.Expr{me}, }, nrf } - if re, ok := arg.(*promql.RollupExpr); ok { - if me, ok := re.Expr.(*promql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() { + if re, ok := arg.(*metricsql.RollupExpr); ok { + if me, ok := re.Expr.(*metricsql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() { return nil, nil } // e = rollupFunc(metricExpr[d]) @@ -345,7 +344,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *promql.AggrFuncExpr) (*promql.FuncExp return nil, nil } -func evalExprs(ec *EvalConfig, es []promql.Expr) ([][]*timeseries, error) { +func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) { var rvs [][]*timeseries for _, e := range es { rv, err := evalExpr(ec, e) @@ -357,8 +356,8 @@ func evalExprs(ec *EvalConfig, es []promql.Expr) ([][]*timeseries, error) { return rvs, nil } -func evalRollupFuncArgs(ec *EvalConfig, fe *promql.FuncExpr) ([]interface{}, *promql.RollupExpr, error) { - var re *promql.RollupExpr +func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) { + var re *metricsql.RollupExpr rollupArgIdx := getRollupArgIdx(fe.Name) args := make([]interface{}, len(fe.Args)) for i, arg := range fe.Args { @@ -376,11 +375,11 @@ func evalRollupFuncArgs(ec *EvalConfig, fe *promql.FuncExpr) ([]interface{}, *pr return args, re, nil } -func getRollupExprArg(arg promql.Expr) *promql.RollupExpr { - re, ok := arg.(*promql.RollupExpr) +func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr { + re, ok := arg.(*metricsql.RollupExpr) if !ok { - // Wrap non-rollup arg into rollupExpr. - return &promql.RollupExpr{ + // Wrap non-rollup arg into metricsql.RollupExpr. + return &metricsql.RollupExpr{ Expr: arg, } } @@ -388,28 +387,28 @@ func getRollupExprArg(arg promql.Expr) *promql.RollupExpr { // Return standard rollup if it doesn't contain subquery. return re } - me, ok := re.Expr.(*promql.MetricExpr) + me, ok := re.Expr.(*metricsql.MetricExpr) if !ok { // arg contains subquery. return re } // Convert me[w:step] -> default_rollup(me)[w:step] reNew := *re - reNew.Expr = &promql.FuncExpr{ + reNew.Expr = &metricsql.FuncExpr{ Name: "default_rollup", - Args: []promql.Expr{ - &promql.RollupExpr{Expr: me}, + Args: []metricsql.Expr{ + &metricsql.RollupExpr{Expr: me}, }, } return &reNew } -func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *promql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) { +func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) { ecNew := ec var offset int64 if len(re.Offset) > 0 { var err error - offset, err = promql.DurationValue(re.Offset, ec.Step) + offset, err = metricsql.DurationValue(re.Offset, ec.Step) if err != nil { return nil, err } @@ -425,7 +424,7 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *promql.Rollu } var rvs []*timeseries var err error - if me, ok := re.Expr.(*promql.MetricExpr); ok { + if me, ok := re.Expr.(*metricsql.MetricExpr); ok { rvs, err = evalRollupFuncWithMetricExpr(ecNew, name, rf, me, iafc, re.Window) } else { if iafc != nil { @@ -450,12 +449,12 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *promql.Rollu return rvs, nil } -func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *promql.RollupExpr) ([]*timeseries, error) { - // Do not use rollupResultCacheV here, since it works only with metricExpr. +func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *metricsql.RollupExpr) ([]*timeseries, error) { + // Do not use rollupResultCacheV here, since it works only with metricsql.MetricExpr. var step int64 if len(re.Step) > 0 { var err error - step, err = promql.DurationValue(re.Step, ec.Step) + step, err = metricsql.PositiveDurationValue(re.Step, ec.Step) if err != nil { return nil, err } @@ -465,7 +464,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re * var window int64 if len(re.Window) > 0 { var err error - window, err = promql.DurationValue(re.Window, ec.Step) + window, err = metricsql.PositiveDurationValue(re.Window, ec.Step) if err != nil { return nil, err } @@ -559,14 +558,14 @@ var ( rollupResultCacheMiss = metrics.NewCounter(`vm_rollup_result_cache_miss_total`) ) -func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me *promql.MetricExpr, iafc *incrementalAggrFuncContext, windowStr string) ([]*timeseries, error) { +func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowStr string) ([]*timeseries, error) { if me.IsEmpty() { return evalNumber(ec, nan), nil } var window int64 if len(windowStr) > 0 { var err error - window, err = promql.DurationValue(windowStr, ec.Step) + window, err = metricsql.PositiveDurationValue(windowStr, ec.Step) if err != nil { return nil, err } @@ -586,12 +585,11 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me } // Fetch the remaining part of the result. + tfs := toTagFilters(me.LabelFilters) sq := &storage.SearchQuery{ MinTimestamp: start - window - maxSilenceInterval, MaxTimestamp: ec.End + ec.Step, - TagFilterss: [][]storage.TagFilter{ - *(*[]storage.TagFilter)(unsafe.Pointer(&me.TagFilters)), - }, + TagFilterss: [][]storage.TagFilter{tfs}, } rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline) if err != nil { @@ -815,3 +813,23 @@ func mulNoOverflow(a, b int64) int64 { } return a * b } + +func toTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter { + tfs := make([]storage.TagFilter, len(lfs)) + for i := range lfs { + toTagFilter(&tfs[i], &lfs[i]) + } + return tfs +} + +func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) { + if src.Label != "__name__" { + dst.Key = []byte(src.Label) + } else { + // This is required for storage.Search. + dst.Key = nil + } + dst.Value = []byte(src.Value) + dst.IsRegexp = src.IsRegexp + dst.IsNegative = src.IsNegative +} diff --git a/app/vmselect/promql/exec.go b/app/vmselect/promql/exec.go index c046c81b3..ca749e8de 100644 --- a/app/vmselect/promql/exec.go +++ b/app/vmselect/promql/exec.go @@ -11,7 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/metrics" ) @@ -86,12 +86,12 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result, return result, err } -func maySortResults(e promql.Expr, tss []*timeseries) bool { +func maySortResults(e metricsql.Expr, tss []*timeseries) bool { if len(tss) > 100 { // There is no sense in sorting a lot of results return false } - fe, ok := e.(*promql.FuncExpr) + fe, ok := e.(*metricsql.FuncExpr) if !ok { return true } @@ -155,19 +155,10 @@ func removeNaNs(tss []*timeseries) []*timeseries { return rvs } -func parsePromQL(q string) (promql.Expr, error) { - e, err := parser.ParsePromQL(q) - if err != nil { - return nil, err - } - - return simplifyConstants(e), nil -} - -func parsePromQLWithCache(q string) (promql.Expr, error) { +func parsePromQLWithCache(q string) (metricsql.Expr, error) { pcv := parseCacheV.Get(q) if pcv == nil { - e, err := parsePromQL(q) + e, err := metricsql.Parse(q) pcv = &parseCacheValue{ e: e, err: err, @@ -180,8 +171,6 @@ func parsePromQLWithCache(q string) (promql.Expr, error) { return pcv.e, nil } -var parser = promql.NewParser(compileRegexpAnchored) - var parseCacheV = func() *parseCache { pc := &parseCache{ m: make(map[string]*parseCacheValue), @@ -201,7 +190,7 @@ var parseCacheV = func() *parseCache { const parseCacheMaxLen = 10e3 type parseCacheValue struct { - e promql.Expr + e metricsql.Expr err error } @@ -261,7 +250,3 @@ func (pc *parseCache) Put(q string, pcv *parseCacheValue) { pc.m[q] = pcv pc.mu.Unlock() } - -func init() { - promql.Panicf = logger.Panicf -} diff --git a/app/vmselect/promql/helper.go b/app/vmselect/promql/parser.go similarity index 50% rename from app/vmselect/promql/helper.go rename to app/vmselect/promql/parser.go index 5820dd4c5..d4b581032 100644 --- a/app/vmselect/promql/helper.go +++ b/app/vmselect/promql/parser.go @@ -2,12 +2,27 @@ package promql import ( "fmt" - "unsafe" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) +// IsRollup verifies whether s is a rollup with non-empty window. +// +// It returns the wrapped query with the corresponding window, step and offset. +func IsRollup(s string) (childQuery string, window, step, offset string) { + expr, err := parsePromQLWithCache(s) + if err != nil { + return + } + re, ok := expr.(*metricsql.RollupExpr) + if !ok || len(re.Window) == 0 { + return + } + wrappedQuery := re.Expr.AppendString(nil) + return string(wrappedQuery), re.Window, re.Step, re.Offset +} + // IsMetricSelectorWithRollup verifies whether s contains PromQL metric selector // wrapped into rollup. // @@ -17,12 +32,12 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset str if err != nil { return } - re, ok := expr.(*promql.RollupExpr) + re, ok := expr.(*metricsql.RollupExpr) if !ok || len(re.Window) == 0 || len(re.Step) > 0 { return } - me, ok := re.Expr.(*promql.MetricExpr) - if !ok || len(me.TagFilters) == 0 { + me, ok := re.Expr.(*metricsql.MetricExpr) + if !ok || len(me.LabelFilters) == 0 { return } wrappedQuery := me.AppendString(nil) @@ -30,18 +45,19 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset str } // ParseMetricSelector parses s containing PromQL metric selector -// and returns the corresponding TagFilters. +// and returns the corresponding LabelFilters. func ParseMetricSelector(s string) ([]storage.TagFilter, error) { expr, err := parsePromQLWithCache(s) if err != nil { return nil, err } - me, ok := expr.(*promql.MetricExpr) + me, ok := expr.(*metricsql.MetricExpr) if !ok { return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil)) } - if len(me.TagFilters) == 0 { - return nil, fmt.Errorf("tagFilters cannot be empty") + if len(me.LabelFilters) == 0 { + return nil, fmt.Errorf("labelFilters cannot be empty") } - return *(*[]storage.TagFilter)(unsafe.Pointer(&me.TagFilters)), nil + tfs := toTagFilters(me.LabelFilters) + return tfs, nil } diff --git a/app/vmselect/promql/helper_test.go b/app/vmselect/promql/parser_test.go similarity index 100% rename from app/vmselect/promql/helper_test.go rename to app/vmselect/promql/parser_test.go diff --git a/app/vmselect/promql/rollup_result_cache.go b/app/vmselect/promql/rollup_result_cache.go index 76c803138..2090ed4e5 100644 --- a/app/vmselect/promql/rollup_result_cache.go +++ b/app/vmselect/promql/rollup_result_cache.go @@ -12,8 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache" "github.com/VictoriaMetrics/fastcache" "github.com/VictoriaMetrics/metrics" @@ -130,7 +129,7 @@ func ResetRollupResultCache() { rollupResultCacheV.c.Reset() } -func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *promql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) (tss []*timeseries, newStart int64) { +func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) (tss []*timeseries, newStart int64) { if *disableCache || !ec.mayCache() { return nil, ec.Start } @@ -211,7 +210,7 @@ func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *promql.Me var resultBufPool bytesutil.ByteBufferPool -func (rrc *rollupResultCache) Put(funcName string, ec *EvalConfig, me *promql.MetricExpr, iafc *incrementalAggrFuncContext, window int64, tss []*timeseries) { +func (rrc *rollupResultCache) Put(funcName string, ec *EvalConfig, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64, tss []*timeseries) { if *disableCache || len(tss) == 0 || !ec.mayCache() { return } @@ -292,7 +291,7 @@ var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total") // Increment this value every time the format of the cache changes. const rollupResultCacheVersion = 6 -func marshalRollupResultCacheKey(dst []byte, funcName string, me *promql.MetricExpr, iafc *incrementalAggrFuncContext, window, step int64) []byte { +func marshalRollupResultCacheKey(dst []byte, funcName string, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, step int64) []byte { dst = append(dst, rollupResultCacheVersion) if iafc == nil { dst = append(dst, 0) @@ -304,9 +303,9 @@ func marshalRollupResultCacheKey(dst []byte, funcName string, me *promql.MetricE dst = append(dst, funcName...) dst = encoding.MarshalInt64(dst, window) dst = encoding.MarshalInt64(dst, step) - for i := range me.TagFilters { - stf := storage.TagFilter(me.TagFilters[i]) - dst = stf.Marshal(dst) + tfs := toTagFilters(me.LabelFilters) + for i := range tfs { + dst = tfs[i].Marshal(dst) } return dst } diff --git a/app/vmselect/promql/rollup_result_cache_test.go b/app/vmselect/promql/rollup_result_cache_test.go index ad521c35a..3ab7eb2aa 100644 --- a/app/vmselect/promql/rollup_result_cache_test.go +++ b/app/vmselect/promql/rollup_result_cache_test.go @@ -3,7 +3,7 @@ package promql import ( "testing" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) @@ -18,14 +18,14 @@ func TestRollupResultCache(t *testing.T) { MayCache: true, } - me := &promql.MetricExpr{ - TagFilters: []promql.TagFilter{{ - Key: []byte("aaa"), - Value: []byte("xxx"), + me := &metricsql.MetricExpr{ + LabelFilters: []metricsql.LabelFilter{{ + Label: "aaa", + Value: "xxx", }}, } iafc := &incrementalAggrFuncContext{ - ae: &promql.AggrFuncExpr{ + ae: &metricsql.AggrFuncExpr{ Name: "foobar", }, } diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index f29c15b5d..f7a274082 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -1,9 +1,10 @@ package promql import ( - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" "math" "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" ) var ( @@ -158,7 +159,7 @@ func TestDerivValues(t *testing.T) { testRowsEqual(t, values, timestamps, valuesExpected, timestamps) } -func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpected *promql.MetricExpr, vExpected float64) { +func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpected *metricsql.MetricExpr, vExpected float64) { t.Helper() nrf := getRollupFunc(funcName) if nrf == nil { @@ -198,8 +199,8 @@ func TestRollupQuantileOverTime(t *testing.T) { Values: []float64{phi}, Timestamps: []int64{123}, }} - var me promql.MetricExpr - args := []interface{}{phis, &promql.RollupExpr{Expr: &me}} + var me metricsql.MetricExpr + args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}} testRollupFunc(t, "quantile_over_time", args, &me, vExpected) } @@ -220,8 +221,8 @@ func TestRollupPredictLinear(t *testing.T) { Values: []float64{sec}, Timestamps: []int64{123}, }} - var me promql.MetricExpr - args := []interface{}{&promql.RollupExpr{Expr: &me}, secs} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}, secs} testRollupFunc(t, "predict_linear", args, &me, vExpected) } @@ -242,8 +243,8 @@ func TestRollupHoltWinters(t *testing.T) { Values: []float64{tf}, Timestamps: []int64{123}, }} - var me promql.MetricExpr - args := []interface{}{&promql.RollupExpr{Expr: &me}, sfs, tfs} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}, sfs, tfs} testRollupFunc(t, "holt_winters", args, &me, vExpected) } @@ -266,8 +267,8 @@ func TestRollupHoltWinters(t *testing.T) { func TestRollupNewRollupFuncSuccess(t *testing.T) { f := func(funcName string, vExpected float64) { t.Helper() - var me promql.MetricExpr - args := []interface{}{&promql.RollupExpr{Expr: &me}} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}} testRollupFunc(t, funcName, args, &me, vExpected) } @@ -328,7 +329,7 @@ func TestRollupNewRollupFuncError(t *testing.T) { Values: []float64{321}, Timestamps: []int64{123}, }} - me := &promql.MetricExpr{} + me := &metricsql.MetricExpr{} f("holt_winters", []interface{}{123, 123, 321}) f("holt_winters", []interface{}{me, 123, 321}) f("holt_winters", []interface{}{me, scalarTs, 321}) diff --git a/app/vmselect/promql/simplify.go b/app/vmselect/promql/simplify.go deleted file mode 100644 index d847281dd..000000000 --- a/app/vmselect/promql/simplify.go +++ /dev/null @@ -1,54 +0,0 @@ -package promql - -import ( - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" -) - -func simplifyConstants(e promql.Expr) promql.Expr { - if re, ok := e.(*promql.RollupExpr); ok { - re.Expr = simplifyConstants(re.Expr) - return re - } - if ae, ok := e.(*promql.AggrFuncExpr); ok { - simplifyConstantsInplace(ae.Args) - return ae - } - if fe, ok := e.(*promql.FuncExpr); ok { - simplifyConstantsInplace(fe.Args) - return fe - } - if pe, ok := e.(*promql.ParensExpr); ok { - if len(*pe) == 1 { - return simplifyConstants((*pe)[0]) - } - simplifyConstantsInplace(*pe) - return pe - } - be, ok := e.(*promql.BinaryOpExpr) - if !ok { - return e - } - - be.Left = simplifyConstants(be.Left) - be.Right = simplifyConstants(be.Right) - - lne, ok := be.Left.(*promql.NumberExpr) - if !ok { - return be - } - rne, ok := be.Right.(*promql.NumberExpr) - if !ok { - return be - } - n := binaryOpConstants(be.Op, lne.N, rne.N, be.Bool) - ne := &promql.NumberExpr{ - N: n, - } - return ne -} - -func simplifyConstantsInplace(args []promql.Expr) { - for i, arg := range args { - args[i] = simplifyConstants(arg) - } -} diff --git a/app/vmselect/promql/simplify_test.go b/app/vmselect/promql/simplify_test.go deleted file mode 100644 index cb002c934..000000000 --- a/app/vmselect/promql/simplify_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package promql - -import ( - "testing" -) - -func TestSimplify(t *testing.T) { - another := func(s string, sExpected string) { - t.Helper() - - e, err := parsePromQL(s) - if err != nil { - t.Fatalf("unexpected error when parsing %q: %s", s, err) - } - res := e.AppendString(nil) - if string(res) != sExpected { - t.Fatalf("unexpected string constructed;\ngot\n%q\nwant\n%q", res, sExpected) - } - } - - // Simplification cases - another(`nan == nan`, `NaN`) - another(`nan ==bool nan`, `1`) - another(`nan !=bool nan`, `0`) - another(`nan !=bool 2`, `1`) - another(`2 !=bool nan`, `1`) - another(`nan >bool nan`, `0`) - another(`nan =bool 2`, `1`) - another(`-1 >bool -inf`, `1`) - another(`-1 2`, `NaN`) - another(`1 > bool 2`, `0`) - another(`3 >= 2`, `3`) - another(`3 <= bool 2`, `0`) - another(`1 + -2 - 3`, `-4`) - another(`1 / 0 + 2`, `+Inf`) - another(`2 + -1 / 0`, `-Inf`) - another(`-1 ^ 0.5`, `NaN`) - another(`512.5 - (1 + 3) * (2 ^ 2) ^ 3`, `256.5`) - another(`1 == bool 1 != bool 24 < bool 4 > bool -1`, `1`) - another(`1 == bOOl 1 != BOOL 24 < Bool 4 > booL -1`, `1`) - another(`1+2 if 2>3`, `NaN`) - another(`1+4 if 2<3`, `5`) - another(`2+6 default 3 if 2>3`, `8`) - another(`2+6 if 2>3 default NaN`, `NaN`) - another(`42 if 3>2 if 2+2<5`, `42`) - another(`42 if 3>2 if 2+2>=5`, `NaN`) - another(`1+2 ifnot 2>3`, `3`) - another(`1+4 ifnot 2<3`, `NaN`) - another(`2+6 default 3 ifnot 2>3`, `8`) - another(`2+6 ifnot 2>3 default NaN`, `8`) - another(`42 if 3>2 ifnot 2+2<5`, `NaN`) - another(`42 if 3>2 ifnot 2+2>=5`, `42`) - another(`5 - 1 + 3 * 2 ^ 2 ^ 3 - 2 OR Metric {Bar= "Baz", aaa!="bb",cc=~"dd" ,zz !~"ff" } `, - `770 or Metric{Bar="Baz", aaa!="bb", cc=~"dd", zz!~"ff"}`) - another(`((foo(bar,baz)), (1+(2)+(3,4)+()))`, `(foo(bar, baz), (3 + (3, 4)) + ())`) - another(` FOO (bar) + f ( m ( ),ff(1 + ( 2.5)) ,M[5m ] , "ff" )`, `FOO(bar) + f(m(), ff(3.5), M[5m], "ff")`) - another(`sum without (a, b) (xx,2+2)`, `sum(xx, 4) without (a, b)`) - another(`Sum WIthout (a, B) (XX,2+2)`, `sum(XX, 4) without (a, B)`) - another(`with (foo = bar{x="x"}) 1+1`, `2`) - another(`with (x(a, b) = a + b) x(foo, x(1, 2))`, `foo + 3`) - another(`WITH ( - x2(x) = x^2, - f(x, y) = x2(x) + x*y + x2(y) - ) - f(a, 3) - `, `((a ^ 2) + (a * 3)) + 9`) - another(`WITH ( - x2(x) = x^2, - f(x, y) = x2(x) + x*y + x2(y) - ) - f(2, 3) - `, `19`) - another(`with(y=123,z=5) union(with(y=3,f(x)=x*y) f(2) + f(3), with(x=5,y=2) x*y*z)`, `union(15, 50)`) -} diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 9e78b0593..f503638e2 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -12,7 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/valyala/histogram" ) @@ -102,7 +102,7 @@ func getTransformFunc(s string) transformFunc { type transformFuncArg struct { ec *EvalConfig - fe *promql.FuncExpr + fe *metricsql.FuncExpr args [][]*timeseries } @@ -123,7 +123,7 @@ func newTransformFuncOneArg(tf func(v float64) float64) transformFunc { } } -func doTransformValues(arg []*timeseries, tf func(values []float64), fe *promql.FuncExpr) ([]*timeseries, error) { +func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) { name := strings.ToLower(fe.Name) keepMetricGroup := transformFuncsKeepMetricGroup[name] for _, ts := range arg { @@ -151,12 +151,13 @@ func transformAbsent(tfa *transformFuncArg) ([]*timeseries, error) { // Copy tags from arg rvs := evalNumber(tfa.ec, 1) rv := rvs[0] - me, ok := tfa.fe.Args[0].(*promql.MetricExpr) + me, ok := tfa.fe.Args[0].(*metricsql.MetricExpr) if !ok { return rvs, nil } - for i := range me.TagFilters { - tf := &me.TagFilters[i] + tfs := toTagFilters(me.LabelFilters) + for i := range tfs { + tf := &tfs[i] if len(tf.Key) == 0 { continue } @@ -1020,7 +1021,7 @@ func transformLabelTransform(tfa *transformFuncArg) ([]*timeseries, error) { return nil, err } - r, err := compileRegexp(regex) + r, err := metricsql.CompileRegexp(regex) if err != nil { return nil, fmt.Errorf(`cannot compile regex %q: %s`, regex, err) } @@ -1049,7 +1050,7 @@ func transformLabelReplace(tfa *transformFuncArg) ([]*timeseries, error) { return nil, err } - r, err := compileRegexpAnchored(regex) + r, err := metricsql.CompileRegexpAnchored(regex) if err != nil { return nil, fmt.Errorf(`cannot compile regex %q: %s`, regex, err) } @@ -1160,7 +1161,7 @@ func transformScalar(tfa *transformFuncArg) ([]*timeseries, error) { // Verify whether the arg is a string. // Then try converting the string to number. - if se, ok := tfa.fe.Args[0].(*promql.StringExpr); ok { + if se, ok := tfa.fe.Args[0].(*metricsql.StringExpr); ok { n, err := strconv.ParseFloat(se.S, 64) if err != nil { n = nan diff --git a/lib/promql/aggr.go b/lib/metricsql/aggr.go similarity index 95% rename from lib/promql/aggr.go rename to lib/metricsql/aggr.go index 917be5ca8..66a8f8b87 100644 --- a/lib/promql/aggr.go +++ b/lib/metricsql/aggr.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "strings" @@ -18,7 +18,7 @@ var aggrFuncs = map[string]bool{ "topk": true, "quantile": true, - // Extended PromQL funcs + // MetricsQL extension funcs "median": true, "limitk": true, "distinct": true, diff --git a/lib/promql/aggr_test.go b/lib/metricsql/aggr_test.go similarity index 96% rename from lib/promql/aggr_test.go rename to lib/metricsql/aggr_test.go index 978baddd0..5dcdf3f02 100644 --- a/lib/promql/aggr_test.go +++ b/lib/metricsql/aggr_test.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "testing" diff --git a/lib/metricsql/binary_op.go b/lib/metricsql/binary_op.go new file mode 100644 index 000000000..453929566 --- /dev/null +++ b/lib/metricsql/binary_op.go @@ -0,0 +1,205 @@ +package metricsql + +import ( + "fmt" + "math" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql/binaryop" +) + +var binaryOps = map[string]bool{ + "+": true, + "-": true, + "*": true, + "/": true, + "%": true, + "^": true, + + // cmp ops + "==": true, + "!=": true, + ">": true, + "<": true, + ">=": true, + "<=": true, + + // logical set ops + "and": true, + "or": true, + "unless": true, + + // New ops for MetricsQL + "if": true, + "ifnot": true, + "default": true, +} + +var binaryOpPriorities = map[string]int{ + "default": -1, + + "if": 0, + "ifnot": 0, + + // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence + "or": 1, + + "and": 2, + "unless": 2, + + "==": 3, + "!=": 3, + "<": 3, + ">": 3, + "<=": 3, + ">=": 3, + + "+": 4, + "-": 4, + + "*": 5, + "/": 5, + "%": 5, + + "^": 6, +} + +func isBinaryOp(op string) bool { + op = strings.ToLower(op) + return binaryOps[op] +} + +func binaryOpPriority(op string) int { + op = strings.ToLower(op) + return binaryOpPriorities[op] +} + +func scanBinaryOpPrefix(s string) int { + n := 0 + for op := range binaryOps { + if len(s) < len(op) { + continue + } + ss := strings.ToLower(s[:len(op)]) + if ss == op && len(op) > n { + n = len(op) + } + } + return n +} + +func isRightAssociativeBinaryOp(op string) bool { + // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence + return op == "^" +} + +func isBinaryOpGroupModifier(s string) bool { + s = strings.ToLower(s) + switch s { + // See https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching + case "on", "ignoring": + return true + default: + return false + } +} + +func isBinaryOpJoinModifier(s string) bool { + s = strings.ToLower(s) + switch s { + case "group_left", "group_right": + return true + default: + return false + } +} + +func isBinaryOpBoolModifier(s string) bool { + s = strings.ToLower(s) + return s == "bool" +} + +// IsBinaryOpCmp returns true if op is comparison operator such as '==', '!=', etc. +func IsBinaryOpCmp(op string) bool { + switch op { + case "==", "!=", ">", "<", ">=", "<=": + return true + default: + return false + } +} + +func isBinaryOpLogicalSet(op string) bool { + op = strings.ToLower(op) + switch op { + case "and", "or", "unless": + return true + default: + return false + } +} + +func binaryOpEval(op string, left, right float64, isBool bool) float64 { + if IsBinaryOpCmp(op) { + evalCmp := func(cf func(left, right float64) bool) float64 { + if isBool { + if cf(left, right) { + return 1 + } + return 0 + } + if cf(left, right) { + return left + } + return nan + } + switch op { + case "==": + left = evalCmp(binaryop.Eq) + case "!=": + left = evalCmp(binaryop.Neq) + case ">": + left = evalCmp(binaryop.Gt) + case "<": + left = evalCmp(binaryop.Lt) + case ">=": + left = evalCmp(binaryop.Gte) + case "<=": + left = evalCmp(binaryop.Lte) + default: + panic(fmt.Errorf("BUG: unexpected comparison binaryOp: %q", op)) + } + } else { + switch op { + case "+": + left = binaryop.Plus(left, right) + case "-": + left = binaryop.Minus(left, right) + case "*": + left = binaryop.Mul(left, right) + case "/": + left = binaryop.Div(left, right) + case "%": + left = binaryop.Mod(left, right) + case "^": + left = binaryop.Pow(left, right) + case "and": + // Nothing to do + case "or": + // Nothing to do + case "unless": + left = nan + case "default": + left = binaryop.Default(left, right) + case "if": + left = binaryop.If(left, right) + case "ifnot": + left = binaryop.Ifnot(left, right) + default: + panic(fmt.Errorf("BUG: unexpected non-comparison binaryOp: %q", op)) + } + } + return left +} + +var nan = math.NaN() diff --git a/lib/promql/binary_op_test.go b/lib/metricsql/binary_op_test.go similarity index 99% rename from lib/promql/binary_op_test.go rename to lib/metricsql/binary_op_test.go index c07bfbb21..b847394ca 100644 --- a/lib/promql/binary_op_test.go +++ b/lib/metricsql/binary_op_test.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "testing" diff --git a/lib/metricsql/binaryop/funcs.go b/lib/metricsql/binaryop/funcs.go new file mode 100644 index 000000000..9a123e24d --- /dev/null +++ b/lib/metricsql/binaryop/funcs.go @@ -0,0 +1,104 @@ +package binaryop + +import ( + "math" +) + +var nan = math.NaN() + +// Eq returns true of left == right. +func Eq(left, right float64) bool { + // Special handling for nan == nan. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 . + if math.IsNaN(left) { + return math.IsNaN(right) + } + return left == right +} + +// Neq returns true of left != right. +func Neq(left, right float64) bool { + // Special handling for comparison with nan. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 . + if math.IsNaN(left) { + return !math.IsNaN(right) + } + if math.IsNaN(right) { + return true + } + return left != right +} + +// Gt returns true of left > right +func Gt(left, right float64) bool { + return left > right +} + +// Lt returns true if left < right +func Lt(left, right float64) bool { + return left < right +} + +// Gte returns true if left >= right +func Gte(left, right float64) bool { + return left >= right +} + +// Lte returns true if left <= right +func Lte(left, right float64) bool { + return left <= right +} + +// Plus returns left + right +func Plus(left, right float64) float64 { + return left + right +} + +// Minus returns left - right +func Minus(left, right float64) float64 { + return left - right +} + +// Mul returns left * right +func Mul(left, right float64) float64 { + return left * right +} + +// Div returns left / right +func Div(left, right float64) float64 { + return left / right +} + +// Mod returns mod(left, right) +func Mod(left, right float64) float64 { + return math.Mod(left, right) +} + +// Pow returns pow(left, right) +func Pow(left, right float64) float64 { + return math.Pow(left, right) +} + +// Default returns left or right if left is NaN. +func Default(left, right float64) float64 { + if math.IsNaN(left) { + return right + } + return left +} + +// If returns left if right is not NaN. Otherwise NaN is returned. +func If(left, right float64) float64 { + if math.IsNaN(right) { + return nan + } + return left +} + +// Ifnot returns left if right is NaN. Otherwise NaN is returned. +func Ifnot(left, right float64) float64 { + if math.IsNaN(right) { + return left + } + return nan +} diff --git a/lib/metricsql/doc.go b/lib/metricsql/doc.go new file mode 100644 index 000000000..f0d6df337 --- /dev/null +++ b/lib/metricsql/doc.go @@ -0,0 +1,15 @@ +// Package metricsql implements MetricsQL parser. +// +// This parser can parse PromQL. Additionally it can parse MetricsQL extensions. +// See https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL for details about MetricsQL extensions. +// +// Usage: +// +// expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (job)`) +// if err != nil { +// // parse error +// } +// // Now expr contains parsed MetricsQL as `*Expr` structs. +// // See metricsql.Parse examples for more details. +// +package metricsql diff --git a/lib/promql/lexer.go b/lib/metricsql/lexer.go similarity index 97% rename from lib/promql/lexer.go rename to lib/metricsql/lexer.go index 35420ce99..446302ac7 100644 --- a/lib/promql/lexer.go +++ b/lib/metricsql/lexer.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "fmt" @@ -220,7 +220,7 @@ func scanIdent(s string) string { } } if i == 0 { - Panicf("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent") + panic("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent") } return s[:i] } @@ -279,7 +279,7 @@ func toHex(n byte) byte { return 'a' + (n - 10) } -func appendEscapedIdent(dst, s []byte) []byte { +func appendEscapedIdent(dst []byte, s string) []byte { for i := 0; i < len(s); i++ { ch := s[i] if isIdentChar(ch) { diff --git a/lib/promql/lexer_test.go b/lib/metricsql/lexer_test.go similarity index 99% rename from lib/promql/lexer_test.go rename to lib/metricsql/lexer_test.go index 6f8fed517..99e4b7483 100644 --- a/lib/promql/lexer_test.go +++ b/lib/metricsql/lexer_test.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "reflect" @@ -28,7 +28,7 @@ func TestUnescapeIdent(t *testing.T) { func TestAppendEscapedIdent(t *testing.T) { f := func(s, resultExpected string) { t.Helper() - result := appendEscapedIdent(nil, []byte(s)) + result := appendEscapedIdent(nil, s) if string(result) != resultExpected { t.Fatalf("unexpected result for appendEscapedIdent(%q); got %q; want %q", s, result, resultExpected) } diff --git a/lib/metricsql/parser.go b/lib/metricsql/parser.go new file mode 100644 index 000000000..15b49ff0e --- /dev/null +++ b/lib/metricsql/parser.go @@ -0,0 +1,1720 @@ +package metricsql + +import ( + "fmt" + "strconv" + "strings" + "sync" +) + +// Parse parses MetricsQL query s. +// +// MetricsQL is backwards-compatible with PromQL. +func Parse(s string) (Expr, error) { + var p parser + p.lex.Init(s) + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf(`cannot find the first token: %s`, err) + } + e, err := p.parseExpr() + if err != nil { + return nil, fmt.Errorf(`%s; unparsed data: %q`, err, p.lex.Context()) + } + if !isEOF(p.lex.Token) { + return nil, fmt.Errorf(`unparsed data left: %q`, p.lex.Context()) + } + was := getDefaultWithArgExprs() + if e, err = expandWithExpr(was, e); err != nil { + return nil, fmt.Errorf(`cannot expand WITH expressions: %s`, err) + } + e = removeParensExpr(e) + e = simplifyConstants(e) + return e, nil +} + +// Expr holds any of *Expr types. +type Expr interface { + // AppendString appends string representation of Expr to dst. + AppendString(dst []byte) []byte +} + +func getDefaultWithArgExprs() []*withArgExpr { + defaultWithArgExprsOnce.Do(func() { + defaultWithArgExprs = prepareWithArgExprs([]string{ + // ru - resource utilization + `ru(freev, maxv) = clamp_min(maxv - clamp_min(freev, 0), 0) / clamp_min(maxv, 0) * 100`, + + // ttf - time to fuckup + `ttf(freev) = smooth_exponential( + clamp_max(clamp_max(-freev, 0) / clamp_max(deriv_fast(freev), 0), 365*24*3600), + clamp_max(step()/300, 1) + )`, + + `median_over_time(m) = quantile_over_time(0.5, m)`, + `range_median(q) = range_quantile(0.5, q)`, + `alias(q, name) = label_set(q, "__name__", name)`, + }) + }) + return defaultWithArgExprs +} + +var ( + defaultWithArgExprs []*withArgExpr + defaultWithArgExprsOnce sync.Once +) + +func prepareWithArgExprs(ss []string) []*withArgExpr { + was := make([]*withArgExpr, len(ss)) + for i, s := range ss { + was[i] = mustParseWithArgExpr(s) + } + if err := checkDuplicateWithArgNames(was); err != nil { + panic(fmt.Errorf("BUG: %s", err)) + } + return was +} + +func checkDuplicateWithArgNames(was []*withArgExpr) error { + m := make(map[string]*withArgExpr, len(was)) + for _, wa := range was { + if waOld := m[wa.Name]; waOld != nil { + return fmt.Errorf("duplicate `with` arg name for: %s; previous one: %s", wa, waOld.AppendString(nil)) + } + m[wa.Name] = wa + } + return nil +} + +func mustParseWithArgExpr(s string) *withArgExpr { + var p parser + p.lex.Init(s) + if err := p.lex.Next(); err != nil { + panic(fmt.Errorf("BUG: cannot find the first token in %q: %s", s, err)) + } + wa, err := p.parseWithArgExpr() + if err != nil { + panic(fmt.Errorf("BUG: cannot parse %q: %s; unparsed data: %q", s, err, p.lex.Context())) + } + return wa +} + +// removeParensExpr removes parensExpr for (Expr) case. +func removeParensExpr(e Expr) Expr { + if re, ok := e.(*RollupExpr); ok { + re.Expr = removeParensExpr(re.Expr) + return re + } + if be, ok := e.(*BinaryOpExpr); ok { + be.Left = removeParensExpr(be.Left) + be.Right = removeParensExpr(be.Right) + return be + } + if ae, ok := e.(*AggrFuncExpr); ok { + for i, arg := range ae.Args { + ae.Args[i] = removeParensExpr(arg) + } + return ae + } + if fe, ok := e.(*FuncExpr); ok { + for i, arg := range fe.Args { + fe.Args[i] = removeParensExpr(arg) + } + return fe + } + if pe, ok := e.(*parensExpr); ok { + args := *pe + for i, arg := range args { + args[i] = removeParensExpr(arg) + } + if len(*pe) == 1 { + return args[0] + } + // Treat parensExpr as a function with empty name, i.e. union() + fe := &FuncExpr{ + Name: "", + Args: args, + } + return fe + } + return e +} + +func simplifyConstants(e Expr) Expr { + if re, ok := e.(*RollupExpr); ok { + re.Expr = simplifyConstants(re.Expr) + return re + } + if ae, ok := e.(*AggrFuncExpr); ok { + simplifyConstantsInplace(ae.Args) + return ae + } + if fe, ok := e.(*FuncExpr); ok { + simplifyConstantsInplace(fe.Args) + return fe + } + if pe, ok := e.(*parensExpr); ok { + if len(*pe) == 1 { + return simplifyConstants((*pe)[0]) + } + simplifyConstantsInplace(*pe) + return pe + } + be, ok := e.(*BinaryOpExpr) + if !ok { + return e + } + + be.Left = simplifyConstants(be.Left) + be.Right = simplifyConstants(be.Right) + + lne, ok := be.Left.(*NumberExpr) + if !ok { + return be + } + rne, ok := be.Right.(*NumberExpr) + if !ok { + return be + } + n := binaryOpEval(be.Op, lne.N, rne.N, be.Bool) + ne := &NumberExpr{ + N: n, + } + return ne +} + +func simplifyConstantsInplace(args []Expr) { + for i, arg := range args { + args[i] = simplifyConstants(arg) + } +} + +// parser parses MetricsQL expression. +// +// preconditions for all parser.parse* funcs: +// - p.lex.Token should point to the first token to parse. +// +// postconditions for all parser.parse* funcs: +// - p.lex.Token should point to the next token after the parsed token. +type parser struct { + lex lexer +} + +func isWith(s string) bool { + s = strings.ToLower(s) + return s == "with" +} + +// parseWithExpr parses `WITH (withArgExpr...) expr`. +func (p *parser) parseWithExpr() (*withExpr, error) { + var we withExpr + if !isWith(p.lex.Token) { + return nil, fmt.Errorf("withExpr: unexpected token %q; want `WITH`", p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token != "(" { + return nil, fmt.Errorf(`withExpr: unexpected token %q; want "("`, p.lex.Token) + } + for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == ")" { + goto end + } + wa, err := p.parseWithArgExpr() + if err != nil { + return nil, err + } + we.Was = append(we.Was, wa) + switch p.lex.Token { + case ",": + continue + case ")": + goto end + default: + return nil, fmt.Errorf(`withExpr: unexpected token %q; want ",", ")"`, p.lex.Token) + } + } + +end: + if err := checkDuplicateWithArgNames(we.Was); err != nil { + return nil, err + } + if err := p.lex.Next(); err != nil { + return nil, err + } + e, err := p.parseExpr() + if err != nil { + return nil, err + } + we.Expr = e + return &we, nil +} + +func (p *parser) parseWithArgExpr() (*withArgExpr, error) { + var wa withArgExpr + if !isIdentPrefix(p.lex.Token) { + return nil, fmt.Errorf(`withArgExpr: unexpected token %q; want "ident"`, p.lex.Token) + } + wa.Name = p.lex.Token + if isAggrFunc(wa.Name) || isRollupFunc(wa.Name) || isTransformFunc(wa.Name) || isWith(wa.Name) { + return nil, fmt.Errorf(`withArgExpr: cannot use reserved name %q`, wa.Name) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == "(" { + // Parse func args. + args, err := p.parseIdentList() + if err != nil { + return nil, fmt.Errorf(`withArgExpr: cannot parse args for %q: %s`, wa.Name, err) + } + // Make sure all the args have different names + m := make(map[string]bool, len(args)) + for _, arg := range args { + if m[arg] { + return nil, fmt.Errorf(`withArgExpr: duplicate func arg found in %q: %q`, wa.Name, arg) + } + m[arg] = true + } + wa.Args = args + } + if p.lex.Token != "=" { + return nil, fmt.Errorf(`withArgExpr: unexpected token %q; want "="`, p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + e, err := p.parseExpr() + if err != nil { + return nil, fmt.Errorf(`withArgExpr: cannot parse %q: %s`, wa.Name, err) + } + wa.Expr = e + return &wa, nil +} + +func (p *parser) parseExpr() (Expr, error) { + e, err := p.parseSingleExpr() + if err != nil { + return nil, err + } + for { + if !isBinaryOp(p.lex.Token) { + return e, nil + } + + var be BinaryOpExpr + be.Op = strings.ToLower(p.lex.Token) + be.Left = e + if err := p.lex.Next(); err != nil { + return nil, err + } + if isBinaryOpBoolModifier(p.lex.Token) { + if !IsBinaryOpCmp(be.Op) { + return nil, fmt.Errorf(`bool modifier cannot be applied to %q`, be.Op) + } + be.Bool = true + if err := p.lex.Next(); err != nil { + return nil, err + } + } + if isBinaryOpGroupModifier(p.lex.Token) { + if err := p.parseModifierExpr(&be.GroupModifier); err != nil { + return nil, err + } + if isBinaryOpJoinModifier(p.lex.Token) { + if isBinaryOpLogicalSet(be.Op) { + return nil, fmt.Errorf(`modifier %q cannot be applied to %q`, p.lex.Token, be.Op) + } + if err := p.parseModifierExpr(&be.JoinModifier); err != nil { + return nil, err + } + } + } + e2, err := p.parseSingleExpr() + if err != nil { + return nil, err + } + be.Right = e2 + e = balanceBinaryOp(&be) + } +} + +func balanceBinaryOp(be *BinaryOpExpr) Expr { + bel, ok := be.Left.(*BinaryOpExpr) + if !ok { + return be + } + lp := binaryOpPriority(bel.Op) + rp := binaryOpPriority(be.Op) + if rp < lp { + return be + } + if rp == lp && !isRightAssociativeBinaryOp(be.Op) { + return be + } + be.Left = bel.Right + bel.Right = balanceBinaryOp(be) + return bel +} + +// parseSingleExpr parses non-binaryOp expressions. +func (p *parser) parseSingleExpr() (Expr, error) { + if isWith(p.lex.Token) { + err := p.lex.Next() + nextToken := p.lex.Token + p.lex.Prev() + if err == nil && nextToken == "(" { + return p.parseWithExpr() + } + } + e, err := p.parseSingleExprWithoutRollupSuffix() + if err != nil { + return nil, err + } + if p.lex.Token != "[" && !isOffset(p.lex.Token) { + // There is no rollup expression. + return e, nil + } + return p.parseRollupExpr(e) +} + +func (p *parser) parseSingleExprWithoutRollupSuffix() (Expr, error) { + if isPositiveNumberPrefix(p.lex.Token) || isInfOrNaN(p.lex.Token) { + return p.parsePositiveNumberExpr() + } + if isStringPrefix(p.lex.Token) { + return p.parseStringExpr() + } + if isIdentPrefix(p.lex.Token) { + return p.parseIdentExpr() + } + switch p.lex.Token { + case "(": + return p.parseParensExpr() + case "{": + return p.parseMetricExpr() + case "-": + // Unary minus. Substitute -expr with (0 - expr) + if err := p.lex.Next(); err != nil { + return nil, err + } + e, err := p.parseSingleExpr() + if err != nil { + return nil, err + } + be := &BinaryOpExpr{ + Op: "-", + Left: &NumberExpr{ + N: 0, + }, + Right: e, + } + pe := parensExpr{be} + return &pe, nil + case "+": + // Unary plus + if err := p.lex.Next(); err != nil { + return nil, err + } + return p.parseSingleExpr() + default: + return nil, fmt.Errorf(`singleExpr: unexpected token %q; want "(", "{", "-", "+"`, p.lex.Token) + } +} + +func (p *parser) parsePositiveNumberExpr() (*NumberExpr, error) { + if !isPositiveNumberPrefix(p.lex.Token) && !isInfOrNaN(p.lex.Token) { + return nil, fmt.Errorf(`positiveNumberExpr: unexpected token %q; want "number"`, p.lex.Token) + } + + n, err := strconv.ParseFloat(p.lex.Token, 64) + if err != nil { + return nil, fmt.Errorf(`positiveNumberExpr: cannot parse %q: %s`, p.lex.Token, err) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + ne := &NumberExpr{ + N: n, + } + return ne, nil +} + +func (p *parser) parseStringExpr() (*StringExpr, error) { + var se StringExpr + + for { + switch { + case isStringPrefix(p.lex.Token) || isIdentPrefix(p.lex.Token): + se.tokens = append(se.tokens, p.lex.Token) + default: + return nil, fmt.Errorf(`StringExpr: unexpected token %q; want "string"`, p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token != "+" { + return &se, nil + } + + // composite StringExpr like `"s1" + "s2"`, `"s" + m()` or `"s" + m{}` or `"s" + unknownToken`. + if err := p.lex.Next(); err != nil { + return nil, err + } + if isStringPrefix(p.lex.Token) { + // "s1" + "s2" + continue + } + if !isIdentPrefix(p.lex.Token) { + // "s" + unknownToken + p.lex.Prev() + return &se, nil + } + // Look after ident + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == "(" || p.lex.Token == "{" { + // `"s" + m(` or `"s" + m{` + p.lex.Prev() + p.lex.Prev() + return &se, nil + } + // "s" + ident + p.lex.Prev() + } +} + +func (p *parser) parseParensExpr() (*parensExpr, error) { + if p.lex.Token != "(" { + return nil, fmt.Errorf(`parensExpr: unexpected token %q; want "("`, p.lex.Token) + } + var exprs []Expr + for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == ")" { + break + } + expr, err := p.parseExpr() + if err != nil { + return nil, err + } + exprs = append(exprs, expr) + if p.lex.Token == "," { + continue + } + if p.lex.Token == ")" { + break + } + return nil, fmt.Errorf(`parensExpr: unexpected token %q; want "," or ")"`, p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return nil, err + } + pe := parensExpr(exprs) + return &pe, nil +} + +func (p *parser) parseAggrFuncExpr() (*AggrFuncExpr, error) { + if !isAggrFunc(p.lex.Token) { + return nil, fmt.Errorf(`AggrFuncExpr: unexpected token %q; want aggregate func`, p.lex.Token) + } + + var ae AggrFuncExpr + ae.Name = strings.ToLower(p.lex.Token) + if err := p.lex.Next(); err != nil { + return nil, err + } + if isIdentPrefix(p.lex.Token) { + goto funcPrefixLabel + } + switch p.lex.Token { + case "(": + goto funcArgsLabel + default: + return nil, fmt.Errorf(`AggrFuncExpr: unexpected token %q; want "("`, p.lex.Token) + } + +funcPrefixLabel: + { + if !isAggrFuncModifier(p.lex.Token) { + return nil, fmt.Errorf(`AggrFuncExpr: unexpected token %q; want aggregate func modifier`, p.lex.Token) + } + if err := p.parseModifierExpr(&ae.Modifier); err != nil { + return nil, err + } + goto funcArgsLabel + } + +funcArgsLabel: + { + args, err := p.parseArgListExpr() + if err != nil { + return nil, err + } + ae.Args = args + + // Verify whether func suffix exists. + if ae.Modifier.Op != "" || !isAggrFuncModifier(p.lex.Token) { + return &ae, nil + } + if err := p.parseModifierExpr(&ae.Modifier); err != nil { + return nil, err + } + return &ae, nil + } +} + +func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { + switch t := e.(type) { + case *BinaryOpExpr: + left, err := expandWithExpr(was, t.Left) + if err != nil { + return nil, err + } + right, err := expandWithExpr(was, t.Right) + if err != nil { + return nil, err + } + groupModifierArgs, err := expandModifierArgs(was, t.GroupModifier.Args) + if err != nil { + return nil, err + } + joinModifierArgs, err := expandModifierArgs(was, t.JoinModifier.Args) + if err != nil { + return nil, err + } + if t.Op == "+" { + lse, lok := left.(*StringExpr) + rse, rok := right.(*StringExpr) + if lok && rok { + se := &StringExpr{ + S: lse.S + rse.S, + } + return se, nil + } + } + be := &BinaryOpExpr{ + Op: t.Op, + Bool: t.Bool, + GroupModifier: t.GroupModifier, + JoinModifier: t.JoinModifier, + Left: left, + Right: right, + } + be.GroupModifier.Args = groupModifierArgs + be.JoinModifier.Args = joinModifierArgs + pe := parensExpr{be} + return &pe, nil + case *FuncExpr: + args, err := expandWithArgs(was, t.Args) + if err != nil { + return nil, err + } + wa := getWithArgExpr(was, t.Name) + if wa == nil { + fe := &FuncExpr{ + Name: t.Name, + Args: args, + } + return fe, nil + } + return expandWithExprExt(was, wa, args) + case *AggrFuncExpr: + args, err := expandWithArgs(was, t.Args) + if err != nil { + return nil, err + } + modifierArgs, err := expandModifierArgs(was, t.Modifier.Args) + if err != nil { + return nil, err + } + ae := &AggrFuncExpr{ + Name: t.Name, + Args: args, + Modifier: t.Modifier, + } + ae.Modifier.Args = modifierArgs + return ae, nil + case *parensExpr: + exprs, err := expandWithArgs(was, *t) + if err != nil { + return nil, err + } + pe := parensExpr(exprs) + return &pe, nil + case *StringExpr: + if len(t.S) > 0 { + // Already expanded. + return t, nil + } + var b []byte + for _, token := range t.tokens { + if isStringPrefix(token) { + s, err := extractStringValue(token) + if err != nil { + return nil, err + } + b = append(b, s...) + continue + } + wa := getWithArgExpr(was, token) + if wa == nil { + return nil, fmt.Errorf("missing %q value inside StringExpr", token) + } + eNew, err := expandWithExprExt(was, wa, nil) + if err != nil { + return nil, err + } + seSrc, ok := eNew.(*StringExpr) + if !ok { + return nil, fmt.Errorf("%q must be string expression; got %q", token, eNew.AppendString(nil)) + } + if len(seSrc.tokens) > 0 { + panic(fmt.Errorf("BUG: seSrc.tokens must be empty; got %q", seSrc.tokens)) + } + b = append(b, seSrc.S...) + } + se := &StringExpr{ + S: string(b), + } + return se, nil + case *RollupExpr: + eNew, err := expandWithExpr(was, t.Expr) + if err != nil { + return nil, err + } + re := *t + re.Expr = eNew + return &re, nil + case *withExpr: + wasNew := make([]*withArgExpr, 0, len(was)+len(t.Was)) + wasNew = append(wasNew, was...) + wasNew = append(wasNew, t.Was...) + eNew, err := expandWithExpr(wasNew, t.Expr) + if err != nil { + return nil, err + } + return eNew, nil + case *MetricExpr: + if len(t.LabelFilters) > 0 { + // Already expanded. + return t, nil + } + { + var me MetricExpr + // Populate me.LabelFilters + for _, lfe := range t.labelFilters { + if lfe.Value == nil { + // Expand lfe.Label into []LabelFilter. + wa := getWithArgExpr(was, lfe.Label) + if wa == nil { + return nil, fmt.Errorf("missing %q value inside %q", lfe.Label, t.AppendString(nil)) + } + eNew, err := expandWithExprExt(was, wa, nil) + if err != nil { + return nil, err + } + wme, ok := eNew.(*MetricExpr) + if !ok || wme.hasNonEmptyMetricGroup() { + return nil, fmt.Errorf("%q must be filters expression inside %q; got %q", lfe.Label, t.AppendString(nil), eNew.AppendString(nil)) + } + if len(wme.labelFilters) > 0 { + panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters)) + } + me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...) + continue + } + + // convert lfe to LabelFilter. + se, err := expandWithExpr(was, lfe.Value) + if err != nil { + return nil, err + } + var lfeNew labelFilterExpr + lfeNew.Label = lfe.Label + lfeNew.Value = se.(*StringExpr) + lfeNew.IsNegative = lfe.IsNegative + lfeNew.IsRegexp = lfe.IsRegexp + lf, err := lfeNew.toLabelFilter() + if err != nil { + return nil, err + } + me.LabelFilters = append(me.LabelFilters, *lf) + } + me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters) + t = &me + } + if !t.hasNonEmptyMetricGroup() { + return t, nil + } + k := string(appendEscapedIdent(nil, t.LabelFilters[0].Value)) + wa := getWithArgExpr(was, k) + if wa == nil { + return t, nil + } + eNew, err := expandWithExprExt(was, wa, nil) + if err != nil { + return nil, err + } + var wme *MetricExpr + re, _ := eNew.(*RollupExpr) + if re != nil { + wme, _ = re.Expr.(*MetricExpr) + } else { + wme, _ = eNew.(*MetricExpr) + } + if wme == nil { + if !t.isOnlyMetricGroup() { + return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) + } + return eNew, nil + } + if len(wme.labelFilters) > 0 { + panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters)) + } + + var me MetricExpr + me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...) + me.LabelFilters = append(me.LabelFilters, t.LabelFilters[1:]...) + me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters) + + if re == nil { + return &me, nil + } + reNew := *re + reNew.Expr = &me + return &reNew, nil + default: + return e, nil + } +} + +func expandWithArgs(was []*withArgExpr, args []Expr) ([]Expr, error) { + dstArgs := make([]Expr, len(args)) + for i, arg := range args { + dstArg, err := expandWithExpr(was, arg) + if err != nil { + return nil, err + } + dstArgs[i] = dstArg + } + return dstArgs, nil +} + +func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) { + if len(args) == 0 { + return nil, nil + } + dstArgs := make([]string, 0, len(args)) + for _, arg := range args { + wa := getWithArgExpr(was, arg) + if wa == nil { + // Leave the arg as is. + dstArgs = append(dstArgs, arg) + continue + } + if len(wa.Args) > 0 { + // Template funcs cannot be used inside modifier list. Leave the arg as is. + dstArgs = append(dstArgs, arg) + continue + } + me, ok := wa.Expr.(*MetricExpr) + if ok { + if !me.isOnlyMetricGroup() { + return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args) + } + dstArg := me.LabelFilters[0].Value + dstArgs = append(dstArgs, dstArg) + continue + } + pe, ok := wa.Expr.(*parensExpr) + if ok { + for _, pArg := range *pe { + me, ok := pArg.(*MetricExpr) + if !ok || !me.isOnlyMetricGroup() { + return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args) + } + dstArg := me.LabelFilters[0].Value + dstArgs = append(dstArgs, dstArg) + } + continue + } + return nil, fmt.Errorf("cannot use %q instead of %q in %s", wa.Expr.AppendString(nil), arg, args) + } + + // Remove duplicate args from dstArgs + m := make(map[string]bool, len(dstArgs)) + filteredArgs := dstArgs[:0] + for _, arg := range dstArgs { + if !m[arg] { + filteredArgs = append(filteredArgs, arg) + m[arg] = true + } + } + return filteredArgs, nil +} + +func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, error) { + if len(wa.Args) != len(args) { + if args == nil { + // Just return MetricExpr with the wa.Name name. + return newMetricExpr(wa.Name), nil + } + return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args)) + } + wasNew := make([]*withArgExpr, 0, len(was)+len(args)) + for _, waTmp := range was { + if waTmp == wa { + break + } + wasNew = append(wasNew, waTmp) + } + for i, arg := range args { + wasNew = append(wasNew, &withArgExpr{ + Name: wa.Args[i], + Expr: arg, + }) + } + return expandWithExpr(wasNew, wa.Expr) +} + +func newMetricExpr(name string) *MetricExpr { + return &MetricExpr{ + LabelFilters: []LabelFilter{{ + Label: "__name__", + Value: name, + }}, + } +} + +func extractStringValue(token string) (string, error) { + if !isStringPrefix(token) { + return "", fmt.Errorf(`StringExpr must contain only string literals; got %q`, token) + } + + // See https://prometheus.io/docs/prometheus/latest/querying/basics/#string-literals + if token[0] == '\'' { + if len(token) < 2 || token[len(token)-1] != '\'' { + return "", fmt.Errorf(`string literal contains unexpected trailing char; got %q`, token) + } + token = token[1 : len(token)-1] + token = strings.Replace(token, "\\'", "'", -1) + token = strings.Replace(token, `"`, `\"`, -1) + token = `"` + token + `"` + } + s, err := strconv.Unquote(token) + if err != nil { + return "", fmt.Errorf(`cannot parse string literal %q: %s`, token, err) + } + return s, nil +} + +func removeDuplicateLabelFilters(lfs []LabelFilter) []LabelFilter { + lfsm := make(map[string]bool, len(lfs)) + lfsNew := lfs[:0] + var buf []byte + for i := range lfs { + lf := &lfs[i] + buf = lf.AppendString(buf[:0]) + if lfsm[string(buf)] { + continue + } + lfsm[string(buf)] = true + lfsNew = append(lfsNew, *lf) + } + return lfsNew +} + +func (p *parser) parseFuncExpr() (*FuncExpr, error) { + if !isIdentPrefix(p.lex.Token) { + return nil, fmt.Errorf(`FuncExpr: unexpected token %q; want "ident"`, p.lex.Token) + } + + var fe FuncExpr + fe.Name = p.lex.Token + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token != "(" { + return nil, fmt.Errorf(`FuncExpr; unexpected token %q; want "("`, p.lex.Token) + } + args, err := p.parseArgListExpr() + if err != nil { + return nil, err + } + fe.Args = args + return &fe, nil +} + +func (p *parser) parseModifierExpr(me *ModifierExpr) error { + if !isIdentPrefix(p.lex.Token) { + return fmt.Errorf(`ModifierExpr: unexpected token %q; want "ident"`, p.lex.Token) + } + + me.Op = strings.ToLower(p.lex.Token) + + if err := p.lex.Next(); err != nil { + return err + } + if isBinaryOpJoinModifier(me.Op) && p.lex.Token != "(" { + // join modifier may miss ident list. + return nil + } + args, err := p.parseIdentList() + if err != nil { + return err + } + me.Args = args + return nil +} + +func (p *parser) parseIdentList() ([]string, error) { + if p.lex.Token != "(" { + return nil, fmt.Errorf(`identList: unexpected token %q; want "("`, p.lex.Token) + } + var idents []string + for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == ")" { + goto closeParensLabel + } + if !isIdentPrefix(p.lex.Token) { + return nil, fmt.Errorf(`identList: unexpected token %q; want "ident"`, p.lex.Token) + } + idents = append(idents, p.lex.Token) + if err := p.lex.Next(); err != nil { + return nil, err + } + switch p.lex.Token { + case ",": + continue + case ")": + goto closeParensLabel + default: + return nil, fmt.Errorf(`identList: unexpected token %q; want ",", ")"`, p.lex.Token) + } + } + +closeParensLabel: + if err := p.lex.Next(); err != nil { + return nil, err + } + return idents, nil +} + +func (p *parser) parseArgListExpr() ([]Expr, error) { + if p.lex.Token != "(" { + return nil, fmt.Errorf(`argList: unexpected token %q; want "("`, p.lex.Token) + } + var args []Expr + for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == ")" { + goto closeParensLabel + } + expr, err := p.parseExpr() + if err != nil { + return nil, err + } + args = append(args, expr) + switch p.lex.Token { + case ",": + continue + case ")": + goto closeParensLabel + default: + return nil, fmt.Errorf(`argList: unexpected token %q; want ",", ")"`, p.lex.Token) + } + } + +closeParensLabel: + if err := p.lex.Next(); err != nil { + return nil, err + } + return args, nil +} + +func getWithArgExpr(was []*withArgExpr, name string) *withArgExpr { + // Scan wes backwards, since certain expressions may override + // previously defined expressions + for i := len(was) - 1; i >= 0; i-- { + wa := was[i] + if wa.Name == name { + return wa + } + } + return nil +} + +func (p *parser) parseLabelFilters() ([]*labelFilterExpr, error) { + if p.lex.Token != "{" { + return nil, fmt.Errorf(`labelFilters: unexpected token %q; want "{"`, p.lex.Token) + } + + var lfes []*labelFilterExpr + for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == "}" { + goto closeBracesLabel + } + lfe, err := p.parseLabelFilterExpr() + if err != nil { + return nil, err + } + lfes = append(lfes, lfe) + switch p.lex.Token { + case ",": + continue + case "}": + goto closeBracesLabel + default: + return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "}"`, p.lex.Token) + } + } + +closeBracesLabel: + if err := p.lex.Next(); err != nil { + return nil, err + } + return lfes, nil +} + +func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) { + if !isIdentPrefix(p.lex.Token) { + return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "ident"`, p.lex.Token) + } + var lfe labelFilterExpr + lfe.Label = p.lex.Token + if err := p.lex.Next(); err != nil { + return nil, err + } + + switch p.lex.Token { + case "=": + // Nothing to do. + case "!=": + lfe.IsNegative = true + case "=~": + lfe.IsRegexp = true + case "!~": + lfe.IsNegative = true + lfe.IsRegexp = true + case ",", "}": + return &lfe, nil + default: + return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "}"`, p.lex.Token) + } + + if err := p.lex.Next(); err != nil { + return nil, err + } + se, err := p.parseStringExpr() + if err != nil { + return nil, err + } + lfe.Value = se + return &lfe, nil +} + +// labelFilterExpr represents `foo "bar"` expression, where is `=`, `!=`, `=~` or `!~`. +// +// This type isn't exported. +type labelFilterExpr struct { + Label string + Value *StringExpr + IsRegexp bool + IsNegative bool +} + +func (lfe *labelFilterExpr) String() string { + return fmt.Sprintf("[label=%q, value=%+v, isRegexp=%v, isNegative=%v]", lfe.Label, lfe.Value, lfe.IsRegexp, lfe.IsNegative) +} + +func (lfe *labelFilterExpr) toLabelFilter() (*LabelFilter, error) { + if lfe.Value == nil || len(lfe.Value.tokens) > 0 { + panic(fmt.Errorf("BUG: lfe.Value must be already expanded; got %v", lfe.Value)) + } + + var lf LabelFilter + lf.Label = unescapeIdent(lfe.Label) + if lf.Label == "__name__" { + lf.Value = unescapeIdent(lfe.Value.S) + } else { + lf.Value = lfe.Value.S + } + lf.IsRegexp = lfe.IsRegexp + lf.IsNegative = lfe.IsNegative + if !lf.IsRegexp { + return &lf, nil + } + + // Verify regexp. + if _, err := CompileRegexpAnchored(lfe.Value.S); err != nil { + return nil, fmt.Errorf("invalid regexp in %s=%q: %s", lf.Label, lf.Value, err) + } + return &lf, nil +} + +func (p *parser) parseWindowAndStep() (string, string, bool, error) { + if p.lex.Token != "[" { + return "", "", false, fmt.Errorf(`windowAndStep: unexpected token %q; want "["`, p.lex.Token) + } + err := p.lex.Next() + if err != nil { + return "", "", false, err + } + var window string + if !strings.HasPrefix(p.lex.Token, ":") { + window, err = p.parsePositiveDuration() + if err != nil { + return "", "", false, err + } + } + var step string + inheritStep := false + if strings.HasPrefix(p.lex.Token, ":") { + // Parse step + p.lex.Token = p.lex.Token[1:] + if p.lex.Token == "" { + if err := p.lex.Next(); err != nil { + return "", "", false, err + } + if p.lex.Token == "]" { + inheritStep = true + } + } + if p.lex.Token != "]" { + step, err = p.parsePositiveDuration() + if err != nil { + return "", "", false, err + } + } + } + if p.lex.Token != "]" { + return "", "", false, fmt.Errorf(`windowAndStep: unexpected token %q; want "]"`, p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return "", "", false, err + } + return window, step, inheritStep, nil +} + +func (p *parser) parseOffset() (string, error) { + if !isOffset(p.lex.Token) { + return "", fmt.Errorf(`offset: unexpected token %q; want "offset"`, p.lex.Token) + } + if err := p.lex.Next(); err != nil { + return "", err + } + d, err := p.parseDuration() + if err != nil { + return "", err + } + return d, nil +} + +func (p *parser) parseDuration() (string, error) { + isNegative := false + if p.lex.Token == "-" { + isNegative = true + if err := p.lex.Next(); err != nil { + return "", err + } + } + if !isPositiveDuration(p.lex.Token) { + return "", fmt.Errorf(`duration: unexpected token %q; want "duration"`, p.lex.Token) + } + d := p.lex.Token + if err := p.lex.Next(); err != nil { + return "", err + } + if isNegative { + d = "-" + d + } + return d, nil +} + +func (p *parser) parsePositiveDuration() (string, error) { + d, err := p.parseDuration() + if err != nil { + return "", err + } + if strings.HasPrefix(d, "-") { + return "", fmt.Errorf("positiveDuration: expecting positive duration; got %q", d) + } + return d, nil +} + +// parseIdentExpr parses expressions starting with `ident` token. +func (p *parser) parseIdentExpr() (Expr, error) { + // Look into the next-next token in order to determine how to parse + // the current expression. + if err := p.lex.Next(); err != nil { + return nil, err + } + if isEOF(p.lex.Token) || isOffset(p.lex.Token) { + p.lex.Prev() + return p.parseMetricExpr() + } + if isIdentPrefix(p.lex.Token) { + p.lex.Prev() + if isAggrFunc(p.lex.Token) { + return p.parseAggrFuncExpr() + } + return p.parseMetricExpr() + } + if isBinaryOp(p.lex.Token) { + p.lex.Prev() + return p.parseMetricExpr() + } + switch p.lex.Token { + case "(": + p.lex.Prev() + if isAggrFunc(p.lex.Token) { + return p.parseAggrFuncExpr() + } + return p.parseFuncExpr() + case "{", "[", ")", ",": + p.lex.Prev() + return p.parseMetricExpr() + default: + return nil, fmt.Errorf(`identExpr: unexpected token %q; want "(", "{", "[", ")", ","`, p.lex.Token) + } +} + +func (p *parser) parseMetricExpr() (*MetricExpr, error) { + var me MetricExpr + if isIdentPrefix(p.lex.Token) { + var lfe labelFilterExpr + lfe.Label = "__name__" + lfe.Value = &StringExpr{ + tokens: []string{strconv.Quote(p.lex.Token)}, + } + me.labelFilters = append(me.labelFilters[:0], &lfe) + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token != "{" { + return &me, nil + } + } + lfes, err := p.parseLabelFilters() + if err != nil { + return nil, err + } + me.labelFilters = append(me.labelFilters, lfes...) + return &me, nil +} + +func (p *parser) parseRollupExpr(arg Expr) (Expr, error) { + var re RollupExpr + re.Expr = arg + if p.lex.Token == "[" { + window, step, inheritStep, err := p.parseWindowAndStep() + if err != nil { + return nil, err + } + re.Window = window + re.Step = step + re.InheritStep = inheritStep + if !isOffset(p.lex.Token) { + return &re, nil + } + } + offset, err := p.parseOffset() + if err != nil { + return nil, err + } + re.Offset = offset + return &re, nil +} + +// StringExpr represents string expression. +type StringExpr struct { + // S contains unquoted value for string expression. + S string + + // Composite string has non-empty tokens. + // They must be converted into S by expandWithExpr. + tokens []string +} + +// AppendString appends string representation of se to dst and returns the result. +func (se *StringExpr) AppendString(dst []byte) []byte { + return strconv.AppendQuote(dst, se.S) +} + +// NumberExpr represents number expression. +type NumberExpr struct { + // N is the parsed number, i.e. `1.23`, `-234`, etc. + N float64 +} + +// AppendString appends string representation of ne to dst and returns the result. +func (ne *NumberExpr) AppendString(dst []byte) []byte { + return strconv.AppendFloat(dst, ne.N, 'g', -1, 64) +} + +// parensExpr represents `(...)`. +// +// It isn't exported. +type parensExpr []Expr + +// AppendString appends string representation of pe to dst and returns the result. +func (pe parensExpr) AppendString(dst []byte) []byte { + return appendStringArgListExpr(dst, pe) +} + +// BinaryOpExpr represents binary operation. +type BinaryOpExpr struct { + // Op is the operation itself, i.e. `+`, `-`, `*`, etc. + Op string + + // Bool indicates whether `bool` modifier is present. + // For example, `foo >bool bar`. + Bool bool + + // GroupModifier contains modifier such as "on" or "ignoring". + GroupModifier ModifierExpr + + // JoinModifier contains modifier such as "group_left" or "group_right". + JoinModifier ModifierExpr + + // Left contains left arg for the `left op right` expression. + Left Expr + + // Right contains right arg for the `left op right` epxression. + Right Expr +} + +// AppendString appends string representation of be to dst and returns the result. +func (be *BinaryOpExpr) AppendString(dst []byte) []byte { + if _, ok := be.Left.(*BinaryOpExpr); ok { + dst = append(dst, '(') + dst = be.Left.AppendString(dst) + dst = append(dst, ')') + } else { + dst = be.Left.AppendString(dst) + } + dst = append(dst, ' ') + dst = append(dst, be.Op...) + if be.Bool { + dst = append(dst, " bool"...) + } + if be.GroupModifier.Op != "" { + dst = append(dst, ' ') + dst = be.GroupModifier.AppendString(dst) + } + if be.JoinModifier.Op != "" { + dst = append(dst, ' ') + dst = be.JoinModifier.AppendString(dst) + } + dst = append(dst, ' ') + if _, ok := be.Right.(*BinaryOpExpr); ok { + dst = append(dst, '(') + dst = be.Right.AppendString(dst) + dst = append(dst, ')') + } else { + dst = be.Right.AppendString(dst) + } + return dst +} + +// ModifierExpr represents MetricsQL modifier such as ` (...)` +type ModifierExpr struct { + // Op is modifier operation. + Op string + + // Args contains modifier args from parens. + Args []string +} + +// AppendString appends string representation of me to dst and returns the result. +func (me *ModifierExpr) AppendString(dst []byte) []byte { + dst = append(dst, me.Op...) + dst = append(dst, " ("...) + for i, arg := range me.Args { + dst = append(dst, arg...) + if i+1 < len(me.Args) { + dst = append(dst, ", "...) + } + } + dst = append(dst, ')') + return dst +} + +func appendStringArgListExpr(dst []byte, args []Expr) []byte { + dst = append(dst, '(') + for i, arg := range args { + dst = arg.AppendString(dst) + if i+1 < len(args) { + dst = append(dst, ", "...) + } + } + dst = append(dst, ')') + return dst +} + +// FuncExpr represetns MetricsQL function such as `foo(...)` +type FuncExpr struct { + // Name is function name. + Name string + + // Args contains function args. + Args []Expr +} + +// AppendString appends string representation of fe to dst and returns the result. +func (fe *FuncExpr) AppendString(dst []byte) []byte { + dst = append(dst, fe.Name...) + dst = appendStringArgListExpr(dst, fe.Args) + return dst +} + +// AggrFuncExpr represents aggregate function such as `sum(...) by (...)` +type AggrFuncExpr struct { + // Name is the function name. + Name string + + // Args is the function args. + Args []Expr + + // Modifier is optional modifier such as `by (...)` or `without (...)`. + Modifier ModifierExpr +} + +// AppendString appends string representation of ae to dst and returns the result. +func (ae *AggrFuncExpr) AppendString(dst []byte) []byte { + dst = append(dst, ae.Name...) + dst = appendStringArgListExpr(dst, ae.Args) + if ae.Modifier.Op != "" { + dst = append(dst, ' ') + dst = ae.Modifier.AppendString(dst) + } + return dst +} + +// withExpr represents `with (...)` extension from MetricsQL. +// +// It isn't exported. +type withExpr struct { + Was []*withArgExpr + Expr Expr +} + +// AppendString appends string representation of we to dst and returns the result. +func (we *withExpr) AppendString(dst []byte) []byte { + dst = append(dst, "WITH ("...) + for i, wa := range we.Was { + dst = wa.AppendString(dst) + if i+1 < len(we.Was) { + dst = append(dst, ',') + } + } + dst = append(dst, ") "...) + dst = we.Expr.AppendString(dst) + return dst +} + +// withArgExpr represents a single entry from WITH expression. +// +// It isn't exported. +type withArgExpr struct { + Name string + Args []string + Expr Expr +} + +// AppendString appends string representation of wa to dst and returns the result. +func (wa *withArgExpr) AppendString(dst []byte) []byte { + dst = append(dst, wa.Name...) + if len(wa.Args) > 0 { + dst = append(dst, '(') + for i, arg := range wa.Args { + dst = append(dst, arg...) + if i+1 < len(wa.Args) { + dst = append(dst, ',') + } + } + dst = append(dst, ')') + } + dst = append(dst, " = "...) + dst = wa.Expr.AppendString(dst) + return dst +} + +// RollupExpr represents MetricsQL expression, which contains at least `offset` or `[...]` part. +type RollupExpr struct { + // The expression for the rollup. Usually it is MetricExpr, but may be arbitrary expr + // if subquery is used. https://prometheus.io/blog/2019/01/28/subquery-support/ + Expr Expr + + // Window contains optional window value from square brackets + // + // For example, `http_requests_total[5m]` will have Window value `5m`. + Window string + + // Offset contains optional value from `offset` part. + // + // For example, `foobar{baz="aa"} offset 5m` will have Offset value `5m`. + Offset string + + // Step contains optional step value from square brackets. + // + // For example, `foobar[1h:3m]` will have Step value '3m'. + Step string + + // If set to true, then `foo[1h:]` would print the same + // instead of `foo[1h]`. + InheritStep bool +} + +// ForSubquery returns true if re represents subquery. +func (re *RollupExpr) ForSubquery() bool { + return len(re.Step) > 0 || re.InheritStep +} + +// AppendString appends string representation of re to dst and returns the result. +func (re *RollupExpr) AppendString(dst []byte) []byte { + needParens := func() bool { + if _, ok := re.Expr.(*RollupExpr); ok { + return true + } + if _, ok := re.Expr.(*BinaryOpExpr); ok { + return true + } + if ae, ok := re.Expr.(*AggrFuncExpr); ok && ae.Modifier.Op != "" { + return true + } + return false + }() + if needParens { + dst = append(dst, '(') + } + dst = re.Expr.AppendString(dst) + if needParens { + dst = append(dst, ')') + } + if len(re.Window) > 0 || re.InheritStep || len(re.Step) > 0 { + dst = append(dst, '[') + if len(re.Window) > 0 { + dst = append(dst, re.Window...) + } + if len(re.Step) > 0 { + dst = append(dst, ':') + dst = append(dst, re.Step...) + } else if re.InheritStep { + dst = append(dst, ':') + } + dst = append(dst, ']') + } + if len(re.Offset) > 0 { + dst = append(dst, " offset "...) + dst = append(dst, re.Offset...) + } + return dst +} + +// LabelFilter represents MetricsQL label filter like `foo="bar"`. +type LabelFilter struct { + // Label contains label name for the filter. + Label string + + // Value contains unquoted value for the filter. + Value string + + // IsNegative reperesents whether the filter is negative, i.e. '!=' or '!~'. + IsNegative bool + + // IsRegexp represents whether the filter is regesp, i.e. `=~` or `!~`. + IsRegexp bool +} + +// AppendString appends string representation of me to dst and returns the result. +func (lf *LabelFilter) AppendString(dst []byte) []byte { + dst = appendEscapedIdent(dst, lf.Label) + var op string + if lf.IsNegative { + if lf.IsRegexp { + op = "!~" + } else { + op = "!=" + } + } else { + if lf.IsRegexp { + op = "=~" + } else { + op = "=" + } + } + dst = append(dst, op...) + dst = strconv.AppendQuote(dst, lf.Value) + return dst +} + +// MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. +type MetricExpr struct { + // LabelFilters contains a list of label filters from curly braces. + // Metric name if present must be the first. + LabelFilters []LabelFilter + + // labelFilters must be expanded to LabelFilters by expandWithExpr. + labelFilters []*labelFilterExpr +} + +// AppendString appends string representation of me to dst and returns the result. +func (me *MetricExpr) AppendString(dst []byte) []byte { + lfs := me.LabelFilters + if len(lfs) > 0 { + lf := &lfs[0] + if lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp { + dst = appendEscapedIdent(dst, lf.Value) + lfs = lfs[1:] + } + } + if len(lfs) > 0 { + dst = append(dst, '{') + for i := range lfs { + dst = lfs[i].AppendString(dst) + if i+1 < len(lfs) { + dst = append(dst, ", "...) + } + } + dst = append(dst, '}') + } else if len(me.LabelFilters) == 0 { + dst = append(dst, "{}"...) + } + return dst +} + +// IsEmpty returns true of me equals to `{}`. +func (me *MetricExpr) IsEmpty() bool { + return len(me.LabelFilters) == 0 +} + +func (me *MetricExpr) isOnlyMetricGroup() bool { + if !me.hasNonEmptyMetricGroup() { + return false + } + return len(me.LabelFilters) == 1 +} + +func (me *MetricExpr) hasNonEmptyMetricGroup() bool { + if len(me.LabelFilters) == 0 { + return false + } + lf := &me.LabelFilters[0] + return lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp +} diff --git a/lib/metricsql/parser_example_test.go b/lib/metricsql/parser_example_test.go new file mode 100644 index 000000000..b9e7540ce --- /dev/null +++ b/lib/metricsql/parser_example_test.go @@ -0,0 +1,35 @@ +package metricsql_test + +import ( + "fmt" + "log" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql" +) + +func ExampleParse() { + expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (x,y)`) + if err != nil { + log.Fatalf("parse error: %s", err) + } + fmt.Printf("parsed expr: %s\n", expr.AppendString(nil)) + + ae := expr.(*metricsql.AggrFuncExpr) + fmt.Printf("aggr func: name=%s, arg=%s, modifier=%s\n", ae.Name, ae.Args[0].AppendString(nil), ae.Modifier.AppendString(nil)) + + fe := ae.Args[0].(*metricsql.FuncExpr) + fmt.Printf("func: name=%s, arg=%s\n", fe.Name, fe.Args[0].AppendString(nil)) + + re := fe.Args[0].(*metricsql.RollupExpr) + fmt.Printf("rollup: expr=%s, window=%s\n", re.Expr.AppendString(nil), re.Window) + + me := re.Expr.(*metricsql.MetricExpr) + fmt.Printf("metric: labelFilter1=%s, labelFilter2=%s", me.LabelFilters[0].AppendString(nil), me.LabelFilters[1].AppendString(nil)) + + // Output: + // parsed expr: sum(rate(foo{bar="baz"}[5m])) by (x, y) + // aggr func: name=sum, arg=rate(foo{bar="baz"}[5m]), modifier=by (x, y) + // func: name=rate, arg=foo{bar="baz"}[5m] + // rollup: expr=foo{bar="baz"}, window=5m + // metric: labelFilter1=__name__="foo", labelFilter2=bar="baz" +} diff --git a/lib/promql/parser_test.go b/lib/metricsql/parser_test.go similarity index 87% rename from lib/promql/parser_test.go rename to lib/metricsql/parser_test.go index 73f951aeb..57a3410eb 100644 --- a/lib/promql/parser_test.go +++ b/lib/metricsql/parser_test.go @@ -1,24 +1,14 @@ -package promql +package metricsql import ( - "regexp" "testing" ) -var testParser = &Parser{ - compileRegexpAnchored: compileRegexpAnchored, -} - -func compileRegexpAnchored(re string) (*regexp.Regexp, error) { - reAnchored := "^(?:" + re + ")$" - return regexp.Compile(reAnchored) -} - -func TestParsePromQLSuccess(t *testing.T) { +func TestParseSuccess(t *testing.T) { another := func(s string, sExpected string) { t.Helper() - e, err := testParser.ParsePromQL(s) + e, err := Parse(s) if err != nil { t.Fatalf("unexpected error when parsing %q: %s", s, err) } @@ -150,24 +140,72 @@ func TestParsePromQLSuccess(t *testing.T) { another(`-inF`, `-Inf`) // binaryOpExpr + another(`nan == nan`, `NaN`) + another(`nan ==bool nan`, `1`) + another(`nan !=bool nan`, `0`) + another(`nan !=bool 2`, `1`) + another(`2 !=bool nan`, `1`) + another(`nan >bool nan`, `0`) + another(`nan =bool 2`, `1`) + another(`-1 >bool -inf`, `1`) + another(`-1 2`, `NaN`) + another(`1 > bool 2`, `0`) + another(`3 >= 2`, `3`) + another(`3 <= bool 2`, `0`) + another(`1 + -2 - 3`, `-4`) + another(`1 / 0 + 2`, `+Inf`) + another(`2 + -1 / 0`, `-Inf`) + another(`-1 ^ 0.5`, `NaN`) + another(`512.5 - (1 + 3) * (2 ^ 2) ^ 3`, `256.5`) + another(`1 == bool 1 != bool 24 < bool 4 > bool -1`, `1`) + another(`1 == bOOl 1 != BOOL 24 < Bool 4 > booL -1`, `1`) another(`m1+on(foo)group_left m2`, `m1 + on (foo) group_left () m2`) another(`M1+ON(FOO)GROUP_left M2`, `M1 + on (FOO) group_left () M2`) same(`m1 + on (foo) group_right () m2`) same(`m1 + on (foo, bar) group_right (x, y) m2`) another(`m1 + on (foo, bar,) group_right (x, y,) m2`, `m1 + on (foo, bar) group_right (x, y) m2`) same(`m1 == bool on (foo, bar) group_right (x, y) m2`) + another(`5 - 1 + 3 * 2 ^ 2 ^ 3 - 2 OR Metric {Bar= "Baz", aaa!="bb",cc=~"dd" ,zz !~"ff" } `, + `770 or Metric{Bar="Baz", aaa!="bb", cc=~"dd", zz!~"ff"}`) same(`"foo" + bar()`) same(`"foo" + bar{x="y"}`) same(`("foo"[3s] + bar{x="y"})[5m:3s] offset 10s`) same(`("foo"[3s] + bar{x="y"})[5i:3i] offset 10i`) same(`bar + "foo" offset 3s`) same(`bar + "foo" offset 3i`) + another(`1+2 if 2>3`, `NaN`) + another(`1+4 if 2<3`, `5`) + another(`2+6 default 3 if 2>3`, `8`) + another(`2+6 if 2>3 default NaN`, `NaN`) + another(`42 if 3>2 if 2+2<5`, `42`) + another(`42 if 3>2 if 2+2>=5`, `NaN`) + another(`1+2 ifnot 2>3`, `3`) + another(`1+4 ifnot 2<3`, `NaN`) + another(`2+6 default 3 ifnot 2>3`, `8`) + another(`2+6 ifnot 2>3 default NaN`, `8`) + another(`42 if 3>2 ifnot 2+2<5`, `NaN`) + another(`42 if 3>2 ifnot 2+2>=5`, `42`) // parensExpr another(`(-foo + ((bar) / (baz))) + ((23))`, `((0 - foo) + (bar / baz)) + 23`) @@ -176,6 +214,7 @@ func TestParsePromQLSuccess(t *testing.T) { another(`((foo, bar),(baz))`, `((foo, bar), baz)`) same(`(foo, (bar, baz), ((x, y), (z, y), xx))`) another(`1+(foo, bar,)`, `1 + (foo, bar)`) + another(`((foo(bar,baz)), (1+(2)+(3,4)+()))`, `(foo(bar, baz), (3 + (3, 4)) + ())`) same(`()`) // funcExpr @@ -192,7 +231,7 @@ func TestParsePromQLSuccess(t *testing.T) { same(`F(HttpServerRequest)`) same(`f(job, foo)`) same(`F(Job, Foo)`) - + another(` FOO (bar) + f ( m ( ),ff(1 + ( 2.5)) ,M[5m ] , "ff" )`, `FOO(bar) + f(m(), ff(3.5), M[5m], "ff")`) // funcName matching keywords same(`by(2)`) same(`BY(2)`) @@ -215,6 +254,8 @@ func TestParsePromQLSuccess(t *testing.T) { another(`sum by () (xx)`, `sum(xx) by ()`) another(`sum by (s) (xx)[5s]`, `(sum(xx) by (s))[5s]`) another(`SUM BY (ZZ, aa) (XX)`, `sum(XX) by (ZZ, aa)`) + another(`sum without (a, b) (xx,2+2)`, `sum(xx, 4) without (a, b)`) + another(`Sum WIthout (a, B) (XX,2+2)`, `sum(XX, 4) without (a, B)`) same(`sum(a) or sum(b)`) same(`sum(a) by () or sum(b) without (x, y)`) same(`sum(a) + sum(b)`) @@ -237,7 +278,7 @@ func TestParsePromQLSuccess(t *testing.T) { another(`with (foo = bar{x="x"}) "x"`, `"x"`) another(`with (f="x") f`, `"x"`) another(`with (foo = bar{x="x"}) x{x="y"}`, `x{x="y"}`) - another(`with (foo = bar{x="x"}) 2`, `2`) + another(`with (foo = bar{x="x"}) 1+1`, `2`) another(`with (foo = bar{x="x"}) f()`, `f()`) another(`with (foo = bar{x="x"}) sum(x)`, `sum(x)`) another(`with (foo = bar{x="x"}) baz{foo="bar"}`, `baz{foo="bar"}`) @@ -270,6 +311,7 @@ func TestParsePromQLSuccess(t *testing.T) { another(`with (x() = y+1) x`, `y + 1`) another(`with (x(foo) = foo+1) x(a)`, `a + 1`) another(`with (x(a, b) = a + b) x(foo, bar)`, `foo + bar`) + another(`with (x(a, b) = a + b) x(foo, x(1, 2))`, `foo + 3`) another(`with (x(a) = sum(a) by (b)) x(xx) / x(y)`, `sum(xx) by (b) / sum(y) by (b)`) another(`with (f(a,f,x)=ff(x,f,a)) f(f(x,y,z),1,2)`, `ff(2, 1, ff(z, y, x))`) another(`with (f(x)=1+f(x)) f(foo{bar="baz"})`, `1 + f(foo{bar="baz"})`) @@ -328,6 +370,18 @@ func TestParsePromQLSuccess(t *testing.T) { ) hitRatio < treshold`, `(sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance) / sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m]) + rate(cache{type="miss", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance)) < 0.9`) + another(`WITH ( + x2(x) = x^2, + f(x, y) = x2(x) + x*y + x2(y) + ) + f(a, 3) + `, `((a ^ 2) + (a * 3)) + 9`) + another(`WITH ( + x2(x) = x^2, + f(x, y) = x2(x) + x*y + x2(y) + ) + f(2, 3) + `, `19`) another(`WITH ( commonFilters = {instance="foo"}, timeToFuckup(currv, maxv) = (maxv - currv) / rate(currv) @@ -341,15 +395,16 @@ func TestParsePromQLSuccess(t *testing.T) { ) hitRate(cacheHits, cacheMisses)`, `sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) / (sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) + sum(rate(cacheMisses{job="foo", instance="bar"})) by (job, instance))`) + another(`with(y=123,z=5) union(with(y=3,f(x)=x*y) f(2) + f(3), with(x=5,y=2) x*y*z)`, `union(15, 50)`) } -func TestParsePromQLError(t *testing.T) { +func TestParseError(t *testing.T) { f := func(s string) { t.Helper() - e, err := testParser.ParsePromQL(s) + e, err := Parse(s) if err == nil { - t.Fatalf("expecting non-nil error when parsing %q (expr=%v)", s, e) + t.Fatalf("expecting non-nil error when parsing %q", s) } if e != nil { t.Fatalf("expecting nil expr when parsing %q", s) diff --git a/app/vmselect/promql/regexp_cache.go b/lib/metricsql/regexp_cache.go similarity index 88% rename from app/vmselect/promql/regexp_cache.go rename to lib/metricsql/regexp_cache.go index df09bb985..327a6dfed 100644 --- a/app/vmselect/promql/regexp_cache.go +++ b/lib/metricsql/regexp_cache.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "regexp" @@ -8,12 +8,14 @@ import ( "github.com/VictoriaMetrics/metrics" ) -func compileRegexpAnchored(re string) (*regexp.Regexp, error) { +// CompileRegexpAnchored returns compiled regexp `^re$`. +func CompileRegexpAnchored(re string) (*regexp.Regexp, error) { reAnchored := "^(?:" + re + ")$" - return compileRegexp(reAnchored) + return CompileRegexp(reAnchored) } -func compileRegexp(re string) (*regexp.Regexp, error) { +// CompileRegexp returns compile regexp re. +func CompileRegexp(re string) (*regexp.Regexp, error) { rcv := regexpCacheV.Get(re) if rcv != nil { return rcv.r, rcv.err diff --git a/lib/promql/rollup.go b/lib/metricsql/rollup.go similarity index 95% rename from lib/promql/rollup.go rename to lib/metricsql/rollup.go index 05d0e745d..178fec783 100644 --- a/lib/promql/rollup.go +++ b/lib/metricsql/rollup.go @@ -1,12 +1,10 @@ -package promql +package metricsql import ( "strings" ) var rollupFuncs = map[string]bool{ - "default_rollup": true, // default rollup func - // Standard rollup funcs from PromQL. // See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ . "changes": true, @@ -30,6 +28,7 @@ var rollupFuncs = map[string]bool{ "stdvar_over_time": true, // Additional rollup funcs. + "default_rollup": true, "sum2_over_time": true, "geomean_over_time": true, "first_over_time": true, diff --git a/lib/promql/transform.go b/lib/metricsql/transform.go similarity index 94% rename from lib/promql/transform.go rename to lib/metricsql/transform.go index fe987d404..44cf09c07 100644 --- a/lib/promql/transform.go +++ b/lib/metricsql/transform.go @@ -1,4 +1,4 @@ -package promql +package metricsql import ( "strings" @@ -36,7 +36,7 @@ var transformFuncs = map[string]bool{ "vector": true, "year": true, - // New funcs + // New funcs from MetricsQL "label_set": true, "label_del": true, "label_keep": true, @@ -45,7 +45,7 @@ var transformFuncs = map[string]bool{ "label_transform": true, "label_value": true, "union": true, - "": true, + "": true, // empty func is a synonim to union "keep_last_value": true, "start": true, "end": true, diff --git a/lib/promql/binary_op.go b/lib/promql/binary_op.go deleted file mode 100644 index 45df3dbd5..000000000 --- a/lib/promql/binary_op.go +++ /dev/null @@ -1,109 +0,0 @@ -package promql - -import ( - "strings" -) - -var binaryOpPriorities = map[string]int{ - "default": -1, - - "if": 0, - "ifnot": 0, - - // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence - "or": 1, - - "and": 2, - "unless": 2, - - "==": 3, - "!=": 3, - "<": 3, - ">": 3, - "<=": 3, - ">=": 3, - - "+": 4, - "-": 4, - - "*": 5, - "/": 5, - "%": 5, - - "^": 6, -} - -func isBinaryOp(op string) bool { - op = strings.ToLower(op) - _, ok := binaryOpPriorities[op] - return ok -} - -func binaryOpPriority(op string) int { - op = strings.ToLower(op) - return binaryOpPriorities[op] -} - -func scanBinaryOpPrefix(s string) int { - n := 0 - for op := range binaryOpPriorities { - if len(s) < len(op) { - continue - } - ss := strings.ToLower(s[:len(op)]) - if ss == op && len(op) > n { - n = len(op) - } - } - return n -} - -func isRightAssociativeBinaryOp(op string) bool { - // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence - return op == "^" -} - -func isBinaryOpGroupModifier(s string) bool { - s = strings.ToLower(s) - switch s { - // See https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching - case "on", "ignoring": - return true - default: - return false - } -} - -func isBinaryOpJoinModifier(s string) bool { - s = strings.ToLower(s) - switch s { - case "group_left", "group_right": - return true - default: - return false - } -} - -func isBinaryOpBoolModifier(s string) bool { - s = strings.ToLower(s) - return s == "bool" -} - -func isBinaryOpCmp(op string) bool { - switch op { - case "==", "!=", ">", "<", ">=", "<=": - return true - default: - return false - } -} - -func isBinaryOpLogicalSet(op string) bool { - op = strings.ToLower(op) - switch op { - case "and", "or", "unless": - return true - default: - return false - } -} diff --git a/lib/promql/expr.go b/lib/promql/expr.go deleted file mode 100644 index 011a50dad..000000000 --- a/lib/promql/expr.go +++ /dev/null @@ -1,423 +0,0 @@ -package promql - -import ( - "fmt" - "strconv" -) - -// An Expr represents a parsed Extended PromQL expression -type Expr interface { - // AppendString appends string representation of expr to dst. - AppendString(dst []byte) []byte -} - -// A StringExpr represents a string -type StringExpr struct { - S string -} - -// AppendString appends string representation of expr to dst. -func (se *StringExpr) AppendString(dst []byte) []byte { - return strconv.AppendQuote(dst, se.S) -} - -// A StringTemplateExpr represents a string prior to applying a With clause -type StringTemplateExpr struct { - // Composite string has non-empty tokens. - Tokens []StringToken -} - -// AppendString appends string representation of expr to dst. -func (ste *StringTemplateExpr) AppendString(dst []byte) []byte { - if ste == nil { - return dst - } - for i, tok := range ste.Tokens { - if i > 0 { - dst = append(dst, " + "...) - } - dst = tok.AppendString(dst) - } - return dst -} - -// A StringToken represents a portion of a string expression -type StringToken struct { - Ident bool - S string -} - -// AppendString appends string representation of st to dst. -func (st *StringToken) AppendString(dst []byte) []byte { - if st.Ident { - return appendEscapedIdent(dst, []byte(st.S)) - } - return strconv.AppendQuote(dst, st.S) -} - -// A NumberExpr represents a number -type NumberExpr struct { - N float64 -} - -// AppendString appends string representation of expr to dst. -func (ne *NumberExpr) AppendString(dst []byte) []byte { - return strconv.AppendFloat(dst, ne.N, 'g', -1, 64) -} - -// A ParensExpr represents a parens expression -type ParensExpr []Expr - -// AppendString appends string representation of expr to dst. -func (pe ParensExpr) AppendString(dst []byte) []byte { - return appendStringArgListExpr(dst, pe) -} - -// A BinaryOpExpr represents a binary operator -type BinaryOpExpr struct { - Op string - - Bool bool - GroupModifier ModifierExpr - JoinModifier ModifierExpr - - Left Expr - Right Expr -} - -// AppendString appends string representation of expr to dst. -func (be *BinaryOpExpr) AppendString(dst []byte) []byte { - if _, ok := be.Left.(*BinaryOpExpr); ok { - dst = append(dst, '(') - dst = be.Left.AppendString(dst) - dst = append(dst, ')') - } else { - dst = be.Left.AppendString(dst) - } - dst = append(dst, ' ') - dst = append(dst, be.Op...) - if be.Bool { - dst = append(dst, " bool"...) - } - if be.GroupModifier.Op != "" { - dst = append(dst, ' ') - dst = be.GroupModifier.AppendString(dst) - } - if be.JoinModifier.Op != "" { - dst = append(dst, ' ') - dst = be.JoinModifier.AppendString(dst) - } - dst = append(dst, ' ') - if _, ok := be.Right.(*BinaryOpExpr); ok { - dst = append(dst, '(') - dst = be.Right.AppendString(dst) - dst = append(dst, ')') - } else { - dst = be.Right.AppendString(dst) - } - return dst -} - -// A ModifierExpr represents a modifier attached to a parent expression -type ModifierExpr struct { - Op string - - Args []string -} - -// AppendString appends string representation of expr to dst. -func (me *ModifierExpr) AppendString(dst []byte) []byte { - dst = append(dst, me.Op...) - dst = append(dst, " ("...) - for i, arg := range me.Args { - dst = append(dst, arg...) - if i+1 < len(me.Args) { - dst = append(dst, ", "...) - } - } - dst = append(dst, ')') - return dst -} - -func appendStringArgListExpr(dst []byte, args []Expr) []byte { - dst = append(dst, '(') - for i, arg := range args { - dst = arg.AppendString(dst) - if i+1 < len(args) { - dst = append(dst, ", "...) - } - } - dst = append(dst, ')') - return dst -} - -// A FuncExpr represents a function invocation -type FuncExpr struct { - Name string - - Args []Expr -} - -// AppendString appends string representation of expr to dst. -func (fe *FuncExpr) AppendString(dst []byte) []byte { - dst = append(dst, fe.Name...) - dst = appendStringArgListExpr(dst, fe.Args) - return dst -} - -// An AggrFuncExpr represents the invocation of an aggregate function -type AggrFuncExpr struct { - Name string - - Args []Expr - - Modifier ModifierExpr -} - -// AppendString appends string representation of expr to dst. -func (ae *AggrFuncExpr) AppendString(dst []byte) []byte { - dst = append(dst, ae.Name...) - dst = appendStringArgListExpr(dst, ae.Args) - if ae.Modifier.Op != "" { - dst = append(dst, ' ') - dst = ae.Modifier.AppendString(dst) - } - return dst -} - -// A WithExpr represents a With expression -type WithExpr struct { - Was []*WithArgExpr - Expr Expr -} - -// AppendString appends string representation of expr to dst. -func (we *WithExpr) AppendString(dst []byte) []byte { - dst = append(dst, "WITH ("...) - for i, wa := range we.Was { - dst = wa.AppendString(dst) - if i+1 < len(we.Was) { - dst = append(dst, ',') - } - } - dst = append(dst, ") "...) - dst = we.Expr.AppendString(dst) - return dst -} - -// A WithArgExpr represents an arg in a With expression -type WithArgExpr struct { - Name string - Args []string - Expr Expr -} - -// AppendString appends string representation of expr to dst. -func (wa *WithArgExpr) AppendString(dst []byte) []byte { - dst = append(dst, wa.Name...) - if len(wa.Args) > 0 { - dst = append(dst, '(') - for i, arg := range wa.Args { - dst = append(dst, arg...) - if i+1 < len(wa.Args) { - dst = append(dst, ',') - } - } - dst = append(dst, ')') - } - dst = append(dst, " = "...) - dst = wa.Expr.AppendString(dst) - return dst -} - -// A RollupExpr represents a rollup expression -type RollupExpr struct { - // The expression for the rollup. Usually it is metricExpr, but may be arbitrary expr - // if subquery is used. https://prometheus.io/blog/2019/01/28/subquery-support/ - Expr Expr - - // Window contains optional window value from square brackets - // - // For example, `http_requests_total[5m]` will have Window value `5m`. - Window string - - // Offset contains optional value from `offset` part. - // - // For example, `foobar{baz="aa"} offset 5m` will have Offset value `5m`. - Offset string - - // Step contains optional step value from square brackets. - // - // For example, `foobar[1h:3m]` will have Step value '3m'. - Step string - - // If set to true, then `foo[1h:]` would print the same - // instead of `foo[1h]`. - InheritStep bool -} - -// ForSubquery returns whether is rollup is for a subquery -func (re *RollupExpr) ForSubquery() bool { - return len(re.Step) > 0 || re.InheritStep -} - -// AppendString appends string representation of expr to dst. -func (re *RollupExpr) AppendString(dst []byte) []byte { - needParens := func() bool { - if _, ok := re.Expr.(*RollupExpr); ok { - return true - } - if _, ok := re.Expr.(*BinaryOpExpr); ok { - return true - } - if ae, ok := re.Expr.(*AggrFuncExpr); ok && ae.Modifier.Op != "" { - return true - } - return false - }() - if needParens { - dst = append(dst, '(') - } - dst = re.Expr.AppendString(dst) - if needParens { - dst = append(dst, ')') - } - if len(re.Window) > 0 || re.InheritStep || len(re.Step) > 0 { - dst = append(dst, '[') - if len(re.Window) > 0 { - dst = append(dst, re.Window...) - } - if len(re.Step) > 0 { - dst = append(dst, ':') - dst = append(dst, re.Step...) - } else if re.InheritStep { - dst = append(dst, ':') - } - dst = append(dst, ']') - } - if len(re.Offset) > 0 { - dst = append(dst, " offset "...) - dst = append(dst, re.Offset...) - } - return dst -} - -// A MetricExpr represents a metric expression -type MetricExpr struct { - // TagFilters contains a list of tag filters from curly braces. - // The first item may be the metric name. - TagFilters []TagFilter -} - -// AppendString appends string representation of expr to dst. -func (me *MetricExpr) AppendString(dst []byte) []byte { - tfs := me.TagFilters - if len(tfs) > 0 { - tf := &tfs[0] - if len(tf.Key) == 0 && !tf.IsNegative && !tf.IsRegexp { - dst = appendEscapedIdent(dst, tf.Value) - tfs = tfs[1:] - } - } - if len(tfs) > 0 { - dst = append(dst, '{') - for i := range tfs { - tf := &tfs[i] - dst = appendStringTagFilter(dst, tf) - if i+1 < len(tfs) { - dst = append(dst, ", "...) - } - } - dst = append(dst, '}') - } else if len(me.TagFilters) == 0 { - dst = append(dst, "{}"...) - } - return dst -} - -// IsEmpty returns whether this is an empty metric expression -func (me *MetricExpr) IsEmpty() bool { - return len(me.TagFilters) == 0 -} - -// IsOnlyMetricGroup returns whether this is a metric group only -func (me *MetricExpr) IsOnlyMetricGroup() bool { - if !me.HasNonEmptyMetricGroup() { - return false - } - return len(me.TagFilters) == 1 -} - -// HasNonEmptyMetricGroup returns whether this has a non-empty metric group -func (me *MetricExpr) HasNonEmptyMetricGroup() bool { - if len(me.TagFilters) == 0 { - return false - } - tf := &me.TagFilters[0] - return len(tf.Key) == 0 && !tf.IsNegative && !tf.IsRegexp -} - -// A TagFilter is a single key value filter tag in a metric filter -// -// Note that this should exactly match the definition in the stroage package -type TagFilter struct { - Key []byte - Value []byte - IsNegative bool - IsRegexp bool -} - -// A MetricTemplateExpr represents a metric expression prior to expansion via -// a with clause -type MetricTemplateExpr struct { - TagFilters []*TagFilterExpr -} - -// AppendString appends string representation of expr to dst. -func (mte *MetricTemplateExpr) AppendString(dst []byte) []byte { - tfs := mte.TagFilters - if len(tfs) > 0 { - tf := tfs[0] - if len(tf.Key) == 0 && !tf.IsNegative && !tf.IsRegexp && len(tf.Value.Tokens) == 1 && !tf.Value.Tokens[0].Ident { - dst = appendEscapedIdent(dst, []byte(tf.Value.Tokens[0].S)) - tfs = tfs[1:] - } - } - if len(tfs) > 0 { - dst = append(dst, '{') - for i := range tfs { - tf := tfs[i] - dst = tf.AppendString(dst) - if i+1 < len(tfs) { - dst = append(dst, ", "...) - } - } - dst = append(dst, '}') - } else if len(mte.TagFilters) == 0 { - dst = append(dst, "{}"...) - } - return dst -} - -// A TagFilterExpr represents a tag filter -type TagFilterExpr struct { - Key string - Value *StringTemplateExpr - IsRegexp bool - IsNegative bool -} - -func (tfe *TagFilterExpr) String() string { - return fmt.Sprintf("[key=%q, value=%+v, isRegexp=%v, isNegative=%v]", tfe.Key, tfe.Value, tfe.IsRegexp, tfe.IsNegative) -} - -// AppendString appends string representation of expr to dst. -func (tfe *TagFilterExpr) AppendString(dst []byte) []byte { - if len(tfe.Key) == 0 { - dst = append(dst, "__name__"...) - } else { - dst = append(dst, tfe.Key...) - } - dst = appendStringTagFilterOp(dst, tfe.IsRegexp, tfe.IsNegative) - return tfe.Value.AppendString(dst) -} diff --git a/lib/promql/parser.go b/lib/promql/parser.go deleted file mode 100644 index 8d325df7f..000000000 --- a/lib/promql/parser.go +++ /dev/null @@ -1,1298 +0,0 @@ -package promql - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "sync" -) - -// Panicf controls how this package reports a runtime error indicative of a -// bug in the implementation. -var Panicf func(format string, args ...interface{}) = func(format string, args ...interface{}) { - panic(fmt.Errorf(format, args...)) -} - -// A Parser is a thread-safe object that can be used to parse Extended PromQL -// into a parsed tree of objects -type Parser struct { - compileRegexpAnchored func(re string) (*regexp.Regexp, error) -} - -// NewParser constructs a new Parser -func NewParser( - compileRegexpAnchored func(re string) (*regexp.Regexp, error), -) *Parser { - return &Parser{ - compileRegexpAnchored: compileRegexpAnchored, - } -} - -// ParsePromQL parses an extended PromQL string into an Expr object -func (p *Parser) ParsePromQL(s string) (Expr, error) { - e, err := p.ParseRawPromQL(s) - if err != nil { - return nil, err - } - was := p.getDefaultWithArgExprs() - if e, err = p.expandWithExpr(was, e); err != nil { - return nil, fmt.Errorf(`cannot expand WITH expressions: %s`, err) - } - e = removeParensExpr(e) - return e, nil -} - -// ParseRawPromQL parses an extended PromQL string into an Expr object, without -// rewriting or expanding with clauses -func (p *Parser) ParseRawPromQL(s string) (Expr, error) { - var ps parseState - ps.lex.Init(s) - if err := ps.lex.Next(); err != nil { - return nil, fmt.Errorf(`cannot find the first token: %s`, err) - } - e, err := ps.parseExpr() - if err != nil { - return nil, fmt.Errorf(`%s; unparsed data: %q`, err, ps.lex.Context()) - } - if !isEOF(ps.lex.Token) { - return nil, fmt.Errorf(`unparsed data left: %q`, ps.lex.Context()) - } - return e, nil -} - -func (p *Parser) expandWithExpr(was []*WithArgExpr, e Expr) (Expr, error) { - switch t := e.(type) { - case *BinaryOpExpr: - left, err := p.expandWithExpr(was, t.Left) - if err != nil { - return nil, err - } - right, err := p.expandWithExpr(was, t.Right) - if err != nil { - return nil, err - } - groupModifierArgs, err := p.expandModifierArgs(was, t.GroupModifier.Args) - if err != nil { - return nil, err - } - joinModifierArgs, err := p.expandModifierArgs(was, t.JoinModifier.Args) - if err != nil { - return nil, err - } - if t.Op == "+" { - lse, lok := left.(*StringExpr) - rse, rok := right.(*StringExpr) - if lok && rok { - se := &StringExpr{ - S: lse.S + rse.S, - } - return se, nil - } - } - be := &BinaryOpExpr{ - Op: t.Op, - Bool: t.Bool, - GroupModifier: t.GroupModifier, - JoinModifier: t.JoinModifier, - Left: left, - Right: right, - } - be.GroupModifier.Args = groupModifierArgs - be.JoinModifier.Args = joinModifierArgs - pe := ParensExpr{be} - return &pe, nil - case *FuncExpr: - args, err := p.expandWithArgs(was, t.Args) - if err != nil { - return nil, err - } - wa := getWithArgExpr(was, t.Name) - if wa == nil { - fe := &FuncExpr{ - Name: t.Name, - Args: args, - } - return fe, nil - } - return p.expandWithExprExt(was, wa, args) - case *AggrFuncExpr: - args, err := p.expandWithArgs(was, t.Args) - if err != nil { - return nil, err - } - modifierArgs, err := p.expandModifierArgs(was, t.Modifier.Args) - if err != nil { - return nil, err - } - ae := &AggrFuncExpr{ - Name: t.Name, - Args: args, - Modifier: t.Modifier, - } - ae.Modifier.Args = modifierArgs - return ae, nil - case *ParensExpr: - exprs, err := p.expandWithArgs(was, *t) - if err != nil { - return nil, err - } - pe := ParensExpr(exprs) - return &pe, nil - case *StringTemplateExpr: - var b []byte - for _, token := range t.Tokens { - if !token.Ident { - b = append(b, token.S...) - continue - } - wa := getWithArgExpr(was, token.S) - if wa == nil { - return nil, fmt.Errorf("missing %q value inside stringExpr", token.S) - } - eNew, err := p.expandWithExprExt(was, wa, nil) - if err != nil { - return nil, err - } - seSrc, ok := eNew.(*StringExpr) - if !ok { - return nil, fmt.Errorf("%q must be string expression; got %q", token.S, eNew.AppendString(nil)) - } - b = append(b, seSrc.S...) - } - se := &StringExpr{ - S: string(b), - } - return se, nil - case *RollupExpr: - eNew, err := p.expandWithExpr(was, t.Expr) - if err != nil { - return nil, err - } - re := *t - re.Expr = eNew - return &re, nil - case *WithExpr: - wasNew := make([]*WithArgExpr, 0, len(was)+len(t.Was)) - wasNew = append(wasNew, was...) - wasNew = append(wasNew, t.Was...) - eNew, err := p.expandWithExpr(wasNew, t.Expr) - if err != nil { - return nil, err - } - return eNew, nil - case *MetricTemplateExpr: - var newMe MetricExpr - // Populate converted tag filters - for _, tfe := range t.TagFilters { - if tfe.Value == nil { - // Expand tfe.Key into TagFilters. - wa := getWithArgExpr(was, tfe.Key) - if wa == nil { - return nil, fmt.Errorf("missing %q value inside %q", tfe.Key, t.AppendString(nil)) - } - eNew, err := p.expandWithExprExt(was, wa, nil) - if err != nil { - return nil, err - } - wme, ok := eNew.(*MetricExpr) - if !ok || wme.HasNonEmptyMetricGroup() { - return nil, fmt.Errorf("%q must be filters expression inside %q; got %q", tfe.Key, t.AppendString(nil), eNew.AppendString(nil)) - } - newMe.TagFilters = append(newMe.TagFilters, wme.TagFilters...) - continue - } - - // convert tfe to TagFilter. - se, err := p.expandWithExpr(was, tfe.Value) - if err != nil { - return nil, err - } - tf, err := p.createTagFilter(tfe.Key, se.(*StringExpr).S, tfe.IsRegexp, tfe.IsNegative) - if err != nil { - return nil, err - } - newMe.TagFilters = append(newMe.TagFilters, *tf) - } - newMe.TagFilters = p.removeDuplicateTagFilters(newMe.TagFilters) - if !newMe.HasNonEmptyMetricGroup() { - return &newMe, nil - } - k := string(appendEscapedIdent(nil, newMe.TagFilters[0].Value)) - wa := getWithArgExpr(was, k) - if wa == nil { - return &newMe, nil - } - eNew, err := p.expandWithExprExt(was, wa, nil) - if err != nil { - return nil, err - } - var wme *MetricExpr - re, _ := eNew.(*RollupExpr) - if re != nil { - wme, _ = re.Expr.(*MetricExpr) - } else { - wme, _ = eNew.(*MetricExpr) - } - if wme == nil { - if !newMe.IsOnlyMetricGroup() { - return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) - } - return eNew, nil - } - - rest := newMe.TagFilters[1:] - newMe.TagFilters = append(make([]TagFilter, 0, len(wme.TagFilters)+len(rest)), wme.TagFilters...) - newMe.TagFilters = append(newMe.TagFilters, rest...) - newMe.TagFilters = p.removeDuplicateTagFilters(newMe.TagFilters) - - if re == nil { - return &newMe, nil - } - reNew := *re - reNew.Expr = &newMe - return &reNew, nil - default: - return e, nil - } -} - -func (p *Parser) expandWithArgs(was []*WithArgExpr, args []Expr) ([]Expr, error) { - dstArgs := make([]Expr, len(args)) - for i, arg := range args { - dstArg, err := p.expandWithExpr(was, arg) - if err != nil { - return nil, err - } - dstArgs[i] = dstArg - } - return dstArgs, nil -} - -func (p *Parser) expandModifierArgs(was []*WithArgExpr, args []string) ([]string, error) { - if len(args) == 0 { - return nil, nil - } - dstArgs := make([]string, 0, len(args)) - for _, arg := range args { - wa := getWithArgExpr(was, arg) - if wa == nil { - // Leave the arg as is. - dstArgs = append(dstArgs, arg) - continue - } - if len(wa.Args) > 0 { - // Template funcs cannot be used inside modifier list. Leave the arg as is. - dstArgs = append(dstArgs, arg) - continue - } - me, ok := wa.Expr.(*MetricExpr) - if ok { - if !me.IsOnlyMetricGroup() { - return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args) - } - dstArg := string(me.TagFilters[0].Value) - dstArgs = append(dstArgs, dstArg) - continue - } - pe, ok := wa.Expr.(*ParensExpr) - if ok { - for _, pArg := range *pe { - me, ok := pArg.(*MetricExpr) - if !ok || !me.IsOnlyMetricGroup() { - return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args) - } - dstArg := string(me.TagFilters[0].Value) - dstArgs = append(dstArgs, dstArg) - } - continue - } - return nil, fmt.Errorf("cannot use %q instead of %q in %s", wa.Expr.AppendString(nil), arg, args) - } - - // Remove duplicate args from dstArgs - m := make(map[string]bool, len(dstArgs)) - filteredArgs := dstArgs[:0] - for _, arg := range dstArgs { - if !m[arg] { - filteredArgs = append(filteredArgs, arg) - m[arg] = true - } - } - return filteredArgs, nil -} - -func (p *Parser) expandWithExprExt(was []*WithArgExpr, wa *WithArgExpr, args []Expr) (Expr, error) { - if len(wa.Args) != len(args) { - if args == nil { - // Just return metricExpr with the wa.Name name. - return newMetricExpr(wa.Name), nil - } - return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args)) - } - wasNew := make([]*WithArgExpr, 0, len(was)+len(args)) - for _, waTmp := range was { - if waTmp == wa { - break - } - wasNew = append(wasNew, waTmp) - } - for i, arg := range args { - wasNew = append(wasNew, &WithArgExpr{ - Name: wa.Args[i], - Expr: arg, - }) - } - return p.expandWithExpr(wasNew, wa.Expr) -} - -func (p *Parser) removeDuplicateTagFilters(tfs []TagFilter) []TagFilter { - tfsm := make(map[string]bool, len(tfs)) - tfsNew := tfs[:0] - var bb []byte - for i := range tfs { - tf := &tfs[i] - bb = appendStringTagFilter(bb[:0], tf) - if tfsm[string(bb)] { - continue - } - tfsm[string(bb)] = true - tfsNew = append(tfsNew, *tf) - } - return tfsNew -} - -func (p *Parser) createTagFilter(key, value string, isRegexp, isNegative bool) (*TagFilter, error) { - var tf TagFilter - tf.Key = []byte(unescapeIdent(key)) - if len(key) == 0 { - tf.Value = []byte(unescapeIdent(value)) - } else { - tf.Value = []byte(value) - } - if string(tf.Key) == "__name__" { - // This is required for storage.Search - tf.Key = nil - } - tf.IsRegexp = isRegexp - tf.IsNegative = isNegative - if !tf.IsRegexp { - return &tf, nil - } - - // Verify regexp. - if _, err := p.compileRegexpAnchored(value); err != nil { - return nil, fmt.Errorf("invalid regexp in %s=%q: %s", tf.Key, tf.Value, err) - } - return &tf, nil -} - -func (p *Parser) getDefaultWithArgExprs() []*WithArgExpr { - defaultWithArgExprsOnce.Do(func() { - defaultWithArgExprs = p.prepareWithArgExprs([]string{ - // ru - resource utilization - `ru(freev, maxv) = clamp_min(maxv - clamp_min(freev, 0), 0) / clamp_min(maxv, 0) * 100`, - - // ttf - time to fuckup - `ttf(freev) = smooth_exponential( - clamp_max(clamp_max(-freev, 0) / clamp_max(deriv_fast(freev), 0), 365*24*3600), - clamp_max(step()/300, 1) - )`, - - `median_over_time(m) = quantile_over_time(0.5, m)`, - `range_median(q) = range_quantile(0.5, q)`, - `alias(q, name) = label_set(q, "__name__", name)`, - }) - }) - return defaultWithArgExprs -} - -var ( - defaultWithArgExprs []*WithArgExpr - defaultWithArgExprsOnce sync.Once -) - -func (p *Parser) prepareWithArgExprs(ss []string) []*WithArgExpr { - was := make([]*WithArgExpr, len(ss)) - for i, s := range ss { - was[i] = p.mustParseWithArgExpr(s) - } - if err := p.checkDuplicateWithArgNames(was); err != nil { - Panicf("BUG: %s", err) - } - return was -} - -func (p *Parser) checkDuplicateWithArgNames(was []*WithArgExpr) error { - m := make(map[string]*WithArgExpr, len(was)) - for _, wa := range was { - if waOld := m[wa.Name]; waOld != nil { - return fmt.Errorf("duplicate `with` arg name for: %s; previous one: %s", wa, waOld.AppendString(nil)) - } - m[wa.Name] = wa - } - return nil -} - -func (p *Parser) mustParseWithArgExpr(s string) *WithArgExpr { - var ps parseState - ps.lex.Init(s) - if err := ps.lex.Next(); err != nil { - Panicf("BUG: cannot find the first token in %q: %s", s, err) - } - wa, err := ps.parseWithArgExpr() - if err != nil { - Panicf("BUG: cannot parse %q: %s; unparsed data: %q", s, err, ps.lex.Context()) - } - return wa -} - -// removeParensExpr removes parensExpr for (expr) case. -func removeParensExpr(e Expr) Expr { - if re, ok := e.(*RollupExpr); ok { - re.Expr = removeParensExpr(re.Expr) - return re - } - if be, ok := e.(*BinaryOpExpr); ok { - be.Left = removeParensExpr(be.Left) - be.Right = removeParensExpr(be.Right) - return be - } - if ae, ok := e.(*AggrFuncExpr); ok { - for i, arg := range ae.Args { - ae.Args[i] = removeParensExpr(arg) - } - return ae - } - if fe, ok := e.(*FuncExpr); ok { - for i, arg := range fe.Args { - fe.Args[i] = removeParensExpr(arg) - } - return fe - } - if pe, ok := e.(*ParensExpr); ok { - args := *pe - for i, arg := range args { - args[i] = removeParensExpr(arg) - } - if len(*pe) == 1 { - return args[0] - } - // Treat parensExpr as a function with empty name, i.e. union() - fe := &FuncExpr{ - Name: "", - Args: args, - } - return fe - } - return e -} - -// parseState parses PromQL expression. -// -// preconditions for all parseState.parse* funcs: -// - p.lex.Token should point to the first token to parse. -// -// postconditions for all parseState.parse* funcs: -// - p.lex.Token should point to the next token after the parsed token. -type parseState struct { - parser *Parser - lex lexer -} - -func isWith(s string) bool { - s = strings.ToLower(s) - return s == "with" -} - -// parseWithExpr parses `WITH (withArgExpr...) expr`. -func (ps *parseState) parseWithExpr() (*WithExpr, error) { - var we WithExpr - if !isWith(ps.lex.Token) { - return nil, fmt.Errorf("withExpr: unexpected token %q; want `WITH`", ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token != "(" { - return nil, fmt.Errorf(`withExpr: unexpected token %q; want "("`, ps.lex.Token) - } - for { - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == ")" { - goto end - } - wa, err := ps.parseWithArgExpr() - if err != nil { - return nil, err - } - we.Was = append(we.Was, wa) - switch ps.lex.Token { - case ",": - continue - case ")": - goto end - default: - return nil, fmt.Errorf(`withExpr: unexpected token %q; want ",", ")"`, ps.lex.Token) - } - } - -end: - if err := ps.parser.checkDuplicateWithArgNames(we.Was); err != nil { - return nil, err - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - e, err := ps.parseExpr() - if err != nil { - return nil, err - } - we.Expr = e - return &we, nil -} - -func (ps *parseState) parseWithArgExpr() (*WithArgExpr, error) { - var wa WithArgExpr - if !isIdentPrefix(ps.lex.Token) { - return nil, fmt.Errorf(`withArgExpr: unexpected token %q; want "ident"`, ps.lex.Token) - } - wa.Name = ps.lex.Token - if isAggrFunc(wa.Name) || isRollupFunc(wa.Name) || isTransformFunc(wa.Name) || isWith(wa.Name) { - return nil, fmt.Errorf(`withArgExpr: cannot use reserved name %q`, wa.Name) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == "(" { - // Parse func args. - args, err := ps.parseIdentList() - if err != nil { - return nil, fmt.Errorf(`withArgExpr: cannot parse args for %q: %s`, wa.Name, err) - } - // Make sure all the args have different names - m := make(map[string]bool, len(args)) - for _, arg := range args { - if m[arg] { - return nil, fmt.Errorf(`withArgExpr: duplicate func arg found in %q: %q`, wa.Name, arg) - } - m[arg] = true - } - wa.Args = args - } - if ps.lex.Token != "=" { - return nil, fmt.Errorf(`withArgExpr: unexpected token %q; want "="`, ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - e, err := ps.parseExpr() - if err != nil { - return nil, fmt.Errorf(`withArgExpr: cannot parse %q: %s`, wa.Name, err) - } - wa.Expr = e - return &wa, nil -} - -// parseExpr parses promql expr -func (ps *parseState) parseExpr() (Expr, error) { - e, err := ps.parseSingleExpr() - if err != nil { - return nil, err - } - for { - if !isBinaryOp(ps.lex.Token) { - return e, nil - } - - var be BinaryOpExpr - be.Op = strings.ToLower(ps.lex.Token) - be.Left = e - if err := ps.lex.Next(); err != nil { - return nil, err - } - if isBinaryOpBoolModifier(ps.lex.Token) { - if !isBinaryOpCmp(be.Op) { - return nil, fmt.Errorf(`bool modifier cannot be applied to %q`, be.Op) - } - be.Bool = true - if err := ps.lex.Next(); err != nil { - return nil, err - } - } - if isBinaryOpGroupModifier(ps.lex.Token) { - if err := ps.parseModifierExpr(&be.GroupModifier); err != nil { - return nil, err - } - if isBinaryOpJoinModifier(ps.lex.Token) { - if isBinaryOpLogicalSet(be.Op) { - return nil, fmt.Errorf(`modifier %q cannot be applied to %q`, ps.lex.Token, be.Op) - } - if err := ps.parseModifierExpr(&be.JoinModifier); err != nil { - return nil, err - } - } - } - e2, err := ps.parseSingleExpr() - if err != nil { - return nil, err - } - be.Right = e2 - e = balanceBinaryOp(&be) - } -} - -func balanceBinaryOp(be *BinaryOpExpr) Expr { - bel, ok := be.Left.(*BinaryOpExpr) - if !ok { - return be - } - lp := binaryOpPriority(bel.Op) - rp := binaryOpPriority(be.Op) - if rp < lp { - return be - } - if rp == lp && !isRightAssociativeBinaryOp(be.Op) { - return be - } - be.Left = bel.Right - bel.Right = balanceBinaryOp(be) - return bel -} - -// parseSingleExpr parses non-binaryOp expressions. -func (ps *parseState) parseSingleExpr() (Expr, error) { - if isWith(ps.lex.Token) { - err := ps.lex.Next() - nextToken := ps.lex.Token - ps.lex.Prev() - if err == nil && nextToken == "(" { - return ps.parseWithExpr() - } - } - e, err := ps.parseSingleExprWithoutRollupSuffix() - if err != nil { - return nil, err - } - if ps.lex.Token != "[" && !isOffset(ps.lex.Token) { - // There is no rollup expression. - return e, nil - } - return ps.parseRollupExpr(e) -} - -func (ps *parseState) parseSingleExprWithoutRollupSuffix() (Expr, error) { - if isPositiveNumberPrefix(ps.lex.Token) || isInfOrNaN(ps.lex.Token) { - return ps.parsePositiveNumberExpr() - } - if isStringPrefix(ps.lex.Token) { - return ps.parseStringTemplateExpr() - } - if isIdentPrefix(ps.lex.Token) { - return ps.parseIdentExpr() - } - switch ps.lex.Token { - case "(": - return ps.parseParensExpr() - case "{": - return ps.parseMetricTemplateExpr() - case "-": - // Unary minus. Substitute -expr with (0 - expr) - if err := ps.lex.Next(); err != nil { - return nil, err - } - e, err := ps.parseSingleExpr() - if err != nil { - return nil, err - } - - // Fall back in the simple - case to a negative number - if ne, ok := e.(*NumberExpr); ok { - ne.N *= -1 - return ne, nil - } - - be := &BinaryOpExpr{ - Op: "-", - Left: &NumberExpr{ - N: 0, - }, - Right: e, - } - pe := ParensExpr{be} - return &pe, nil - case "+": - // Unary plus - if err := ps.lex.Next(); err != nil { - return nil, err - } - return ps.parseSingleExpr() - default: - return nil, fmt.Errorf(`singleExpr: unexpected token %q; want "(", "{", "-", "+"`, ps.lex.Token) - } -} - -func (ps *parseState) parsePositiveNumberExpr() (*NumberExpr, error) { - if !isPositiveNumberPrefix(ps.lex.Token) && !isInfOrNaN(ps.lex.Token) { - return nil, fmt.Errorf(`positiveNumberExpr: unexpected token %q; want "number"`, ps.lex.Token) - } - - n, err := strconv.ParseFloat(ps.lex.Token, 64) - if err != nil { - return nil, fmt.Errorf(`positiveNumberExpr: cannot parse %q: %s`, ps.lex.Token, err) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - ne := &NumberExpr{ - N: n, - } - return ne, nil -} - -func (ps *parseState) parseStringTemplateExpr() (*StringTemplateExpr, error) { - var se StringTemplateExpr - - for { - switch { - case isStringPrefix(ps.lex.Token): - s, err := extractStringValue(ps.lex.Token) - if err != nil { - return nil, err - } - se.Tokens = append(se.Tokens, StringToken{Ident: false, S: s}) - case isIdentPrefix(ps.lex.Token): - se.Tokens = append(se.Tokens, StringToken{Ident: true, S: ps.lex.Token}) - default: - return nil, fmt.Errorf(`stringExpr: unexpected token %q; want "string"`, ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token != "+" { - return &se, nil - } - - // composite stringExpr like `"s1" + "s2"`, `"s" + m()` or `"s" + m{}` or `"s" + unknownToken`. - if err := ps.lex.Next(); err != nil { - return nil, err - } - if isStringPrefix(ps.lex.Token) { - // "s1" + "s2" - continue - } - if !isIdentPrefix(ps.lex.Token) { - // "s" + unknownToken - ps.lex.Prev() - return &se, nil - } - // Look after ident - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == "(" || ps.lex.Token == "{" { - // `"s" + m(` or `"s" + m{` - ps.lex.Prev() - ps.lex.Prev() - return &se, nil - } - // "s" + ident - ps.lex.Prev() - } -} - -func (ps *parseState) parseParensExpr() (*ParensExpr, error) { - if ps.lex.Token != "(" { - return nil, fmt.Errorf(`parensExpr: unexpected token %q; want "("`, ps.lex.Token) - } - var exprs []Expr - for { - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == ")" { - break - } - expr, err := ps.parseExpr() - if err != nil { - return nil, err - } - exprs = append(exprs, expr) - if ps.lex.Token == "," { - continue - } - if ps.lex.Token == ")" { - break - } - return nil, fmt.Errorf(`parensExpr: unexpected token %q; want "," or ")"`, ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return nil, err - } - pe := ParensExpr(exprs) - return &pe, nil -} - -func (ps *parseState) parseAggrFuncExpr() (*AggrFuncExpr, error) { - if !isAggrFunc(ps.lex.Token) { - return nil, fmt.Errorf(`aggrFuncExpr: unexpected token %q; want aggregate func`, ps.lex.Token) - } - - var ae AggrFuncExpr - ae.Name = strings.ToLower(ps.lex.Token) - if err := ps.lex.Next(); err != nil { - return nil, err - } - if isIdentPrefix(ps.lex.Token) { - goto funcPrefixLabel - } - switch ps.lex.Token { - case "(": - goto funcArgsLabel - default: - return nil, fmt.Errorf(`aggrFuncExpr: unexpected token %q; want "("`, ps.lex.Token) - } - -funcPrefixLabel: - { - if !isAggrFuncModifier(ps.lex.Token) { - return nil, fmt.Errorf(`aggrFuncExpr: unexpected token %q; want aggregate func modifier`, ps.lex.Token) - } - if err := ps.parseModifierExpr(&ae.Modifier); err != nil { - return nil, err - } - goto funcArgsLabel - } - -funcArgsLabel: - { - args, err := ps.parseArgListExpr() - if err != nil { - return nil, err - } - ae.Args = args - - // Verify whether func suffix exists. - if ae.Modifier.Op != "" || !isAggrFuncModifier(ps.lex.Token) { - return &ae, nil - } - if err := ps.parseModifierExpr(&ae.Modifier); err != nil { - return nil, err - } - return &ae, nil - } -} - -func newMetricExpr(name string) *MetricExpr { - return &MetricExpr{ - TagFilters: []TagFilter{{ - Value: []byte(name), - }}, - } -} - -func extractStringValue(token string) (string, error) { - if !isStringPrefix(token) { - return "", fmt.Errorf(`stringExpr must contain only string literals; got %q`, token) - } - - // See https://prometheus.io/docs/prometheus/latest/querying/basics/#string-literals - if token[0] == '\'' { - if len(token) < 2 || token[len(token)-1] != '\'' { - return "", fmt.Errorf(`string literal contains unexpected trailing char; got %q`, token) - } - token = token[1 : len(token)-1] - token = strings.Replace(token, "\\'", "'", -1) - token = strings.Replace(token, `"`, `\"`, -1) - token = `"` + token + `"` - } - s, err := strconv.Unquote(token) - if err != nil { - return "", fmt.Errorf(`cannot parse string literal %q: %s`, token, err) - } - return s, nil -} - -func (ps *parseState) parseFuncExpr() (*FuncExpr, error) { - if !isIdentPrefix(ps.lex.Token) { - return nil, fmt.Errorf(`funcExpr: unexpected token %q; want "ident"`, ps.lex.Token) - } - - var fe FuncExpr - fe.Name = ps.lex.Token - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token != "(" { - return nil, fmt.Errorf(`funcExpr; unexpected token %q; want "("`, ps.lex.Token) - } - args, err := ps.parseArgListExpr() - if err != nil { - return nil, err - } - fe.Args = args - return &fe, nil -} - -func (ps *parseState) parseModifierExpr(me *ModifierExpr) error { - if !isIdentPrefix(ps.lex.Token) { - return fmt.Errorf(`modifierExpr: unexpected token %q; want "ident"`, ps.lex.Token) - } - - me.Op = strings.ToLower(ps.lex.Token) - - if err := ps.lex.Next(); err != nil { - return err - } - if isBinaryOpJoinModifier(me.Op) && ps.lex.Token != "(" { - // join modifier may miss ident list. - return nil - } - args, err := ps.parseIdentList() - if err != nil { - return err - } - me.Args = args - return nil -} - -func (ps *parseState) parseIdentList() ([]string, error) { - if ps.lex.Token != "(" { - return nil, fmt.Errorf(`identList: unexpected token %q; want "("`, ps.lex.Token) - } - var idents []string - for { - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == ")" { - goto closeParensLabel - } - if !isIdentPrefix(ps.lex.Token) { - return nil, fmt.Errorf(`identList: unexpected token %q; want "ident"`, ps.lex.Token) - } - idents = append(idents, ps.lex.Token) - if err := ps.lex.Next(); err != nil { - return nil, err - } - switch ps.lex.Token { - case ",": - continue - case ")": - goto closeParensLabel - default: - return nil, fmt.Errorf(`identList: unexpected token %q; want ",", ")"`, ps.lex.Token) - } - } - -closeParensLabel: - if err := ps.lex.Next(); err != nil { - return nil, err - } - return idents, nil -} - -func (ps *parseState) parseArgListExpr() ([]Expr, error) { - if ps.lex.Token != "(" { - return nil, fmt.Errorf(`argList: unexpected token %q; want "("`, ps.lex.Token) - } - var args []Expr - for { - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == ")" { - goto closeParensLabel - } - expr, err := ps.parseExpr() - if err != nil { - return nil, err - } - args = append(args, expr) - switch ps.lex.Token { - case ",": - continue - case ")": - goto closeParensLabel - default: - return nil, fmt.Errorf(`argList: unexpected token %q; want ",", ")"`, ps.lex.Token) - } - } - -closeParensLabel: - if err := ps.lex.Next(); err != nil { - return nil, err - } - return args, nil -} - -func getWithArgExpr(was []*WithArgExpr, name string) *WithArgExpr { - // Scan wes backwards, since certain expressions may override - // previously defined expressions - for i := len(was) - 1; i >= 0; i-- { - wa := was[i] - if wa.Name == name { - return wa - } - } - return nil -} - -func (ps *parseState) parseTagFilters() ([]*TagFilterExpr, error) { - if ps.lex.Token != "{" { - return nil, fmt.Errorf(`tagFilters: unexpected token %q; want "{"`, ps.lex.Token) - } - - var tfes []*TagFilterExpr - for { - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token == "}" { - goto closeBracesLabel - } - tfe, err := ps.parseTagFilterExpr() - if err != nil { - return nil, err - } - tfes = append(tfes, tfe) - switch ps.lex.Token { - case ",": - continue - case "}": - goto closeBracesLabel - default: - return nil, fmt.Errorf(`tagFilters: unexpected token %q; want ",", "}"`, ps.lex.Token) - } - } - -closeBracesLabel: - if err := ps.lex.Next(); err != nil { - return nil, err - } - return tfes, nil -} - -func (ps *parseState) parseTagFilterExpr() (*TagFilterExpr, error) { - if !isIdentPrefix(ps.lex.Token) { - return nil, fmt.Errorf(`tagFilterExpr: unexpected token %q; want "ident"`, ps.lex.Token) - } - var tfe TagFilterExpr - tfe.Key = ps.lex.Token - if err := ps.lex.Next(); err != nil { - return nil, err - } - - switch ps.lex.Token { - case "=": - // Nothing to do. - case "!=": - tfe.IsNegative = true - case "=~": - tfe.IsRegexp = true - case "!~": - tfe.IsNegative = true - tfe.IsRegexp = true - case ",", "}": - return &tfe, nil - default: - return nil, fmt.Errorf(`tagFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "}"`, ps.lex.Token) - } - - if err := ps.lex.Next(); err != nil { - return nil, err - } - se, err := ps.parseStringTemplateExpr() - if err != nil { - return nil, err - } - tfe.Value = se - return &tfe, nil -} - -func (ps *parseState) parseWindowAndStep() (string, string, bool, error) { - if ps.lex.Token != "[" { - return "", "", false, fmt.Errorf(`windowAndStep: unexpected token %q; want "["`, ps.lex.Token) - } - err := ps.lex.Next() - if err != nil { - return "", "", false, err - } - var window string - if !strings.HasPrefix(ps.lex.Token, ":") { - window, err = ps.parseDuration() - if err != nil { - return "", "", false, err - } - } - var step string - inheritStep := false - if strings.HasPrefix(ps.lex.Token, ":") { - // Parse step - ps.lex.Token = ps.lex.Token[1:] - if ps.lex.Token == "" { - if err := ps.lex.Next(); err != nil { - return "", "", false, err - } - if ps.lex.Token == "]" { - inheritStep = true - } - } - if ps.lex.Token != "]" { - step, err = ps.parseDuration() - if err != nil { - return "", "", false, err - } - } - } - if ps.lex.Token != "]" { - return "", "", false, fmt.Errorf(`windowAndStep: unexpected token %q; want "]"`, ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return "", "", false, err - } - return window, step, inheritStep, nil -} - -func (ps *parseState) parseOffset() (string, error) { - if !isOffset(ps.lex.Token) { - return "", fmt.Errorf(`offset: unexpected token %q; want "offset"`, ps.lex.Token) - } - if err := ps.lex.Next(); err != nil { - return "", err - } - d, err := ps.parseDuration() - if err != nil { - return "", err - } - return d, nil -} - -func (ps *parseState) parseDuration() (string, error) { - if !isDuration(ps.lex.Token) { - return "", fmt.Errorf(`duration: unexpected token %q; want "duration"`, ps.lex.Token) - } - d := ps.lex.Token - if err := ps.lex.Next(); err != nil { - return "", err - } - return d, nil -} - -// parseIdentExpr parses expressions starting with `ident` token. -func (ps *parseState) parseIdentExpr() (Expr, error) { - // Look into the next-next token in order to determine how to parse - // the current expression. - if err := ps.lex.Next(); err != nil { - return nil, err - } - if isEOF(ps.lex.Token) || isOffset(ps.lex.Token) { - ps.lex.Prev() - return ps.parseMetricTemplateExpr() - } - if isIdentPrefix(ps.lex.Token) { - ps.lex.Prev() - if isAggrFunc(ps.lex.Token) { - return ps.parseAggrFuncExpr() - } - return ps.parseMetricTemplateExpr() - } - if isBinaryOp(ps.lex.Token) { - ps.lex.Prev() - return ps.parseMetricTemplateExpr() - } - switch ps.lex.Token { - case "(": - ps.lex.Prev() - if isAggrFunc(ps.lex.Token) { - return ps.parseAggrFuncExpr() - } - return ps.parseFuncExpr() - case "{", "[", ")", ",": - ps.lex.Prev() - return ps.parseMetricTemplateExpr() - default: - return nil, fmt.Errorf(`identExpr: unexpected token %q; want "(", "{", "[", ")", ","`, ps.lex.Token) - } -} - -func (ps *parseState) parseMetricTemplateExpr() (*MetricTemplateExpr, error) { - var me MetricTemplateExpr - if isIdentPrefix(ps.lex.Token) { - var tfe TagFilterExpr - tfe.Value = &StringTemplateExpr{ - Tokens: []StringToken{ - { - Ident: false, - S: ps.lex.Token, - }, - }, - } - me.TagFilters = append(me.TagFilters[:0], &tfe) - if err := ps.lex.Next(); err != nil { - return nil, err - } - if ps.lex.Token != "{" { - return &me, nil - } - } - tfes, err := ps.parseTagFilters() - if err != nil { - return nil, err - } - me.TagFilters = append(me.TagFilters, tfes...) - return &me, nil -} - -func (ps *parseState) parseRollupExpr(arg Expr) (Expr, error) { - var re RollupExpr - re.Expr = arg - if ps.lex.Token == "[" { - window, step, inheritStep, err := ps.parseWindowAndStep() - if err != nil { - return nil, err - } - re.Window = window - re.Step = step - re.InheritStep = inheritStep - if !isOffset(ps.lex.Token) { - return &re, nil - } - } - offset, err := ps.parseOffset() - if err != nil { - return nil, err - } - re.Offset = offset - return &re, nil -} - -func appendStringTagFilter(dst []byte, tf *TagFilter) []byte { - if len(tf.Key) == 0 { - dst = append(dst, "__name__"...) - } else { - dst = appendEscapedIdent(dst, []byte(tf.Key)) - } - dst = appendStringTagFilterOp(dst, tf.IsRegexp, tf.IsNegative) - return strconv.AppendQuote(dst, string(tf.Value)) -} - -func appendStringTagFilterOp(dst []byte, isRegexp, isNegative bool) []byte { - var op string - if isNegative { - if isRegexp { - op = "!~" - } else { - op = "!=" - } - } else { - if isRegexp { - op = "=~" - } else { - op = "=" - } - } - return append(dst, op...) -} diff --git a/lib/promql/visitor.go b/lib/promql/visitor.go deleted file mode 100644 index d15aa4873..000000000 --- a/lib/promql/visitor.go +++ /dev/null @@ -1,47 +0,0 @@ -package promql - -// A Visitor is used to walk a parsed query -type Visitor interface { - Visit(expr Expr) Visitor -} - -// Walk invokes Visit on v for each node in the parsed query tree -func Walk(expr Expr, v Visitor) { - nv := v.Visit(expr) - if nv == nil { - return - } - switch t := expr.(type) { - case *ParensExpr: - for _, e := range *t { - Walk(e, nv) - } - case *BinaryOpExpr: - Walk(t.Left, nv) - Walk(t.Right, nv) - case *FuncExpr: - for _, ae := range t.Args { - Walk(ae, nv) - } - case *AggrFuncExpr: - for _, ae := range t.Args { - Walk(ae, nv) - } - Walk(&t.Modifier, nv) - case *WithExpr: - for _, wa := range t.Was { - Walk(wa, nv) - } - Walk(t.Expr, nv) - case *WithArgExpr: - Walk(t.Expr, nv) - case *RollupExpr: - Walk(t.Expr, nv) - case *MetricTemplateExpr: - for _, tfe := range t.TagFilters { - Walk(tfe, nv) - } - case *TagFilterExpr: - Walk(t.Value, nv) - } -} diff --git a/lib/promql/visitor_test.go b/lib/promql/visitor_test.go deleted file mode 100644 index 0a10004e4..000000000 --- a/lib/promql/visitor_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package promql - -import ( - "fmt" - "strings" - "testing" -) - -func TestWalk(t *testing.T) { - e, err := testParser.ParseRawPromQL(` - WITH ( - rf(a, b) = a + b - ) - rf(metric1{foo="bar"}, metric2) or (sum(abs(changes(metric3)))) - `) - if err != nil { - t.Fatalf("unexpected error when parsing: %s", err) - } - var cv collectVisitor - Walk(e, &cv) - expected := []string{ - `*promql.WithExpr WITH (rf(a,b) = a + b) rf(metric1{foo="bar"}, metric2) or (sum(abs(changes(metric3))))`, - `*promql.WithArgExpr rf(a,b) = a + b`, - `*promql.BinaryOpExpr a + b`, - `*promql.MetricTemplateExpr a`, - `*promql.TagFilterExpr __name__="a"`, - `*promql.StringTemplateExpr "a"`, - `*promql.MetricTemplateExpr b`, - `*promql.TagFilterExpr __name__="b"`, - `*promql.StringTemplateExpr "b"`, - `*promql.BinaryOpExpr rf(metric1{foo="bar"}, metric2) or (sum(abs(changes(metric3))))`, - `*promql.FuncExpr rf(metric1{foo="bar"}, metric2)`, - `*promql.MetricTemplateExpr metric1{foo="bar"}`, - `*promql.TagFilterExpr __name__="metric1"`, - `*promql.StringTemplateExpr "metric1"`, - `*promql.TagFilterExpr foo="bar"`, - `*promql.StringTemplateExpr "bar"`, - `*promql.MetricTemplateExpr metric2`, - `*promql.TagFilterExpr __name__="metric2"`, - `*promql.StringTemplateExpr "metric2"`, - `*promql.ParensExpr (sum(abs(changes(metric3))))`, - `*promql.AggrFuncExpr sum(abs(changes(metric3)))`, - `*promql.FuncExpr abs(changes(metric3))`, - `*promql.FuncExpr changes(metric3)`, - `*promql.MetricTemplateExpr metric3`, - `*promql.TagFilterExpr __name__="metric3"`, - `*promql.StringTemplateExpr "metric3"`, - `*promql.ModifierExpr ()`, - } - if len(cv.visited) != len(expected) { - t.Fatal("Expected", len(expected), "elements visited, got", len(cv.visited)) - } - for i, v := range cv.visited { - if strings.TrimSpace(v) != expected[i] { - t.Fatalf("Expected %s, got %s at position %v", expected[i], v, i) - } - } -} - -type collectVisitor struct { - visited []string -} - -func (cv *collectVisitor) Visit(e Expr) Visitor { - sb := e.AppendString(nil) - s := fmt.Sprintf("%T %v\n", e, string(sb)) - cv.visited = append(cv.visited, s) - return cv -}