app/vmselect/promql: allow using SI and IEC suffixes in numeric values inside queries

For example, 10Ki is equivalent to 10*1024, while 5.3M is equivalent to 5.3*1000*1000

docs/CHANGELOG.md

@@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 ## tip
 
 * FEATURE: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): add `-storageNode.filter` command-line flag for filtering the [discovered vmstorage nodes](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery) with arbitrary regular expressions. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3353).
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow using numeric values with `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes inside MetricsQL queries. For example `8Ki` equals to `8*1024`, while `8.2M` equals to `8.2*1000*1000`.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_linear_regression](https://docs.victoriametrics.com/MetricsQL.html#range_linear_regression) function for calculating [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression) over the input time series on the selected time range. This function is useful for predictions and capacity planning. For example, `range_linear_regression(process_resident_memory_bytes)` can predict future memory usage based on the past memory usage.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_stddev](https://docs.victoriametrics.com/MetricsQL.html#range_stddev) and [range_stdvar](https://docs.victoriametrics.com/MetricsQL.html#range_stdvar) functions.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): optimize `expr1 op expr2` query when `expr1` returns an empty result. In this case there is no sense in executing `expr2` for `op` not equal to `or`, since the end result will be empty according to [PromQL series matching rules](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349). Thanks to @jianglinjian for pointing to this case.

docs/MetricsQL.md

@@ -73,6 +73,7 @@ The list of MetricsQL features:
 * The duration suffix is optional. The duration is in seconds if the suffix is missing.
   For example, `rate(m[300] offset 1800)` is equivalent to `rate(m[5m]) offset 30m`.
 * The duration can be placed anywhere in the query. For example, `sum_over_time(m[1h]) / 1h` is equivalent to `sum_over_time(m[1h]) / 3600`.
+* Numeric values can have `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes. For example, `8K` is equivalent to `8000`, while `1.2Mi` is equvalent to `1.2*1024*1024`.
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.

go.mod

@@ -12,7 +12,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.1.0
 	github.com/VictoriaMetrics/metrics v1.23.0
-	github.com/VictoriaMetrics/metricsql v0.47.0
+	github.com/VictoriaMetrics/metricsql v0.48.0
 	github.com/aws/aws-sdk-go-v2 v1.17.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.1
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.40

go.sum

@@ -100,8 +100,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
 github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
 github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA=
 github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
-github.com/VictoriaMetrics/metricsql v0.47.0 h1:PQwadjoQnKKkaUiupkDq0ZbCAHX2qP8OOexJ9oJwupo=
+github.com/VictoriaMetrics/metricsql v0.48.0 h1:rq8ULfIDJ0QyDbyQWRuWrMTffEqL2sevU2Zs3Vx1pfw=
-github.com/VictoriaMetrics/metricsql v0.47.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
+github.com/VictoriaMetrics/metricsql v0.48.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=

vendor/github.com/VictoriaMetrics/metricsql/lexer.go

@@ -149,6 +149,73 @@ func scanString(s string) (string, error) {
 	}
 }
 
+func parsePositiveNumber(s string) (float64, error) {
+	if isSpecialIntegerPrefix(s) {
+		n, err := strconv.ParseInt(s, 0, 64)
+		if err != nil {
+			return 0, err
+		}
+		return float64(n), nil
+	}
+	s = strings.ToLower(s)
+	m := float64(1)
+	switch true {
+	case strings.HasSuffix(s, "kib"):
+		s = s[:len(s)-3]
+		m = 1024
+	case strings.HasSuffix(s, "ki"):
+		s = s[:len(s)-2]
+		m = 1024
+	case strings.HasSuffix(s, "kb"):
+		s = s[:len(s)-2]
+		m = 1000
+	case strings.HasSuffix(s, "k"):
+		s = s[:len(s)-1]
+		m = 1000
+	case strings.HasSuffix(s, "mib"):
+		s = s[:len(s)-3]
+		m = 1024 * 1024
+	case strings.HasSuffix(s, "mi"):
+		s = s[:len(s)-2]
+		m = 1024 * 1024
+	case strings.HasSuffix(s, "mb"):
+		s = s[:len(s)-2]
+		m = 1000 * 1000
+	case strings.HasSuffix(s, "m"):
+		s = s[:len(s)-1]
+		m = 1000 * 1000
+	case strings.HasSuffix(s, "gib"):
+		s = s[:len(s)-3]
+		m = 1024 * 1024 * 1024
+	case strings.HasSuffix(s, "gi"):
+		s = s[:len(s)-2]
+		m = 1024 * 1024 * 1024
+	case strings.HasSuffix(s, "gb"):
+		s = s[:len(s)-2]
+		m = 1000 * 1000 * 1000
+	case strings.HasSuffix(s, "g"):
+		s = s[:len(s)-1]
+		m = 1000 * 1000 * 1000
+	case strings.HasSuffix(s, "tib"):
+		s = s[:len(s)-3]
+		m = 1024 * 1024 * 1024 * 1024
+	case strings.HasSuffix(s, "ti"):
+		s = s[:len(s)-2]
+		m = 1024 * 1024 * 1024 * 1024
+	case strings.HasSuffix(s, "tb"):
+		s = s[:len(s)-2]
+		m = 1000 * 1000 * 1000 * 1000
+	case strings.HasSuffix(s, "t"):
+		s = s[:len(s)-1]
+		m = 1000 * 1000 * 1000 * 1000
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return 0, err
+	}
+	return v * m, nil
+}
+
 func scanPositiveNumber(s string) (string, error) {
 	// Scan integer part. It may be empty if fractional part exists.
 	i := 0
@@ -171,7 +238,14 @@ func scanPositiveNumber(s string) (string, error) {
 		}
 		return s, nil
 	}
+	if sLen := scanNumMultiplier(s[i:]); sLen > 0 {
+		i += sLen
+		return s[:i], nil
+	}
 	if s[i] != '.' && s[i] != 'e' && s[i] != 'E' {
+		if i == 0 {
+			return "", fmt.Errorf("missing positive number")
+		}
 		return s[:i], nil
 	}
 
@@ -182,14 +256,15 @@ func scanPositiveNumber(s string) (string, error) {
 		for j < len(s) && isDecimalChar(s[j]) {
 			j++
 		}
-		if j == i {
-			return "", fmt.Errorf("missing fractional part in %q", s)
-		}
 		i = j
 		if i == len(s) {
 			return s, nil
 		}
 	}
+	if sLen := scanNumMultiplier(s[i:]); sLen > 0 {
+		i += sLen
+		return s[:i], nil
+	}
 
 	if s[i] != 'e' && s[i] != 'E' {
 		return s[:i], nil
@@ -213,6 +288,46 @@ func scanPositiveNumber(s string) (string, error) {
 	return s[:j], nil
 }
 
+func scanNumMultiplier(s string) int {
+	s = strings.ToLower(s)
+	switch true {
+	case strings.HasPrefix(s, "kib"):
+		return 3
+	case strings.HasPrefix(s, "ki"):
+		return 2
+	case strings.HasPrefix(s, "kb"):
+		return 2
+	case strings.HasPrefix(s, "k"):
+		return 1
+	case strings.HasPrefix(s, "mib"):
+		return 3
+	case strings.HasPrefix(s, "mi"):
+		return 2
+	case strings.HasPrefix(s, "mb"):
+		return 2
+	case strings.HasPrefix(s, "m"):
+		return 1
+	case strings.HasPrefix(s, "gib"):
+		return 3
+	case strings.HasPrefix(s, "gi"):
+		return 2
+	case strings.HasPrefix(s, "gb"):
+		return 2
+	case strings.HasPrefix(s, "g"):
+		return 1
+	case strings.HasPrefix(s, "tib"):
+		return 3
+	case strings.HasPrefix(s, "ti"):
+		return 2
+	case strings.HasPrefix(s, "tb"):
+		return 2
+	case strings.HasPrefix(s, "t"):
+		return 1
+	default:
+		return 0
+	}
+}
+
 func scanIdent(s string) string {
 	i := 0
 	for i < len(s) {

vendor/github.com/VictoriaMetrics/metricsql/parser.go

@@ -479,24 +479,19 @@ func (p *parser) parsePositiveNumberExpr() (*NumberExpr, error) {
 	if !isPositiveNumberPrefix(p.lex.Token) && !isInfOrNaN(p.lex.Token) {
 		return nil, fmt.Errorf(`positiveNumberExpr: unexpected token %q; want "number"`, p.lex.Token)
 	}
-	var ne NumberExpr
+	s := p.lex.Token
-	if isSpecialIntegerPrefix(p.lex.Token) {
+	n, err := parsePositiveNumber(s)
-		in, err := strconv.ParseInt(p.lex.Token, 0, 64)
+	if err != nil {
-		if err != nil {
+		return nil, fmt.Errorf(`positivenumberExpr: cannot parse %q: %s`, s, err)
-			return nil, fmt.Errorf(`positiveNumberExpr: cannot parse integer %q: %s`, p.lex.Token, err)
-		}
-		ne.N = float64(in)
-	} else {
-		n, err := strconv.ParseFloat(p.lex.Token, 64)
-		if err != nil {
-			return nil, fmt.Errorf(`positiveNumberExpr: cannot parse %q: %s`, p.lex.Token, err)
-		}
-		ne.N = n
 	}
 	if err := p.lex.Next(); err != nil {
 		return nil, err
 	}
-	return &ne, nil
+	ne := &NumberExpr{
+		N: n,
+		s: s,
+	}
+	return ne, nil
 }
 
 func (p *parser) parseStringExpr() (*StringExpr, error) {
@@ -1499,10 +1494,16 @@ func (se *StringExpr) AppendString(dst []byte) []byte {
 type NumberExpr struct {
 	// N is the parsed number, i.e. `1.23`, `-234`, etc.
 	N float64
+
+	// s contains the original string representation for N.
+	s string
 }
 
 // AppendString appends string representation of ne to dst and returns the result.
 func (ne *NumberExpr) AppendString(dst []byte) []byte {
+	if ne.s != "" {
+		return append(dst, ne.s...)
+	}
 	return strconv.AppendFloat(dst, ne.N, 'g', -1, 64)
 }
 

vendor/modules.txt

@@ -69,7 +69,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.23.0
 ## explicit; go 1.15
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.47.0
+# github.com/VictoriaMetrics/metricsql v0.48.0
 ## explicit; go 1.13
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop