From 05ed98c98b63ba147784542f919075c623e53e1f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 21 Nov 2022 21:27:52 +0200 Subject: [PATCH] app/vmselect/promql: allow using SI and IEC suffixes in numeric values inside queries For example, 10Ki is equivalent to 10*1024, while 5.3M is equivalent to 5.3*1000*1000 --- docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 1 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/lexer.go | 121 +++++++++++++++++- .../VictoriaMetrics/metricsql/parser.go | 29 +++-- vendor/modules.txt | 2 +- 7 files changed, 139 insertions(+), 21 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 930882d70..70aca2453 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): add `-storageNode.filter` command-line flag for filtering the [discovered vmstorage nodes](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery) with arbitrary regular expressions. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3353). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow using numeric values with `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes inside MetricsQL queries. For example `8Ki` equals to `8*1024`, while `8.2M` equals to `8.2*1000*1000`. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_linear_regression](https://docs.victoriametrics.com/MetricsQL.html#range_linear_regression) function for calculating [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression) over the input time series on the selected time range. This function is useful for predictions and capacity planning. For example, `range_linear_regression(process_resident_memory_bytes)` can predict future memory usage based on the past memory usage. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_stddev](https://docs.victoriametrics.com/MetricsQL.html#range_stddev) and [range_stdvar](https://docs.victoriametrics.com/MetricsQL.html#range_stdvar) functions. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): optimize `expr1 op expr2` query when `expr1` returns an empty result. In this case there is no sense in executing `expr2` for `op` not equal to `or`, since the end result will be empty according to [PromQL series matching rules](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349). Thanks to @jianglinjian for pointing to this case. diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 33d1b9c92..217c6f8da 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -73,6 +73,7 @@ The list of MetricsQL features: * The duration suffix is optional. The duration is in seconds if the suffix is missing. For example, `rate(m[300] offset 1800)` is equivalent to `rate(m[5m]) offset 30m`. * The duration can be placed anywhere in the query. For example, `sum_over_time(m[1h]) / 1h` is equivalent to `sum_over_time(m[1h]) / 3600`. +* Numeric values can have `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes. For example, `8K` is equivalent to `8000`, while `1.2Mi` is equvalent to `1.2*1024*1024`. * Trailing commas on all the lists are allowed - label filters, function args and with expressions. For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`. This simplifies maintenance of multi-line queries. diff --git a/go.mod b/go.mod index 0c8db875c..0eeed2444 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.23.0 - github.com/VictoriaMetrics/metricsql v0.47.0 + github.com/VictoriaMetrics/metricsql v0.48.0 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.18.1 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.40 diff --git a/go.sum b/go.sum index 62b35fb06..6f9c862bc 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA= github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.47.0 h1:PQwadjoQnKKkaUiupkDq0ZbCAHX2qP8OOexJ9oJwupo= -github.com/VictoriaMetrics/metricsql v0.47.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.48.0 h1:rq8ULfIDJ0QyDbyQWRuWrMTffEqL2sevU2Zs3Vx1pfw= +github.com/VictoriaMetrics/metricsql v0.48.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go index 947da2c53..c1a0ec69b 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go @@ -149,6 +149,73 @@ func scanString(s string) (string, error) { } } +func parsePositiveNumber(s string) (float64, error) { + if isSpecialIntegerPrefix(s) { + n, err := strconv.ParseInt(s, 0, 64) + if err != nil { + return 0, err + } + return float64(n), nil + } + s = strings.ToLower(s) + m := float64(1) + switch true { + case strings.HasSuffix(s, "kib"): + s = s[:len(s)-3] + m = 1024 + case strings.HasSuffix(s, "ki"): + s = s[:len(s)-2] + m = 1024 + case strings.HasSuffix(s, "kb"): + s = s[:len(s)-2] + m = 1000 + case strings.HasSuffix(s, "k"): + s = s[:len(s)-1] + m = 1000 + case strings.HasSuffix(s, "mib"): + s = s[:len(s)-3] + m = 1024 * 1024 + case strings.HasSuffix(s, "mi"): + s = s[:len(s)-2] + m = 1024 * 1024 + case strings.HasSuffix(s, "mb"): + s = s[:len(s)-2] + m = 1000 * 1000 + case strings.HasSuffix(s, "m"): + s = s[:len(s)-1] + m = 1000 * 1000 + case strings.HasSuffix(s, "gib"): + s = s[:len(s)-3] + m = 1024 * 1024 * 1024 + case strings.HasSuffix(s, "gi"): + s = s[:len(s)-2] + m = 1024 * 1024 * 1024 + case strings.HasSuffix(s, "gb"): + s = s[:len(s)-2] + m = 1000 * 1000 * 1000 + case strings.HasSuffix(s, "g"): + s = s[:len(s)-1] + m = 1000 * 1000 * 1000 + case strings.HasSuffix(s, "tib"): + s = s[:len(s)-3] + m = 1024 * 1024 * 1024 * 1024 + case strings.HasSuffix(s, "ti"): + s = s[:len(s)-2] + m = 1024 * 1024 * 1024 * 1024 + case strings.HasSuffix(s, "tb"): + s = s[:len(s)-2] + m = 1000 * 1000 * 1000 * 1000 + case strings.HasSuffix(s, "t"): + s = s[:len(s)-1] + m = 1000 * 1000 * 1000 * 1000 + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, err + } + return v * m, nil +} + func scanPositiveNumber(s string) (string, error) { // Scan integer part. It may be empty if fractional part exists. i := 0 @@ -171,7 +238,14 @@ func scanPositiveNumber(s string) (string, error) { } return s, nil } + if sLen := scanNumMultiplier(s[i:]); sLen > 0 { + i += sLen + return s[:i], nil + } if s[i] != '.' && s[i] != 'e' && s[i] != 'E' { + if i == 0 { + return "", fmt.Errorf("missing positive number") + } return s[:i], nil } @@ -182,14 +256,15 @@ func scanPositiveNumber(s string) (string, error) { for j < len(s) && isDecimalChar(s[j]) { j++ } - if j == i { - return "", fmt.Errorf("missing fractional part in %q", s) - } i = j if i == len(s) { return s, nil } } + if sLen := scanNumMultiplier(s[i:]); sLen > 0 { + i += sLen + return s[:i], nil + } if s[i] != 'e' && s[i] != 'E' { return s[:i], nil @@ -213,6 +288,46 @@ func scanPositiveNumber(s string) (string, error) { return s[:j], nil } +func scanNumMultiplier(s string) int { + s = strings.ToLower(s) + switch true { + case strings.HasPrefix(s, "kib"): + return 3 + case strings.HasPrefix(s, "ki"): + return 2 + case strings.HasPrefix(s, "kb"): + return 2 + case strings.HasPrefix(s, "k"): + return 1 + case strings.HasPrefix(s, "mib"): + return 3 + case strings.HasPrefix(s, "mi"): + return 2 + case strings.HasPrefix(s, "mb"): + return 2 + case strings.HasPrefix(s, "m"): + return 1 + case strings.HasPrefix(s, "gib"): + return 3 + case strings.HasPrefix(s, "gi"): + return 2 + case strings.HasPrefix(s, "gb"): + return 2 + case strings.HasPrefix(s, "g"): + return 1 + case strings.HasPrefix(s, "tib"): + return 3 + case strings.HasPrefix(s, "ti"): + return 2 + case strings.HasPrefix(s, "tb"): + return 2 + case strings.HasPrefix(s, "t"): + return 1 + default: + return 0 + } +} + func scanIdent(s string) string { i := 0 for i < len(s) { diff --git a/vendor/github.com/VictoriaMetrics/metricsql/parser.go b/vendor/github.com/VictoriaMetrics/metricsql/parser.go index ac64fc438..9b92dbabb 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/parser.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/parser.go @@ -479,24 +479,19 @@ func (p *parser) parsePositiveNumberExpr() (*NumberExpr, error) { if !isPositiveNumberPrefix(p.lex.Token) && !isInfOrNaN(p.lex.Token) { return nil, fmt.Errorf(`positiveNumberExpr: unexpected token %q; want "number"`, p.lex.Token) } - var ne NumberExpr - if isSpecialIntegerPrefix(p.lex.Token) { - in, err := strconv.ParseInt(p.lex.Token, 0, 64) - if err != nil { - return nil, fmt.Errorf(`positiveNumberExpr: cannot parse integer %q: %s`, p.lex.Token, err) - } - ne.N = float64(in) - } else { - n, err := strconv.ParseFloat(p.lex.Token, 64) - if err != nil { - return nil, fmt.Errorf(`positiveNumberExpr: cannot parse %q: %s`, p.lex.Token, err) - } - ne.N = n + s := p.lex.Token + n, err := parsePositiveNumber(s) + if err != nil { + return nil, fmt.Errorf(`positivenumberExpr: cannot parse %q: %s`, s, err) } if err := p.lex.Next(); err != nil { return nil, err } - return &ne, nil + ne := &NumberExpr{ + N: n, + s: s, + } + return ne, nil } func (p *parser) parseStringExpr() (*StringExpr, error) { @@ -1499,10 +1494,16 @@ func (se *StringExpr) AppendString(dst []byte) []byte { type NumberExpr struct { // N is the parsed number, i.e. `1.23`, `-234`, etc. N float64 + + // s contains the original string representation for N. + s string } // AppendString appends string representation of ne to dst and returns the result. func (ne *NumberExpr) AppendString(dst []byte) []byte { + if ne.s != "" { + return append(dst, ne.s...) + } return strconv.AppendFloat(dst, ne.N, 'g', -1, 64) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 17b3e4dce..ad20c9371 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -69,7 +69,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.23.0 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.47.0 +# github.com/VictoriaMetrics/metricsql v0.48.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop