diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index f0c5c6007..a33a2f671 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -7367,7 +7367,7 @@ func TestExecSuccess(t *testing.T) { }) t.Run(`rollup_scrape_interval()`, func(t *testing.T) { t.Parallel() - q := `sort_by_label(rollup_scrape_interval(1[5m:10s]), "rollup")` + q := `sort_by_label(rollup_scrape_interval(1[5M:10s]), "rollup")` r1 := netstorage.Result{ MetricName: metricNameExpected, Values: []float64{10, 10, 10, 10, 10, 10}, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 80b8d0313..6d105d2ac 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for aggregation of incoming [samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) by time and by labels. See [these docs](https://docs.victoriametrics.com/stream-aggregation.html) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3460). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386). * FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow using unicode letters in identifiers. For example, `температура{город="Киев"}` is a valid MetricsQL expression now. Previously every non-ascii letters should be escaped with `\` char when used inside MetricsQL expression: `\т\е\м\п\е\р\а\т\у\р\а{\г\о\р\о\д="Киев"}`. Now both expressions are equivalent. Thanks to @hzwwww for the [pull request](https://github.com/VictoriaMetrics/metricsql/pull/7). * FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`. * FEATURE: [csvimport](https://docs.victoriametrics.com/#how-to-import-csv-data): support empty values for imported metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540). * FEATURE: [vmalert](httpпоs://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556). @@ -25,6 +26,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update tooltip when quickly hovering multiple lines on the graph. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3530). * BUGFIX: properly parse floating-point numbers without integer or fractional parts such as `.123` and `20.` during [data import](https://docs.victoriametrics.com/#how-to-import-time-series-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3544). +* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly parse durations with uppercase suffixes such as `10S`, `5MS`, `1W`, etc. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3589). ## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3) diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index f06e38a96..b8edc4451 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -77,9 +77,12 @@ The list of MetricsQL features: * Trailing commas on all the lists are allowed - label filters, function args and with expressions. For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`. This simplifies maintenance of multi-line queries. -* Metric names and metric labels may contain escaped chars. For instance, `foo\-bar{baz\=aa="b"}` is valid expression. +* Metric names and label names may contain any unicode letter. For example `температура{город="Киев"}` is a value MetricsQL expression. +* Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression. It returns time series with name `foo-bar` containing label `baz=aa` with value `b`. - Additionally, `\xXX` escape sequence is supported, where `XX` is hexadecimal representation of escaped char. + Additionally, the following escape sequences are supported: + - `\xXX`, where `XX` is hexadecimal representation of the escaped ascii char. + - `\uXXXX`, where `XXXX` is a hexadecimal representation of the escaped unicode char. * Aggregate functions support optional `limit N` suffix in order to limit the number of output series. For example, `sum(x) by (y) limit 3` limits the number of output time series after the aggregation to 3. All the other time series are dropped. diff --git a/go.mod b/go.mod index 9e4de7594..eee96d5a5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.23.0 - github.com/VictoriaMetrics/metricsql v0.50.0 + github.com/VictoriaMetrics/metricsql v0.51.1 github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.18.7 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 diff --git a/go.sum b/go.sum index 7e1a5543d..d93bf8279 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA= github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.50.0 h1:MCBhjn1qlfMqPGP6HiR9JgmEw7oTRGm/O8YwSeoaI1E= -github.com/VictoriaMetrics/metricsql v0.50.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.51.1 h1:gmh3ZGCDrqUTdhUrr87eJOXMOputDYs1PtLwTfySTsI= +github.com/VictoriaMetrics/metricsql v0.51.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go index c91f86381..c3aebd80d 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go @@ -331,19 +331,22 @@ func scanNumMultiplier(s string) int { func scanIdent(s string) string { i := 0 for i < len(s) { - if isIdentChar(s[i]) { - i++ + r, size := utf8.DecodeRuneInString(s[i:]) + if i == 0 && isFirstIdentChar(r) || i > 0 && isIdentChar(r) { + i += size continue } - if s[i] != '\\' { + if r != '\\' { break } - i++ - - // Do not verify the next char, since it is escaped. - // The next char may be encoded as multi-byte UTF8 sequence. See https://en.wikipedia.org/wiki/UTF-8#Encoding - _, size := utf8.DecodeRuneInString(s[i:]) i += size + r, n := decodeEscapeSequence(s[i:]) + if r == utf8.RuneError { + // Invalid escape sequence + i -= size + break + } + i += n } if i == 0 { panic("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent") @@ -360,23 +363,12 @@ func unescapeIdent(s string) string { for { dst = append(dst, s[:n]...) s = s[n+1:] - if len(s) == 0 { - return string(dst) - } - if s[0] == 'x' && len(s) >= 3 { - h1 := fromHex(s[1]) - h2 := fromHex(s[2]) - if h1 >= 0 && h2 >= 0 { - dst = append(dst, byte((h1<<4)|h2)) - s = s[3:] - } else { - dst = append(dst, s[0]) - s = s[1:] - } + r, size := decodeEscapeSequence(s) + if r == utf8.RuneError { + // Cannot decode escape sequence. Put it in the output as is + dst = append(dst, '\\') } else { - // UTF8 char. See https://en.wikipedia.org/wiki/UTF-8#Encoding - _, size := utf8.DecodeRuneInString(s) - dst = append(dst, s[:size]...) + dst = utf8.AppendRune(dst, r) s = s[size:] } n = strings.IndexByte(s, '\\') @@ -387,49 +379,16 @@ func unescapeIdent(s string) string { } } -func fromHex(ch byte) int { - if ch >= '0' && ch <= '9' { - return int(ch - '0') - } - if ch >= 'a' && ch <= 'f' { - return int((ch - 'a') + 10) - } - if ch >= 'A' && ch <= 'F' { - return int((ch - 'A') + 10) - } - return -1 -} - -func toHex(n byte) byte { - if n < 10 { - return '0' + n - } - return 'a' + (n - 10) -} - func appendEscapedIdent(dst []byte, s string) []byte { - for i := 0; i < len(s); i++ { - ch := s[i] - if isIdentChar(ch) { - if i == 0 && !isFirstIdentChar(ch) { - // hex-encode the first char - dst = append(dst, '\\', 'x', toHex(ch>>4), toHex(ch&0xf)) - } else { - dst = append(dst, ch) - } - continue - } - - // escape ch - dst = append(dst, '\\') + i := 0 + for i < len(s) { r, size := utf8.DecodeRuneInString(s[i:]) - if r != utf8.RuneError && unicode.IsPrint(r) { - dst = append(dst, s[i:i+size]...) - i += size - 1 + if i == 0 && isFirstIdentChar(r) || i > 0 && isIdentChar(r) { + dst = utf8.AppendRune(dst, r) } else { - // hex-encode non-printable chars - dst = append(dst, 'x', toHex(ch>>4), toHex(ch&0xf)) + dst = appendEscapeSequence(dst, r) } + i += size } return dst } @@ -597,6 +556,7 @@ func DurationValue(s string, step int64) (int64, error) { } func parseSingleDuration(s string, step int64) (float64, error) { + s = strings.ToLower(s) numPart := s[:len(s)-1] if strings.HasSuffix(numPart, "m") { // Duration in ms @@ -656,6 +616,7 @@ func scanSingleDuration(s string, canBeNegative bool) int { if len(s) == 0 { return -1 } + s = strings.ToLower(s) i := 0 if s[0] == '-' && canBeNegative { i++ @@ -703,25 +664,26 @@ func isIdentPrefix(s string) bool { if len(s) == 0 { return false } - if s[0] == '\\' { - // Assume this is an escape char for the next char. - return true + r, size := utf8.DecodeRuneInString(s) + if r == '\\' { + r, _ = decodeEscapeSequence(s[size:]) + return r != utf8.RuneError } - return isFirstIdentChar(s[0]) + return isFirstIdentChar(r) } -func isFirstIdentChar(ch byte) bool { - if ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' { +func isFirstIdentChar(r rune) bool { + if unicode.IsLetter(r) { return true } - return ch == '_' || ch == ':' + return r == '_' || r == ':' } -func isIdentChar(ch byte) bool { - if isFirstIdentChar(ch) { +func isIdentChar(r rune) bool { + if isFirstIdentChar(r) { return true } - return isDecimalChar(ch) || ch == '.' + return r < 256 && isDecimalChar(byte(r)) || r == '.' } func isSpaceChar(ch byte) bool { @@ -732,3 +694,67 @@ func isSpaceChar(ch byte) bool { return false } } + +func appendEscapeSequence(dst []byte, r rune) []byte { + dst = append(dst, '\\') + if unicode.IsPrint(r) { + return utf8.AppendRune(dst, r) + } + // hex-encode non-printable chars + if r < 256 { + return append(dst, 'x', toHex(byte(r>>4)), toHex(byte(r&0xf))) + } + return append(dst, 'u', toHex(byte(r>>12)), toHex(byte((r>>8)&0xf)), toHex(byte(r>>4)), toHex(byte(r&0xf))) +} + +func decodeEscapeSequence(s string) (rune, int) { + if strings.HasPrefix(s, "x") || strings.HasPrefix(s, "X") { + if len(s) >= 3 { + h1 := fromHex(s[1]) + h2 := fromHex(s[2]) + if h1 >= 0 && h2 >= 0 { + r := rune((h1 << 4) | h2) + return r, 3 + } + } + return utf8.RuneError, 0 + } + if strings.HasPrefix(s, "u") || strings.HasPrefix(s, "U") { + if len(s) >= 5 { + h1 := fromHex(s[1]) + h2 := fromHex(s[2]) + h3 := fromHex(s[3]) + h4 := fromHex(s[4]) + if h1 >= 0 && h2 >= 0 && h3 >= 0 && h4 >= 0 { + return rune((h1 << 12) | (h2 << 8) | (h3 << 4) | h4), 5 + } + } + return utf8.RuneError, 0 + } + r, size := utf8.DecodeRuneInString(s) + if unicode.IsPrint(r) { + return r, size + } + // Improperly escaped non-printable char + return utf8.RuneError, 0 +} + +func fromHex(ch byte) int { + if ch >= '0' && ch <= '9' { + return int(ch - '0') + } + if ch >= 'a' && ch <= 'f' { + return int((ch - 'a') + 10) + } + if ch >= 'A' && ch <= 'F' { + return int((ch - 'A') + 10) + } + return -1 +} + +func toHex(n byte) byte { + if n < 10 { + return '0' + n + } + return 'a' + (n - 10) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d0a39197f..747954076 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -70,7 +70,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.23.0 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.50.0 +# github.com/VictoriaMetrics/metricsql v0.51.1 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop