diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 07d3fb266d..c6622d0aaf 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,9 +30,11 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow selecting time series matching at least one of multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries). See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to preserve metric names for binary operation results via `keep_metric_names` modifier. For example, `({__name__=~"foo|bar"} / 10) keep_metric_names` leaves `foo` and `bar` metric names in division results. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#keep_metric_names). This helps to address issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3710). -* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to copy all the labels from `one` side of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches) by specifying `*` inside `group_left()` or `group_right()`. Also allow adding a prefix for copied label names via `group_left(*) prefix "..."` syntax. For example, the following query copies Kubernetes namespace labels to `kube_pod_info` series and adds `ns_` prefix for the copied label names: `kube_pod_info * on(namespace) group_left(*) prefix "ns_" kube_namespace_labels`. The labels from `on()` list aren't prefixed. - This feature resolves [this](https://stackoverflow.com/questions/76661818/how-to-add-namespace-labels-to-pod-labels-in-prometheus) - and [that](https://stackoverflow.com/questions/76653997/how-can-i-make-a-new-copy-of-kube-namespace-labels-metric-with-a-different-name) questions at StackOverflow. +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to copy all the labels from `one` side of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches) by specifying `*` inside `group_left()` or `group_right()`. Also allow adding a prefix for copied label names via `group_left(*) prefix "..."` syntax. For example, the following query copies Kubernetes namespace labels to `kube_pod_info` series and adds `ns_` prefix for the copied label names: `kube_pod_info * on(namespace) group_left(*) prefix "ns_" kube_namespace_labels`. The labels from `on()` list aren't prefixed. This feature resolves [this](https://stackoverflow.com/questions/76661818/how-to-add-namespace-labels-to-pod-labels-in-prometheus) and [that](https://stackoverflow.com/questions/76653997/how-can-i-make-a-new-copy-of-kube-namespace-labels-metric-with-a-different-name) questions at StackOverflow. +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to specify durations via [`WITH` templates](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/expand-with-exprs). Examples: + - `WITH (w = 5m) m[w]` is automatically transformed to `m[5m]` + - `WITH (f(window, step, off) = m[window:step] offset off) f(5m, 10s, 1h)` is automatically transformed to `m[5m:10s] offset 1h` + Thanks to @lujiajing1126 for the initial idea and [implementation](https://github.com/VictoriaMetrics/metricsql/pull/13). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4025). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402). diff --git a/go.mod b/go.mod index d9f96a639b..89d35a6754 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.60.0 + github.com/VictoriaMetrics/metricsql v0.61.1 github.com/aws/aws-sdk-go-v2 v1.19.0 github.com/aws/aws-sdk-go-v2/config v1.18.28 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 diff --git a/go.sum b/go.sum index a2b4f20803..53837d8d01 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.60.0 h1:KlVWTabXGu/50U3Dp72TyJYo01WTaxSsHwMLiE9yHuA= -github.com/VictoriaMetrics/metricsql v0.60.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.61.1 h1:vYkVDVa+ROvrDkhlrCzBLKB273tE6N681ftfZP+Lav8= +github.com/VictoriaMetrics/metricsql v0.61.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go index 915552564d..f0a2a34663 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go @@ -37,6 +37,11 @@ func (lex *lexer) Init(s string) { lex.sTail = s } +func (lex *lexer) PushBack(currToken, sHead string) { + lex.Token = currToken + lex.sTail = sHead + lex.sTail +} + func (lex *lexer) Next() error { if lex.err != nil { return lex.err diff --git a/vendor/github.com/VictoriaMetrics/metricsql/parser.go b/vendor/github.com/VictoriaMetrics/metricsql/parser.go index 55339f06c1..8c520d1637 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/parser.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/parser.go @@ -784,6 +784,18 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { } re := *t re.Expr = eNew + re.Window, err = expandDuration(was, re.Window) + if err != nil { + return nil, fmt.Errorf("cannot parse window for %s: %w", re.Expr.AppendString(nil), err) + } + re.Step, err = expandDuration(was, re.Step) + if err != nil { + return nil, fmt.Errorf("cannot parse step in %s: %w", re.Expr.AppendString(nil), err) + } + re.Offset, err = expandDuration(was, re.Offset) + if err != nil { + return nil, fmt.Errorf("cannot parse offset in %s: %w", re.Expr.AppendString(nil), err) + } if t.At != nil { atNew, err := expandWithExpr(was, t.At) if err != nil { @@ -828,7 +840,7 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { lfe.Label, t.AppendString(nil), eNew.AppendString(nil)) } if len(wme.labelFilterss) > 0 { - panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH template expansion; got %s", wme.labelFilterss)) + panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH template expansion; got %s", wme.AppendString(nil))) } lfssSrc := wme.LabelFilterss if len(lfssSrc) > 1 { @@ -888,7 +900,7 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) } if len(wme.labelFilterss) > 0 { - panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH templates expansion; got %s", wme.labelFilterss)) + panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH templates expansion; got %s", wme.AppendString(nil))) } lfssSrc := wme.LabelFilterss var lfssNew [][]LabelFilter @@ -945,6 +957,38 @@ func expandWithArgs(was []*withArgExpr, args []Expr) ([]Expr, error) { return dstArgs, nil } +func expandDuration(was []*withArgExpr, d *DurationExpr) (*DurationExpr, error) { + if d == nil { + return nil, nil + } + if !d.needsParsing { + return d, nil + } + wa := getWithArgExpr(was, d.s) + if wa == nil { + return nil, fmt.Errorf("cannot find WITH template for %q", d.s) + } + e, err := expandWithExprExt(was, wa, []Expr{}) + if err != nil { + return nil, err + } + switch t := e.(type) { + case *DurationExpr: + if t.needsParsing { + panic(fmt.Errorf("BUG: DurationExpr %q must be already parsed", t.s)) + } + return t, nil + case *NumberExpr: + // Convert number of seconds to DurationExpr + de := &DurationExpr{ + s: t.s, + } + return de, nil + default: + return nil, fmt.Errorf("unexpected value for WITH template %q; got %s; want duration", d.s, e.AppendString(nil)) + } +} + func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) { if len(args) == 0 { return nil, nil @@ -1335,8 +1379,24 @@ type labelFilterExpr struct { IsNegative bool } -func (lfe *labelFilterExpr) String() string { - return fmt.Sprintf("[label=%q, value=%+v, isRegexp=%v, isNegative=%v]", lfe.Label, lfe.Value, lfe.IsRegexp, lfe.IsNegative) +func (lfe *labelFilterExpr) AppendString(dst []byte) []byte { + dst = appendEscapedIdent(dst, lfe.Label) + if lfe.Value == nil { + return dst + } + dst = appendLabelFilterOp(dst, lfe.IsNegative, lfe.IsRegexp) + tokens := lfe.Value.tokens + if len(tokens) == 0 { + dst = strconv.AppendQuote(dst, lfe.Value.S) + return dst + } + for i, token := range tokens { + dst = append(dst, token...) + if i+1 < len(tokens) { + dst = append(dst, '+') + } + } + return dst } func (lfe *labelFilterExpr) toLabelFilter() (*LabelFilter, error) { @@ -1452,13 +1512,28 @@ func (p *parser) parseDuration() (*DurationExpr, error) { func (p *parser) parsePositiveDuration() (*DurationExpr, error) { s := p.lex.Token + if isIdentPrefix(s) { + n := strings.IndexByte(s, ':') + if n >= 0 { + p.lex.PushBack(s[:n], s[n:]) + s = s[:n] + } + if err := p.lex.Next(); err != nil { + return nil, err + } + de := &DurationExpr{ + s: s, + needsParsing: true, + } + return de, nil + } if isPositiveDuration(s) { if err := p.lex.Next(); err != nil { return nil, err } } else { if !isPositiveNumberPrefix(s) { - return nil, fmt.Errorf(`duration: unexpected token %q; want "duration"`, s) + return nil, fmt.Errorf(`duration: unexpected token %q; want valid duration`, s) } // Verify the duration in seconds without explicit suffix. if _, err := p.parsePositiveNumberExpr(); err != nil { @@ -1478,6 +1553,9 @@ func (p *parser) parsePositiveDuration() (*DurationExpr, error) { // DurationExpr contains the duration type DurationExpr struct { s string + + // needsParsing is set to true if s isn't parsed yet with expandWithExpr() + needsParsing bool } // AppendString appends string representation of de to dst and returns the result. @@ -1485,6 +1563,9 @@ func (de *DurationExpr) AppendString(dst []byte) []byte { if de == nil { return dst } + if de.needsParsing { + panic(fmt.Errorf("BUG: duration %q must be already parsed with expandWithExpr()", de.s)) + } return append(dst, de.s...) } @@ -1493,6 +1574,9 @@ func (de *DurationExpr) Duration(step int64) int64 { if de == nil { return 0 } + if de.needsParsing { + panic(fmt.Errorf("BUG: duration %q must be already parsed", de.s)) + } d, err := DurationValue(de.s, step) if err != nil { panic(fmt.Errorf("BUG: cannot parse duration %q: %s", de.s, err)) @@ -1617,6 +1701,9 @@ type StringExpr struct { // AppendString appends string representation of se to dst and returns the result. func (se *StringExpr) AppendString(dst []byte) []byte { + if len(se.tokens) > 0 { + panic(fmt.Errorf("BUG: StringExpr=%q must be already parsed with expandWithExpr()", se.tokens)) + } return strconv.AppendQuote(dst, se.S) } @@ -2037,25 +2124,24 @@ type LabelFilter struct { // AppendString appends string representation of me to dst and returns the result. func (lf *LabelFilter) AppendString(dst []byte) []byte { dst = appendEscapedIdent(dst, lf.Label) - var op string - if lf.IsNegative { - if lf.IsRegexp { - op = "!~" - } else { - op = "!=" - } - } else { - if lf.IsRegexp { - op = "=~" - } else { - op = "=" - } - } - dst = append(dst, op...) + dst = appendLabelFilterOp(dst, lf.IsNegative, lf.IsRegexp) dst = strconv.AppendQuote(dst, lf.Value) return dst } +func appendLabelFilterOp(dst []byte, isNegative, isRegexp bool) []byte { + if isNegative { + if isRegexp { + return append(dst, "!~"...) + } + return append(dst, "!="...) + } + if isRegexp { + return append(dst, "=~"...) + } + return append(dst, '=') +} + // MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. // // Curly braces may contain or-delimited list of filters. For example: @@ -2082,8 +2168,28 @@ type MetricExpr struct { labelFilterss [][]*labelFilterExpr } +func appendLabelFilterss(dst []byte, lfss [][]*labelFilterExpr) []byte { + dst = append(dst, '{') + for i, lfs := range lfss { + for j, lf := range lfs { + dst = lf.AppendString(dst) + if j+1 < len(lfs) { + dst = append(dst, ',') + } + } + if i+1 < len(lfss) { + dst = append(dst, " or "...) + } + } + dst = append(dst, '}') + return dst +} + // AppendString appends string representation of me to dst and returns the result. func (me *MetricExpr) AppendString(dst []byte) []byte { + if len(me.labelFilterss) > 0 { + return appendLabelFilterss(dst, me.labelFilterss) + } lfss := me.LabelFilterss if len(lfss) == 0 { dst = append(dst, "{}"...) diff --git a/vendor/modules.txt b/vendor/modules.txt index 82c8a37ba4..8f1a08a770 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.60.0 +# github.com/VictoriaMetrics/metricsql v0.61.1 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop