From 4cb024d8a3a2cdd0b14ee6850770ba114e3537e7 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 15 Jul 2023 23:48:21 -0700 Subject: [PATCH] all: add support for `or` filters in series selectors This commit adds ability to select series matching distinct filters via a single series selector. For example, the following selector selects series with either {env="prod",job="a"} or {env="dev",job="b"} labels: {env="prod",job="a" or env="dev",job="b"} The `or` filter is supported in all the VictoriaMetrics tools now. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3997 Uses https://github.com/VictoriaMetrics/metricsql/pull/14 --- app/vmagent/README.md | 4 +- app/vmctl/vm_native.go | 23 +- app/vmselect/prometheus/prometheus.go | 8 +- app/vmselect/promql/eval.go | 4 +- app/vmselect/promql/eval_test.go | 7 +- app/vmselect/promql/exec.go | 10 +- app/vmselect/promql/exec_test.go | 2 +- app/vmselect/promql/parser.go | 2 +- .../promql/rollup_result_cache_test.go | 12 +- app/vmselect/promql/transform.go | 6 +- app/vmselect/searchutils/searchutils.go | 34 +- app/vmselect/searchutils/searchutils_test.go | 54 +-- docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 3 + docs/keyConcepts.md | 40 +- docs/vmagent.md | 4 +- go.mod | 2 +- go.sum | 6 +- lib/promrelabel/if_expression.go | 39 +- lib/promrelabel/if_expression_test.go | 12 + .../VictoriaMetrics/metricsql/lexer.go | 6 +- .../VictoriaMetrics/metricsql/optimizer.go | 24 +- .../VictoriaMetrics/metricsql/parser.go | 358 ++++++++++++------ vendor/modules.txt | 2 +- 24 files changed, 441 insertions(+), 222 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 3814edd36..8e0d15c26 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -512,8 +512,8 @@ The following articles contain useful information about Prometheus relabeling: {% endraw %} * An optional `if` filter can be used for conditional relabeling. The `if` filter may contain - arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). - For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: + arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). + For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics: ```yaml - if: 'foo{bar="baz"}' diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index 683ad6150..3c8392b9a 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -339,23 +339,26 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) { if filter == metricName { return filter, nil } + nameFilter := fmt.Sprintf("__name__=%q", metricName) - labels, err := searchutils.ParseMetricSelector(filter) + tfss, err := searchutils.ParseMetricSelector(filter) if err != nil { return "", err } - str := make([]string, 0, len(labels)) - for _, label := range labels { - if len(label.Key) == 0 { - continue + var filters []string + for _, tfs := range tfss { + var a []string + for _, tf := range tfs { + if len(tf.Key) == 0 { + continue + } + a = append(a, tf.String()) } - str = append(str, label.String()) + a = append(a, nameFilter) + filters = append(filters, strings.Join(a, ",")) } - nameFilter := fmt.Sprintf("__name__=%q", metricName) - str = append(str, nameFilter) - - match := fmt.Sprintf("{%s}", strings.Join(str, ",")) + match := "{" + strings.Join(filters, " or ") + "}" return match, nil } diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 1f47a18bc..e072d8034 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -1005,15 +1005,15 @@ func getMaxLookback(r *http.Request) (int64, error) { } func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) { - tagFilterss := make([][]storage.TagFilter, 0, len(matches)) + tfss := make([][]storage.TagFilter, 0, len(matches)) for _, match := range matches { - tagFilters, err := searchutils.ParseMetricSelector(match) + tfssLocal, err := searchutils.ParseMetricSelector(match) if err != nil { return nil, fmt.Errorf("cannot parse matches[]=%s: %w", match, err) } - tagFilterss = append(tagFilterss, tagFilters) + tfss = append(tfss, tfssLocal...) } - return tagFilterss, nil + return tfss, nil } func getRoundDigits(r *http.Request) int { diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index 2bab4cb0c..f83296ed5 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -1075,8 +1075,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa } // Fetch the remaining part of the result. - tfs := searchutils.ToTagFilters(me.LabelFilters) - tfss := searchutils.JoinTagFilterss([][]storage.TagFilter{tfs}, ec.EnforcedTagFilterss) + tfss := searchutils.ToTagFilterss(me.LabelFilterss) + tfss = searchutils.JoinTagFilterss(tfss, ec.EnforcedTagFilterss) minTimestamp := start - maxSilenceInterval if window > ec.Step { minTimestamp -= window diff --git a/app/vmselect/promql/eval_test.go b/app/vmselect/promql/eval_test.go index 46e54eda1..30a1d9d71 100644 --- a/app/vmselect/promql/eval_test.go +++ b/app/vmselect/promql/eval_test.go @@ -29,8 +29,9 @@ func TestGetCommonLabelFilters(t *testing.T) { tss = append(tss, &ts) } lfs := getCommonLabelFilters(tss) - me := &metricsql.MetricExpr{ - LabelFilters: lfs, + var me metricsql.MetricExpr + if len(lfs) > 0 { + me.LabelFilterss = [][]metricsql.LabelFilter{lfs} } lfsMarshaled := me.AppendString(nil) if string(lfsMarshaled) != lfsExpected { @@ -40,7 +41,7 @@ func TestGetCommonLabelFilters(t *testing.T) { f(``, `{}`) f(`m 1`, `{}`) f(`m{a="b"} 1`, `{a="b"}`) - f(`m{c="d",a="b"} 1`, `{a="b", c="d"}`) + f(`m{c="d",a="b"} 1`, `{a="b",c="d"}`) f(`m1{a="foo"} 1 m2{a="bar"} 1`, `{a=~"bar|foo"}`) f(`m1{a="foo"} 1 diff --git a/app/vmselect/promql/exec.go b/app/vmselect/promql/exec.go index f2ce4e1b5..7d70754d7 100644 --- a/app/vmselect/promql/exec.go +++ b/app/vmselect/promql/exec.go @@ -277,10 +277,12 @@ func escapeDotsInRegexpLabelFilters(e metricsql.Expr) metricsql.Expr { if !ok { return } - for i := range me.LabelFilters { - f := &me.LabelFilters[i] - if f.IsRegexp { - f.Value = escapeDots(f.Value) + for _, lfs := range me.LabelFilterss { + for i := range lfs { + f := &lfs[i] + if f.IsRegexp { + f.Value = escapeDots(f.Value) + } } } }) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 1a70971df..b2a53acc0 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -45,7 +45,7 @@ func TestEscapeDotsInRegexpLabelFilters(t *testing.T) { f("2", "2") f(`foo.bar + 123`, `foo.bar + 123`) f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`) - f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e", x=~"a\\.b.+[\\.a]", y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`) + f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e",x=~"a\\.b.+[\\.a]",y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`) } func TestExecSuccess(t *testing.T) { diff --git a/app/vmselect/promql/parser.go b/app/vmselect/promql/parser.go index ec6c3041e..7123fd48e 100644 --- a/app/vmselect/promql/parser.go +++ b/app/vmselect/promql/parser.go @@ -34,7 +34,7 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset *me return } me, ok := re.Expr.(*metricsql.MetricExpr) - if !ok || len(me.LabelFilters) == 0 { + if !ok || len(me.LabelFilterss) == 0 { return } wrappedQuery := me.AppendString(nil) diff --git a/app/vmselect/promql/rollup_result_cache_test.go b/app/vmselect/promql/rollup_result_cache_test.go index 45cc568c5..d72a9f8d4 100644 --- a/app/vmselect/promql/rollup_result_cache_test.go +++ b/app/vmselect/promql/rollup_result_cache_test.go @@ -41,10 +41,14 @@ func TestRollupResultCache(t *testing.T) { MayCache: true, } me := &metricsql.MetricExpr{ - LabelFilters: []metricsql.LabelFilter{{ - Label: "aaa", - Value: "xxx", - }}, + LabelFilterss: [][]metricsql.LabelFilter{ + { + { + Label: "aaa", + Value: "xxx", + }, + }, + }, } fe := &metricsql.FuncExpr{ Name: "foo", diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 539775173..28acb36bf 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -240,7 +240,11 @@ func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries { if !ok { return rvs } - tfs := searchutils.ToTagFilters(me.LabelFilters) + tfss := searchutils.ToTagFilterss(me.LabelFilterss) + if len(tfss) != 1 { + return rvs + } + tfs := tfss[0] for i := range tfs { tf := &tfs[i] if len(tf.Key) == 0 { diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index 8d301b346..1eb887f1c 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -140,12 +140,14 @@ func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) { } var etfs [][]storage.TagFilter for _, extraFilter := range extraFilters { - tfs, err := ParseMetricSelector(extraFilter) + tfss, err := ParseMetricSelector(extraFilter) if err != nil { return nil, fmt.Errorf("cannot parse extra_filters=%s: %w", extraFilter, err) } - tfs = append(tfs, tagFilters...) - etfs = append(etfs, tfs) + for i := range tfss { + tfss[i] = append(tfss[i], tagFilters...) + } + etfs = append(etfs, tfss...) } return etfs, nil } @@ -170,7 +172,7 @@ func JoinTagFilterss(src, etfs [][]storage.TagFilter) [][]storage.TagFilter { } // ParseMetricSelector parses s containing PromQL metric selector and returns the corresponding LabelFilters. -func ParseMetricSelector(s string) ([]storage.TagFilter, error) { +func ParseMetricSelector(s string) ([][]storage.TagFilter, error) { expr, err := metricsql.Parse(s) if err != nil { return nil, err @@ -179,20 +181,24 @@ func ParseMetricSelector(s string) ([]storage.TagFilter, error) { if !ok { return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil)) } - if len(me.LabelFilters) == 0 { - return nil, fmt.Errorf("labelFilters cannot be empty") + if len(me.LabelFilterss) == 0 { + return nil, fmt.Errorf("labelFilterss cannot be empty") } - tfs := ToTagFilters(me.LabelFilters) - return tfs, nil + tfss := ToTagFilterss(me.LabelFilterss) + return tfss, nil } -// ToTagFilters converts lfs to a slice of storage.TagFilter -func ToTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter { - tfs := make([]storage.TagFilter, len(lfs)) - for i := range lfs { - toTagFilter(&tfs[i], &lfs[i]) +// ToTagFilterss converts lfss to or-delimited slices of storage.TagFilter +func ToTagFilterss(lfss [][]metricsql.LabelFilter) [][]storage.TagFilter { + tfss := make([][]storage.TagFilter, len(lfss)) + for i, lfs := range lfss { + tfs := make([]storage.TagFilter, len(lfs)) + for j := range lfs { + toTagFilter(&tfs[j], &lfs[j]) + } + tfss[i] = tfs } - return tfs + return tfss } func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) { diff --git a/app/vmselect/searchutils/searchutils_test.go b/app/vmselect/searchutils/searchutils_test.go index b763b8b76..9eecded0f 100644 --- a/app/vmselect/searchutils/searchutils_test.go +++ b/app/vmselect/searchutils/searchutils_test.go @@ -135,69 +135,69 @@ func TestJoinTagFilterss(t *testing.T) { } } // Single tag filter - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), - }, nil, []string{ + ), nil, []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, }) // Miltiple tag filters - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k5=~"v5"}`), - }, nil, []string{ + ), nil, []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k5=~"v5"}`, }) // Single extra filter - f(t, nil, [][]storage.TagFilter{ + f(t, nil, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, }) // Multiple extra filters - f(t, nil, [][]storage.TagFilter{ + f(t, nil, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k5=~"v5"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k5=~"v5"}`, }) // Single tag filter and a single extra filter - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), - }, [][]storage.TagFilter{ + ), joinTagFilters( mustParseMetricSelector(`{k5=~"v5"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`, }) // Multiple tag filters and a single extra filter - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k5=~"v5"}`), - }, [][]storage.TagFilter{ + ), joinTagFilters( mustParseMetricSelector(`{k6=~"v6"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, `{k5=~"v5",k6=~"v6"}`, }) // Single tag filter and multiple extra filters - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), - }, [][]storage.TagFilter{ + ), joinTagFilters( mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k6=~"v6"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, }) // Multiple tag filters and multiple extra filters - f(t, [][]storage.TagFilter{ + f(t, joinTagFilters( mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k5=~"v5"}`), - }, [][]storage.TagFilter{ + ), joinTagFilters( mustParseMetricSelector(`{k6=~"v6"}`), mustParseMetricSelector(`{k7=~"v7"}`), - }, []string{ + ), []string{ `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k7=~"v7"}`, `{k5=~"v5",k6=~"v6"}`, @@ -205,12 +205,20 @@ func TestJoinTagFilterss(t *testing.T) { }) } -func mustParseMetricSelector(s string) []storage.TagFilter { - tf, err := ParseMetricSelector(s) +func joinTagFilters(args ...[][]storage.TagFilter) [][]storage.TagFilter { + result := append([][]storage.TagFilter{}, args[0]...) + for _, tfss := range args[1:] { + result = append(result, tfss...) + } + return result +} + +func mustParseMetricSelector(s string) [][]storage.TagFilter { + tfss, err := ParseMetricSelector(s) if err != nil { panic(fmt.Errorf("cannot parse %q: %w", s, err)) } - return tf + return tfss } func tagFilterssToStrings(tfss [][]storage.TagFilter) []string { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 979a835c6..e6e8ec2ac 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved). * FETURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details. +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow selecting time series matching at least one of multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries). See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index c146019c8..66c4af59f 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -67,6 +67,9 @@ The list of MetricsQL features: depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)). For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`. It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana. +* [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` + selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. + See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details. * [Aggregate functions](#aggregate-functions) accept arbitrary number of args. For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`. * [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query. diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md index 549d7d254..5f0a6a002 100644 --- a/docs/keyConcepts.md +++ b/docs/keyConcepts.md @@ -785,15 +785,15 @@ requests_total{path="/", code="200"} requests_total{path="/", code="403"} ``` -To select only time series with specific label value specify the matching condition in curly braces: +To select only time series with specific label value specify the matching filter in curly braces: ```metricsql requests_total{code="200"} ``` -The query above will return all time series with the name `requests_total` and `code="200"`. We use the operator `=` to -match a label value. For negative match use `!=` operator. Filters also support regex matching `=~` for positive -and `!~` for negative matching: +The query above returns all time series with the name `requests_total` and label `code="200"`. We use the operator `=` to +match label value. For negative match use `!=` operator. Filters also support positive regex matching via `=~` +and negative regex matching via `!~`: ```metricsql requests_total{code=~"2.*"} @@ -802,23 +802,45 @@ requests_total{code=~"2.*"} Filters can also be combined: ```metricsql -requests_total{code=~"200|204", path="/home"} +requests_total{code=~"200", path="/home"} ``` -The query above will return all time series with a name `requests_total`, status `code` `200` or `204`and `path="/home"` -. +The query above returns all time series with `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`. #### Filtering by name Sometimes it is required to return all the time series for multiple metric names. As was mentioned in -the [data model section](#data-model), the metric name is just an ordinary label with a special name — `__name__`. So +the [data model section](#data-model), the metric name is just an ordinary label with a special name - `__name__`. So filtering by multiple metric names may be performed by applying regexps on metric names: ```metricsql {__name__=~"requests_(error|success)_total"} ``` -The query above is supposed to return series for two metrics: `requests_error_total` and `requests_success_total`. +The query above returns series for two metrics: `requests_error_total` and `requests_success_total`. + +#### Filtering by multiple "or" filters + +[MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) supports selecting time series, which match at least one of multiple "or" filters. +Such filters must be delimited by `or` inside curly braces. For example, the following query selects time series with +either `{job="app1",env="prod"}` or `{job="app2",env="dev"}` labels: + +```metricsql +{job="app1",env="prod" or job="app2",env="dev"} +``` + +The number of `or` filters can be arbitrary. This functionality allows passing the selected series +to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) such as [rate()](https://docs.victoriametrics.com/MetricsQL.html#rate) +without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries): + +```metricsql +rate({job="app1",env="prod" or job="app2",env="dev"}[5m]) + +``` + +If you need to select series matching multiple filters for the same label, then it is better from performance PoV +to use regexp filter `{label=~"value1|...|valueN"}` instead of `{label="value1" or ... or label="valueN"}`. + #### Arithmetic operations diff --git a/docs/vmagent.md b/docs/vmagent.md index 0a7eeedfb..4447d0688 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -523,8 +523,8 @@ The following articles contain useful information about Prometheus relabeling: {% endraw %} * An optional `if` filter can be used for conditional relabeling. The `if` filter may contain - arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). - For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: + arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). + For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics: ```yaml - if: 'foo{bar="baz"}' diff --git a/go.mod b/go.mod index 7adc532da..d39516ece 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.56.2 + github.com/VictoriaMetrics/metricsql v0.57.1 github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 diff --git a/go.sum b/go.sum index b0047ee5d..b8e0ae53c 100644 --- a/go.sum +++ b/go.sum @@ -67,11 +67,10 @@ github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bw github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I= github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= -github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.56.2 h1:quBAbYOlWMhmdgzFSCr1yjtVcdZYZrVQJ7nR9zor7ZM= -github.com/VictoriaMetrics/metricsql v0.56.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.57.1 h1:ryMe7w95s80BilS8RlV3G/25BLlmMBVRtTq2GnLB/4o= +github.com/VictoriaMetrics/metricsql v0.57.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -631,6 +630,7 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/lib/promrelabel/if_expression.go b/lib/promrelabel/if_expression.go index a0a6542cc..e001c91ab 100644 --- a/lib/promrelabel/if_expression.go +++ b/lib/promrelabel/if_expression.go @@ -14,8 +14,8 @@ import ( // // The `if` expression can contain arbitrary PromQL-like label filters such as `metric_name{filters...}` type IfExpression struct { - s string - lfs []*labelFilter + s string + lfss [][]*labelFilter } // String returns string representation of ie. @@ -36,12 +36,12 @@ func (ie *IfExpression) Parse(s string) error { if !ok { return fmt.Errorf("expecting series selector; got %q", expr.AppendString(nil)) } - lfs, err := metricExprToLabelFilters(me) + lfss, err := metricExprToLabelFilterss(me) if err != nil { return fmt.Errorf("cannot parse series selector: %w", err) } ie.s = s - ie.lfs = lfs + ie.lfss = lfss return nil } @@ -81,7 +81,16 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool { if ie == nil { return true } - for _, lf := range ie.lfs { + for _, lfs := range ie.lfss { + if matchLabelFilters(lfs, labels) { + return true + } + } + return false +} + +func matchLabelFilters(lfs []*labelFilter, labels []prompbmarshal.Label) bool { + for _, lf := range lfs { if !lf.match(labels) { return false } @@ -89,16 +98,20 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool { return true } -func metricExprToLabelFilters(me *metricsql.MetricExpr) ([]*labelFilter, error) { - lfs := make([]*labelFilter, len(me.LabelFilters)) - for i := range me.LabelFilters { - lf, err := newLabelFilter(&me.LabelFilters[i]) - if err != nil { - return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err) +func metricExprToLabelFilterss(me *metricsql.MetricExpr) ([][]*labelFilter, error) { + lfssNew := make([][]*labelFilter, len(me.LabelFilterss)) + for i, lfs := range me.LabelFilterss { + lfsNew := make([]*labelFilter, len(lfs)) + for j := range lfs { + lf, err := newLabelFilter(&lfs[j]) + if err != nil { + return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err) + } + lfsNew[j] = lf } - lfs[i] = lf + lfssNew[i] = lfsNew } - return lfs, nil + return lfssNew, nil } // labelFilter contains PromQL filter for `{label op "value"}` diff --git a/lib/promrelabel/if_expression_test.go b/lib/promrelabel/if_expression_test.go index 40962c70a..88296fe2d 100644 --- a/lib/promrelabel/if_expression_test.go +++ b/lib/promrelabel/if_expression_test.go @@ -20,6 +20,7 @@ func TestIfExpressionParseFailure(t *testing.T) { f(`{`) f(`{foo`) f(`foo{`) + f(`foo{bar="a" or}`) } func TestIfExpressionParseSuccess(t *testing.T) { @@ -33,6 +34,12 @@ func TestIfExpressionParseSuccess(t *testing.T) { f(`foo`) f(`{foo="bar"}`) f(`foo{bar=~"baz", x!="y"}`) + f(`{a="b" or c="d",e="x"}`) + f(`foo{ + bar="a",x="y" or + x="a",a="b" or + a="x" +}`) } func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) { @@ -63,6 +70,7 @@ func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) { } f("foo", `"foo"`) f(`{foo="bar",baz=~"x.*"}`, `"{foo=\"bar\",baz=~\"x.*\"}"`) + f(`{a="b" or c="d",x="z"}`, `"{a=\"b\" or c=\"d\",x=\"z\"}"`) } func TestIfExpressionUnmarshalFailure(t *testing.T) { @@ -113,6 +121,7 @@ func TestIfExpressionUnmarshalSuccess(t *testing.T) { f(`foo{bar="baz"}`) f(`'{a="b", c!="d", e=~"g", h!~"d"}'`) f(`foo{bar="zs",a=~"b|c"}`) + f(`foo{z="y" or bar="zs",a=~"b|c"}`) } func TestIfExpressionMatch(t *testing.T) { @@ -130,6 +139,8 @@ func TestIfExpressionMatch(t *testing.T) { f(`foo`, `foo`) f(`foo`, `foo{bar="baz",a="b"}`) f(`foo{bar="a"}`, `foo{bar="a"}`) + f(`foo{bar="a" or baz="x"}`, `foo{bar="a"}`) + f(`foo{baz="x" or bar="a"}`, `foo{bar="a"}`) f(`foo{bar="a"}`, `foo{x="y",bar="a",baz="b"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc",y="qwe"}`) @@ -164,6 +175,7 @@ func TestIfExpressionMismatch(t *testing.T) { f(`foo`, `bar`) f(`foo`, `a{foo="bar"}`) f(`foo{bar="a"}`, `foo`) + f(`foo{bar="a" or baz="a"}`, `foo`) f(`foo{bar="a"}`, `foo{bar="b"}`) f(`foo{bar="a"}`, `foo{baz="b",a="b"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="xabc"}`) diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go index f0dec0a82..915552564 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go @@ -561,10 +561,8 @@ func DurationValue(s string, step int64) (int64, error) { func parseSingleDuration(s string, step int64) (float64, error) { s = strings.ToLower(s) numPart := s[:len(s)-1] - if strings.HasSuffix(numPart, "m") { - // Duration in ms - numPart = numPart[:len(numPart)-1] - } + // Strip trailing m if the duration is in ms + numPart = strings.TrimSuffix(numPart, "m") f, err := strconv.ParseFloat(numPart, 64) if err != nil { return 0, fmt.Errorf("cannot parse duration %q: %s", s, err) diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index ff5ad3524..8072667cd 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -78,7 +78,7 @@ func optimizeInplace(e Expr) { func getCommonLabelFilters(e Expr) []LabelFilter { switch t := e.(type) { case *MetricExpr: - return getLabelFiltersWithoutMetricName(t.LabelFilters) + return getCommonLabelFiltersWithoutMetricName(t.LabelFilterss) case *RollupExpr: return getCommonLabelFilters(t.Expr) case *FuncExpr: @@ -180,6 +180,21 @@ func TrimFiltersByGroupModifier(lfs []LabelFilter, be *BinaryOpExpr) []LabelFilt } } +func getCommonLabelFiltersWithoutMetricName(lfss [][]LabelFilter) []LabelFilter { + if len(lfss) == 0 { + return nil + } + lfsA := getLabelFiltersWithoutMetricName(lfss[0]) + for _, lfs := range lfss[1:] { + if len(lfsA) == 0 { + return nil + } + lfsB := getLabelFiltersWithoutMetricName(lfs) + lfsA = intersectLabelFilters(lfsA, lfsB) + } + return lfsA +} + func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter { lfsNew := make([]LabelFilter, 0, len(lfs)) for _, lf := range lfs { @@ -213,8 +228,11 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) { } switch t := e.(type) { case *MetricExpr: - t.LabelFilters = unionLabelFilters(t.LabelFilters, lfs) - sortLabelFilters(t.LabelFilters) + for i, lfsLocal := range t.LabelFilterss { + lfsLocal = unionLabelFilters(lfsLocal, lfs) + sortLabelFilters(lfsLocal) + t.LabelFilterss[i] = lfsLocal + } case *RollupExpr: pushdownBinaryOpFiltersInplace(t.Expr, lfs) case *FuncExpr: diff --git a/vendor/github.com/VictoriaMetrics/metricsql/parser.go b/vendor/github.com/VictoriaMetrics/metricsql/parser.go index 57f63bcb0..5e6f7be97 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/parser.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/parser.go @@ -765,59 +765,71 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { } return eNew, nil case *MetricExpr: - if len(t.LabelFilters) > 0 { + if len(t.labelFilterss) == 0 { // Already expanded. return t, nil } { var me MetricExpr - // Populate me.LabelFilters - for _, lfe := range t.labelFilters { - if lfe.Value == nil { - // Expand lfe.Label into []LabelFilter. - wa := getWithArgExpr(was, lfe.Label) - if wa == nil { - return nil, fmt.Errorf("missing %q value inside %q", lfe.Label, t.AppendString(nil)) + // Populate me.LabelFilterss + for _, lfes := range t.labelFilterss { + var lfsNew []LabelFilter + for _, lfe := range lfes { + if lfe.Value == nil { + // Expand lfe.Label into lfsNew. + wa := getWithArgExpr(was, lfe.Label) + if wa == nil { + return nil, fmt.Errorf("cannot find WITH template for %q inside %q", lfe.Label, t.AppendString(nil)) + } + eNew, err := expandWithExprExt(was, wa, []Expr{}) + if err != nil { + return nil, err + } + wme, ok := eNew.(*MetricExpr) + if !ok || wme.getMetricName() != "" { + return nil, fmt.Errorf("WITH template %q inside %q must be {...}; got %q", + lfe.Label, t.AppendString(nil), eNew.AppendString(nil)) + } + if len(wme.labelFilterss) > 0 { + panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH template expansion; got %s", wme.labelFilterss)) + } + lfssSrc := wme.LabelFilterss + if len(lfssSrc) > 1 { + return nil, fmt.Errorf("WITH template %q at %q must be {...} without 'or'; got %s", + lfe.Label, t.AppendString(nil), wme.AppendString(nil)) + } + if len(lfssSrc) == 1 { + lfsNew = append(lfsNew, lfssSrc[0]...) + } + continue } - eNew, err := expandWithExprExt(was, wa, nil) + + // convert lfe to LabelFilter. + se, err := expandWithExpr(was, lfe.Value) if err != nil { return nil, err } - wme, ok := eNew.(*MetricExpr) - if !ok || wme.hasNonEmptyMetricGroup() { - return nil, fmt.Errorf("%q must be filters expression inside %q; got %q", lfe.Label, t.AppendString(nil), eNew.AppendString(nil)) + var lfeNew labelFilterExpr + lfeNew.Label = lfe.Label + lfeNew.Value = se.(*StringExpr) + lfeNew.IsNegative = lfe.IsNegative + lfeNew.IsRegexp = lfe.IsRegexp + lf, err := lfeNew.toLabelFilter() + if err != nil { + return nil, err } - if len(wme.labelFilters) > 0 { - panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters)) - } - me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...) - continue + lfsNew = append(lfsNew, *lf) } - - // convert lfe to LabelFilter. - se, err := expandWithExpr(was, lfe.Value) - if err != nil { - return nil, err - } - var lfeNew labelFilterExpr - lfeNew.Label = lfe.Label - lfeNew.Value = se.(*StringExpr) - lfeNew.IsNegative = lfe.IsNegative - lfeNew.IsRegexp = lfe.IsRegexp - lf, err := lfeNew.toLabelFilter() - if err != nil { - return nil, err - } - me.LabelFilters = append(me.LabelFilters, *lf) + lfsNew = removeDuplicateLabelFilters(lfsNew) + me.LabelFilterss = append(me.LabelFilterss, lfsNew) } - me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters) t = &me } - if !t.hasNonEmptyMetricGroup() { + metricName := t.getMetricName() + if metricName == "" { return t, nil } - k := t.LabelFilters[0].Value - wa := getWithArgExpr(was, k) + wa := getWithArgExpr(was, metricName) if wa == nil { return t, nil } @@ -833,25 +845,51 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) { wme, _ = eNew.(*MetricExpr) } if wme == nil { - if !t.isOnlyMetricGroup() { - return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) + if t.isOnlyMetricName() { + return eNew, nil } - return eNew, nil + return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) } - if len(wme.labelFilters) > 0 { - panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters)) + if len(wme.labelFilterss) > 0 { + panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH templates expansion; got %s", wme.labelFilterss)) + } + lfssSrc := wme.LabelFilterss + var lfssNew [][]LabelFilter + if len(lfssSrc) != 1 { + // template_name{filters} where template_name is {... or ...} + if t.isOnlyMetricName() { + // {filters} is empty. Return {... or ...} + return eNew, nil + } + if len(t.LabelFilterss) != 1 { + // {filters} contain {... or ...}. It cannot be merged with {... or ...} + return nil, fmt.Errorf("%q mustn't contain 'or' filters; got %s", metricName, wme.AppendString(nil)) + } + // {filters} doesn't contain `or`. Merge it with {... or ...} into {...,filters or ...,filters} + for _, lfs := range lfssSrc { + lfsNew := append([]LabelFilter{}, lfs...) + lfsNew = append(lfsNew, t.LabelFilterss[0][1:]...) + lfsNew = removeDuplicateLabelFilters(lfsNew) + lfssNew = append(lfssNew, lfsNew) + } + } else { + // template_name{... or ...} where template_name is an ordinary {filters} without 'or'. + // Merge it into {filters,... or filters,...} + for _, lfs := range t.LabelFilterss { + lfsNew := append([]LabelFilter{}, lfssSrc[0]...) + lfsNew = append(lfsNew, lfs[1:]...) + lfsNew = removeDuplicateLabelFilters(lfsNew) + lfssNew = append(lfssNew, lfsNew) + } + } + me := &MetricExpr{ + LabelFilterss: lfssNew, } - - var me MetricExpr - me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...) - me.LabelFilters = append(me.LabelFilters, t.LabelFilters[1:]...) - me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters) - if re == nil { - return &me, nil + return me, nil } reNew := *re - reNew.Expr = &me + reNew.Expr = me return &reNew, nil default: return e, nil @@ -889,22 +927,22 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) { } me, ok := wa.Expr.(*MetricExpr) if ok { - if !me.isOnlyMetricGroup() { + if !me.isOnlyMetricName() { return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args) } - dstArg := me.LabelFilters[0].Value - dstArgs = append(dstArgs, dstArg) + metricName := me.getMetricName() + dstArgs = append(dstArgs, metricName) continue } pe, ok := wa.Expr.(*parensExpr) if ok { for _, pArg := range *pe { me, ok := pArg.(*MetricExpr) - if !ok || !me.isOnlyMetricGroup() { + if !ok || !me.isOnlyMetricName() { return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args) } - dstArg := me.LabelFilters[0].Value - dstArgs = append(dstArgs, dstArg) + metricName := me.getMetricName() + dstArgs = append(dstArgs, metricName) } continue } @@ -926,7 +964,9 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) { func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, error) { if len(wa.Args) != len(args) { if args == nil { - // Just return MetricExpr with the wa.Name name. + // This case is possible if metric name clashes with one of the WITH template name. + // + // In this case just return MetricExpr with the wa.Name name. return newMetricExpr(wa.Name), nil } return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args)) @@ -949,10 +989,14 @@ func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, func newMetricExpr(name string) *MetricExpr { return &MetricExpr{ - LabelFilters: []LabelFilter{{ - Label: "__name__", - Value: name, - }}, + LabelFilterss: [][]LabelFilter{ + { + { + Label: "__name__", + Value: name, + }, + }, + }, } } @@ -1130,39 +1174,70 @@ func getWithArgExpr(was []*withArgExpr, name string) *withArgExpr { return nil } -func (p *parser) parseLabelFilters() ([]*labelFilterExpr, error) { +func (p *parser) parseLabelFilterss(mf *labelFilterExpr) ([][]*labelFilterExpr, error) { if p.lex.Token != "{" { return nil, fmt.Errorf(`labelFilters: unexpected token %q; want "{"`, p.lex.Token) } - - var lfes []*labelFilterExpr - for { + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == "}" { if err := p.lex.Next(); err != nil { return nil, err } - if p.lex.Token == "}" { - goto closeBracesLabel + if mf != nil { + return [][]*labelFilterExpr{{mf}}, nil } + return nil, nil + } + + var lfess [][]*labelFilterExpr + for { + lfes, err := p.parseLabelFilters(mf) + if err != nil { + return nil, err + } + lfess = append(lfess, lfes) + switch strings.ToLower(p.lex.Token) { + case "}": + if err := p.lex.Next(); err != nil { + return nil, err + } + return lfess, nil + case "or": + if err := p.lex.Next(); err != nil { + return nil, err + } + } + } +} + +func (p *parser) parseLabelFilters(mf *labelFilterExpr) ([]*labelFilterExpr, error) { + var lfes []*labelFilterExpr + if mf != nil { + lfes = append(lfes, mf) + } + for { lfe, err := p.parseLabelFilterExpr() if err != nil { return nil, err } lfes = append(lfes, lfe) - switch p.lex.Token { + switch strings.ToLower(p.lex.Token) { case ",": + if err := p.lex.Next(); err != nil { + return nil, err + } + if p.lex.Token == "}" { + return lfes, nil + } continue - case "}": - goto closeBracesLabel + case "or", "}": + return lfes, nil default: - return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "}"`, p.lex.Token) + return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "or", "}"`, p.lex.Token) } } - -closeBracesLabel: - if err := p.lex.Next(); err != nil { - return nil, err - } - return lfes, nil } func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) { @@ -1175,7 +1250,7 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) { return nil, err } - switch p.lex.Token { + switch strings.ToLower(p.lex.Token) { case "=": // Nothing to do. case "!=": @@ -1185,10 +1260,10 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) { case "!~": lfe.IsNegative = true lfe.IsRegexp = true - case ",", "}": + case ",", "}", "or": return &lfe, nil default: - return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "}"`, p.lex.Token) + return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "or", "}"`, p.lex.Token) } if err := p.lex.Next(); err != nil { @@ -1415,26 +1490,28 @@ func (p *parser) parseIdentExpr() (Expr, error) { } func (p *parser) parseMetricExpr() (*MetricExpr, error) { + var mf *labelFilterExpr var me MetricExpr if isIdentPrefix(p.lex.Token) { - var lfe labelFilterExpr - lfe.Label = "__name__" - lfe.Value = &StringExpr{ - tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))}, + mf = &labelFilterExpr{ + Label: "__name__", + Value: &StringExpr{ + tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))}, + }, } - me.labelFilters = append(me.labelFilters[:0], &lfe) if err := p.lex.Next(); err != nil { return nil, err } if p.lex.Token != "{" { + me.labelFilterss = append(me.labelFilterss[:0], []*labelFilterExpr{mf}) return &me, nil } } - lfes, err := p.parseLabelFilters() + lfess, err := p.parseLabelFilterss(mf) if err != nil { return nil, err } - me.labelFilters = append(me.labelFilters, lfes...) + me.labelFilterss = append(me.labelFilterss, lfess...) return &me, nil } @@ -1841,57 +1918,104 @@ func (lf *LabelFilter) AppendString(dst []byte) []byte { } // MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. +// +// Curly braces may contain or-delimited list of filters. For example: +// +// x{job="foo",instance="bar" or job="x",instance="baz"} +// +// In this case the filter returns all the series, which match at least one of the following filters: +// +// x{job="foo",instance="bar"} +// x{job="x",instance="baz"} +// +// This allows using or-delimited list of filters inside rollup functions. For example, +// the following query calculates rate per each matching series for the given or-delimited filters: +// +// rate(x{job="foo",instance="bar" or job="x",instance="baz"}[5m]) type MetricExpr struct { - // LabelFilters contains a list of label filters from curly braces. - // Filter or metric name must be the first if present. - LabelFilters []LabelFilter + // LabelFilters contains a list of or-delimited groups of label filters from curly braces. + // Filter for metric name (aka __name__ label) must go first in every group. + LabelFilterss [][]LabelFilter + // labelFilters contain non-expanded label filters joined by 'or' operator. + // // labelFilters must be expanded to LabelFilters by expandWithExpr. - labelFilters []*labelFilterExpr + labelFilterss [][]*labelFilterExpr } // AppendString appends string representation of me to dst and returns the result. func (me *MetricExpr) AppendString(dst []byte) []byte { - lfs := me.LabelFilters - if len(lfs) > 0 { - lf := &lfs[0] - if lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp { - dst = appendEscapedIdent(dst, lf.Value) - lfs = lfs[1:] - } - } - if len(lfs) > 0 { - dst = append(dst, '{') - for i := range lfs { - dst = lfs[i].AppendString(dst) - if i+1 < len(lfs) { - dst = append(dst, ", "...) - } - } - dst = append(dst, '}') - } else if len(me.LabelFilters) == 0 { + lfss := me.LabelFilterss + if len(lfss) == 0 { dst = append(dst, "{}"...) + return dst + } + offset := 0 + metricName := me.getMetricName() + if metricName != "" { + offset = 1 + dst = appendEscapedIdent(dst, metricName) + } + if len(lfss) == 1 && len(lfss[0]) == offset { + return dst + } + dst = append(dst, '{') + lfs := lfss[0] + dst = appendLabelFilters(dst, lfs[offset:]) + for _, lfs := range lfss[1:] { + dst = append(dst, " or "...) + dst = appendLabelFilters(dst, lfs[offset:]) + } + dst = append(dst, '}') + return dst +} + +func appendLabelFilters(dst []byte, lfs []LabelFilter) []byte { + if len(lfs) == 0 { + return dst + } + dst = lfs[0].AppendString(dst) + lfs = lfs[1:] + for i := range lfs { + dst = append(dst, ',') + dst = lfs[i].AppendString(dst) } return dst } // IsEmpty returns true of me equals to `{}`. func (me *MetricExpr) IsEmpty() bool { - return len(me.LabelFilters) == 0 + return len(me.LabelFilterss) == 0 } -func (me *MetricExpr) isOnlyMetricGroup() bool { - if !me.hasNonEmptyMetricGroup() { +func (me *MetricExpr) isOnlyMetricName() bool { + if me.getMetricName() == "" { return false } - return len(me.LabelFilters) == 1 + for _, lfs := range me.LabelFilterss { + if len(lfs) > 1 { + return false + } + } + return true } -func (me *MetricExpr) hasNonEmptyMetricGroup() bool { - if len(me.LabelFilters) == 0 { - return false +func (me *MetricExpr) getMetricName() string { + lfss := me.LabelFilterss + if len(lfss) == 0 { + return "" } - return me.LabelFilters[0].isMetricNameFilter() + lfs := lfss[0] + if len(lfs) == 0 || !lfs[0].isMetricNameFilter() { + return "" + } + metricName := lfs[0].Value + for _, lfs := range lfss[1:] { + if len(lfs) == 0 || !lfs[0].isMetricNameFilter() || lfs[0].Value != metricName { + return "" + } + } + return metricName } func (lf *LabelFilter) isMetricNameFilter() bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index 42e84c0f1..29fa2291a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.56.2 +# github.com/VictoriaMetrics/metricsql v0.57.1 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop