github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

all: add support for or filters in series selectors

Browse source 
This commit adds ability to select series matching distinct filters via a single series selector.
For example, the following selector selects series with either {env="prod",job="a"}
or {env="dev",job="b"} labels:

  {env="prod",job="a" or env="dev",job="b"}

The `or` filter is supported in all the VictoriaMetrics tools now.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3997
Uses https://github.com/VictoriaMetrics/metricsql/pull/14
This commit is contained in:
Aliaksandr Valialkin 2023-07-15 23:48:21 -07:00
parent bc4b6f2cb4
commit 4cb024d8a3
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
24 changed files with 441 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/vmagent/README.md
View file

@ -512,8 +512,8 @@ The following articles contain useful information about Prometheus relabeling:
{% endraw %} {% endraw %}
* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain * An optional `if` filter can be used for conditional relabeling. The `if` filter may contain
arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics:
```yaml ```yaml
- if: 'foo{bar="baz"}' - if: 'foo{bar="baz"}'

23
app/vmctl/vm_native.go
View file

@ -339,23 +339,26 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
if filter == metricName { if filter == metricName {
return filter, nil return filter, nil
} }
nameFilter := fmt.Sprintf("__name__=%q", metricName)
labels, err := searchutils.ParseMetricSelector(filter) tfss, err := searchutils.ParseMetricSelector(filter)
if err != nil { if err != nil {
return "", err return "", err
} }
str := make([]string, 0, len(labels)) var filters []string
for _, label := range labels { for _, tfs := range tfss {
if len(label.Key) == 0 { var a []string
continue for _, tf := range tfs {
if len(tf.Key) == 0 {
continue
}
a = append(a, tf.String())
} }
str = append(str, label.String()) a = append(a, nameFilter)
filters = append(filters, strings.Join(a, ","))
} }
nameFilter := fmt.Sprintf("__name__=%q", metricName) match := "{" + strings.Join(filters, " or ") + "}"
str = append(str, nameFilter)
match := fmt.Sprintf("{%s}", strings.Join(str, ","))
return match, nil return match, nil
} }

8
app/vmselect/prometheus/prometheus.go
View file

@ -1005,15 +1005,15 @@ func getMaxLookback(r *http.Request) (int64, error) {
} }
func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) { func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) {
tagFilterss := make([][]storage.TagFilter, 0, len(matches)) tfss := make([][]storage.TagFilter, 0, len(matches))
for _, match := range matches { for _, match := range matches {
tagFilters, err := searchutils.ParseMetricSelector(match) tfssLocal, err := searchutils.ParseMetricSelector(match)
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot parse matches[]=%s: %w", match, err) return nil, fmt.Errorf("cannot parse matches[]=%s: %w", match, err)
} }
tagFilterss = append(tagFilterss, tagFilters) tfss = append(tfss, tfssLocal...)
} }
return tagFilterss, nil return tfss, nil
} }
func getRoundDigits(r *http.Request) int { func getRoundDigits(r *http.Request) int {

4
app/vmselect/promql/eval.go
View file

@ -1075,8 +1075,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
} }
// Fetch the remaining part of the result. // Fetch the remaining part of the result.
tfs := searchutils.ToTagFilters(me.LabelFilters) tfss := searchutils.ToTagFilterss(me.LabelFilterss)
tfss := searchutils.JoinTagFilterss([][]storage.TagFilter{tfs}, ec.EnforcedTagFilterss) tfss = searchutils.JoinTagFilterss(tfss, ec.EnforcedTagFilterss)
minTimestamp := start - maxSilenceInterval minTimestamp := start - maxSilenceInterval
if window > ec.Step { if window > ec.Step {
minTimestamp -= window minTimestamp -= window

7
app/vmselect/promql/eval_test.go
View file

@ -29,8 +29,9 @@ func TestGetCommonLabelFilters(t *testing.T) {
tss = append(tss, &ts) tss = append(tss, &ts)
} }
lfs := getCommonLabelFilters(tss) lfs := getCommonLabelFilters(tss)
me := &metricsql.MetricExpr{ var me metricsql.MetricExpr
LabelFilters: lfs, if len(lfs) > 0 {
me.LabelFilterss = [][]metricsql.LabelFilter{lfs}
} }
lfsMarshaled := me.AppendString(nil) lfsMarshaled := me.AppendString(nil)
if string(lfsMarshaled) != lfsExpected { if string(lfsMarshaled) != lfsExpected {
@ -40,7 +41,7 @@ func TestGetCommonLabelFilters(t *testing.T) {
f(``, `{}`) f(``, `{}`)
f(`m 1`, `{}`) f(`m 1`, `{}`)
f(`m{a="b"} 1`, `{a="b"}`) f(`m{a="b"} 1`, `{a="b"}`)
f(`m{c="d",a="b"} 1`, `{a="b", c="d"}`) f(`m{c="d",a="b"} 1`, `{a="b",c="d"}`)
f(`m1{a="foo"} 1 f(`m1{a="foo"} 1
m2{a="bar"} 1`, `{a=~"bar|foo"}`) m2{a="bar"} 1`, `{a=~"bar|foo"}`)
f(`m1{a="foo"} 1 f(`m1{a="foo"} 1

10
app/vmselect/promql/exec.go
View file

@ -277,10 +277,12 @@ func escapeDotsInRegexpLabelFilters(e metricsql.Expr) metricsql.Expr {
if !ok { if !ok {
return return
} }
for i := range me.LabelFilters { for _, lfs := range me.LabelFilterss {
f := &me.LabelFilters[i] for i := range lfs {
if f.IsRegexp { f := &lfs[i]
f.Value = escapeDots(f.Value) if f.IsRegexp {
f.Value = escapeDots(f.Value)
}
} }
} }
}) })

2
app/vmselect/promql/exec_test.go
View file

@ -45,7 +45,7 @@ func TestEscapeDotsInRegexpLabelFilters(t *testing.T) {
f("2", "2") f("2", "2")
f(`foo.bar + 123`, `foo.bar + 123`) f(`foo.bar + 123`, `foo.bar + 123`)
f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`) f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`)
f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e", x=~"a\\.b.+[\\.a]", y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`) f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e",x=~"a\\.b.+[\\.a]",y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
} }
func TestExecSuccess(t *testing.T) { func TestExecSuccess(t *testing.T) {

2
app/vmselect/promql/parser.go
View file

@ -34,7 +34,7 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset *me
return return
} }
me, ok := re.Expr.(*metricsql.MetricExpr) me, ok := re.Expr.(*metricsql.MetricExpr)
if !ok || len(me.LabelFilters) == 0 { if !ok || len(me.LabelFilterss) == 0 {
return return
} }
wrappedQuery := me.AppendString(nil) wrappedQuery := me.AppendString(nil)

12
app/vmselect/promql/rollup_result_cache_test.go
View file

@ -41,10 +41,14 @@ func TestRollupResultCache(t *testing.T) {
MayCache: true, MayCache: true,
} }
me := &metricsql.MetricExpr{ me := &metricsql.MetricExpr{
LabelFilters: []metricsql.LabelFilter{{ LabelFilterss: [][]metricsql.LabelFilter{
Label: "aaa", {
Value: "xxx", {
}}, Label: "aaa",
Value: "xxx",
},
},
},
} }
fe := &metricsql.FuncExpr{ fe := &metricsql.FuncExpr{
Name: "foo", Name: "foo",

6
app/vmselect/promql/transform.go
View file

@ -240,7 +240,11 @@ func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries {
if !ok { if !ok {
return rvs return rvs
} }
tfs := searchutils.ToTagFilters(me.LabelFilters) tfss := searchutils.ToTagFilterss(me.LabelFilterss)
if len(tfss) != 1 {
return rvs
}
tfs := tfss[0]
for i := range tfs { for i := range tfs {
tf := &tfs[i] tf := &tfs[i]
if len(tf.Key) == 0 { if len(tf.Key) == 0 {

34
app/vmselect/searchutils/searchutils.go
View file

@ -140,12 +140,14 @@ func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) {
} }
var etfs [][]storage.TagFilter var etfs [][]storage.TagFilter
for _, extraFilter := range extraFilters { for _, extraFilter := range extraFilters {
tfs, err := ParseMetricSelector(extraFilter) tfss, err := ParseMetricSelector(extraFilter)
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot parse extra_filters=%s: %w", extraFilter, err) return nil, fmt.Errorf("cannot parse extra_filters=%s: %w", extraFilter, err)
} }
tfs = append(tfs, tagFilters...) for i := range tfss {
etfs = append(etfs, tfs) tfss[i] = append(tfss[i], tagFilters...)
}
etfs = append(etfs, tfss...)
} }
return etfs, nil return etfs, nil
} }
@ -170,7 +172,7 @@ func JoinTagFilterss(src, etfs [][]storage.TagFilter) [][]storage.TagFilter {
} }
// ParseMetricSelector parses s containing PromQL metric selector and returns the corresponding LabelFilters. // ParseMetricSelector parses s containing PromQL metric selector and returns the corresponding LabelFilters.
func ParseMetricSelector(s string) ([]storage.TagFilter, error) { func ParseMetricSelector(s string) ([][]storage.TagFilter, error) {
expr, err := metricsql.Parse(s) expr, err := metricsql.Parse(s)
if err != nil { if err != nil {
return nil, err return nil, err
@ -179,20 +181,24 @@ func ParseMetricSelector(s string) ([]storage.TagFilter, error) {
if !ok { if !ok {
return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil)) return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil))
} }
if len(me.LabelFilters) == 0 { if len(me.LabelFilterss) == 0 {
return nil, fmt.Errorf("labelFilters cannot be empty") return nil, fmt.Errorf("labelFilterss cannot be empty")
} }
tfs := ToTagFilters(me.LabelFilters) tfss := ToTagFilterss(me.LabelFilterss)
return tfs, nil return tfss, nil
} }
// ToTagFilters converts lfs to a slice of storage.TagFilter // ToTagFilterss converts lfss to or-delimited slices of storage.TagFilter
func ToTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter { func ToTagFilterss(lfss [][]metricsql.LabelFilter) [][]storage.TagFilter {
tfs := make([]storage.TagFilter, len(lfs)) tfss := make([][]storage.TagFilter, len(lfss))
for i := range lfs { for i, lfs := range lfss {
toTagFilter(&tfs[i], &lfs[i]) tfs := make([]storage.TagFilter, len(lfs))
for j := range lfs {
toTagFilter(&tfs[j], &lfs[j])
}
tfss[i] = tfs
} }
return tfs return tfss
} }
func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) { func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {

54
app/vmselect/searchutils/searchutils_test.go
View file

@ -135,69 +135,69 @@ func TestJoinTagFilterss(t *testing.T) {
} }
} }
// Single tag filter // Single tag filter
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
}, nil, []string{ ), nil, []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
}) })
// Miltiple tag filters // Miltiple tag filters
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
}, nil, []string{ ), nil, []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
`{k5=~"v5"}`, `{k5=~"v5"}`,
}) })
// Single extra filter // Single extra filter
f(t, nil, [][]storage.TagFilter{ f(t, nil, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
}) })
// Multiple extra filters // Multiple extra filters
f(t, nil, [][]storage.TagFilter{ f(t, nil, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
`{k5=~"v5"}`, `{k5=~"v5"}`,
}) })
// Single tag filter and a single extra filter // Single tag filter and a single extra filter
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
}, [][]storage.TagFilter{ ), joinTagFilters(
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`,
}) })
// Multiple tag filters and a single extra filter // Multiple tag filters and a single extra filter
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
}, [][]storage.TagFilter{ ), joinTagFilters(
mustParseMetricSelector(`{k6=~"v6"}`), mustParseMetricSelector(`{k6=~"v6"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
`{k5=~"v5",k6=~"v6"}`, `{k5=~"v5",k6=~"v6"}`,
}) })
// Single tag filter and multiple extra filters // Single tag filter and multiple extra filters
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
}, [][]storage.TagFilter{ ), joinTagFilters(
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
mustParseMetricSelector(`{k6=~"v6"}`), mustParseMetricSelector(`{k6=~"v6"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`,
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
}) })
// Multiple tag filters and multiple extra filters // Multiple tag filters and multiple extra filters
f(t, [][]storage.TagFilter{ f(t, joinTagFilters(
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`), mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
mustParseMetricSelector(`{k5=~"v5"}`), mustParseMetricSelector(`{k5=~"v5"}`),
}, [][]storage.TagFilter{ ), joinTagFilters(
mustParseMetricSelector(`{k6=~"v6"}`), mustParseMetricSelector(`{k6=~"v6"}`),
mustParseMetricSelector(`{k7=~"v7"}`), mustParseMetricSelector(`{k7=~"v7"}`),
}, []string{ ), []string{
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k7=~"v7"}`, `{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k7=~"v7"}`,
`{k5=~"v5",k6=~"v6"}`, `{k5=~"v5",k6=~"v6"}`,
@ -205,12 +205,20 @@ func TestJoinTagFilterss(t *testing.T) {
}) })
} }
func mustParseMetricSelector(s string) []storage.TagFilter { func joinTagFilters(args ...[][]storage.TagFilter) [][]storage.TagFilter {
tf, err := ParseMetricSelector(s) result := append([][]storage.TagFilter{}, args[0]...)
for _, tfss := range args[1:] {
result = append(result, tfss...)
}
return result
}
func mustParseMetricSelector(s string) [][]storage.TagFilter {
tfss, err := ParseMetricSelector(s)
if err != nil { if err != nil {
panic(fmt.Errorf("cannot parse %q: %w", s, err)) panic(fmt.Errorf("cannot parse %q: %w", s, err))
} }
return tf return tfss
} }
func tagFilterssToStrings(tfss [][]storage.TagFilter) []string { func tagFilterssToStrings(tfss [][]storage.TagFilter) []string {

1
docs/CHANGELOG.md
View file

@ -28,6 +28,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved). * SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved).
* FETURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details. * FETURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details.
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow selecting time series matching at least one of multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries). See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402).

3
docs/MetricsQL.md
View file

@ -67,6 +67,9 @@ The list of MetricsQL features:
depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)). depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)).
For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`. For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana. It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
* [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
* [Aggregate functions](#aggregate-functions) accept arbitrary number of args. * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`. For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query. * [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.

40
docs/keyConcepts.md
View file

@ -785,15 +785,15 @@ requests_total{path="/", code="200"}
requests_total{path="/", code="403"} requests_total{path="/", code="403"}
``` ```
To select only time series with specific label value specify the matching condition in curly braces: To select only time series with specific label value specify the matching filter in curly braces:
```metricsql ```metricsql
requests_total{code="200"} requests_total{code="200"}
``` ```
The query above will return all time series with the name `requests_total` and `code="200"`. We use the operator `=` to The query above returns all time series with the name `requests_total` and label `code="200"`. We use the operator `=` to
match a label value. For negative match use `!=` operator. Filters also support regex matching `=~` for positive match label value. For negative match use `!=` operator. Filters also support positive regex matching via `=~`
and `!~` for negative matching: and negative regex matching via `!~`:
```metricsql ```metricsql
requests_total{code=~"2.*"} requests_total{code=~"2.*"}
@ -802,23 +802,45 @@ requests_total{code=~"2.*"}
Filters can also be combined: Filters can also be combined:
```metricsql ```metricsql
requests_total{code=~"200|204", path="/home"} requests_total{code=~"200", path="/home"}
``` ```
The query above will return all time series with a name `requests_total`, status `code` `200` or `204`and `path="/home"` The query above returns all time series with `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
.
#### Filtering by name #### Filtering by name
Sometimes it is required to return all the time series for multiple metric names. As was mentioned in Sometimes it is required to return all the time series for multiple metric names. As was mentioned in
the [data model section](#data-model), the metric name is just an ordinary label with a special name — `__name__`. So the [data model section](#data-model), the metric name is just an ordinary label with a special name - `__name__`. So
filtering by multiple metric names may be performed by applying regexps on metric names: filtering by multiple metric names may be performed by applying regexps on metric names:
```metricsql ```metricsql
{__name__=~"requests_(error|success)_total"} {__name__=~"requests_(error|success)_total"}
``` ```
The query above is supposed to return series for two metrics: `requests_error_total` and `requests_success_total`. The query above returns series for two metrics: `requests_error_total` and `requests_success_total`.
#### Filtering by multiple "or" filters
[MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) supports selecting time series, which match at least one of multiple "or" filters.
Such filters must be delimited by `or` inside curly braces. For example, the following query selects time series with
either `{job="app1",env="prod"}` or `{job="app2",env="dev"}` labels:
```metricsql
{job="app1",env="prod" or job="app2",env="dev"}
```
The number of `or` filters can be arbitrary. This functionality allows passing the selected series
to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) such as [rate()](https://docs.victoriametrics.com/MetricsQL.html#rate)
without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries):
```metricsql
rate({job="app1",env="prod" or job="app2",env="dev"}[5m])
```
If you need to select series matching multiple filters for the same label, then it is better from performance PoV
to use regexp filter `{label=~"value1|...|valueN"}` instead of `{label="value1" or ... or label="valueN"}`.
#### Arithmetic operations #### Arithmetic operations

4
docs/vmagent.md
View file

@ -523,8 +523,8 @@ The following articles contain useful information about Prometheus relabeling:
{% endraw %} {% endraw %}
* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain * An optional `if` filter can be used for conditional relabeling. The `if` filter may contain
arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics:
```yaml ```yaml
- if: 'foo{bar="baz"}' - if: 'foo{bar="baz"}'

2
go.mod
View file

@ -12,7 +12,7 @@ require (
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/fasthttp v1.2.0
github.com/VictoriaMetrics/metrics v1.24.0 github.com/VictoriaMetrics/metrics v1.24.0
github.com/VictoriaMetrics/metricsql v0.56.2 github.com/VictoriaMetrics/metricsql v0.57.1
github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/config v1.18.27
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71

6
go.sum
View file

@ -67,11 +67,10 @@ github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bw
github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I= github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I=
github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE=
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
github.com/VictoriaMetrics/metricsql v0.56.2 h1:quBAbYOlWMhmdgzFSCr1yjtVcdZYZrVQJ7nR9zor7ZM= github.com/VictoriaMetrics/metricsql v0.57.1 h1:ryMe7w95s80BilS8RlV3G/25BLlmMBVRtTq2GnLB/4o=
github.com/VictoriaMetrics/metricsql v0.56.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VictoriaMetrics/metricsql v0.57.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -631,6 +630,7 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

39
lib/promrelabel/if_expression.go
View file

@ -14,8 +14,8 @@ import (
// //
// The `if` expression can contain arbitrary PromQL-like label filters such as `metric_name{filters...}` // The `if` expression can contain arbitrary PromQL-like label filters such as `metric_name{filters...}`
type IfExpression struct { type IfExpression struct {
s string s string
lfs []*labelFilter lfss [][]*labelFilter
} }
// String returns string representation of ie. // String returns string representation of ie.
@ -36,12 +36,12 @@ func (ie *IfExpression) Parse(s string) error {
if !ok { if !ok {
return fmt.Errorf("expecting series selector; got %q", expr.AppendString(nil)) return fmt.Errorf("expecting series selector; got %q", expr.AppendString(nil))
} }
lfs, err := metricExprToLabelFilters(me) lfss, err := metricExprToLabelFilterss(me)
if err != nil { if err != nil {
return fmt.Errorf("cannot parse series selector: %w", err) return fmt.Errorf("cannot parse series selector: %w", err)
} }
ie.s = s ie.s = s
ie.lfs = lfs ie.lfss = lfss
return nil return nil
} }
@ -81,7 +81,16 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool {
if ie == nil { if ie == nil {
return true return true
} }
for _, lf := range ie.lfs { for _, lfs := range ie.lfss {
if matchLabelFilters(lfs, labels) {
return true
}
}
return false
}
func matchLabelFilters(lfs []*labelFilter, labels []prompbmarshal.Label) bool {
for _, lf := range lfs {
if !lf.match(labels) { if !lf.match(labels) {
return false return false
} }
@ -89,16 +98,20 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool {
return true return true
} }
func metricExprToLabelFilters(me *metricsql.MetricExpr) ([]*labelFilter, error) { func metricExprToLabelFilterss(me *metricsql.MetricExpr) ([][]*labelFilter, error) {
lfs := make([]*labelFilter, len(me.LabelFilters)) lfssNew := make([][]*labelFilter, len(me.LabelFilterss))
for i := range me.LabelFilters { for i, lfs := range me.LabelFilterss {
lf, err := newLabelFilter(&me.LabelFilters[i]) lfsNew := make([]*labelFilter, len(lfs))
if err != nil { for j := range lfs {
return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err) lf, err := newLabelFilter(&lfs[j])
if err != nil {
return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err)
}
lfsNew[j] = lf
} }
lfs[i] = lf lfssNew[i] = lfsNew
} }
return lfs, nil return lfssNew, nil
} }
// labelFilter contains PromQL filter for `{label op "value"}` // labelFilter contains PromQL filter for `{label op "value"}`

12
lib/promrelabel/if_expression_test.go
View file

@ -20,6 +20,7 @@ func TestIfExpressionParseFailure(t *testing.T) {
f(`{`) f(`{`)
f(`{foo`) f(`{foo`)
f(`foo{`) f(`foo{`)
f(`foo{bar="a" or}`)
} }
func TestIfExpressionParseSuccess(t *testing.T) { func TestIfExpressionParseSuccess(t *testing.T) {
@ -33,6 +34,12 @@ func TestIfExpressionParseSuccess(t *testing.T) {
f(`foo`) f(`foo`)
f(`{foo="bar"}`) f(`{foo="bar"}`)
f(`foo{bar=~"baz", x!="y"}`) f(`foo{bar=~"baz", x!="y"}`)
f(`{a="b" or c="d",e="x"}`)
f(`foo{
bar="a",x="y" or
x="a",a="b" or
a="x"
}`)
} }
func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) { func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) {
@ -63,6 +70,7 @@ func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) {
} }
f("foo", `"foo"`) f("foo", `"foo"`)
f(`{foo="bar",baz=~"x.*"}`, `"{foo=\"bar\",baz=~\"x.*\"}"`) f(`{foo="bar",baz=~"x.*"}`, `"{foo=\"bar\",baz=~\"x.*\"}"`)
f(`{a="b" or c="d",x="z"}`, `"{a=\"b\" or c=\"d\",x=\"z\"}"`)
} }
func TestIfExpressionUnmarshalFailure(t *testing.T) { func TestIfExpressionUnmarshalFailure(t *testing.T) {
@ -113,6 +121,7 @@ func TestIfExpressionUnmarshalSuccess(t *testing.T) {
f(`foo{bar="baz"}`) f(`foo{bar="baz"}`)
f(`'{a="b", c!="d", e=~"g", h!~"d"}'`) f(`'{a="b", c!="d", e=~"g", h!~"d"}'`)
f(`foo{bar="zs",a=~"b|c"}`) f(`foo{bar="zs",a=~"b|c"}`)
f(`foo{z="y" or bar="zs",a=~"b|c"}`)
} }
func TestIfExpressionMatch(t *testing.T) { func TestIfExpressionMatch(t *testing.T) {
@ -130,6 +139,8 @@ func TestIfExpressionMatch(t *testing.T) {
f(`foo`, `foo`) f(`foo`, `foo`)
f(`foo`, `foo{bar="baz",a="b"}`) f(`foo`, `foo{bar="baz",a="b"}`)
f(`foo{bar="a"}`, `foo{bar="a"}`) f(`foo{bar="a"}`, `foo{bar="a"}`)
f(`foo{bar="a" or baz="x"}`, `foo{bar="a"}`)
f(`foo{baz="x" or bar="a"}`, `foo{bar="a"}`)
f(`foo{bar="a"}`, `foo{x="y",bar="a",baz="b"}`) f(`foo{bar="a"}`, `foo{x="y",bar="a",baz="b"}`)
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc"}`)
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc",y="qwe"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc",y="qwe"}`)
@ -164,6 +175,7 @@ func TestIfExpressionMismatch(t *testing.T) {
f(`foo`, `bar`) f(`foo`, `bar`)
f(`foo`, `a{foo="bar"}`) f(`foo`, `a{foo="bar"}`)
f(`foo{bar="a"}`, `foo`) f(`foo{bar="a"}`, `foo`)
f(`foo{bar="a" or baz="a"}`, `foo`)
f(`foo{bar="a"}`, `foo{bar="b"}`) f(`foo{bar="a"}`, `foo{bar="b"}`)
f(`foo{bar="a"}`, `foo{baz="b",a="b"}`) f(`foo{bar="a"}`, `foo{baz="b",a="b"}`)
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="xabc"}`) f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="xabc"}`)

6
vendor/github.com/VictoriaMetrics/metricsql/lexer.go generated vendored
View file

@ -561,10 +561,8 @@ func DurationValue(s string, step int64) (int64, error) {
func parseSingleDuration(s string, step int64) (float64, error) { func parseSingleDuration(s string, step int64) (float64, error) {
s = strings.ToLower(s) s = strings.ToLower(s)
numPart := s[:len(s)-1] numPart := s[:len(s)-1]
if strings.HasSuffix(numPart, "m") { // Strip trailing m if the duration is in ms
// Duration in ms numPart = strings.TrimSuffix(numPart, "m")
numPart = numPart[:len(numPart)-1]
}
f, err := strconv.ParseFloat(numPart, 64) f, err := strconv.ParseFloat(numPart, 64)
if err != nil { if err != nil {
return 0, fmt.Errorf("cannot parse duration %q: %s", s, err) return 0, fmt.Errorf("cannot parse duration %q: %s", s, err)

24
vendor/github.com/VictoriaMetrics/metricsql/optimizer.go generated vendored
View file

@ -78,7 +78,7 @@ func optimizeInplace(e Expr) {
func getCommonLabelFilters(e Expr) []LabelFilter { func getCommonLabelFilters(e Expr) []LabelFilter {
switch t := e.(type) { switch t := e.(type) {
case *MetricExpr: case *MetricExpr:
return getLabelFiltersWithoutMetricName(t.LabelFilters) return getCommonLabelFiltersWithoutMetricName(t.LabelFilterss)
case *RollupExpr: case *RollupExpr:
return getCommonLabelFilters(t.Expr) return getCommonLabelFilters(t.Expr)
case *FuncExpr: case *FuncExpr:
@ -180,6 +180,21 @@ func TrimFiltersByGroupModifier(lfs []LabelFilter, be *BinaryOpExpr) []LabelFilt
} }
} }
func getCommonLabelFiltersWithoutMetricName(lfss [][]LabelFilter) []LabelFilter {
if len(lfss) == 0 {
return nil
}
lfsA := getLabelFiltersWithoutMetricName(lfss[0])
for _, lfs := range lfss[1:] {
if len(lfsA) == 0 {
return nil
}
lfsB := getLabelFiltersWithoutMetricName(lfs)
lfsA = intersectLabelFilters(lfsA, lfsB)
}
return lfsA
}
func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter { func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter {
lfsNew := make([]LabelFilter, 0, len(lfs)) lfsNew := make([]LabelFilter, 0, len(lfs))
for _, lf := range lfs { for _, lf := range lfs {
@ -213,8 +228,11 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) {
} }
switch t := e.(type) { switch t := e.(type) {
case *MetricExpr: case *MetricExpr:
t.LabelFilters = unionLabelFilters(t.LabelFilters, lfs) for i, lfsLocal := range t.LabelFilterss {
sortLabelFilters(t.LabelFilters) lfsLocal = unionLabelFilters(lfsLocal, lfs)
sortLabelFilters(lfsLocal)
t.LabelFilterss[i] = lfsLocal
}
case *RollupExpr: case *RollupExpr:
pushdownBinaryOpFiltersInplace(t.Expr, lfs) pushdownBinaryOpFiltersInplace(t.Expr, lfs)
case *FuncExpr: case *FuncExpr:

358
vendor/github.com/VictoriaMetrics/metricsql/parser.go generated vendored
View file

@ -765,59 +765,71 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
} }
return eNew, nil return eNew, nil
case *MetricExpr: case *MetricExpr:
if len(t.LabelFilters) > 0 { if len(t.labelFilterss) == 0 {
// Already expanded. // Already expanded.
return t, nil return t, nil
} }
{ {
var me MetricExpr var me MetricExpr
// Populate me.LabelFilters // Populate me.LabelFilterss
for _, lfe := range t.labelFilters { for _, lfes := range t.labelFilterss {
if lfe.Value == nil { var lfsNew []LabelFilter
// Expand lfe.Label into []LabelFilter. for _, lfe := range lfes {
wa := getWithArgExpr(was, lfe.Label) if lfe.Value == nil {
if wa == nil { // Expand lfe.Label into lfsNew.
return nil, fmt.Errorf("missing %q value inside %q", lfe.Label, t.AppendString(nil)) wa := getWithArgExpr(was, lfe.Label)
if wa == nil {
return nil, fmt.Errorf("cannot find WITH template for %q inside %q", lfe.Label, t.AppendString(nil))
}
eNew, err := expandWithExprExt(was, wa, []Expr{})
if err != nil {
return nil, err
}
wme, ok := eNew.(*MetricExpr)
if !ok || wme.getMetricName() != "" {
return nil, fmt.Errorf("WITH template %q inside %q must be {...}; got %q",
lfe.Label, t.AppendString(nil), eNew.AppendString(nil))
}
if len(wme.labelFilterss) > 0 {
panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH template expansion; got %s", wme.labelFilterss))
}
lfssSrc := wme.LabelFilterss
if len(lfssSrc) > 1 {
return nil, fmt.Errorf("WITH template %q at %q must be {...} without 'or'; got %s",
lfe.Label, t.AppendString(nil), wme.AppendString(nil))
}
if len(lfssSrc) == 1 {
lfsNew = append(lfsNew, lfssSrc[0]...)
}
continue
} }
eNew, err := expandWithExprExt(was, wa, nil)
// convert lfe to LabelFilter.
se, err := expandWithExpr(was, lfe.Value)
if err != nil { if err != nil {
return nil, err return nil, err
} }
wme, ok := eNew.(*MetricExpr) var lfeNew labelFilterExpr
if !ok || wme.hasNonEmptyMetricGroup() { lfeNew.Label = lfe.Label
return nil, fmt.Errorf("%q must be filters expression inside %q; got %q", lfe.Label, t.AppendString(nil), eNew.AppendString(nil)) lfeNew.Value = se.(*StringExpr)
lfeNew.IsNegative = lfe.IsNegative
lfeNew.IsRegexp = lfe.IsRegexp
lf, err := lfeNew.toLabelFilter()
if err != nil {
return nil, err
} }
if len(wme.labelFilters) > 0 { lfsNew = append(lfsNew, *lf)
panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters))
}
me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...)
continue
} }
lfsNew = removeDuplicateLabelFilters(lfsNew)
// convert lfe to LabelFilter. me.LabelFilterss = append(me.LabelFilterss, lfsNew)
se, err := expandWithExpr(was, lfe.Value)
if err != nil {
return nil, err
}
var lfeNew labelFilterExpr
lfeNew.Label = lfe.Label
lfeNew.Value = se.(*StringExpr)
lfeNew.IsNegative = lfe.IsNegative
lfeNew.IsRegexp = lfe.IsRegexp
lf, err := lfeNew.toLabelFilter()
if err != nil {
return nil, err
}
me.LabelFilters = append(me.LabelFilters, *lf)
} }
me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters)
t = &me t = &me
} }
if !t.hasNonEmptyMetricGroup() { metricName := t.getMetricName()
if metricName == "" {
return t, nil return t, nil
} }
k := t.LabelFilters[0].Value wa := getWithArgExpr(was, metricName)
wa := getWithArgExpr(was, k)
if wa == nil { if wa == nil {
return t, nil return t, nil
} }
@ -833,25 +845,51 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
wme, _ = eNew.(*MetricExpr) wme, _ = eNew.(*MetricExpr)
} }
if wme == nil { if wme == nil {
if !t.isOnlyMetricGroup() { if t.isOnlyMetricName() {
return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil)) return eNew, nil
} }
return eNew, nil return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil))
} }
if len(wme.labelFilters) > 0 { if len(wme.labelFilterss) > 0 {
panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters)) panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH templates expansion; got %s", wme.labelFilterss))
}
lfssSrc := wme.LabelFilterss
var lfssNew [][]LabelFilter
if len(lfssSrc) != 1 {
// template_name{filters} where template_name is {... or ...}
if t.isOnlyMetricName() {
// {filters} is empty. Return {... or ...}
return eNew, nil
}
if len(t.LabelFilterss) != 1 {
// {filters} contain {... or ...}. It cannot be merged with {... or ...}
return nil, fmt.Errorf("%q mustn't contain 'or' filters; got %s", metricName, wme.AppendString(nil))
}
// {filters} doesn't contain `or`. Merge it with {... or ...} into {...,filters or ...,filters}
for _, lfs := range lfssSrc {
lfsNew := append([]LabelFilter{}, lfs...)
lfsNew = append(lfsNew, t.LabelFilterss[0][1:]...)
lfsNew = removeDuplicateLabelFilters(lfsNew)
lfssNew = append(lfssNew, lfsNew)
}
} else {
// template_name{... or ...} where template_name is an ordinary {filters} without 'or'.
// Merge it into {filters,... or filters,...}
for _, lfs := range t.LabelFilterss {
lfsNew := append([]LabelFilter{}, lfssSrc[0]...)
lfsNew = append(lfsNew, lfs[1:]...)
lfsNew = removeDuplicateLabelFilters(lfsNew)
lfssNew = append(lfssNew, lfsNew)
}
}
me := &MetricExpr{
LabelFilterss: lfssNew,
} }
var me MetricExpr
me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...)
me.LabelFilters = append(me.LabelFilters, t.LabelFilters[1:]...)
me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters)
if re == nil { if re == nil {
return &me, nil return me, nil
} }
reNew := *re reNew := *re
reNew.Expr = &me reNew.Expr = me
return &reNew, nil return &reNew, nil
default: default:
return e, nil return e, nil
@ -889,22 +927,22 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) {
} }
me, ok := wa.Expr.(*MetricExpr) me, ok := wa.Expr.(*MetricExpr)
if ok { if ok {
if !me.isOnlyMetricGroup() { if !me.isOnlyMetricName() {
return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args) return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args)
} }
dstArg := me.LabelFilters[0].Value metricName := me.getMetricName()
dstArgs = append(dstArgs, dstArg) dstArgs = append(dstArgs, metricName)
continue continue
} }
pe, ok := wa.Expr.(*parensExpr) pe, ok := wa.Expr.(*parensExpr)
if ok { if ok {
for _, pArg := range *pe { for _, pArg := range *pe {
me, ok := pArg.(*MetricExpr) me, ok := pArg.(*MetricExpr)
if !ok || !me.isOnlyMetricGroup() { if !ok || !me.isOnlyMetricName() {
return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args) return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args)
} }
dstArg := me.LabelFilters[0].Value metricName := me.getMetricName()
dstArgs = append(dstArgs, dstArg) dstArgs = append(dstArgs, metricName)
} }
continue continue
} }
@ -926,7 +964,9 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) {
func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, error) { func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, error) {
if len(wa.Args) != len(args) { if len(wa.Args) != len(args) {
if args == nil { if args == nil {
// Just return MetricExpr with the wa.Name name. // This case is possible if metric name clashes with one of the WITH template name.
//
// In this case just return MetricExpr with the wa.Name name.
return newMetricExpr(wa.Name), nil return newMetricExpr(wa.Name), nil
} }
return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args)) return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args))
@ -949,10 +989,14 @@ func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr,
func newMetricExpr(name string) *MetricExpr { func newMetricExpr(name string) *MetricExpr {
return &MetricExpr{ return &MetricExpr{
LabelFilters: []LabelFilter{{ LabelFilterss: [][]LabelFilter{
Label: "__name__", {
Value: name, {
}}, Label: "__name__",
Value: name,
},
},
},
} }
} }
@ -1130,39 +1174,70 @@ func getWithArgExpr(was []*withArgExpr, name string) *withArgExpr {
return nil return nil
} }
func (p *parser) parseLabelFilters() ([]*labelFilterExpr, error) { func (p *parser) parseLabelFilterss(mf *labelFilterExpr) ([][]*labelFilterExpr, error) {
if p.lex.Token != "{" { if p.lex.Token != "{" {
return nil, fmt.Errorf(`labelFilters: unexpected token %q; want "{"`, p.lex.Token) return nil, fmt.Errorf(`labelFilters: unexpected token %q; want "{"`, p.lex.Token)
} }
if err := p.lex.Next(); err != nil {
var lfes []*labelFilterExpr return nil, err
for { }
if p.lex.Token == "}" {
if err := p.lex.Next(); err != nil { if err := p.lex.Next(); err != nil {
return nil, err return nil, err
} }
if p.lex.Token == "}" { if mf != nil {
goto closeBracesLabel return [][]*labelFilterExpr{{mf}}, nil
} }
return nil, nil
}
var lfess [][]*labelFilterExpr
for {
lfes, err := p.parseLabelFilters(mf)
if err != nil {
return nil, err
}
lfess = append(lfess, lfes)
switch strings.ToLower(p.lex.Token) {
case "}":
if err := p.lex.Next(); err != nil {
return nil, err
}
return lfess, nil
case "or":
if err := p.lex.Next(); err != nil {
return nil, err
}
}
}
}
func (p *parser) parseLabelFilters(mf *labelFilterExpr) ([]*labelFilterExpr, error) {
var lfes []*labelFilterExpr
if mf != nil {
lfes = append(lfes, mf)
}
for {
lfe, err := p.parseLabelFilterExpr() lfe, err := p.parseLabelFilterExpr()
if err != nil { if err != nil {
return nil, err return nil, err
} }
lfes = append(lfes, lfe) lfes = append(lfes, lfe)
switch p.lex.Token { switch strings.ToLower(p.lex.Token) {
case ",": case ",":
if err := p.lex.Next(); err != nil {
return nil, err
}
if p.lex.Token == "}" {
return lfes, nil
}
continue continue
case "}": case "or", "}":
goto closeBracesLabel return lfes, nil
default: default:
return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "}"`, p.lex.Token) return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "or", "}"`, p.lex.Token)
} }
} }
closeBracesLabel:
if err := p.lex.Next(); err != nil {
return nil, err
}
return lfes, nil
} }
func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) { func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
@ -1175,7 +1250,7 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
return nil, err return nil, err
} }
switch p.lex.Token { switch strings.ToLower(p.lex.Token) {
case "=": case "=":
// Nothing to do. // Nothing to do.
case "!=": case "!=":
@ -1185,10 +1260,10 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
case "!~": case "!~":
lfe.IsNegative = true lfe.IsNegative = true
lfe.IsRegexp = true lfe.IsRegexp = true
case ",", "}": case ",", "}", "or":
return &lfe, nil return &lfe, nil
default: default:
return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "}"`, p.lex.Token) return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "or", "}"`, p.lex.Token)
} }
if err := p.lex.Next(); err != nil { if err := p.lex.Next(); err != nil {
@ -1415,26 +1490,28 @@ func (p *parser) parseIdentExpr() (Expr, error) {
} }
func (p *parser) parseMetricExpr() (*MetricExpr, error) { func (p *parser) parseMetricExpr() (*MetricExpr, error) {
var mf *labelFilterExpr
var me MetricExpr var me MetricExpr
if isIdentPrefix(p.lex.Token) { if isIdentPrefix(p.lex.Token) {
var lfe labelFilterExpr mf = &labelFilterExpr{
lfe.Label = "__name__" Label: "__name__",
lfe.Value = &StringExpr{ Value: &StringExpr{
tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))}, tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))},
},
} }
me.labelFilters = append(me.labelFilters[:0], &lfe)
if err := p.lex.Next(); err != nil { if err := p.lex.Next(); err != nil {
return nil, err return nil, err
} }
if p.lex.Token != "{" { if p.lex.Token != "{" {
me.labelFilterss = append(me.labelFilterss[:0], []*labelFilterExpr{mf})
return &me, nil return &me, nil
} }
} }
lfes, err := p.parseLabelFilters() lfess, err := p.parseLabelFilterss(mf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
me.labelFilters = append(me.labelFilters, lfes...) me.labelFilterss = append(me.labelFilterss, lfess...)
return &me, nil return &me, nil
} }
@ -1841,57 +1918,104 @@ func (lf *LabelFilter) AppendString(dst []byte) []byte {
} }
// MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. // MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`.
//
// Curly braces may contain or-delimited list of filters. For example:
//
// x{job="foo",instance="bar" or job="x",instance="baz"}
//
// In this case the filter returns all the series, which match at least one of the following filters:
//
// x{job="foo",instance="bar"}
// x{job="x",instance="baz"}
//
// This allows using or-delimited list of filters inside rollup functions. For example,
// the following query calculates rate per each matching series for the given or-delimited filters:
//
// rate(x{job="foo",instance="bar" or job="x",instance="baz"}[5m])
type MetricExpr struct { type MetricExpr struct {
// LabelFilters contains a list of label filters from curly braces. // LabelFilters contains a list of or-delimited groups of label filters from curly braces.
// Filter or metric name must be the first if present. // Filter for metric name (aka __name__ label) must go first in every group.
LabelFilters []LabelFilter LabelFilterss [][]LabelFilter
// labelFilters contain non-expanded label filters joined by 'or' operator.
//
// labelFilters must be expanded to LabelFilters by expandWithExpr. // labelFilters must be expanded to LabelFilters by expandWithExpr.
labelFilters []*labelFilterExpr labelFilterss [][]*labelFilterExpr
} }
// AppendString appends string representation of me to dst and returns the result. // AppendString appends string representation of me to dst and returns the result.
func (me *MetricExpr) AppendString(dst []byte) []byte { func (me *MetricExpr) AppendString(dst []byte) []byte {
lfs := me.LabelFilters lfss := me.LabelFilterss
if len(lfs) > 0 { if len(lfss) == 0 {
lf := &lfs[0]
if lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp {
dst = appendEscapedIdent(dst, lf.Value)
lfs = lfs[1:]
}
}
if len(lfs) > 0 {
dst = append(dst, '{')
for i := range lfs {
dst = lfs[i].AppendString(dst)
if i+1 < len(lfs) {
dst = append(dst, ", "...)
}
}
dst = append(dst, '}')
} else if len(me.LabelFilters) == 0 {
dst = append(dst, "{}"...) dst = append(dst, "{}"...)
return dst
}
offset := 0
metricName := me.getMetricName()
if metricName != "" {
offset = 1
dst = appendEscapedIdent(dst, metricName)
}
if len(lfss) == 1 && len(lfss[0]) == offset {
return dst
}
dst = append(dst, '{')
lfs := lfss[0]
dst = appendLabelFilters(dst, lfs[offset:])
for _, lfs := range lfss[1:] {
dst = append(dst, " or "...)
dst = appendLabelFilters(dst, lfs[offset:])
}
dst = append(dst, '}')
return dst
}
func appendLabelFilters(dst []byte, lfs []LabelFilter) []byte {
if len(lfs) == 0 {
return dst
}
dst = lfs[0].AppendString(dst)
lfs = lfs[1:]
for i := range lfs {
dst = append(dst, ',')
dst = lfs[i].AppendString(dst)
} }
return dst return dst
} }
// IsEmpty returns true of me equals to `{}`. // IsEmpty returns true of me equals to `{}`.
func (me *MetricExpr) IsEmpty() bool { func (me *MetricExpr) IsEmpty() bool {
return len(me.LabelFilters) == 0 return len(me.LabelFilterss) == 0
} }
func (me *MetricExpr) isOnlyMetricGroup() bool { func (me *MetricExpr) isOnlyMetricName() bool {
if !me.hasNonEmptyMetricGroup() { if me.getMetricName() == "" {
return false return false
} }
return len(me.LabelFilters) == 1 for _, lfs := range me.LabelFilterss {
if len(lfs) > 1 {
return false
}
}
return true
} }
func (me *MetricExpr) hasNonEmptyMetricGroup() bool { func (me *MetricExpr) getMetricName() string {
if len(me.LabelFilters) == 0 { lfss := me.LabelFilterss
return false if len(lfss) == 0 {
return ""
} }
return me.LabelFilters[0].isMetricNameFilter() lfs := lfss[0]
if len(lfs) == 0 || !lfs[0].isMetricNameFilter() {
return ""
}
metricName := lfs[0].Value
for _, lfs := range lfss[1:] {
if len(lfs) == 0 || !lfs[0].isMetricNameFilter() || lfs[0].Value != metricName {
return ""
}
}
return metricName
} }
func (lf *LabelFilter) isMetricNameFilter() bool { func (lf *LabelFilter) isMetricNameFilter() bool {

2
vendor/modules.txt vendored
View file

@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
# github.com/VictoriaMetrics/metrics v1.24.0 # github.com/VictoriaMetrics/metrics v1.24.0
## explicit; go 1.20 ## explicit; go 1.20
github.com/VictoriaMetrics/metrics github.com/VictoriaMetrics/metrics
# github.com/VictoriaMetrics/metricsql v0.56.2 # github.com/VictoriaMetrics/metricsql v0.57.1
## explicit; go 1.13 ## explicit; go 1.13
github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql
github.com/VictoriaMetrics/metricsql/binaryop github.com/VictoriaMetrics/metricsql/binaryop