mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
all: add support for or filters in series selectors
This commit adds ability to select series matching distinct filters via a single series selector.
For example, the following selector selects series with either {env="prod",job="a"}
or {env="dev",job="b"} labels:
{env="prod",job="a" or env="dev",job="b"}
The `or` filter is supported in all the VictoriaMetrics tools now.
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3997
Uses https://github.com/VictoriaMetrics/metricsql/pull/14
This commit is contained in:
Aliaksandr Valialkin
parent
bc4b6f2cb4
commit
4cb024d8a3
24 changed files with 441 additions and 222 deletions
|
|
@ -512,8 +512,8 @@ The following articles contain useful information about Prometheus relabeling:
|
|||
{% endraw %}
|
||||
|
||||
* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain
|
||||
arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
|
||||
For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics:
|
||||
arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
|
||||
For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics:
|
||||
|
||||
```yaml
|
||||
- if: 'foo{bar="baz"}'
|
||||
|
|
|
|||
|
|
@ -339,23 +339,26 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
|
|||
if filter == metricName {
|
||||
return filter, nil
|
||||
}
|
||||
nameFilter := fmt.Sprintf("__name__=%q", metricName)
|
||||
|
||||
labels, err := searchutils.ParseMetricSelector(filter)
|
||||
tfss, err := searchutils.ParseMetricSelector(filter)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
str := make([]string, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
if len(label.Key) == 0 {
|
||||
continue
|
||||
var filters []string
|
||||
for _, tfs := range tfss {
|
||||
var a []string
|
||||
for _, tf := range tfs {
|
||||
if len(tf.Key) == 0 {
|
||||
continue
|
||||
}
|
||||
a = append(a, tf.String())
|
||||
}
|
||||
str = append(str, label.String())
|
||||
a = append(a, nameFilter)
|
||||
filters = append(filters, strings.Join(a, ","))
|
||||
}
|
||||
|
||||
nameFilter := fmt.Sprintf("__name__=%q", metricName)
|
||||
str = append(str, nameFilter)
|
||||
|
||||
match := fmt.Sprintf("{%s}", strings.Join(str, ","))
|
||||
match := "{" + strings.Join(filters, " or ") + "}"
|
||||
return match, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1005,15 +1005,15 @@ func getMaxLookback(r *http.Request) (int64, error) {
|
|||
}
|
||||
|
||||
func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) {
|
||||
tagFilterss := make([][]storage.TagFilter, 0, len(matches))
|
||||
tfss := make([][]storage.TagFilter, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
tagFilters, err := searchutils.ParseMetricSelector(match)
|
||||
tfssLocal, err := searchutils.ParseMetricSelector(match)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse matches[]=%s: %w", match, err)
|
||||
}
|
||||
tagFilterss = append(tagFilterss, tagFilters)
|
||||
tfss = append(tfss, tfssLocal...)
|
||||
}
|
||||
return tagFilterss, nil
|
||||
return tfss, nil
|
||||
}
|
||||
|
||||
func getRoundDigits(r *http.Request) int {
|
||||
|
|
|
|||
|
|
@ -1075,8 +1075,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
|
|||
}
|
||||
|
||||
// Fetch the remaining part of the result.
|
||||
tfs := searchutils.ToTagFilters(me.LabelFilters)
|
||||
tfss := searchutils.JoinTagFilterss([][]storage.TagFilter{tfs}, ec.EnforcedTagFilterss)
|
||||
tfss := searchutils.ToTagFilterss(me.LabelFilterss)
|
||||
tfss = searchutils.JoinTagFilterss(tfss, ec.EnforcedTagFilterss)
|
||||
minTimestamp := start - maxSilenceInterval
|
||||
if window > ec.Step {
|
||||
minTimestamp -= window
|
||||
|
|
|
|||
|
|
@ -29,8 +29,9 @@ func TestGetCommonLabelFilters(t *testing.T) {
|
|||
tss = append(tss, &ts)
|
||||
}
|
||||
lfs := getCommonLabelFilters(tss)
|
||||
me := &metricsql.MetricExpr{
|
||||
LabelFilters: lfs,
|
||||
var me metricsql.MetricExpr
|
||||
if len(lfs) > 0 {
|
||||
me.LabelFilterss = [][]metricsql.LabelFilter{lfs}
|
||||
}
|
||||
lfsMarshaled := me.AppendString(nil)
|
||||
if string(lfsMarshaled) != lfsExpected {
|
||||
|
|
@ -40,7 +41,7 @@ func TestGetCommonLabelFilters(t *testing.T) {
|
|||
f(``, `{}`)
|
||||
f(`m 1`, `{}`)
|
||||
f(`m{a="b"} 1`, `{a="b"}`)
|
||||
f(`m{c="d",a="b"} 1`, `{a="b", c="d"}`)
|
||||
f(`m{c="d",a="b"} 1`, `{a="b",c="d"}`)
|
||||
f(`m1{a="foo"} 1
|
||||
m2{a="bar"} 1`, `{a=~"bar|foo"}`)
|
||||
f(`m1{a="foo"} 1
|
||||
|
|
|
|||
|
|
@ -277,10 +277,12 @@ func escapeDotsInRegexpLabelFilters(e metricsql.Expr) metricsql.Expr {
|
|||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := range me.LabelFilters {
|
||||
f := &me.LabelFilters[i]
|
||||
if f.IsRegexp {
|
||||
f.Value = escapeDots(f.Value)
|
||||
for _, lfs := range me.LabelFilterss {
|
||||
for i := range lfs {
|
||||
f := &lfs[i]
|
||||
if f.IsRegexp {
|
||||
f.Value = escapeDots(f.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ func TestEscapeDotsInRegexpLabelFilters(t *testing.T) {
|
|||
f("2", "2")
|
||||
f(`foo.bar + 123`, `foo.bar + 123`)
|
||||
f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`)
|
||||
f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e", x=~"a\\.b.+[\\.a]", y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
|
||||
f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e",x=~"a\\.b.+[\\.a]",y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
|
||||
}
|
||||
|
||||
func TestExecSuccess(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset *me
|
|||
return
|
||||
}
|
||||
me, ok := re.Expr.(*metricsql.MetricExpr)
|
||||
if !ok || len(me.LabelFilters) == 0 {
|
||||
if !ok || len(me.LabelFilterss) == 0 {
|
||||
return
|
||||
}
|
||||
wrappedQuery := me.AppendString(nil)
|
||||
|
|
|
|||
|
|
@ -41,10 +41,14 @@ func TestRollupResultCache(t *testing.T) {
|
|||
MayCache: true,
|
||||
}
|
||||
me := &metricsql.MetricExpr{
|
||||
LabelFilters: []metricsql.LabelFilter{{
|
||||
Label: "aaa",
|
||||
Value: "xxx",
|
||||
}},
|
||||
LabelFilterss: [][]metricsql.LabelFilter{
|
||||
{
|
||||
{
|
||||
Label: "aaa",
|
||||
Value: "xxx",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
fe := &metricsql.FuncExpr{
|
||||
Name: "foo",
|
||||
|
|
|
|||
|
|
@ -240,7 +240,11 @@ func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries {
|
|||
if !ok {
|
||||
return rvs
|
||||
}
|
||||
tfs := searchutils.ToTagFilters(me.LabelFilters)
|
||||
tfss := searchutils.ToTagFilterss(me.LabelFilterss)
|
||||
if len(tfss) != 1 {
|
||||
return rvs
|
||||
}
|
||||
tfs := tfss[0]
|
||||
for i := range tfs {
|
||||
tf := &tfs[i]
|
||||
if len(tf.Key) == 0 {
|
||||
|
|
|
|||
|
|
@ -140,12 +140,14 @@ func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) {
|
|||
}
|
||||
var etfs [][]storage.TagFilter
|
||||
for _, extraFilter := range extraFilters {
|
||||
tfs, err := ParseMetricSelector(extraFilter)
|
||||
tfss, err := ParseMetricSelector(extraFilter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse extra_filters=%s: %w", extraFilter, err)
|
||||
}
|
||||
tfs = append(tfs, tagFilters...)
|
||||
etfs = append(etfs, tfs)
|
||||
for i := range tfss {
|
||||
tfss[i] = append(tfss[i], tagFilters...)
|
||||
}
|
||||
etfs = append(etfs, tfss...)
|
||||
}
|
||||
return etfs, nil
|
||||
}
|
||||
|
|
@ -170,7 +172,7 @@ func JoinTagFilterss(src, etfs [][]storage.TagFilter) [][]storage.TagFilter {
|
|||
}
|
||||
|
||||
// ParseMetricSelector parses s containing PromQL metric selector and returns the corresponding LabelFilters.
|
||||
func ParseMetricSelector(s string) ([]storage.TagFilter, error) {
|
||||
func ParseMetricSelector(s string) ([][]storage.TagFilter, error) {
|
||||
expr, err := metricsql.Parse(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -179,20 +181,24 @@ func ParseMetricSelector(s string) ([]storage.TagFilter, error) {
|
|||
if !ok {
|
||||
return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil))
|
||||
}
|
||||
if len(me.LabelFilters) == 0 {
|
||||
return nil, fmt.Errorf("labelFilters cannot be empty")
|
||||
if len(me.LabelFilterss) == 0 {
|
||||
return nil, fmt.Errorf("labelFilterss cannot be empty")
|
||||
}
|
||||
tfs := ToTagFilters(me.LabelFilters)
|
||||
return tfs, nil
|
||||
tfss := ToTagFilterss(me.LabelFilterss)
|
||||
return tfss, nil
|
||||
}
|
||||
|
||||
// ToTagFilters converts lfs to a slice of storage.TagFilter
|
||||
func ToTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter {
|
||||
tfs := make([]storage.TagFilter, len(lfs))
|
||||
for i := range lfs {
|
||||
toTagFilter(&tfs[i], &lfs[i])
|
||||
// ToTagFilterss converts lfss to or-delimited slices of storage.TagFilter
|
||||
func ToTagFilterss(lfss [][]metricsql.LabelFilter) [][]storage.TagFilter {
|
||||
tfss := make([][]storage.TagFilter, len(lfss))
|
||||
for i, lfs := range lfss {
|
||||
tfs := make([]storage.TagFilter, len(lfs))
|
||||
for j := range lfs {
|
||||
toTagFilter(&tfs[j], &lfs[j])
|
||||
}
|
||||
tfss[i] = tfs
|
||||
}
|
||||
return tfs
|
||||
return tfss
|
||||
}
|
||||
|
||||
func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {
|
||||
|
|
|
|||
|
|
@ -135,69 +135,69 @@ func TestJoinTagFilterss(t *testing.T) {
|
|||
}
|
||||
}
|
||||
// Single tag filter
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
}, nil, []string{
|
||||
), nil, []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
|
||||
})
|
||||
// Miltiple tag filters
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
}, nil, []string{
|
||||
), nil, []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
|
||||
`{k5=~"v5"}`,
|
||||
})
|
||||
// Single extra filter
|
||||
f(t, nil, [][]storage.TagFilter{
|
||||
f(t, nil, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
|
||||
})
|
||||
// Multiple extra filters
|
||||
f(t, nil, [][]storage.TagFilter{
|
||||
f(t, nil, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
|
||||
`{k5=~"v5"}`,
|
||||
})
|
||||
// Single tag filter and a single extra filter
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
}, [][]storage.TagFilter{
|
||||
), joinTagFilters(
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`,
|
||||
})
|
||||
// Multiple tag filters and a single extra filter
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
}, [][]storage.TagFilter{
|
||||
), joinTagFilters(
|
||||
mustParseMetricSelector(`{k6=~"v6"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
|
||||
`{k5=~"v5",k6=~"v6"}`,
|
||||
})
|
||||
// Single tag filter and multiple extra filters
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
}, [][]storage.TagFilter{
|
||||
), joinTagFilters(
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
mustParseMetricSelector(`{k6=~"v6"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k5=~"v5"}`,
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
|
||||
})
|
||||
// Multiple tag filters and multiple extra filters
|
||||
f(t, [][]storage.TagFilter{
|
||||
f(t, joinTagFilters(
|
||||
mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
|
||||
mustParseMetricSelector(`{k5=~"v5"}`),
|
||||
}, [][]storage.TagFilter{
|
||||
), joinTagFilters(
|
||||
mustParseMetricSelector(`{k6=~"v6"}`),
|
||||
mustParseMetricSelector(`{k7=~"v7"}`),
|
||||
}, []string{
|
||||
), []string{
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k6=~"v6"}`,
|
||||
`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4",k7=~"v7"}`,
|
||||
`{k5=~"v5",k6=~"v6"}`,
|
||||
|
|
@ -205,12 +205,20 @@ func TestJoinTagFilterss(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func mustParseMetricSelector(s string) []storage.TagFilter {
|
||||
tf, err := ParseMetricSelector(s)
|
||||
func joinTagFilters(args ...[][]storage.TagFilter) [][]storage.TagFilter {
|
||||
result := append([][]storage.TagFilter{}, args[0]...)
|
||||
for _, tfss := range args[1:] {
|
||||
result = append(result, tfss...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mustParseMetricSelector(s string) [][]storage.TagFilter {
|
||||
tfss, err := ParseMetricSelector(s)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot parse %q: %w", s, err))
|
||||
}
|
||||
return tf
|
||||
return tfss
|
||||
}
|
||||
|
||||
func tagFilterssToStrings(tfss [][]storage.TagFilter) []string {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
* SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved).
|
||||
|
||||
* FETURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details.
|
||||
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow selecting time series matching at least one of multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries). See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402).
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@ The list of MetricsQL features:
|
|||
depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)).
|
||||
For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
|
||||
It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
|
||||
* [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
|
||||
selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
|
||||
See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
|
||||
* [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
|
||||
For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
|
||||
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
|
||||
|
|
|
|||
|
|
@ -785,15 +785,15 @@ requests_total{path="/", code="200"}
|
|||
requests_total{path="/", code="403"}
|
||||
```
|
||||
|
||||
To select only time series with specific label value specify the matching condition in curly braces:
|
||||
To select only time series with specific label value specify the matching filter in curly braces:
|
||||
|
||||
```metricsql
|
||||
requests_total{code="200"}
|
||||
```
|
||||
|
||||
The query above will return all time series with the name `requests_total` and `code="200"`. We use the operator `=` to
|
||||
match a label value. For negative match use `!=` operator. Filters also support regex matching `=~` for positive
|
||||
and `!~` for negative matching:
|
||||
The query above returns all time series with the name `requests_total` and label `code="200"`. We use the operator `=` to
|
||||
match label value. For negative match use `!=` operator. Filters also support positive regex matching via `=~`
|
||||
and negative regex matching via `!~`:
|
||||
|
||||
```metricsql
|
||||
requests_total{code=~"2.*"}
|
||||
|
|
@ -802,23 +802,45 @@ requests_total{code=~"2.*"}
|
|||
Filters can also be combined:
|
||||
|
||||
```metricsql
|
||||
requests_total{code=~"200|204", path="/home"}
|
||||
requests_total{code=~"200", path="/home"}
|
||||
```
|
||||
|
||||
The query above will return all time series with a name `requests_total`, status `code` `200` or `204`and `path="/home"`
|
||||
.
|
||||
The query above returns all time series with `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
|
||||
|
||||
#### Filtering by name
|
||||
|
||||
Sometimes it is required to return all the time series for multiple metric names. As was mentioned in
|
||||
the [data model section](#data-model), the metric name is just an ordinary label with a special name — `__name__`. So
|
||||
the [data model section](#data-model), the metric name is just an ordinary label with a special name - `__name__`. So
|
||||
filtering by multiple metric names may be performed by applying regexps on metric names:
|
||||
|
||||
```metricsql
|
||||
{__name__=~"requests_(error|success)_total"}
|
||||
```
|
||||
|
||||
The query above is supposed to return series for two metrics: `requests_error_total` and `requests_success_total`.
|
||||
The query above returns series for two metrics: `requests_error_total` and `requests_success_total`.
|
||||
|
||||
#### Filtering by multiple "or" filters
|
||||
|
||||
[MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) supports selecting time series, which match at least one of multiple "or" filters.
|
||||
Such filters must be delimited by `or` inside curly braces. For example, the following query selects time series with
|
||||
either `{job="app1",env="prod"}` or `{job="app2",env="dev"}` labels:
|
||||
|
||||
```metricsql
|
||||
{job="app1",env="prod" or job="app2",env="dev"}
|
||||
```
|
||||
|
||||
The number of `or` filters can be arbitrary. This functionality allows passing the selected series
|
||||
to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) such as [rate()](https://docs.victoriametrics.com/MetricsQL.html#rate)
|
||||
without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries):
|
||||
|
||||
```metricsql
|
||||
rate({job="app1",env="prod" or job="app2",env="dev"}[5m])
|
||||
|
||||
```
|
||||
|
||||
If you need to select series matching multiple filters for the same label, then it is better from performance PoV
|
||||
to use regexp filter `{label=~"value1|...|valueN"}` instead of `{label="value1" or ... or label="valueN"}`.
|
||||
|
||||
|
||||
#### Arithmetic operations
|
||||
|
||||
|
|
|
|||
|
|
@ -523,8 +523,8 @@ The following articles contain useful information about Prometheus relabeling:
|
|||
{% endraw %}
|
||||
|
||||
* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain
|
||||
arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
|
||||
For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics:
|
||||
arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
|
||||
For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics:
|
||||
|
||||
```yaml
|
||||
- if: 'foo{bar="baz"}'
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -12,7 +12,7 @@ require (
|
|||
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
|
||||
github.com/VictoriaMetrics/fasthttp v1.2.0
|
||||
github.com/VictoriaMetrics/metrics v1.24.0
|
||||
github.com/VictoriaMetrics/metricsql v0.56.2
|
||||
github.com/VictoriaMetrics/metricsql v0.57.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.27
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -67,11 +67,10 @@ github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bw
|
|||
github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
|
||||
github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I=
|
||||
github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE=
|
||||
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
|
||||
github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
|
||||
github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
|
||||
github.com/VictoriaMetrics/metricsql v0.56.2 h1:quBAbYOlWMhmdgzFSCr1yjtVcdZYZrVQJ7nR9zor7ZM=
|
||||
github.com/VictoriaMetrics/metricsql v0.56.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
|
||||
github.com/VictoriaMetrics/metricsql v0.57.1 h1:ryMe7w95s80BilS8RlV3G/25BLlmMBVRtTq2GnLB/4o=
|
||||
github.com/VictoriaMetrics/metricsql v0.57.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
|
|
@ -631,6 +630,7 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ import (
|
|||
//
|
||||
// The `if` expression can contain arbitrary PromQL-like label filters such as `metric_name{filters...}`
|
||||
type IfExpression struct {
|
||||
s string
|
||||
lfs []*labelFilter
|
||||
s string
|
||||
lfss [][]*labelFilter
|
||||
}
|
||||
|
||||
// String returns string representation of ie.
|
||||
|
|
@ -36,12 +36,12 @@ func (ie *IfExpression) Parse(s string) error {
|
|||
if !ok {
|
||||
return fmt.Errorf("expecting series selector; got %q", expr.AppendString(nil))
|
||||
}
|
||||
lfs, err := metricExprToLabelFilters(me)
|
||||
lfss, err := metricExprToLabelFilterss(me)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse series selector: %w", err)
|
||||
}
|
||||
ie.s = s
|
||||
ie.lfs = lfs
|
||||
ie.lfss = lfss
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -81,7 +81,16 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool {
|
|||
if ie == nil {
|
||||
return true
|
||||
}
|
||||
for _, lf := range ie.lfs {
|
||||
for _, lfs := range ie.lfss {
|
||||
if matchLabelFilters(lfs, labels) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func matchLabelFilters(lfs []*labelFilter, labels []prompbmarshal.Label) bool {
|
||||
for _, lf := range lfs {
|
||||
if !lf.match(labels) {
|
||||
return false
|
||||
}
|
||||
|
|
@ -89,16 +98,20 @@ func (ie *IfExpression) Match(labels []prompbmarshal.Label) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func metricExprToLabelFilters(me *metricsql.MetricExpr) ([]*labelFilter, error) {
|
||||
lfs := make([]*labelFilter, len(me.LabelFilters))
|
||||
for i := range me.LabelFilters {
|
||||
lf, err := newLabelFilter(&me.LabelFilters[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err)
|
||||
func metricExprToLabelFilterss(me *metricsql.MetricExpr) ([][]*labelFilter, error) {
|
||||
lfssNew := make([][]*labelFilter, len(me.LabelFilterss))
|
||||
for i, lfs := range me.LabelFilterss {
|
||||
lfsNew := make([]*labelFilter, len(lfs))
|
||||
for j := range lfs {
|
||||
lf, err := newLabelFilter(&lfs[j])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse %s: %w", me.AppendString(nil), err)
|
||||
}
|
||||
lfsNew[j] = lf
|
||||
}
|
||||
lfs[i] = lf
|
||||
lfssNew[i] = lfsNew
|
||||
}
|
||||
return lfs, nil
|
||||
return lfssNew, nil
|
||||
}
|
||||
|
||||
// labelFilter contains PromQL filter for `{label op "value"}`
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ func TestIfExpressionParseFailure(t *testing.T) {
|
|||
f(`{`)
|
||||
f(`{foo`)
|
||||
f(`foo{`)
|
||||
f(`foo{bar="a" or}`)
|
||||
}
|
||||
|
||||
func TestIfExpressionParseSuccess(t *testing.T) {
|
||||
|
|
@ -33,6 +34,12 @@ func TestIfExpressionParseSuccess(t *testing.T) {
|
|||
f(`foo`)
|
||||
f(`{foo="bar"}`)
|
||||
f(`foo{bar=~"baz", x!="y"}`)
|
||||
f(`{a="b" or c="d",e="x"}`)
|
||||
f(`foo{
|
||||
bar="a",x="y" or
|
||||
x="a",a="b" or
|
||||
a="x"
|
||||
}`)
|
||||
}
|
||||
|
||||
func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) {
|
||||
|
|
@ -63,6 +70,7 @@ func TestIfExpressionMarshalUnmarshalJSON(t *testing.T) {
|
|||
}
|
||||
f("foo", `"foo"`)
|
||||
f(`{foo="bar",baz=~"x.*"}`, `"{foo=\"bar\",baz=~\"x.*\"}"`)
|
||||
f(`{a="b" or c="d",x="z"}`, `"{a=\"b\" or c=\"d\",x=\"z\"}"`)
|
||||
}
|
||||
|
||||
func TestIfExpressionUnmarshalFailure(t *testing.T) {
|
||||
|
|
@ -113,6 +121,7 @@ func TestIfExpressionUnmarshalSuccess(t *testing.T) {
|
|||
f(`foo{bar="baz"}`)
|
||||
f(`'{a="b", c!="d", e=~"g", h!~"d"}'`)
|
||||
f(`foo{bar="zs",a=~"b|c"}`)
|
||||
f(`foo{z="y" or bar="zs",a=~"b|c"}`)
|
||||
}
|
||||
|
||||
func TestIfExpressionMatch(t *testing.T) {
|
||||
|
|
@ -130,6 +139,8 @@ func TestIfExpressionMatch(t *testing.T) {
|
|||
f(`foo`, `foo`)
|
||||
f(`foo`, `foo{bar="baz",a="b"}`)
|
||||
f(`foo{bar="a"}`, `foo{bar="a"}`)
|
||||
f(`foo{bar="a" or baz="x"}`, `foo{bar="a"}`)
|
||||
f(`foo{baz="x" or bar="a"}`, `foo{bar="a"}`)
|
||||
f(`foo{bar="a"}`, `foo{x="y",bar="a",baz="b"}`)
|
||||
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc"}`)
|
||||
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="abc",y="qwe"}`)
|
||||
|
|
@ -164,6 +175,7 @@ func TestIfExpressionMismatch(t *testing.T) {
|
|||
f(`foo`, `bar`)
|
||||
f(`foo`, `a{foo="bar"}`)
|
||||
f(`foo{bar="a"}`, `foo`)
|
||||
f(`foo{bar="a" or baz="a"}`, `foo`)
|
||||
f(`foo{bar="a"}`, `foo{bar="b"}`)
|
||||
f(`foo{bar="a"}`, `foo{baz="b",a="b"}`)
|
||||
f(`'{a=~"x|abc",y!="z"}'`, `m{x="aa",a="xabc"}`)
|
||||
|
|
|
|||
6
vendor/github.com/VictoriaMetrics/metricsql/lexer.go
generated
vendored
6
vendor/github.com/VictoriaMetrics/metricsql/lexer.go
generated
vendored
|
|
@ -561,10 +561,8 @@ func DurationValue(s string, step int64) (int64, error) {
|
|||
func parseSingleDuration(s string, step int64) (float64, error) {
|
||||
s = strings.ToLower(s)
|
||||
numPart := s[:len(s)-1]
|
||||
if strings.HasSuffix(numPart, "m") {
|
||||
// Duration in ms
|
||||
numPart = numPart[:len(numPart)-1]
|
||||
}
|
||||
// Strip trailing m if the duration is in ms
|
||||
numPart = strings.TrimSuffix(numPart, "m")
|
||||
f, err := strconv.ParseFloat(numPart, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse duration %q: %s", s, err)
|
||||
|
|
|
|||
24
vendor/github.com/VictoriaMetrics/metricsql/optimizer.go
generated
vendored
24
vendor/github.com/VictoriaMetrics/metricsql/optimizer.go
generated
vendored
|
|
@ -78,7 +78,7 @@ func optimizeInplace(e Expr) {
|
|||
func getCommonLabelFilters(e Expr) []LabelFilter {
|
||||
switch t := e.(type) {
|
||||
case *MetricExpr:
|
||||
return getLabelFiltersWithoutMetricName(t.LabelFilters)
|
||||
return getCommonLabelFiltersWithoutMetricName(t.LabelFilterss)
|
||||
case *RollupExpr:
|
||||
return getCommonLabelFilters(t.Expr)
|
||||
case *FuncExpr:
|
||||
|
|
@ -180,6 +180,21 @@ func TrimFiltersByGroupModifier(lfs []LabelFilter, be *BinaryOpExpr) []LabelFilt
|
|||
}
|
||||
}
|
||||
|
||||
func getCommonLabelFiltersWithoutMetricName(lfss [][]LabelFilter) []LabelFilter {
|
||||
if len(lfss) == 0 {
|
||||
return nil
|
||||
}
|
||||
lfsA := getLabelFiltersWithoutMetricName(lfss[0])
|
||||
for _, lfs := range lfss[1:] {
|
||||
if len(lfsA) == 0 {
|
||||
return nil
|
||||
}
|
||||
lfsB := getLabelFiltersWithoutMetricName(lfs)
|
||||
lfsA = intersectLabelFilters(lfsA, lfsB)
|
||||
}
|
||||
return lfsA
|
||||
}
|
||||
|
||||
func getLabelFiltersWithoutMetricName(lfs []LabelFilter) []LabelFilter {
|
||||
lfsNew := make([]LabelFilter, 0, len(lfs))
|
||||
for _, lf := range lfs {
|
||||
|
|
@ -213,8 +228,11 @@ func pushdownBinaryOpFiltersInplace(e Expr, lfs []LabelFilter) {
|
|||
}
|
||||
switch t := e.(type) {
|
||||
case *MetricExpr:
|
||||
t.LabelFilters = unionLabelFilters(t.LabelFilters, lfs)
|
||||
sortLabelFilters(t.LabelFilters)
|
||||
for i, lfsLocal := range t.LabelFilterss {
|
||||
lfsLocal = unionLabelFilters(lfsLocal, lfs)
|
||||
sortLabelFilters(lfsLocal)
|
||||
t.LabelFilterss[i] = lfsLocal
|
||||
}
|
||||
case *RollupExpr:
|
||||
pushdownBinaryOpFiltersInplace(t.Expr, lfs)
|
||||
case *FuncExpr:
|
||||
|
|
|
|||
358
vendor/github.com/VictoriaMetrics/metricsql/parser.go
generated
vendored
358
vendor/github.com/VictoriaMetrics/metricsql/parser.go
generated
vendored
|
|
@ -765,59 +765,71 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
|
|||
}
|
||||
return eNew, nil
|
||||
case *MetricExpr:
|
||||
if len(t.LabelFilters) > 0 {
|
||||
if len(t.labelFilterss) == 0 {
|
||||
// Already expanded.
|
||||
return t, nil
|
||||
}
|
||||
{
|
||||
var me MetricExpr
|
||||
// Populate me.LabelFilters
|
||||
for _, lfe := range t.labelFilters {
|
||||
if lfe.Value == nil {
|
||||
// Expand lfe.Label into []LabelFilter.
|
||||
wa := getWithArgExpr(was, lfe.Label)
|
||||
if wa == nil {
|
||||
return nil, fmt.Errorf("missing %q value inside %q", lfe.Label, t.AppendString(nil))
|
||||
// Populate me.LabelFilterss
|
||||
for _, lfes := range t.labelFilterss {
|
||||
var lfsNew []LabelFilter
|
||||
for _, lfe := range lfes {
|
||||
if lfe.Value == nil {
|
||||
// Expand lfe.Label into lfsNew.
|
||||
wa := getWithArgExpr(was, lfe.Label)
|
||||
if wa == nil {
|
||||
return nil, fmt.Errorf("cannot find WITH template for %q inside %q", lfe.Label, t.AppendString(nil))
|
||||
}
|
||||
eNew, err := expandWithExprExt(was, wa, []Expr{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wme, ok := eNew.(*MetricExpr)
|
||||
if !ok || wme.getMetricName() != "" {
|
||||
return nil, fmt.Errorf("WITH template %q inside %q must be {...}; got %q",
|
||||
lfe.Label, t.AppendString(nil), eNew.AppendString(nil))
|
||||
}
|
||||
if len(wme.labelFilterss) > 0 {
|
||||
panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH template expansion; got %s", wme.labelFilterss))
|
||||
}
|
||||
lfssSrc := wme.LabelFilterss
|
||||
if len(lfssSrc) > 1 {
|
||||
return nil, fmt.Errorf("WITH template %q at %q must be {...} without 'or'; got %s",
|
||||
lfe.Label, t.AppendString(nil), wme.AppendString(nil))
|
||||
}
|
||||
if len(lfssSrc) == 1 {
|
||||
lfsNew = append(lfsNew, lfssSrc[0]...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
eNew, err := expandWithExprExt(was, wa, nil)
|
||||
|
||||
// convert lfe to LabelFilter.
|
||||
se, err := expandWithExpr(was, lfe.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wme, ok := eNew.(*MetricExpr)
|
||||
if !ok || wme.hasNonEmptyMetricGroup() {
|
||||
return nil, fmt.Errorf("%q must be filters expression inside %q; got %q", lfe.Label, t.AppendString(nil), eNew.AppendString(nil))
|
||||
var lfeNew labelFilterExpr
|
||||
lfeNew.Label = lfe.Label
|
||||
lfeNew.Value = se.(*StringExpr)
|
||||
lfeNew.IsNegative = lfe.IsNegative
|
||||
lfeNew.IsRegexp = lfe.IsRegexp
|
||||
lf, err := lfeNew.toLabelFilter()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(wme.labelFilters) > 0 {
|
||||
panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters))
|
||||
}
|
||||
me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...)
|
||||
continue
|
||||
lfsNew = append(lfsNew, *lf)
|
||||
}
|
||||
|
||||
// convert lfe to LabelFilter.
|
||||
se, err := expandWithExpr(was, lfe.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lfeNew labelFilterExpr
|
||||
lfeNew.Label = lfe.Label
|
||||
lfeNew.Value = se.(*StringExpr)
|
||||
lfeNew.IsNegative = lfe.IsNegative
|
||||
lfeNew.IsRegexp = lfe.IsRegexp
|
||||
lf, err := lfeNew.toLabelFilter()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
me.LabelFilters = append(me.LabelFilters, *lf)
|
||||
lfsNew = removeDuplicateLabelFilters(lfsNew)
|
||||
me.LabelFilterss = append(me.LabelFilterss, lfsNew)
|
||||
}
|
||||
me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters)
|
||||
t = &me
|
||||
}
|
||||
if !t.hasNonEmptyMetricGroup() {
|
||||
metricName := t.getMetricName()
|
||||
if metricName == "" {
|
||||
return t, nil
|
||||
}
|
||||
k := t.LabelFilters[0].Value
|
||||
wa := getWithArgExpr(was, k)
|
||||
wa := getWithArgExpr(was, metricName)
|
||||
if wa == nil {
|
||||
return t, nil
|
||||
}
|
||||
|
|
@ -833,25 +845,51 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
|
|||
wme, _ = eNew.(*MetricExpr)
|
||||
}
|
||||
if wme == nil {
|
||||
if !t.isOnlyMetricGroup() {
|
||||
return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil))
|
||||
if t.isOnlyMetricName() {
|
||||
return eNew, nil
|
||||
}
|
||||
return eNew, nil
|
||||
return nil, fmt.Errorf("cannot expand %q to non-metric expression %q", t.AppendString(nil), eNew.AppendString(nil))
|
||||
}
|
||||
if len(wme.labelFilters) > 0 {
|
||||
panic(fmt.Errorf("BUG: wme.labelFilters must be empty; got %s", wme.labelFilters))
|
||||
if len(wme.labelFilterss) > 0 {
|
||||
panic(fmt.Errorf("BUG: wme.labelFilterss must be empty after WITH templates expansion; got %s", wme.labelFilterss))
|
||||
}
|
||||
lfssSrc := wme.LabelFilterss
|
||||
var lfssNew [][]LabelFilter
|
||||
if len(lfssSrc) != 1 {
|
||||
// template_name{filters} where template_name is {... or ...}
|
||||
if t.isOnlyMetricName() {
|
||||
// {filters} is empty. Return {... or ...}
|
||||
return eNew, nil
|
||||
}
|
||||
if len(t.LabelFilterss) != 1 {
|
||||
// {filters} contain {... or ...}. It cannot be merged with {... or ...}
|
||||
return nil, fmt.Errorf("%q mustn't contain 'or' filters; got %s", metricName, wme.AppendString(nil))
|
||||
}
|
||||
// {filters} doesn't contain `or`. Merge it with {... or ...} into {...,filters or ...,filters}
|
||||
for _, lfs := range lfssSrc {
|
||||
lfsNew := append([]LabelFilter{}, lfs...)
|
||||
lfsNew = append(lfsNew, t.LabelFilterss[0][1:]...)
|
||||
lfsNew = removeDuplicateLabelFilters(lfsNew)
|
||||
lfssNew = append(lfssNew, lfsNew)
|
||||
}
|
||||
} else {
|
||||
// template_name{... or ...} where template_name is an ordinary {filters} without 'or'.
|
||||
// Merge it into {filters,... or filters,...}
|
||||
for _, lfs := range t.LabelFilterss {
|
||||
lfsNew := append([]LabelFilter{}, lfssSrc[0]...)
|
||||
lfsNew = append(lfsNew, lfs[1:]...)
|
||||
lfsNew = removeDuplicateLabelFilters(lfsNew)
|
||||
lfssNew = append(lfssNew, lfsNew)
|
||||
}
|
||||
}
|
||||
me := &MetricExpr{
|
||||
LabelFilterss: lfssNew,
|
||||
}
|
||||
|
||||
var me MetricExpr
|
||||
me.LabelFilters = append(me.LabelFilters, wme.LabelFilters...)
|
||||
me.LabelFilters = append(me.LabelFilters, t.LabelFilters[1:]...)
|
||||
me.LabelFilters = removeDuplicateLabelFilters(me.LabelFilters)
|
||||
|
||||
if re == nil {
|
||||
return &me, nil
|
||||
return me, nil
|
||||
}
|
||||
reNew := *re
|
||||
reNew.Expr = &me
|
||||
reNew.Expr = me
|
||||
return &reNew, nil
|
||||
default:
|
||||
return e, nil
|
||||
|
|
@ -889,22 +927,22 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) {
|
|||
}
|
||||
me, ok := wa.Expr.(*MetricExpr)
|
||||
if ok {
|
||||
if !me.isOnlyMetricGroup() {
|
||||
if !me.isOnlyMetricName() {
|
||||
return nil, fmt.Errorf("cannot use %q instead of %q in %s", me.AppendString(nil), arg, args)
|
||||
}
|
||||
dstArg := me.LabelFilters[0].Value
|
||||
dstArgs = append(dstArgs, dstArg)
|
||||
metricName := me.getMetricName()
|
||||
dstArgs = append(dstArgs, metricName)
|
||||
continue
|
||||
}
|
||||
pe, ok := wa.Expr.(*parensExpr)
|
||||
if ok {
|
||||
for _, pArg := range *pe {
|
||||
me, ok := pArg.(*MetricExpr)
|
||||
if !ok || !me.isOnlyMetricGroup() {
|
||||
if !ok || !me.isOnlyMetricName() {
|
||||
return nil, fmt.Errorf("cannot use %q instead of %q in %s", pe.AppendString(nil), arg, args)
|
||||
}
|
||||
dstArg := me.LabelFilters[0].Value
|
||||
dstArgs = append(dstArgs, dstArg)
|
||||
metricName := me.getMetricName()
|
||||
dstArgs = append(dstArgs, metricName)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
|
@ -926,7 +964,9 @@ func expandModifierArgs(was []*withArgExpr, args []string) ([]string, error) {
|
|||
func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr, error) {
|
||||
if len(wa.Args) != len(args) {
|
||||
if args == nil {
|
||||
// Just return MetricExpr with the wa.Name name.
|
||||
// This case is possible if metric name clashes with one of the WITH template name.
|
||||
//
|
||||
// In this case just return MetricExpr with the wa.Name name.
|
||||
return newMetricExpr(wa.Name), nil
|
||||
}
|
||||
return nil, fmt.Errorf("invalid number of args for %q; got %d; want %d", wa.Name, len(args), len(wa.Args))
|
||||
|
|
@ -949,10 +989,14 @@ func expandWithExprExt(was []*withArgExpr, wa *withArgExpr, args []Expr) (Expr,
|
|||
|
||||
func newMetricExpr(name string) *MetricExpr {
|
||||
return &MetricExpr{
|
||||
LabelFilters: []LabelFilter{{
|
||||
Label: "__name__",
|
||||
Value: name,
|
||||
}},
|
||||
LabelFilterss: [][]LabelFilter{
|
||||
{
|
||||
{
|
||||
Label: "__name__",
|
||||
Value: name,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1130,39 +1174,70 @@ func getWithArgExpr(was []*withArgExpr, name string) *withArgExpr {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (p *parser) parseLabelFilters() ([]*labelFilterExpr, error) {
|
||||
func (p *parser) parseLabelFilterss(mf *labelFilterExpr) ([][]*labelFilterExpr, error) {
|
||||
if p.lex.Token != "{" {
|
||||
return nil, fmt.Errorf(`labelFilters: unexpected token %q; want "{"`, p.lex.Token)
|
||||
}
|
||||
|
||||
var lfes []*labelFilterExpr
|
||||
for {
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.lex.Token == "}" {
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.lex.Token == "}" {
|
||||
goto closeBracesLabel
|
||||
if mf != nil {
|
||||
return [][]*labelFilterExpr{{mf}}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var lfess [][]*labelFilterExpr
|
||||
for {
|
||||
lfes, err := p.parseLabelFilters(mf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lfess = append(lfess, lfes)
|
||||
switch strings.ToLower(p.lex.Token) {
|
||||
case "}":
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lfess, nil
|
||||
case "or":
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseLabelFilters(mf *labelFilterExpr) ([]*labelFilterExpr, error) {
|
||||
var lfes []*labelFilterExpr
|
||||
if mf != nil {
|
||||
lfes = append(lfes, mf)
|
||||
}
|
||||
for {
|
||||
lfe, err := p.parseLabelFilterExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lfes = append(lfes, lfe)
|
||||
switch p.lex.Token {
|
||||
switch strings.ToLower(p.lex.Token) {
|
||||
case ",":
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.lex.Token == "}" {
|
||||
return lfes, nil
|
||||
}
|
||||
continue
|
||||
case "}":
|
||||
goto closeBracesLabel
|
||||
case "or", "}":
|
||||
return lfes, nil
|
||||
default:
|
||||
return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "}"`, p.lex.Token)
|
||||
return nil, fmt.Errorf(`labelFilters: unexpected token %q; want ",", "or", "}"`, p.lex.Token)
|
||||
}
|
||||
}
|
||||
|
||||
closeBracesLabel:
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lfes, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
|
||||
|
|
@ -1175,7 +1250,7 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
switch p.lex.Token {
|
||||
switch strings.ToLower(p.lex.Token) {
|
||||
case "=":
|
||||
// Nothing to do.
|
||||
case "!=":
|
||||
|
|
@ -1185,10 +1260,10 @@ func (p *parser) parseLabelFilterExpr() (*labelFilterExpr, error) {
|
|||
case "!~":
|
||||
lfe.IsNegative = true
|
||||
lfe.IsRegexp = true
|
||||
case ",", "}":
|
||||
case ",", "}", "or":
|
||||
return &lfe, nil
|
||||
default:
|
||||
return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "}"`, p.lex.Token)
|
||||
return nil, fmt.Errorf(`labelFilterExpr: unexpected token %q; want "=", "!=", "=~", "!~", ",", "or", "}"`, p.lex.Token)
|
||||
}
|
||||
|
||||
if err := p.lex.Next(); err != nil {
|
||||
|
|
@ -1415,26 +1490,28 @@ func (p *parser) parseIdentExpr() (Expr, error) {
|
|||
}
|
||||
|
||||
func (p *parser) parseMetricExpr() (*MetricExpr, error) {
|
||||
var mf *labelFilterExpr
|
||||
var me MetricExpr
|
||||
if isIdentPrefix(p.lex.Token) {
|
||||
var lfe labelFilterExpr
|
||||
lfe.Label = "__name__"
|
||||
lfe.Value = &StringExpr{
|
||||
tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))},
|
||||
mf = &labelFilterExpr{
|
||||
Label: "__name__",
|
||||
Value: &StringExpr{
|
||||
tokens: []string{strconv.Quote(unescapeIdent(p.lex.Token))},
|
||||
},
|
||||
}
|
||||
me.labelFilters = append(me.labelFilters[:0], &lfe)
|
||||
if err := p.lex.Next(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.lex.Token != "{" {
|
||||
me.labelFilterss = append(me.labelFilterss[:0], []*labelFilterExpr{mf})
|
||||
return &me, nil
|
||||
}
|
||||
}
|
||||
lfes, err := p.parseLabelFilters()
|
||||
lfess, err := p.parseLabelFilterss(mf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
me.labelFilters = append(me.labelFilters, lfes...)
|
||||
me.labelFilterss = append(me.labelFilterss, lfess...)
|
||||
return &me, nil
|
||||
}
|
||||
|
||||
|
|
@ -1841,57 +1918,104 @@ func (lf *LabelFilter) AppendString(dst []byte) []byte {
|
|||
}
|
||||
|
||||
// MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`.
|
||||
//
|
||||
// Curly braces may contain or-delimited list of filters. For example:
|
||||
//
|
||||
// x{job="foo",instance="bar" or job="x",instance="baz"}
|
||||
//
|
||||
// In this case the filter returns all the series, which match at least one of the following filters:
|
||||
//
|
||||
// x{job="foo",instance="bar"}
|
||||
// x{job="x",instance="baz"}
|
||||
//
|
||||
// This allows using or-delimited list of filters inside rollup functions. For example,
|
||||
// the following query calculates rate per each matching series for the given or-delimited filters:
|
||||
//
|
||||
// rate(x{job="foo",instance="bar" or job="x",instance="baz"}[5m])
|
||||
type MetricExpr struct {
|
||||
// LabelFilters contains a list of label filters from curly braces.
|
||||
// Filter or metric name must be the first if present.
|
||||
LabelFilters []LabelFilter
|
||||
// LabelFilters contains a list of or-delimited groups of label filters from curly braces.
|
||||
// Filter for metric name (aka __name__ label) must go first in every group.
|
||||
LabelFilterss [][]LabelFilter
|
||||
|
||||
// labelFilters contain non-expanded label filters joined by 'or' operator.
|
||||
//
|
||||
// labelFilters must be expanded to LabelFilters by expandWithExpr.
|
||||
labelFilters []*labelFilterExpr
|
||||
labelFilterss [][]*labelFilterExpr
|
||||
}
|
||||
|
||||
// AppendString appends string representation of me to dst and returns the result.
|
||||
func (me *MetricExpr) AppendString(dst []byte) []byte {
|
||||
lfs := me.LabelFilters
|
||||
if len(lfs) > 0 {
|
||||
lf := &lfs[0]
|
||||
if lf.Label == "__name__" && !lf.IsNegative && !lf.IsRegexp {
|
||||
dst = appendEscapedIdent(dst, lf.Value)
|
||||
lfs = lfs[1:]
|
||||
}
|
||||
}
|
||||
if len(lfs) > 0 {
|
||||
dst = append(dst, '{')
|
||||
for i := range lfs {
|
||||
dst = lfs[i].AppendString(dst)
|
||||
if i+1 < len(lfs) {
|
||||
dst = append(dst, ", "...)
|
||||
}
|
||||
}
|
||||
dst = append(dst, '}')
|
||||
} else if len(me.LabelFilters) == 0 {
|
||||
lfss := me.LabelFilterss
|
||||
if len(lfss) == 0 {
|
||||
dst = append(dst, "{}"...)
|
||||
return dst
|
||||
}
|
||||
offset := 0
|
||||
metricName := me.getMetricName()
|
||||
if metricName != "" {
|
||||
offset = 1
|
||||
dst = appendEscapedIdent(dst, metricName)
|
||||
}
|
||||
if len(lfss) == 1 && len(lfss[0]) == offset {
|
||||
return dst
|
||||
}
|
||||
dst = append(dst, '{')
|
||||
lfs := lfss[0]
|
||||
dst = appendLabelFilters(dst, lfs[offset:])
|
||||
for _, lfs := range lfss[1:] {
|
||||
dst = append(dst, " or "...)
|
||||
dst = appendLabelFilters(dst, lfs[offset:])
|
||||
}
|
||||
dst = append(dst, '}')
|
||||
return dst
|
||||
}
|
||||
|
||||
func appendLabelFilters(dst []byte, lfs []LabelFilter) []byte {
|
||||
if len(lfs) == 0 {
|
||||
return dst
|
||||
}
|
||||
dst = lfs[0].AppendString(dst)
|
||||
lfs = lfs[1:]
|
||||
for i := range lfs {
|
||||
dst = append(dst, ',')
|
||||
dst = lfs[i].AppendString(dst)
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// IsEmpty returns true of me equals to `{}`.
|
||||
func (me *MetricExpr) IsEmpty() bool {
|
||||
return len(me.LabelFilters) == 0
|
||||
return len(me.LabelFilterss) == 0
|
||||
}
|
||||
|
||||
func (me *MetricExpr) isOnlyMetricGroup() bool {
|
||||
if !me.hasNonEmptyMetricGroup() {
|
||||
func (me *MetricExpr) isOnlyMetricName() bool {
|
||||
if me.getMetricName() == "" {
|
||||
return false
|
||||
}
|
||||
return len(me.LabelFilters) == 1
|
||||
for _, lfs := range me.LabelFilterss {
|
||||
if len(lfs) > 1 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (me *MetricExpr) hasNonEmptyMetricGroup() bool {
|
||||
if len(me.LabelFilters) == 0 {
|
||||
return false
|
||||
func (me *MetricExpr) getMetricName() string {
|
||||
lfss := me.LabelFilterss
|
||||
if len(lfss) == 0 {
|
||||
return ""
|
||||
}
|
||||
return me.LabelFilters[0].isMetricNameFilter()
|
||||
lfs := lfss[0]
|
||||
if len(lfs) == 0 || !lfs[0].isMetricNameFilter() {
|
||||
return ""
|
||||
}
|
||||
metricName := lfs[0].Value
|
||||
for _, lfs := range lfss[1:] {
|
||||
if len(lfs) == 0 || !lfs[0].isMetricNameFilter() || lfs[0].Value != metricName {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return metricName
|
||||
}
|
||||
|
||||
func (lf *LabelFilter) isMetricNameFilter() bool {
|
||||
|
|
|
|||
2
vendor/modules.txt
vendored
2
vendor/modules.txt
vendored
|
|
@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
|
|||
# github.com/VictoriaMetrics/metrics v1.24.0
|
||||
## explicit; go 1.20
|
||||
github.com/VictoriaMetrics/metrics
|
||||
# github.com/VictoriaMetrics/metricsql v0.56.2
|
||||
# github.com/VictoriaMetrics/metricsql v0.57.1
|
||||
## explicit; go 1.13
|
||||
github.com/VictoriaMetrics/metricsql
|
||||
github.com/VictoriaMetrics/metricsql/binaryop
|
||||
|
|
|
|||
Loading…
Reference in a new issue