diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 253d907f6..067834650 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -95,6 +95,13 @@ name: extra_filter_labels: [ : ... ] +# Optional list of labels added to every rule within a group. +# It has priority over the external labels. +# Labels are commonly used for adding environment +# or tenant-specific tag. +labels: + [ : ... ] + rules: [ - ... ] ``` diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 76d92cb15..8a21cc90b 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -31,6 +31,9 @@ type Group struct { // request withing a group. Is compatible only with VM datasources. // See https://docs.victoriametrics.com#prometheus-querying-api-enhancements ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"` + // Labels is a set of label value pairs, that will be added to every rule. + // It has priority over the external labels. + Labels map[string]string `yaml:"labels"` // Checksum stores the hash of yaml definition for this group. // May be used to detect any changes like rules re-ordering etc. Checksum string diff --git a/app/vmalert/config/testdata/dir/rules-update0-good.rules b/app/vmalert/config/testdata/dir/rules-update0-good.rules index ffbfdf30f..e91277ad0 100644 --- a/app/vmalert/config/testdata/dir/rules-update0-good.rules +++ b/app/vmalert/config/testdata/dir/rules-update0-good.rules @@ -3,6 +3,8 @@ groups: interval: 2s concurrency: 2 type: prometheus + labels: + cluster: main rules: - alert: up expr: up == 0 diff --git a/app/vmalert/config/testdata/dir/rules1-good.rules b/app/vmalert/config/testdata/dir/rules1-good.rules index 1d04b1a6c..7f09278ed 100644 --- a/app/vmalert/config/testdata/dir/rules1-good.rules +++ b/app/vmalert/config/testdata/dir/rules1-good.rules @@ -1,5 +1,7 @@ groups: - name: duplicatedGroupDiffFiles + labels: + dc: gcp rules: - alert: VMRows for: 5m diff --git a/app/vmalert/group.go b/app/vmalert/group.go index f9f2b957a..24b19193b 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -18,15 +18,17 @@ import ( // Group is an entity for grouping rules type Group struct { - mu sync.RWMutex - Name string - File string - Rules []Rule - Type datasource.Type - Interval time.Duration - Concurrency int - Checksum string + mu sync.RWMutex + Name string + File string + Rules []Rule + Type datasource.Type + Interval time.Duration + Concurrency int + Checksum string + ExtraFilterLabels map[string]string + Labels map[string]string doneCh chan struct{} finishedCh chan struct{} @@ -50,6 +52,23 @@ func newGroupMetrics(name, file string) *groupMetrics { return m } +// merges group rule labels into result map +// set2 has priority over set1. +func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string { + r := map[string]string{} + for k, v := range set1 { + r[k] = v + } + for k, v := range set2 { + if prevV, ok := r[k]; ok { + logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q", + k, prevV, groupName, ruleName, k, v) + } + r[k] = v + } + return r +} + func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group { g := &Group{ Type: cfg.Type, @@ -59,6 +78,7 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti Concurrency: cfg.Concurrency, Checksum: cfg.Checksum, ExtraFilterLabels: cfg.ExtraFilterLabels, + Labels: cfg.Labels, doneCh: make(chan struct{}), finishedCh: make(chan struct{}), @@ -73,17 +93,20 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti } rules := make([]Rule, len(cfg.Rules)) for i, r := range cfg.Rules { - // override rule labels with external labels - for k, v := range labels { - if prevV, ok := r.Labels[k]; ok { - logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q", - k, prevV, g.Name, r.Name(), k, v) - } - if r.Labels == nil { - r.Labels = map[string]string{} - } - r.Labels[k] = v + var extraLabels map[string]string + // apply external labels + if len(labels) > 0 { + extraLabels = labels } + // apply group labels, it has priority on external labels + if len(cfg.Labels) > 0 { + extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels) + } + // apply rules labels, it has priority on other labels + if len(extraLabels) > 0 { + r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels) + } + rules[i] = g.newRule(qb, r) } g.Rules = rules @@ -110,6 +133,7 @@ func (g *Group) ID() uint64 { // Restore restores alerts state for group rules func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error { + labels = mergeLabels(g.Name, "", labels, g.Labels) for _, rule := range g.Rules { rr, ok := rule.(*AlertingRule) if !ok { @@ -169,6 +193,7 @@ func (g *Group) updateWith(newGroup *Group) error { g.Type = newGroup.Type g.Concurrency = newGroup.Concurrency g.ExtraFilterLabels = newGroup.ExtraFilterLabels + g.Labels = newGroup.Labels g.Checksum = newGroup.Checksum g.Rules = newRules return nil diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go index e73d399fb..630a9a271 100644 --- a/app/vmalert/manager.go +++ b/app/vmalert/manager.go @@ -148,6 +148,7 @@ func (g *Group) toAPI() APIGroup { Interval: g.Interval.String(), Concurrency: g.Concurrency, ExtraFilterLabels: g.ExtraFilterLabels, + Labels: g.Labels, } for _, r := range g.Rules { switch v := r.(type) { diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go index bd5859fab..31d885369 100644 --- a/app/vmalert/manager_test.go +++ b/app/vmalert/manager_test.go @@ -148,7 +148,7 @@ func TestManagerUpdate(t *testing.T) { Name: "VMRows", Expr: "vm_rows > 0", For: 5 * time.Minute, - Labels: map[string]string{"label": "bar"}, + Labels: map[string]string{"dc": "gcp", "label": "bar"}, Annotations: map[string]string{ "summary": "{{ $value }}", "description": "{{$labels}}", diff --git a/app/vmalert/web_types.go b/app/vmalert/web_types.go index 6b677ef8c..bcf673222 100644 --- a/app/vmalert/web_types.go +++ b/app/vmalert/web_types.go @@ -27,6 +27,7 @@ type APIGroup struct { Interval string `json:"interval"` Concurrency int `json:"concurrency"` ExtraFilterLabels map[string]string `json:"extra_filter_labels"` + Labels map[string]string `json:"labels,omitempty"` AlertingRules []APIAlertingRule `json:"alerting_rules"` RecordingRules []APIRecordingRule `json:"recording_rules"` } diff --git a/docs/vmalert.md b/docs/vmalert.md index d3072bd34..5e2f51678 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -99,6 +99,13 @@ name: extra_filter_labels: [ : ... ] +# Optional list of labels added to every rule within a group. +# It has priority over the external labels. +# Labels are commonly used for adding environment +# or tenant-specific tag. +labels: + [ : ... ] + rules: [ - ... ] ``` @@ -509,7 +516,9 @@ The shortlist of configuration flags is the following: -remoteWrite.tlsServerName string Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used -remoteWrite.url string - Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend + Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend + -remoteWrite.disablePathAppend + Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url. -replay.maxDatapointsPerQuery int Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000) -replay.ruleRetryAttempts int