adds external_labels per group for vmalert (#1485)

* adds external_label per group for vmalert
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1471

app/vmalert/README.md

@@ -95,6 +95,13 @@ name: <string>
 extra_filter_labels:
   [ <labelname>: <labelvalue> ... ]
 
+# Optional list of labels added to every rule within a group.
+# It has priority over the external labels.
+# Labels are commonly used for adding environment
+# or tenant-specific tag.
+labels:
+  [ <labelname>: <labelvalue> ... ]
+
 rules:
   [ - <rule> ... ]
 ```

app/vmalert/config/config.go

@@ -31,6 +31,9 @@ type Group struct {
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	// It has priority over the external labels.
+	Labels map[string]string `yaml:"labels"`
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -3,6 +3,8 @@ groups:
     interval: 2s
     concurrency: 2
     type: prometheus
+    labels:
+      cluster: main
     rules:
       - alert: up
         expr: up == 0

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: duplicatedGroupDiffFiles
+    labels:
+       dc: gcp
     rules:
       - alert: VMRows
         for: 5m

app/vmalert/group.go

@@ -18,15 +18,17 @@ import (
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu                sync.RWMutex
+	mu          sync.RWMutex
-	Name              string
+	Name        string
-	File              string
+	File        string
-	Rules             []Rule
+	Rules       []Rule
-	Type              datasource.Type
+	Type        datasource.Type
-	Interval          time.Duration
+	Interval    time.Duration
-	Concurrency       int
+	Concurrency int
-	Checksum          string
+	Checksum    string
+
 	ExtraFilterLabels map[string]string
+	Labels            map[string]string
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -50,6 +52,23 @@ func newGroupMetrics(name, file string) *groupMetrics {
 	return m
 }
 
+// merges group rule labels into result map
+// set2 has priority over set1.
+func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
+	r := map[string]string{}
+	for k, v := range set1 {
+		r[k] = v
+	}
+	for k, v := range set2 {
+		if prevV, ok := r[k]; ok {
+			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+				k, prevV, groupName, ruleName, k, v)
+		}
+		r[k] = v
+	}
+	return r
+}
+
 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
 		Type:              cfg.Type,
@@ -59,6 +78,7 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Concurrency:       cfg.Concurrency,
 		Checksum:          cfg.Checksum,
 		ExtraFilterLabels: cfg.ExtraFilterLabels,
+		Labels:            cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -73,17 +93,20 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	}
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
-		// override rule labels with external labels
+		var extraLabels map[string]string
-		for k, v := range labels {
+		// apply external labels
-			if prevV, ok := r.Labels[k]; ok {
+		if len(labels) > 0 {
-				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+			extraLabels = labels
-					k, prevV, g.Name, r.Name(), k, v)
-			}
-			if r.Labels == nil {
-				r.Labels = map[string]string{}
-			}
-			r.Labels[k] = v
 		}
+		// apply group labels, it has priority on external labels
+		if len(cfg.Labels) > 0 {
+			extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels)
+		}
+		// apply rules labels, it has priority on other labels
+		if len(extraLabels) > 0 {
+			r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels)
+		}
+
 		rules[i] = g.newRule(qb, r)
 	}
 	g.Rules = rules
@@ -110,6 +133,7 @@ func (g *Group) ID() uint64 {
 
 // Restore restores alerts state for group rules
 func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
+	labels = mergeLabels(g.Name, "", labels, g.Labels)
 	for _, rule := range g.Rules {
 		rr, ok := rule.(*AlertingRule)
 		if !ok {
@@ -169,6 +193,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
 	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil

app/vmalert/manager.go

@@ -148,6 +148,7 @@ func (g *Group) toAPI() APIGroup {
 		Interval:          g.Interval.String(),
 		Concurrency:       g.Concurrency,
 		ExtraFilterLabels: g.ExtraFilterLabels,
+		Labels:            g.Labels,
 	}
 	for _, r := range g.Rules {
 		switch v := r.(type) {

app/vmalert/manager_test.go

@@ -148,7 +148,7 @@ func TestManagerUpdate(t *testing.T) {
 							Name:   "VMRows",
 							Expr:   "vm_rows > 0",
 							For:    5 * time.Minute,
-							Labels: map[string]string{"label": "bar"},
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
 							Annotations: map[string]string{
 								"summary":     "{{ $value }}",
 								"description": "{{$labels}}",

app/vmalert/web_types.go

@@ -27,6 +27,7 @@ type APIGroup struct {
 	Interval          string             `json:"interval"`
 	Concurrency       int                `json:"concurrency"`
 	ExtraFilterLabels map[string]string  `json:"extra_filter_labels"`
+	Labels            map[string]string  `json:"labels,omitempty"`
 	AlertingRules     []APIAlertingRule  `json:"alerting_rules"`
 	RecordingRules    []APIRecordingRule `json:"recording_rules"`
 }

docs/vmalert.md

@@ -99,6 +99,13 @@ name: <string>
 extra_filter_labels:
   [ <labelname>: <labelvalue> ... ]
 
+# Optional list of labels added to every rule within a group.
+# It has priority over the external labels.
+# Labels are commonly used for adding environment
+# or tenant-specific tag.
+labels:
+  [ <labelname>: <labelvalue> ... ]
+
 rules:
   [ - <rule> ... ]
 ```
@@ -509,7 +516,9 @@ The shortlist of configuration flags is the following:
   -remoteWrite.tlsServerName string
     	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-    	Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+      Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+  -remoteWrite.disablePathAppend
+      Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
   -replay.maxDatapointsPerQuery int
     	Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int