vmalert: allow to blackhole alerting notifications (#4639)

vmalert: support option to blackhole alerting notifications

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4122

---------

Co-authored-by: Rao, B V Chalapathi <b_v_chalapathi.rao@nokia.com>

app/vmalert/README.md

@@ -1088,6 +1088,8 @@ The shortlist of configuration flags is the following:
   -notifier.url array
      Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
      Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.blackhole bool
+       Whether to blackhole alerting notifications. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). `-notifier.url`, `-notifier.config` and `-notifier.blackhole` are mutually exclusive.
   -pprofAuthKey string
      Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.consul.waitTime duration

app/vmalert/manager.go

@@ -120,7 +120,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
 	}
 	if arPresent && m.notifiers == nil {
-		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
+		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` nor `-notifier.blackhole` aren't set")
 	}
 
 	type updateItem struct {

app/vmalert/manager_test.go

@@ -340,3 +340,21 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	}
 	return cfg
 }
+
+func TestUrlValuesToStrings(t *testing.T) {
+	mapQueryParams := map[string][]string{
+		"param1": {"param1"},
+		"param2": {"anotherparam"},
+	}
+	expectedRes := []string{"param1=param1", "param2=anotherparam"}
+	res := urlValuesToStrings(mapQueryParams)
+
+	if len(res) != len(expectedRes) {
+		t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
+	}
+	for ind, val := range expectedRes {
+		if val != res[ind] {
+			t.Errorf("Expected %v; but got %v", val, res[ind])
+		}
+	}
+}

app/vmalert/notifier/init.go

@@ -19,6 +19,7 @@ var (
 
 	addrs = flagutil.NewArrayString("notifier.url", "Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. "+
 		"List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.")
+	blackHole = flag.Bool("notifier.blackhole", false, "Don't send any notifications to anywhere. -notifier.url and -notifier.blackhole and -notifier.config are mutually exclusive")
 
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
@@ -90,6 +91,17 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 
 	templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
 
+	if *blackHole {
+		if len(*addrs) > 0 || *configPath != "" {
+			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
+		}
+
+		staticNotifiersFn = func() []Notifier {
+			return []Notifier{NewBlackHoleNotifier()}
+		}
+		return staticNotifiersFn, nil
+	}
+
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
 	}

app/vmalert/notifier/init_test.go

@@ -1,8 +1,9 @@
 package notifier
 
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestInit(t *testing.T) {
@@ -35,3 +36,89 @@ func TestInit(t *testing.T) {
 		t.Fatalf("expected to get \"127.0.0.2/api/v2/alerts\"; got %q instead", nf2.Addr())
 	}
 }
+
+func TestInitBlackHole(t *testing.T) {
+	oldBlackHole := *blackHole
+	defer func() { *blackHole = oldBlackHole }()
+
+	*blackHole = true
+
+	fn, err := Init(nil, nil, "")
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+
+	nfs := fn()
+	if len(nfs) != 1 {
+		t.Fatalf("expected to get 1 notifiers; got %d", len(nfs))
+	}
+
+	targets := GetTargets()
+	if targets == nil || targets[TargetStatic] == nil {
+		t.Fatalf("expected to get static targets in response")
+	}
+	if len(targets[TargetStatic]) != 1 {
+		t.Fatalf("expected to get 1 static targets in response; but got %d", len(targets[TargetStatic]))
+	}
+	nf1 := targets[TargetStatic][0]
+	if nf1.Addr() != "blackhole" {
+		t.Fatalf("expected to get \"blackhole\"; got %q instead", nf1.Addr())
+	}
+}
+
+func TestInitBlackHoleWithNotifierUrl(t *testing.T) {
+	oldAddrs := *addrs
+	oldBlackHole := *blackHole
+	defer func() {
+		*addrs = oldAddrs
+		*blackHole = oldBlackHole
+	}()
+
+	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
+	*blackHole = true
+
+	_, err := Init(nil, nil, "")
+	if err == nil {
+		t.Fatalf("Expect Init to return error; instead got no error")
+	}
+
+}
+
+func TestInitBlackHoleWithNotifierConfig(t *testing.T) {
+	oldConfigPath := *configPath
+	oldBlackHole := *blackHole
+	defer func() {
+		*configPath = oldConfigPath
+		*blackHole = oldBlackHole
+	}()
+
+	*configPath = "/dummy/path"
+	*blackHole = true
+
+	fn, err := Init(nil, nil, "")
+	if err == nil {
+		t.Fatalf("Expect Init to return error; instead got no error")
+	}
+
+	if fn != nil {
+		t.Fatalf("expected no notifiers to be returned;but got %v instead", fn())
+	}
+}
+
+func TestInitWithNotifierConfigAndAddr(t *testing.T) {
+	oldConfigPath := *configPath
+	oldAddrs := *addrs
+
+	defer func() {
+		*configPath = oldConfigPath
+		*addrs = oldAddrs
+	}()
+
+	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
+	*configPath = "/dummy/path"
+
+	_, err := Init(nil, nil, "")
+	if err == nil {
+		t.Fatalf("Expect Init to return error; instead got no error")
+	}
+}

app/vmalert/notifier/notifier_blackhole.go

@@ -0,0 +1,35 @@
+package notifier
+
+import "context"
+
+// BlackHoleNotifier is used when no notifications needs to be sent
+type BlackHoleNotifier struct {
+	addr    string
+	metrics *metrics
+}
+
+// Send will not send any notifications. Only increase the alerts sent number.
+func (bh *BlackHoleNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error { //nolint:revive
+	bh.metrics.alertsSent.Add(len(alerts))
+	return nil
+}
+
+// Addr of black hole notifier
+func (bh BlackHoleNotifier) Addr() string {
+	return bh.addr
+}
+
+// Close unregister the metrics
+func (bh *BlackHoleNotifier) Close() {
+	bh.metrics.alertsSent.Unregister()
+	bh.metrics.alertsSendErrors.Unregister()
+}
+
+// NewBlackHoleNotifier Create a new BlackHoleNotifier
+func NewBlackHoleNotifier() *BlackHoleNotifier {
+	address := "blackhole"
+	return &BlackHoleNotifier{
+		addr:    address,
+		metrics: newMetrics(address),
+	}
+}

app/vmalert/notifier/notifier_blackhole_test.go

@@ -0,0 +1,53 @@
+package notifier
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	metricset "github.com/VictoriaMetrics/metrics"
+)
+
+func TestBlackHoleNotifier_Send(t *testing.T) {
+	bh := NewBlackHoleNotifier()
+
+	if err := bh.Send(context.Background(), []Alert{{
+		GroupID:     0,
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	alertCount := bh.metrics.alertsSent.Get()
+
+	if alertCount != 1 {
+		t.Errorf("expect value 1; instead got %d", alertCount)
+	}
+}
+
+func TestBlackHoleNotifier_Close(t *testing.T) {
+	bh := NewBlackHoleNotifier()
+
+	if err := bh.Send(context.Background(), []Alert{{
+		GroupID:     0,
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+
+	bh.Close()
+
+	defaultMetrics := metricset.GetDefaultSet()
+	alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
+	for _, name := range defaultMetrics.ListMetricNames() {
+		if name == alertMetricName {
+			t.Errorf("Metric name should have unregistered.But still present")
+		}
+
+	}
+}