mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-03-11 15:34:56 +00:00
vmalert: allow to blackhole alerting notifications (#4639)
vmalert: support option to blackhole alerting notifications https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4122 --------- Co-authored-by: Rao, B V Chalapathi <b_v_chalapathi.rao@nokia.com>
This commit is contained in:
parent
456a2e70fd
commit
bd2a37429c
7 changed files with 209 additions and 2 deletions
|
@ -1088,6 +1088,8 @@ The shortlist of configuration flags is the following:
|
|||
-notifier.url array
|
||||
Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-notifier.blackhole bool
|
||||
Whether to blackhole alerting notifications. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). `-notifier.url`, `-notifier.config` and `-notifier.blackhole` are mutually exclusive.
|
||||
-pprofAuthKey string
|
||||
Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
|
||||
-promscrape.consul.waitTime duration
|
||||
|
|
|
@ -120,7 +120,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
|
|||
return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
|
||||
}
|
||||
if arPresent && m.notifiers == nil {
|
||||
return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
|
||||
return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` nor `-notifier.blackhole` aren't set")
|
||||
}
|
||||
|
||||
type updateItem struct {
|
||||
|
|
|
@ -340,3 +340,21 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
|
|||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestUrlValuesToStrings(t *testing.T) {
|
||||
mapQueryParams := map[string][]string{
|
||||
"param1": {"param1"},
|
||||
"param2": {"anotherparam"},
|
||||
}
|
||||
expectedRes := []string{"param1=param1", "param2=anotherparam"}
|
||||
res := urlValuesToStrings(mapQueryParams)
|
||||
|
||||
if len(res) != len(expectedRes) {
|
||||
t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
|
||||
}
|
||||
for ind, val := range expectedRes {
|
||||
if val != res[ind] {
|
||||
t.Errorf("Expected %v; but got %v", val, res[ind])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ var (
|
|||
|
||||
addrs = flagutil.NewArrayString("notifier.url", "Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. "+
|
||||
"List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.")
|
||||
blackHole = flag.Bool("notifier.blackhole", false, "Don't send any notifications to anywhere. -notifier.url and -notifier.blackhole and -notifier.config are mutually exclusive")
|
||||
|
||||
basicAuthUsername = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
|
||||
basicAuthPassword = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
|
||||
|
@ -90,6 +91,17 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
|
|||
|
||||
templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
|
||||
|
||||
if *blackHole {
|
||||
if len(*addrs) > 0 || *configPath != "" {
|
||||
return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
|
||||
}
|
||||
|
||||
staticNotifiersFn = func() []Notifier {
|
||||
return []Notifier{NewBlackHoleNotifier()}
|
||||
}
|
||||
return staticNotifiersFn, nil
|
||||
}
|
||||
|
||||
if *configPath == "" && len(*addrs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
package notifier
|
||||
|
||||
import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
)
|
||||
|
||||
func TestInit(t *testing.T) {
|
||||
|
@ -35,3 +36,89 @@ func TestInit(t *testing.T) {
|
|||
t.Fatalf("expected to get \"127.0.0.2/api/v2/alerts\"; got %q instead", nf2.Addr())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitBlackHole(t *testing.T) {
|
||||
oldBlackHole := *blackHole
|
||||
defer func() { *blackHole = oldBlackHole }()
|
||||
|
||||
*blackHole = true
|
||||
|
||||
fn, err := Init(nil, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("%s", err)
|
||||
}
|
||||
|
||||
nfs := fn()
|
||||
if len(nfs) != 1 {
|
||||
t.Fatalf("expected to get 1 notifiers; got %d", len(nfs))
|
||||
}
|
||||
|
||||
targets := GetTargets()
|
||||
if targets == nil || targets[TargetStatic] == nil {
|
||||
t.Fatalf("expected to get static targets in response")
|
||||
}
|
||||
if len(targets[TargetStatic]) != 1 {
|
||||
t.Fatalf("expected to get 1 static targets in response; but got %d", len(targets[TargetStatic]))
|
||||
}
|
||||
nf1 := targets[TargetStatic][0]
|
||||
if nf1.Addr() != "blackhole" {
|
||||
t.Fatalf("expected to get \"blackhole\"; got %q instead", nf1.Addr())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitBlackHoleWithNotifierUrl(t *testing.T) {
|
||||
oldAddrs := *addrs
|
||||
oldBlackHole := *blackHole
|
||||
defer func() {
|
||||
*addrs = oldAddrs
|
||||
*blackHole = oldBlackHole
|
||||
}()
|
||||
|
||||
*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
|
||||
*blackHole = true
|
||||
|
||||
_, err := Init(nil, nil, "")
|
||||
if err == nil {
|
||||
t.Fatalf("Expect Init to return error; instead got no error")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestInitBlackHoleWithNotifierConfig(t *testing.T) {
|
||||
oldConfigPath := *configPath
|
||||
oldBlackHole := *blackHole
|
||||
defer func() {
|
||||
*configPath = oldConfigPath
|
||||
*blackHole = oldBlackHole
|
||||
}()
|
||||
|
||||
*configPath = "/dummy/path"
|
||||
*blackHole = true
|
||||
|
||||
fn, err := Init(nil, nil, "")
|
||||
if err == nil {
|
||||
t.Fatalf("Expect Init to return error; instead got no error")
|
||||
}
|
||||
|
||||
if fn != nil {
|
||||
t.Fatalf("expected no notifiers to be returned;but got %v instead", fn())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithNotifierConfigAndAddr(t *testing.T) {
|
||||
oldConfigPath := *configPath
|
||||
oldAddrs := *addrs
|
||||
|
||||
defer func() {
|
||||
*configPath = oldConfigPath
|
||||
*addrs = oldAddrs
|
||||
}()
|
||||
|
||||
*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
|
||||
*configPath = "/dummy/path"
|
||||
|
||||
_, err := Init(nil, nil, "")
|
||||
if err == nil {
|
||||
t.Fatalf("Expect Init to return error; instead got no error")
|
||||
}
|
||||
}
|
||||
|
|
35
app/vmalert/notifier/notifier_blackhole.go
Normal file
35
app/vmalert/notifier/notifier_blackhole.go
Normal file
|
@ -0,0 +1,35 @@
|
|||
package notifier
|
||||
|
||||
import "context"
|
||||
|
||||
// BlackHoleNotifier is used when no notifications needs to be sent
|
||||
type BlackHoleNotifier struct {
|
||||
addr string
|
||||
metrics *metrics
|
||||
}
|
||||
|
||||
// Send will not send any notifications. Only increase the alerts sent number.
|
||||
func (bh *BlackHoleNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error { //nolint:revive
|
||||
bh.metrics.alertsSent.Add(len(alerts))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Addr of black hole notifier
|
||||
func (bh BlackHoleNotifier) Addr() string {
|
||||
return bh.addr
|
||||
}
|
||||
|
||||
// Close unregister the metrics
|
||||
func (bh *BlackHoleNotifier) Close() {
|
||||
bh.metrics.alertsSent.Unregister()
|
||||
bh.metrics.alertsSendErrors.Unregister()
|
||||
}
|
||||
|
||||
// NewBlackHoleNotifier Create a new BlackHoleNotifier
|
||||
func NewBlackHoleNotifier() *BlackHoleNotifier {
|
||||
address := "blackhole"
|
||||
return &BlackHoleNotifier{
|
||||
addr: address,
|
||||
metrics: newMetrics(address),
|
||||
}
|
||||
}
|
53
app/vmalert/notifier/notifier_blackhole_test.go
Normal file
53
app/vmalert/notifier/notifier_blackhole_test.go
Normal file
|
@ -0,0 +1,53 @@
|
|||
package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metricset "github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
func TestBlackHoleNotifier_Send(t *testing.T) {
|
||||
bh := NewBlackHoleNotifier()
|
||||
|
||||
if err := bh.Send(context.Background(), []Alert{{
|
||||
GroupID: 0,
|
||||
Name: "alert0",
|
||||
Start: time.Now().UTC(),
|
||||
End: time.Now().UTC(),
|
||||
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
|
||||
}}, nil); err != nil {
|
||||
t.Errorf("unexpected error %s", err)
|
||||
}
|
||||
alertCount := bh.metrics.alertsSent.Get()
|
||||
|
||||
if alertCount != 1 {
|
||||
t.Errorf("expect value 1; instead got %d", alertCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlackHoleNotifier_Close(t *testing.T) {
|
||||
bh := NewBlackHoleNotifier()
|
||||
|
||||
if err := bh.Send(context.Background(), []Alert{{
|
||||
GroupID: 0,
|
||||
Name: "alert0",
|
||||
Start: time.Now().UTC(),
|
||||
End: time.Now().UTC(),
|
||||
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
|
||||
}}, nil); err != nil {
|
||||
t.Errorf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
bh.Close()
|
||||
|
||||
defaultMetrics := metricset.GetDefaultSet()
|
||||
alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
|
||||
for _, name := range defaultMetrics.ListMetricNames() {
|
||||
if name == alertMetricName {
|
||||
t.Errorf("Metric name should have unregistered.But still present")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue