VictoriaMetrics/app/vmalert/notifier/init.go
Roman Khavronenko 5da71eb685
vmalert: support configuration file for notifiers (#2127)
vmalert: support configuration file for notifiers

* vmalert notifiers now can be configured via file
see https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file
* add support of Consul service discovery for notifiers config
see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1947
* add UI section for currently loaded/discovered notifiers
* deprecate `-rule.configCheckInterval` in favour of `-configCheckInterval`
* add ability to suppress logs for duplicated targets for notifiers discovery
* change behaviour of `vmalert_alerts_send_errors_total` - it now accounts
for failed alerts, not HTTP calls.
2022-02-02 14:11:41 +02:00

143 lines
4.9 KiB
Go

package notifier
import (
"flag"
"fmt"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
)
var (
configPath = flag.String("notifier.config", "", "Path to configuration file for notifiers")
suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
tlsCertFile = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
tlsKeyFile = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
tlsCAFile = flagutil.NewArray("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
"By default system CA is used")
tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
"By default the server name from -notifier.url is used")
)
// cw holds a configWatcher for configPath configuration file
// configWatcher provides a list of Notifier objects discovered
// from static config or via service discovery.
// cw is not nil only if configPath is provided.
var cw *configWatcher
// Reload checks the changes in configPath configuration file
// and applies changes if any.
func Reload() error {
if cw == nil {
return nil
}
return cw.reload(*configPath)
}
var staticNotifiersFn func() []Notifier
// Init returns a function for retrieving actual list of Notifier objects.
// Init works in two mods:
// * configuration via flags (for backward compatibility). Is always static
// and don't support live reloads.
// * configuration via file. Supports live reloads and service discovery.
// Init returns an error if both mods are used.
func Init(gen AlertURLGenerator) (func() []Notifier, error) {
if *configPath == "" && len(*addrs) == 0 {
return nil, nil
}
if *configPath != "" && len(*addrs) > 0 {
return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
}
if len(*addrs) > 0 {
notifiers, err := notifiersFromFlags(gen)
if err != nil {
return nil, fmt.Errorf("failed to create notifier from flag values: %s", err)
}
staticNotifiersFn = func() []Notifier {
return notifiers
}
return staticNotifiersFn, nil
}
var err error
cw, err = newWatcher(*configPath, gen)
if err != nil {
return nil, fmt.Errorf("failed to init config watcher: %s", err)
}
return cw.notifiers, nil
}
func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
var notifiers []Notifier
for i, addr := range *addrs {
authCfg := promauth.HTTPClientConfig{
TLSConfig: &promauth.TLSConfig{
CAFile: tlsCAFile.GetOptionalArg(i),
CertFile: tlsCertFile.GetOptionalArg(i),
KeyFile: tlsKeyFile.GetOptionalArg(i),
ServerName: tlsServerName.GetOptionalArg(i),
InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i),
},
BasicAuth: &promauth.BasicAuthConfig{
Username: basicAuthUsername.GetOptionalArg(i),
Password: promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
},
}
am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
if err != nil {
return nil, err
}
notifiers = append(notifiers, am)
}
return notifiers, nil
}
// Target represents a Notifier and optional
// list of labels added during discovery.
type Target struct {
Notifier
Labels []prompbmarshal.Label
}
// TargetType defines how the Target was discovered
type TargetType string
const (
// TargetStatic is for targets configured statically
TargetStatic TargetType = "static"
// TargetConsul is for targets discovered via Consul
TargetConsul TargetType = "consulSD"
)
// GetTargets returns list of static or discovered targets
// via notifier configuration.
func GetTargets() map[TargetType][]Target {
var targets = make(map[TargetType][]Target)
if staticNotifiersFn != nil {
for _, ns := range staticNotifiersFn() {
targets[TargetStatic] = append(targets[TargetStatic], Target{
Notifier: ns,
})
}
}
if cw != nil {
cw.targetsMu.RLock()
for key, ns := range cw.targets {
targets[key] = append(targets[key], ns...)
}
cw.targetsMu.RUnlock()
}
return targets
}