mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
5da71eb685
vmalert: support configuration file for notifiers * vmalert notifiers now can be configured via file see https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file * add support of Consul service discovery for notifiers config see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1947 * add UI section for currently loaded/discovered notifiers * deprecate `-rule.configCheckInterval` in favour of `-configCheckInterval` * add ability to suppress logs for duplicated targets for notifiers discovery * change behaviour of `vmalert_alerts_send_errors_total` - it now accounts for failed alerts, not HTTP calls.
143 lines
4.9 KiB
Go
143 lines
4.9 KiB
Go
package notifier
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
)
|
|
|
|
var (
|
|
configPath = flag.String("notifier.config", "", "Path to configuration file for notifiers")
|
|
suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
|
|
|
|
addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
|
|
basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
|
|
basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
|
|
|
|
tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
|
|
tlsCertFile = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
|
|
tlsKeyFile = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
|
|
tlsCAFile = flagutil.NewArray("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
|
|
"By default system CA is used")
|
|
tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
|
|
"By default the server name from -notifier.url is used")
|
|
)
|
|
|
|
// cw holds a configWatcher for configPath configuration file
|
|
// configWatcher provides a list of Notifier objects discovered
|
|
// from static config or via service discovery.
|
|
// cw is not nil only if configPath is provided.
|
|
var cw *configWatcher
|
|
|
|
// Reload checks the changes in configPath configuration file
|
|
// and applies changes if any.
|
|
func Reload() error {
|
|
if cw == nil {
|
|
return nil
|
|
}
|
|
return cw.reload(*configPath)
|
|
}
|
|
|
|
var staticNotifiersFn func() []Notifier
|
|
|
|
// Init returns a function for retrieving actual list of Notifier objects.
|
|
// Init works in two mods:
|
|
// * configuration via flags (for backward compatibility). Is always static
|
|
// and don't support live reloads.
|
|
// * configuration via file. Supports live reloads and service discovery.
|
|
// Init returns an error if both mods are used.
|
|
func Init(gen AlertURLGenerator) (func() []Notifier, error) {
|
|
if *configPath == "" && len(*addrs) == 0 {
|
|
return nil, nil
|
|
}
|
|
if *configPath != "" && len(*addrs) > 0 {
|
|
return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
|
|
}
|
|
|
|
if len(*addrs) > 0 {
|
|
notifiers, err := notifiersFromFlags(gen)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create notifier from flag values: %s", err)
|
|
}
|
|
staticNotifiersFn = func() []Notifier {
|
|
return notifiers
|
|
}
|
|
return staticNotifiersFn, nil
|
|
}
|
|
|
|
var err error
|
|
cw, err = newWatcher(*configPath, gen)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to init config watcher: %s", err)
|
|
}
|
|
return cw.notifiers, nil
|
|
}
|
|
|
|
func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
|
|
var notifiers []Notifier
|
|
for i, addr := range *addrs {
|
|
authCfg := promauth.HTTPClientConfig{
|
|
TLSConfig: &promauth.TLSConfig{
|
|
CAFile: tlsCAFile.GetOptionalArg(i),
|
|
CertFile: tlsCertFile.GetOptionalArg(i),
|
|
KeyFile: tlsKeyFile.GetOptionalArg(i),
|
|
ServerName: tlsServerName.GetOptionalArg(i),
|
|
InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i),
|
|
},
|
|
BasicAuth: &promauth.BasicAuthConfig{
|
|
Username: basicAuthUsername.GetOptionalArg(i),
|
|
Password: promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
|
|
},
|
|
}
|
|
am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
notifiers = append(notifiers, am)
|
|
}
|
|
return notifiers, nil
|
|
}
|
|
|
|
// Target represents a Notifier and optional
|
|
// list of labels added during discovery.
|
|
type Target struct {
|
|
Notifier
|
|
Labels []prompbmarshal.Label
|
|
}
|
|
|
|
// TargetType defines how the Target was discovered
|
|
type TargetType string
|
|
|
|
const (
|
|
// TargetStatic is for targets configured statically
|
|
TargetStatic TargetType = "static"
|
|
// TargetConsul is for targets discovered via Consul
|
|
TargetConsul TargetType = "consulSD"
|
|
)
|
|
|
|
// GetTargets returns list of static or discovered targets
|
|
// via notifier configuration.
|
|
func GetTargets() map[TargetType][]Target {
|
|
var targets = make(map[TargetType][]Target)
|
|
|
|
if staticNotifiersFn != nil {
|
|
for _, ns := range staticNotifiersFn() {
|
|
targets[TargetStatic] = append(targets[TargetStatic], Target{
|
|
Notifier: ns,
|
|
})
|
|
}
|
|
}
|
|
|
|
if cw != nil {
|
|
cw.targetsMu.RLock()
|
|
for key, ns := range cw.targets {
|
|
targets[key] = append(targets[key], ns...)
|
|
}
|
|
cw.targetsMu.RUnlock()
|
|
}
|
|
return targets
|
|
}
|