From dde750e7c105bdd43ad8787153b6ca33dc3af83f Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Fri, 30 Dec 2022 09:54:03 +0100 Subject: [PATCH] vmalert: mention specifics of Alertmanager HA mode (#3573) Stress the importance of specifying of all Alertmanager URLs in vmalert's `-notifier.url` or `notifier.config` if it runs in cluster mode. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3547 Signed-off-by: hagen1778 --- app/vmalert/README.md | 7 +++++-- app/vmalert/notifier/init.go | 3 ++- docs/vmalert.md | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index cce14e58d..7ff7d747a 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -480,7 +480,8 @@ Alertmanager will automatically deduplicate alerts with identical labels, so ens all `vmalert`s are having the same config. Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/) -for Alertmanagers for better reliability. +for Alertmanagers for better reliability. List all Alertmanager URLs in vmalert's `-notifier.url` +to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability). This example uses single-node VM server for the sake of simplicity. Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. @@ -969,7 +970,7 @@ The shortlist of configuration flags is the following: Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used Supports an array of values separated by comma or specified via multiple flags. -notifier.url array - Prometheus alertmanager URL, e.g. http://127.0.0.1:9093 + Prometheus alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability. Supports an array of values separated by comma or specified via multiple flags. -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings @@ -1201,6 +1202,8 @@ dns_sd_configs: ``` The list of configured or discovered Notifiers can be explored via [UI](#Web). +If Alertmanager runs in cluster mode then all its URLs needs to be available during discovery +to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability). The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go) is the following: diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index 39edcd631..e85bc47f1 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -17,7 +17,8 @@ var ( configPath = flag.String("notifier.config", "", "Path to configuration file for notifiers") suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery") - addrs = flagutil.NewArrayString("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093") + addrs = flagutil.NewArrayString("notifier.url", "Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. "+ + "List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.") basicAuthUsername = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url") basicAuthPassword = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url") diff --git a/docs/vmalert.md b/docs/vmalert.md index 06753e58c..1afa9a977 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -484,7 +484,8 @@ Alertmanager will automatically deduplicate alerts with identical labels, so ens all `vmalert`s are having the same config. Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/) -for Alertmanagers for better reliability. +for Alertmanagers for better reliability. List all Alertmanager URLs in vmalert's `-notifier.url` +to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability). This example uses single-node VM server for the sake of simplicity. Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. @@ -973,7 +974,7 @@ The shortlist of configuration flags is the following: Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used Supports an array of values separated by comma or specified via multiple flags. -notifier.url array - Prometheus alertmanager URL, e.g. http://127.0.0.1:9093 + Prometheus alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability. Supports an array of values separated by comma or specified via multiple flags. -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings @@ -1205,6 +1206,8 @@ dns_sd_configs: ``` The list of configured or discovered Notifiers can be explored via [UI](#Web). +If Alertmanager runs in cluster mode then all its URLs needs to be available during discovery +to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability). The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go) is the following: