From e8f5dbd59831850c8f043a12964609b7d38196b4 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Tue, 20 Aug 2024 03:40:57 +0800 Subject: [PATCH] vmalert: add command line flag `-notifier.headers` (#6751) to allow configuring additional headers in each request to the corresponding notifier. Other flags like `-datasource.headers`, `-remoteWrite.headers` already use `^^` as delimiter, it's consistent to use it in `-notifier.headers` as well. related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260 vmalert can integrate with alertmanager that supports multi-tenant by adding tenantID header`X-Scope-OrgID` in requests. In multitenancy, vmalert can also filter alerts which send to different notifier addresses(or with different header settings) using `alert_relabel_configs`. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260 --------- Signed-off-by: hagen1778 Co-authored-by: hagen1778 (cherry picked from commit 0f1ec33892e2b5522b868c932025bd46f075f021) --- app/vmalert/notifier/init.go | 4 ++++ docs/CHANGELOG.md | 1 + docs/vmalert.md | 2 ++ 3 files changed, 7 insertions(+) diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go index 6d29ef238..3f5c79a88 100644 --- a/app/vmalert/notifier/init.go +++ b/app/vmalert/notifier/init.go @@ -25,6 +25,9 @@ var ( "Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+ "-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.") + headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+ + "For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+ + "Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'") basicAuthUsername = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url") basicAuthPassword = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url") basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url") @@ -171,6 +174,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) { Scopes: strings.Split(oauth2Scopes.GetOptionalArg(i), ";"), TokenURL: oauth2TokenURL.GetOptionalArg(i), }, + Headers: []string{headers.GetOptionalArg(i)}, } addr = strings.TrimSuffix(addr, "/") diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e7e426a25..52f508b6a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--vm-backoff-retries`, `--vm-backoff-factor`, `--vm-backoff-min-duration` and `--vm-native-backoff-retries`, `--vm-native-backoff-factor`, `--vm-native-backoff-min-duration` command-line flags. These flags allow to change backoff policy config for import requests to VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6622). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): allow overriding the `sample_limit` option at [scrape_configs](https://docs.victoriametrics.com/sd_configs/#scrape_configs) when a label `__sample_limit__` is specified for target. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6665). Thanks to @zoglam for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6666). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): reduce memory usage when scraping targets with big response body. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6759). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): add command-line flag `-notifier.headers` to allow configuring additional headers for all requests sent to the corresponding `-notifier.url`. * FEATURE: [vmalert-tool](https://docs.victoriametrics.com/vmalert-tool/): add `-external.label` and `-external.url` command-line flags, in the same way as these flags are supported by vmalert. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735). * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): use exponential backoff for retries when uploading or downloading data from S3. This should reduce the number of failed uploads and downloads when S3 is temporarily unavailable. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6732). * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): do not allow enabling `-stream.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config), as it may result in time series collision. diff --git a/docs/vmalert.md b/docs/vmalert.md index e2be95c61..ca693306e 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1212,6 +1212,8 @@ The shortlist of configuration flags is the following: Whether to blackhole alerting notifications. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). -notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive. -notifier.config string Path to configuration file for notifiers + -notifier.headers array + Optional HTTP headers to send with each request to the corresponding -notifier.url. For example, -notifier.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'. -notifier.oauth2.clientID array Optional OAuth2 clientID to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url Supports an array of values separated by comma or specified via multiple flags.