From 4b529562ce4ce62e51548e0653bbbec1d6da5bac Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 17 Dec 2023 19:56:46 +0200 Subject: [PATCH] lib/pushmetrics: add -pushmetrics.header and -pushmetrics.disableCompression command-line flags --- README.md | 16 +- docs/CHANGELOG.md | 1 + docs/Cluster-VictoriaMetrics.md | 27 ++- docs/README.md | 19 +- docs/Single-server-VictoriaMetrics.md | 19 +- docs/vmagent.md | 9 +- docs/vmalert.md | 9 +- docs/vmauth.md | 9 +- docs/vmbackup.md | 9 +- docs/vmbackupmanager.md | 9 +- docs/vmgateway.md | 9 +- docs/vmrestore.md | 9 +- go.mod | 2 +- go.sum | 4 +- lib/pushmetrics/pushmetrics.go | 17 +- .../VictoriaMetrics/metrics/go_metrics.go | 31 ++- .../VictoriaMetrics/metrics/push.go | 190 +++++++++++++++--- vendor/modules.txt | 4 +- 18 files changed, 320 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index f896c6ff2..6b4a9288e 100644 --- a/README.md +++ b/README.md @@ -2288,11 +2288,14 @@ The following command-line flags are related to pushing metrics from VictoriaMet The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form `http://user:pass@hostname/api/v1/import/prometheus`. Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. - This allows reducing the required network bandwidth for metrics push. -* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to `-pushmetrics.url`. Each label must be specified in the format `label="value"`. + This allows reducing the required network bandwidth for metrics push. The compression can be disabled by passing `-pushmetrics.disableCompression` command-line flag. +* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to every `-pushmetrics.url`. Each label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics before sending them to all the configured `-pushmetrics.url` addresses. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. +* `-pushmetrics.header` - an optional HTTP header to send to every `-pushmetrics.url`. For example, `-pushmetrics.header='Authorization: Basic foo'` instructs to send + `Authorization: Basic foo` HTTP header with every request to every `-pushmetrics.url`. It is possible to set multiple `-pushmetrics.header` command-line flags + for sending multiple different HTTP headers to `-pushmetrics.url`. For example, the following command instructs VictoriaMetrics to push metrics from `/metrics` page to `https://maas.victoriametrics.com/api/v1/import/prometheus` with `user:pass` [Basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication). The `instance="foobar"` and `job="vm"` labels @@ -2838,11 +2841,16 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors -promscrape.yandexcloudSDCheckInterval duration Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#yandexcloud_sd_configs for details (default 30s) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2fcf40557..6c8c65aa2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning. +* FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics). * BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped. * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 6a5b2f815..afcda9e19 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -1109,11 +1109,16 @@ Below is the output for `/path/to/vminsert -help`: Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings -prevCacheRemovalPercent float Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. @@ -1289,11 +1294,16 @@ Below is the output for `/path/to/vmselect -help`: Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings -prevCacheRemovalPercent float Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. @@ -1552,11 +1562,16 @@ Below is the output for `/path/to/vmstorage -help`: The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss (default 64) -prevCacheRemovalPercent float Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/README.md b/docs/README.md index 813363e58..81e3e2b95 100644 --- a/docs/README.md +++ b/docs/README.md @@ -124,9 +124,8 @@ Case studies: * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode) * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind) * [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver) -* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay) -* [RELEX Solutions](https://docs.victoriametrics.com/CaseStudies.html#relex-solutions) +* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox) * [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia) * [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets) @@ -2292,11 +2291,14 @@ The following command-line flags are related to pushing metrics from VictoriaMet The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form `http://user:pass@hostname/api/v1/import/prometheus`. Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. - This allows reducing the required network bandwidth for metrics push. -* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to `-pushmetrics.url`. Each label must be specified in the format `label="value"`. + This allows reducing the required network bandwidth for metrics push. The compression can be disabled by passing `-pushmetrics.disableCompression` command-line flag. +* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to every `-pushmetrics.url`. Each label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics before sending them to all the configured `-pushmetrics.url` addresses. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. +* `-pushmetrics.header` - an optional HTTP header to send to every `-pushmetrics.url`. For example, `-pushmetrics.header='Authorization: Basic foo'` instructs to send + `Authorization: Basic foo` HTTP header with every request to every `-pushmetrics.url`. It is possible to set multiple `-pushmetrics.header` command-line flags + for sending multiple different HTTP headers to `-pushmetrics.url`. For example, the following command instructs VictoriaMetrics to push metrics from `/metrics` page to `https://maas.victoriametrics.com/api/v1/import/prometheus` with `user:pass` [Basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication). The `instance="foobar"` and `job="vm"` labels @@ -2842,11 +2844,16 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors -promscrape.yandexcloudSDCheckInterval duration Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#yandexcloud_sd_configs for details (default 30s) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 9a5348d93..584936219 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -132,9 +132,8 @@ Case studies: * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode) * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind) * [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver) -* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay) -* [RELEX Solutions](https://docs.victoriametrics.com/CaseStudies.html#relex-solutions) +* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox) * [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia) * [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets) @@ -2300,11 +2299,14 @@ The following command-line flags are related to pushing metrics from VictoriaMet The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form `http://user:pass@hostname/api/v1/import/prometheus`. Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. - This allows reducing the required network bandwidth for metrics push. -* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to `-pushmetrics.url`. Each label must be specified in the format `label="value"`. + This allows reducing the required network bandwidth for metrics push. The compression can be disabled by passing `-pushmetrics.disableCompression` command-line flag. +* `-pushmetrics.extraLabel` - labels to add to all the metrics before sending them to every `-pushmetrics.url`. Each label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics before sending them to all the configured `-pushmetrics.url` addresses. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. +* `-pushmetrics.header` - an optional HTTP header to send to every `-pushmetrics.url`. For example, `-pushmetrics.header='Authorization: Basic foo'` instructs to send + `Authorization: Basic foo` HTTP header with every request to every `-pushmetrics.url`. It is possible to set multiple `-pushmetrics.header` command-line flags + for sending multiple different HTTP headers to `-pushmetrics.url`. For example, the following command instructs VictoriaMetrics to push metrics from `/metrics` page to `https://maas.victoriametrics.com/api/v1/import/prometheus` with `user:pass` [Basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication). The `instance="foobar"` and `job="vm"` labels @@ -2850,11 +2852,16 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors -promscrape.yandexcloudSDCheckInterval duration Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#yandexcloud_sd_configs for details (default 30s) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmagent.md b/docs/vmagent.md index d622e8128..04a29c24a 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1794,11 +1794,16 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors -promscrape.yandexcloudSDCheckInterval duration Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#yandexcloud_sd_configs for details (default 30s) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmalert.md b/docs/vmalert.md index 1eb9c0625..cfb29fc43 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1194,11 +1194,16 @@ The shortlist of configuration flags is the following: The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -promscrape.dnsSDCheckInterval duration Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#dns_sd_configs for details (default 30s) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmauth.md b/docs/vmauth.md index e20be290f..a6538d1ff 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -885,11 +885,16 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmbackup.md b/docs/vmbackup.md index 8a0663fda..1b758d533 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -391,11 +391,16 @@ Run `vmbackup -help` in order to see all the available options: Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index 18c65cc6e..934cfaaca 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -530,11 +530,16 @@ command-line flags: Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmgateway.md b/docs/vmgateway.md index 46dd243d0..cef6ef9a7 100644 --- a/docs/vmgateway.md +++ b/docs/vmgateway.md @@ -425,11 +425,16 @@ The shortlist of configuration flags include the following: Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/vmrestore.md b/docs/vmrestore.md index 405e0e5e5..81bca4680 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -186,11 +186,16 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -pprofAuthKey string Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. diff --git a/go.mod b/go.mod index e44b98be2..c32a74c9c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 - github.com/VictoriaMetrics/metrics v1.25.3 + github.com/VictoriaMetrics/metrics v1.26.1 github.com/VictoriaMetrics/metricsql v0.70.0 github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2/config v1.26.1 diff --git a/go.sum b/go.sum index 192cd8311..59450ece6 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I= github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metrics v1.25.3 h1:Zcxyj8JbAB6CQU51Er3D7RBRupcP55DevVQi9cFqo2Q= -github.com/VictoriaMetrics/metrics v1.25.3/go.mod h1:ZKmlI+QN6b9LUC0OiHNp2LiGQGlBy4U1re6Slooln1o= +github.com/VictoriaMetrics/metrics v1.26.1 h1:zJmbXMED+mnirSpJeVJKxXNaFKJaX3Bj/VuV2j2uKKY= +github.com/VictoriaMetrics/metrics v1.26.1/go.mod h1:EQ7JHshszbJYGuC1IIqkL+K73bTTTurJMQvUT5NMT5Q= github.com/VictoriaMetrics/metricsql v0.70.0 h1:G0k/m1yAF6pmk0dM3VT9/XI5PZ8dL7EbcLhREf4bgeI= github.com/VictoriaMetrics/metricsql v0.70.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index a6d2dd0b5..ebb17bea4 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -1,6 +1,7 @@ package pushmetrics import ( + "context" "flag" "strings" "time" @@ -14,9 +15,12 @@ import ( var ( pushURL = flagutil.NewArrayString("pushmetrics.url", "Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . "+ "By default, metrics exposed at /metrics page aren't pushed to any remote storage") - pushInterval = flag.Duration("pushmetrics.interval", 10*time.Second, "Interval for pushing metrics to -pushmetrics.url") - pushExtraLabel = flagutil.NewArrayString("pushmetrics.extraLabel", "Optional labels to add to metrics pushed to -pushmetrics.url . "+ - `For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url`) + pushInterval = flag.Duration("pushmetrics.interval", 10*time.Second, "Interval for pushing metrics to every -pushmetrics.url") + pushExtraLabel = flagutil.NewArrayString("pushmetrics.extraLabel", "Optional labels to add to metrics pushed to every -pushmetrics.url . "+ + `For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url`) + pushHeader = flagutil.NewArrayString("pushmetrics.header", "Optional HTTP request header to send to every -pushmetrics.url . "+ + "For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url") + disableCompression = flag.Bool("pushmetrics.disableCompression", false, "Whether to disable request body compression when pushing metrics to every -pushmetrics.url") ) func init() { @@ -28,7 +32,12 @@ func init() { func Init() { extraLabels := strings.Join(*pushExtraLabel, ",") for _, pu := range *pushURL { - if err := metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics); err != nil { + opts := &metrics.PushOptions{ + ExtraLabels: extraLabels, + Headers: *pushHeader, + DisableCompression: *disableCompression, + } + if err := metrics.InitPushExtWithOptions(context.Background(), pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) } } diff --git a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go index 5de7cb81d..1d0c7e24d 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "io" + "log" "math" "runtime" runtimemetrics "runtime/metrics" @@ -22,6 +23,25 @@ var runtimeMetrics = [][2]string{ {"/gc/gomemlimit:bytes", "go_memlimit_bytes"}, } +var supportedRuntimeMetrics = initSupportedRuntimeMetrics(runtimeMetrics) + +func initSupportedRuntimeMetrics(rms [][2]string) [][2]string { + exposedMetrics := make(map[string]struct{}) + for _, d := range runtimemetrics.All() { + exposedMetrics[d.Name] = struct{}{} + } + var supportedMetrics [][2]string + for _, rm := range rms { + metricName := rm[0] + if _, ok := exposedMetrics[metricName]; ok { + supportedMetrics = append(supportedMetrics, rm) + } else { + log.Printf("github.com/VictoriaMetrics/metrics: do not expose %s metric, since the corresponding metric %s isn't supported in the current Go runtime", rm[1], metricName) + } + } + return supportedMetrics +} + func writeGoMetrics(w io.Writer) { writeRuntimeMetrics(w) @@ -81,18 +101,19 @@ func writeGoMetrics(w io.Writer) { } func writeRuntimeMetrics(w io.Writer) { - samples := make([]runtimemetrics.Sample, len(runtimeMetrics)) - for i, rm := range runtimeMetrics { + samples := make([]runtimemetrics.Sample, len(supportedRuntimeMetrics)) + for i, rm := range supportedRuntimeMetrics { samples[i].Name = rm[0] } runtimemetrics.Read(samples) - for i, rm := range runtimeMetrics { + for i, rm := range supportedRuntimeMetrics { writeRuntimeMetric(w, rm[1], &samples[i]) } } func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample) { - switch sample.Value.Kind() { + kind := sample.Value.Kind() + switch kind { case runtimemetrics.KindBad: panic(fmt.Errorf("BUG: unexpected runtimemetrics.KindBad for sample.Name=%q", sample.Name)) case runtimemetrics.KindUint64: @@ -101,6 +122,8 @@ func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample) fmt.Fprintf(w, "%s %g\n", name, sample.Value.Float64()) case runtimemetrics.KindFloat64Histogram: writeRuntimeHistogramMetric(w, name, sample.Value.Float64Histogram()) + default: + panic(fmt.Errorf("unexpected metric kind=%d", kind)) } } diff --git a/vendor/github.com/VictoriaMetrics/metrics/push.go b/vendor/github.com/VictoriaMetrics/metrics/push.go index 4215f48ab..c2f93ab0f 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/push.go +++ b/vendor/github.com/VictoriaMetrics/metrics/push.go @@ -2,17 +2,59 @@ package metrics import ( "bytes" + "context" + "errors" "fmt" "io" "io/ioutil" "log" "net/http" "net/url" + "strings" "time" "compress/gzip" ) +// PushOptions is the list of options, which may be applied to InitPushWithOptions(). +type PushOptions struct { + // ExtraLabels is an optional comma-separated list of `label="value"` labels, which must be added to all the metrics before pushing them to pushURL. + ExtraLabels string + + // Headers is an optional list of HTTP headers to add to every push request to pushURL. + // + // Every item in the list must have the form `Header: value`. For example, `Authorization: Custom my-top-secret`. + Headers []string + + // Whether to disable HTTP request body compression before sending the metrics to pushURL. + // + // By default the compression is enabled. + DisableCompression bool +} + +// InitPushWithOptions sets up periodic push for globally registered metrics to the given pushURL with the given interval. +// +// The periodic push is stopped when ctx is canceled. +// +// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL. +// +// opts may contain additional configuration options if non-nil. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func InitPushWithOptions(ctx context.Context, pushURL string, interval time.Duration, pushProcessMetrics bool, opts *PushOptions) error { + writeMetrics := func(w io.Writer) { + WritePrometheus(w, pushProcessMetrics) + } + return InitPushExtWithOptions(ctx, pushURL, interval, writeMetrics, opts) +} + // InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added @@ -38,7 +80,7 @@ func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // -// If pushProcessMetrics is set to true, then 'process_*' metrics are also pushed to pushURL. +// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL. // // The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format @@ -55,12 +97,33 @@ func InitPush(pushURL string, interval time.Duration, extraLabels string, pushPr return InitPushExt(pushURL, interval, extraLabels, writeMetrics) } +// InitPushWithOptions sets up periodic push for metrics from s to the given pushURL with the given interval. +// +// The periodic push is stopped when the ctx is canceled. +// +// opts may contain additional configuration options if non-nil. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func (s *Set) InitPushWithOptions(ctx context.Context, pushURL string, interval time.Duration, opts *PushOptions) error { + writeMetrics := func(w io.Writer) { + s.WritePrometheus(w) + } + return InitPushExtWithOptions(ctx, pushURL, interval, writeMetrics, opts) +} + // InitPush sets up periodic push for metrics from s to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // -// / The metrics are pushed to pushURL in Prometheus text exposition format. +// The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to @@ -90,14 +153,33 @@ func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels strin // in this case metrics are pushed to all the provided pushURL urls. // // It is OK calling InitPushExt multiple times with different writeMetrics - -// in this case all the metrics generated by writeMetrics callbacks are writte to pushURL. +// in this case all the metrics generated by writeMetrics callbacks are written to pushURL. func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error { - if interval <= 0 { - return fmt.Errorf("interval must be positive; got %s", interval) - } - if err := validateTags(extraLabels); err != nil { - return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err) + opts := &PushOptions{ + ExtraLabels: extraLabels, } + return InitPushExtWithOptions(context.Background(), pushURL, interval, writeMetrics, opts) +} + +// InitPushExtWithOptions sets up periodic push for metrics obtained by calling writeMetrics with the given interval. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// The periodic push is stopped when the ctx is canceled. +// +// opts may contain additional configuration options if non-nil. +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushExtWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +// +// It is OK calling InitPushExtWithOptions multiple times with different writeMetrics - +// in this case all the metrics generated by writeMetrics callbacks are written to pushURL. +func InitPushExtWithOptions(ctx context.Context, pushURL string, interval time.Duration, writeMetrics func(w io.Writer), opts *PushOptions) error { + // validate pushURL pu, err := url.Parse(pushURL) if err != nil { return fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err) @@ -108,22 +190,66 @@ func InitPushExt(pushURL string, interval time.Duration, extraLabels string, wri if pu.Host == "" { return fmt.Errorf("missing host in pushURL=%q", pushURL) } - pushURLRedacted := pu.Redacted() - c := &http.Client{ - Timeout: interval, + + // validate interval + if interval <= 0 { + return fmt.Errorf("interval must be positive; got %s", interval) } + + // validate ExtraLabels + var extraLabels string + if opts != nil { + extraLabels = opts.ExtraLabels + } + if err := validateTags(extraLabels); err != nil { + return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err) + } + + // validate Headers + headers := make(http.Header) + if opts != nil { + for _, h := range opts.Headers { + n := strings.IndexByte(h, ':') + if n < 0 { + return fmt.Errorf("missing `:` delimiter in the header %q", h) + } + name := strings.TrimSpace(h[:n]) + value := strings.TrimSpace(h[n+1:]) + headers.Add(name, value) + } + } + + // validate DisableCompression + disableCompression := false + if opts != nil { + disableCompression = opts.DisableCompression + } + + // Initialize metrics for the given pushURL + pushURLRedacted := pu.Redacted() pushesTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted)) pushErrorsTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted)) bytesPushedTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted)) pushDuration := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted)) pushBlockSize := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted)) pushMetrics.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pushURLRedacted)).Set(interval.Seconds()) + + c := &http.Client{ + Timeout: interval, + } go func() { ticker := time.NewTicker(interval) var bb bytes.Buffer var tmpBuf []byte zw := gzip.NewWriter(&bb) - for range ticker.C { + stopCh := ctx.Done() + for { + select { + case <-ticker.C: + case <-stopCh: + return + } + bb.Reset() writeMetrics(&bb) if len(extraLabels) > 0 { @@ -133,31 +259,47 @@ func InitPushExt(pushURL string, interval time.Duration, extraLabels string, wri panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err)) } } - tmpBuf = append(tmpBuf[:0], bb.Bytes()...) - bb.Reset() - zw.Reset(&bb) - if _, err := zw.Write(tmpBuf); err != nil { - panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err)) - } - if err := zw.Close(); err != nil { - panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) + if !disableCompression { + tmpBuf = append(tmpBuf[:0], bb.Bytes()...) + bb.Reset() + zw.Reset(&bb) + if _, err := zw.Write(tmpBuf); err != nil { + panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err)) + } + if err := zw.Close(); err != nil { + panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) + } } pushesTotal.Inc() blockLen := bb.Len() bytesPushedTotal.Add(blockLen) pushBlockSize.Update(float64(blockLen)) - req, err := http.NewRequest("GET", pushURL, &bb) + req, err := http.NewRequestWithContext(ctx, "GET", pushURL, &bb) if err != nil { panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pushURLRedacted, err)) } + + // Set the needed headers + for name, values := range headers { + for _, value := range values { + req.Header.Add(name, value) + } + } req.Header.Set("Content-Type", "text/plain") - req.Header.Set("Content-Encoding", "gzip") + + if !disableCompression { + req.Header.Set("Content-Encoding", "gzip") + } + + // Perform the request startTime := time.Now() resp, err := c.Do(req) pushDuration.UpdateDuration(startTime) if err != nil { - log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) - pushErrorsTotal.Inc() + if !errors.Is(err, context.Canceled) { + log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) + pushErrorsTotal.Inc() + } continue } if resp.StatusCode/100 != 2 { diff --git a/vendor/modules.txt b/vendor/modules.txt index 82c917a43..930539a92 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -97,8 +97,8 @@ github.com/VictoriaMetrics/fastcache github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless -# github.com/VictoriaMetrics/metrics v1.25.3 -## explicit; go 1.20 +# github.com/VictoriaMetrics/metrics v1.26.1 +## explicit; go 1.16 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.70.0 ## explicit; go 1.13