From 7afcc4245425381f8d582e3d1db837a79a30a4be Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 25 Jul 2022 10:42:26 +0300 Subject: [PATCH] all: push metrics to -pusmetrics.url in gzip-compressed form in order to reduce the needed network bandwidth --- README.md | 2 +- docs/README.md | 3 ++- docs/Single-server-VictoriaMetrics.md | 3 ++- go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metrics/push.go | 27 +++++++++++++++++-- vendor/modules.txt | 2 +- 7 files changed, 34 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3949b4c06..845696072 100644 --- a/README.md +++ b/README.md @@ -1637,7 +1637,7 @@ See also [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting All the VictoriaMetrics apps support pushing their metrics exposed at `/metrics` page to remote storage in Prometheus text exposition format. This can be done by specifying the following command-line flags: -* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . +* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. This allows reducing the required network bandwidth for metrics push. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. * `-pushmetrics.extraLabel` - label to add to all the metrics before sending them to `-pushmetrics.url`. The label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics sending them to `-pushmetrics.url`. diff --git a/docs/README.md b/docs/README.md index 1e7dd0975..845696072 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,7 @@ VictoriaMetrics has the following prominent features: * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter). * It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/). * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster). +* It can store data on [NFS-based storages](https://en.wikipedia.org/wiki/Network_File_System) such as [Amazon EFS](https://aws.amazon.com/efs/) and [Google Filestore](https://cloud.google.com/filestore). See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html). @@ -1636,7 +1637,7 @@ See also [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting All the VictoriaMetrics apps support pushing their metrics exposed at `/metrics` page to remote storage in Prometheus text exposition format. This can be done by specifying the following command-line flags: -* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . +* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. This allows reducing the required network bandwidth for metrics push. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. * `-pushmetrics.extraLabel` - label to add to all the metrics before sending them to `-pushmetrics.url`. The label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics sending them to `-pushmetrics.url`. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 1ab5670b6..a45a583f6 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -70,6 +70,7 @@ VictoriaMetrics has the following prominent features: * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter). * It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/). * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster). +* It can store data on [NFS-based storages](https://en.wikipedia.org/wiki/Network_File_System) such as [Amazon EFS](https://aws.amazon.com/efs/) and [Google Filestore](https://cloud.google.com/filestore). See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html). @@ -1640,7 +1641,7 @@ See also [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting All the VictoriaMetrics apps support pushing their metrics exposed at `/metrics` page to remote storage in Prometheus text exposition format. This can be done by specifying the following command-line flags: -* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . +* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . Metrics are pushed to the provided `-pushmetrics.url` in a compressed form with `Content-Encoding: gzip` request header. This allows reducing the required network bandwidth for metrics push. * `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. * `-pushmetrics.extraLabel` - label to add to all the metrics before sending them to `-pushmetrics.url`. The label must be specified in the format `label="value"`. It is OK to specify multiple `-pushmetrics.extraLabel` command-line flags. In this case all the specified labels are added to all the metrics sending them to `-pushmetrics.url`. diff --git a/go.mod b/go.mod index 35c97601a..d89dbde38 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 - github.com/VictoriaMetrics/metrics v1.19.3 + github.com/VictoriaMetrics/metrics v1.20.1 github.com/VictoriaMetrics/metricsql v0.44.1 github.com/aws/aws-sdk-go v1.44.59 github.com/cespare/xxhash/v2 v2.1.2 diff --git a/go.sum b/go.sum index 2899bbee7..b53ffe603 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJ github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0= github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metrics v1.19.3 h1:cr7yyS6fHSzjvwCAYsJbvh8qaRfFzilkcqgHgO97e6Y= -github.com/VictoriaMetrics/metrics v1.19.3/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= +github.com/VictoriaMetrics/metrics v1.20.1 h1:XqQbRKYzwkmo0DKKDbvp6V7upUqErlqd0vXPoeBsEbU= +github.com/VictoriaMetrics/metrics v1.20.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metricsql v0.44.1 h1:qGoRt0g84uMUscVjS7P3uDZKmjJubWKaIx9v0iHKgck= github.com/VictoriaMetrics/metricsql v0.44.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= diff --git a/vendor/github.com/VictoriaMetrics/metrics/push.go b/vendor/github.com/VictoriaMetrics/metrics/push.go index a63c93693..c592ae69b 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/push.go +++ b/vendor/github.com/VictoriaMetrics/metrics/push.go @@ -9,6 +9,8 @@ import ( "net/http" "net/url" "time" + + "compress/gzip" ) // InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval. @@ -86,6 +88,9 @@ func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels strin // // It is OK calling InitPushExt multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. +// +// It is OK calling InitPushExt multiple times with different writeMetrics - +// in this case all the metrics generated by writeMetrics callbacks are writte to pushURL. func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error { if interval <= 0 { return fmt.Errorf("interval must be positive; got %s", interval) @@ -111,15 +116,33 @@ func InitPushExt(pushURL string, interval time.Duration, extraLabels string, wri ticker := time.NewTicker(interval) var bb bytes.Buffer var tmpBuf []byte + zw := gzip.NewWriter(&bb) for range ticker.C { bb.Reset() writeMetrics(&bb) if len(extraLabels) > 0 { tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels) bb.Reset() - bb.Write(tmpBuf) + if _, err := bb.Write(tmpBuf); err != nil { + panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err)) + } } - resp, err := c.Post(pushURL, "text/plain", &bb) + tmpBuf = append(tmpBuf[:0], bb.Bytes()...) + bb.Reset() + zw.Reset(&bb) + if _, err := zw.Write(tmpBuf); err != nil { + panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err)) + } + if err := zw.Close(); err != nil { + panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) + } + req, err := http.NewRequest("GET", pushURL, &bb) + if err != nil { + log.Printf("ERROR: metrics.push: cannot initialize request for metrics push to %q: %s", pushURLRedacted, err) + } + req.Header.Set("Content-Type", "text/plain") + req.Header.Set("Content-Encoding", "gzip") + resp, err := c.Do(req) if err != nil { log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) continue diff --git a/vendor/modules.txt b/vendor/modules.txt index e006b34b9..f46584e3b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -25,7 +25,7 @@ github.com/VictoriaMetrics/fastcache github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless -# github.com/VictoriaMetrics/metrics v1.19.3 +# github.com/VictoriaMetrics/metrics v1.20.1 ## explicit; go 1.12 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.44.1