all: add ability to push internal metrics to remote storage system specified via -pushmetrics.url

app/victoria-metrics/self_scraper.go

@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -60,7 +60,7 @@ func selfScraper(scrapeInterval time.Duration) {
 			currentTimestamp = currentTime.UnixNano() / 1e6
 		}
 		bb.Reset()
-		httpserver.WritePrometheusMetrics(&bb)
+		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
 		rows.Unmarshal(s)

app/vminsert/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -63,6 +64,7 @@ var staticServer = http.FileServer(http.FS(staticFiles))
 
 // Init initializes vminsert.
 func Init() {
+	pushmetrics.Init()
 	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
 	storage.SetMaxLabelValueLen(*maxLabelValueLen)

docs/CHANGELOG.md

@@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring additional headers for `datasource.url`, `remoteWrite.url` and `remoteRead.url` URLs. Headers also can be set on group level via `headers` param. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2860) for details.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): execute left and right sides of certain operations in parallel. For example, `q1 or q2`, `aggr_func(q1) <op> q2`, `q1 <op> aggr_func(q1)`. This may improve query performance if VictoriaMetrics has enough free resources for parallel processing of both sides of the operation. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmagent.html): allow duplicate username records with different passwords at configuration file. It should allow password rotation without username change.
+* FEATURE: add ability to push internal metrics (e.g. metrics exposed at `/metrics` page) to the configured remote storage from all the VictoriaMetrics components. See [these docs](https://docs.victoriametrics.com/#push-metrics).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): restart all the scrape jobs during [config reload](https://docs.victoriametrics.com/vmagent.html#configuration-update) after `global` section is changed inside `-promscrape.config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with AWS ECS credentials. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2875). Thanks to @transacid for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2876).

docs/Single-server-VictoriaMetrics.md

@@ -1636,6 +1636,20 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
 
 See also [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html).
 
+## Push metrics
+
+All the VictoriaMetrics apps support pushing their metrics exposed at `/metrics` page to remote storage in Prometheus text exposition format. This can be done by specifying the following command-line flags:
+
+* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus .
+* `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds.
+* `-pushmetrics.extraLabels` - the list of labels to add to all the metrics before sending them to `-pushmetrics.url`.
+
+For example, the following command instructs VictoriaMetrics to push metrics from `/metrics` page to `https://maas.victoriametrics.com/api/v1/import/prometheus` with `user:pass` [Basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication):
+
+```console
+/path/to/victoria-metrics -pushmetrics.url=https://user:pass@maas.victoriametrics.com/api/v1/import/prometheus
+```
+
 ## Cache removal
 
 VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown (e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup. Sometimes it is needed to remove such caches on the next startup. This can be performed by placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.

go.mod

@@ -9,7 +9,7 @@ require (
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.1.0
-	github.com/VictoriaMetrics/metrics v1.18.1
+	github.com/VictoriaMetrics/metrics v1.19.3
 	github.com/VictoriaMetrics/metricsql v0.44.1
 	github.com/aws/aws-sdk-go v1.44.56
 	github.com/cespare/xxhash/v2 v2.1.2

go.sum

@@ -107,8 +107,9 @@ github.com/VictoriaMetrics/fastcache v1.10.0 h1:5hDJnLsKLpnUEToub7ETuRu8RCkb40wo
 github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8=
 github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0=
 github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ=
-github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
 github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
+github.com/VictoriaMetrics/metrics v1.19.3 h1:cr7yyS6fHSzjvwCAYsJbvh8qaRfFzilkcqgHgO97e6Y=
+github.com/VictoriaMetrics/metrics v1.19.3/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
 github.com/VictoriaMetrics/metricsql v0.44.1 h1:qGoRt0g84uMUscVjS7P3uDZKmjJubWKaIx9v0iHKgck=
 github.com/VictoriaMetrics/metricsql v0.44.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=

lib/appmetrics/appmetrics.go

@@ -1,4 +1,4 @@
-package httpserver
+package appmetrics
 
 import (
 	"flag"

lib/httpserver/httpserver.go

@@ -20,6 +20,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -281,7 +282,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		}
 		startTime := time.Now()
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		WritePrometheusMetrics(w)
+		appmetrics.WritePrometheusMetrics(w)
 		metricsHandlerDuration.UpdateDuration(startTime)
 		return
 	case "/flags":

lib/pushmetrics/pushmetrics.go

@@ -0,0 +1,32 @@
+package pushmetrics
+
+import (
+	"flag"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	pushURL = flagutil.NewArray("pushmetrics.url", "Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . "+
+		"By default metrics exposed at /metrics page aren't pushed to any remote storage")
+	pushInterval    = flag.Duration("pushmetrics.interval", 10*time.Second, "Interval for pushing metrics to -pushmetrics.url")
+	pushExtraLabels = flagutil.NewArray("pushmetrics.extraLabels", "Optional labels to add to metrics pushed to -pushmetrics.url . "+
+		`For example, -pushmetrics.extraLabels='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url`)
+)
+
+func init() {
+	// The -pushmetrics.url flag can contain basic auth creds, so it mustn't be visible when exposing the flags.
+	flagutil.RegisterSecretFlag("pushmetrics.url")
+}
+
+// Init must be called after flag.Parse.
+func Init() {
+	extraLabels := strings.Join(*pushExtraLabels, ",")
+	for _, pu := range *pushURL {
+		metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics)
+	}
+}

vendor/github.com/VictoriaMetrics/metrics/README.md

@@ -16,6 +16,9 @@
 * Allows exporting distinct metric sets via distinct endpoints. See [Set](http://godoc.org/github.com/VictoriaMetrics/metrics#Set).
 * Supports [easy-to-use histograms](http://godoc.org/github.com/VictoriaMetrics/metrics#Histogram), which just work without any tuning.
   Read more about VictoriaMetrics histograms at [this article](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350).
+* Can push metrics to VictoriaMetrics or to any other remote storage, which accepts metrics
+  in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format).
+  See [these docs](http://godoc.org/github.com/VictoriaMetrics/metrics#InitPush).
 
 
 ### Limitations
@@ -28,8 +31,8 @@
 ```go
 import "github.com/VictoriaMetrics/metrics"
 
-// Register various time series.
-// Time series name may contain labels in Prometheus format - see below.
+// Register various metrics.
+// Metric name may contain labels in Prometheus format - see below.
 var (
 	// Register counter without labels.
 	requestsTotal = metrics.NewCounter("requests_total")
@@ -64,6 +67,10 @@ func requestHandler() {
 http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
 	metrics.WritePrometheus(w, true)
 })
+
+// ... or push registered metrics every 10 seconds to http://victoria-metrics:8428/api/v1/import/prometheus
+// with the added `instance="foobar"` label to all the pushed metrics.
+metrics.InitPush("http://victoria-metrics:8428/api/v1/import/prometheus", 10*time.Second, `instance="foobar"`, true)
 ```
 
 See [docs](http://godoc.org/github.com/VictoriaMetrics/metrics) for more info.
@@ -86,8 +93,8 @@ Because the `github.com/prometheus/client_golang` is too complex and is hard to
 #### Why the `metrics.WritePrometheus` doesn't expose documentation for each metric?
 
 Because this documentation is ignored by Prometheus. The documentation is for users.
-Just give meaningful names to the exported metrics or add comments in the source code
-or in other suitable place explaining each metric exposed from your application.
+Just give [meaningful names to the exported metrics](https://prometheus.io/docs/practices/naming/#metric-names)
+or add comments in the source code or in other suitable place explaining each metric exposed from your application.
 
 
 #### How to implement [CounterVec](https://godoc.org/github.com/prometheus/client_golang/prometheus#CounterVec) in `metrics`?

vendor/github.com/VictoriaMetrics/metrics/metrics.go

@@ -110,3 +110,8 @@ func WriteFDMetrics(w io.Writer) {
 func UnregisterMetric(name string) bool {
 	return defaultSet.UnregisterMetric(name)
 }
+
+// ListMetricNames returns a list of all the metric names from default set.
+func ListMetricNames() []string {
+	return defaultSet.ListMetricNames()
+}

vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go

@@ -45,13 +45,13 @@ func writeProcessMetrics(w io.Writer) {
 	statFilepath := "/proc/self/stat"
 	data, err := ioutil.ReadFile(statFilepath)
 	if err != nil {
-		log.Printf("ERROR: cannot open %s: %s", statFilepath, err)
+		log.Printf("ERROR: metrics: cannot open %s: %s", statFilepath, err)
 		return
 	}
 	// Search for the end of command.
 	n := bytes.LastIndex(data, []byte(") "))
 	if n < 0 {
-		log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath)
+		log.Printf("ERROR: metrics: cannot find command in parentheses in %q read from %s", data, statFilepath)
 		return
 	}
 	data = data[n+2:]
@@ -62,7 +62,7 @@ func writeProcessMetrics(w io.Writer) {
 		&p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt,
 		&p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss)
 	if err != nil {
-		log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err)
+		log.Printf("ERROR: metrics: cannot parse %q read from %s: %s", data, statFilepath, err)
 		return
 	}
 
@@ -89,17 +89,17 @@ func writeIOMetrics(w io.Writer) {
 	ioFilepath := "/proc/self/io"
 	data, err := ioutil.ReadFile(ioFilepath)
 	if err != nil {
-		log.Printf("ERROR: cannot open %q: %s", ioFilepath, err)
+		log.Printf("ERROR: metrics: cannot open %q: %s", ioFilepath, err)
 	}
 	getInt := func(s string) int64 {
 		n := strings.IndexByte(s, ' ')
 		if n < 0 {
-			log.Printf("ERROR: cannot find whitespace in %q at %q", s, ioFilepath)
+			log.Printf("ERROR: metrics: cannot find whitespace in %q at %q", s, ioFilepath)
 			return 0
 		}
 		v, err := strconv.ParseInt(s[n+1:], 10, 64)
 		if err != nil {
-			log.Printf("ERROR: cannot parse %q at %q: %s", s, ioFilepath, err)
+			log.Printf("ERROR: metrics: cannot parse %q at %q: %s", s, ioFilepath, err)
 			return 0
 		}
 		return v
@@ -137,12 +137,12 @@ var startTimeSeconds = time.Now().Unix()
 func writeFDMetrics(w io.Writer) {
 	totalOpenFDs, err := getOpenFDsCount("/proc/self/fd")
 	if err != nil {
-		log.Printf("ERROR: cannot determine open file descriptors count: %s", err)
+		log.Printf("ERROR: metrics: cannot determine open file descriptors count: %s", err)
 		return
 	}
 	maxOpenFDs, err := getMaxFilesLimit("/proc/self/limits")
 	if err != nil {
-		log.Printf("ERROR: cannot determine the limit on open file descritors: %s", err)
+		log.Printf("ERROR: metrics: cannot determine the limit on open file descritors: %s", err)
 		return
 	}
 	fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs)
@@ -211,7 +211,7 @@ type memStats struct {
 func writeProcessMemMetrics(w io.Writer) {
 	ms, err := getMemStats("/proc/self/status")
 	if err != nil {
-		log.Printf("ERROR: cannot determine memory status: %s", err)
+		log.Printf("ERROR: metrics: cannot determine memory status: %s", err)
 		return
 	}
 	fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak)

vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package metrics

vendor/github.com/VictoriaMetrics/metrics/push.go

@@ -0,0 +1,184 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval.
+//
+// extraLabels may contain comma-separated list of `label="value"` labels, which will be added
+// to all the metrics before pushing them to pushURL.
+//
+// The metrics are pushed to pushURL in Prometheus text exposition format.
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+//
+// It is OK calling InitPushProcessMetrics multiple times with different pushURL -
+// in this case metrics are pushed to all the provided pushURL urls.
+func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels string) error {
+	writeMetrics := func(w io.Writer) {
+		WriteProcessMetrics(w)
+	}
+	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
+}
+
+// InitPush sets up periodic push for globally registered metrics to the given pushURL with the given interval.
+//
+// extraLabels may contain comma-separated list of `label="value"` labels, which will be added
+// to all the metrics before pushing them to pushURL.
+//
+// If pushProcessMetrics is set to true, then 'process_*' metrics are also pushed to pushURL.
+//
+// The metrics are pushed to pushURL in Prometheus text exposition format.
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+//
+// It is OK calling InitPush multiple times with different pushURL -
+// in this case metrics are pushed to all the provided pushURL urls.
+func InitPush(pushURL string, interval time.Duration, extraLabels string, pushProcessMetrics bool) error {
+	writeMetrics := func(w io.Writer) {
+		WritePrometheus(w, pushProcessMetrics)
+	}
+	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
+}
+
+// InitPush sets up periodic push for metrics from s to the given pushURL with the given interval.
+//
+// extraLabels may contain comma-separated list of `label="value"` labels, which will be added
+// to all the metrics before pushing them to pushURL.
+//
+/// The metrics are pushed to pushURL in Prometheus text exposition format.
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+//
+// It is OK calling InitPush multiple times with different pushURL -
+// in this case metrics are pushed to all the provided pushURL urls.
+func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels string) error {
+	writeMetrics := func(w io.Writer) {
+		s.WritePrometheus(w)
+	}
+	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
+}
+
+// InitPushExt sets up periodic push for metrics obtained by calling writeMetrics with the given interval.
+//
+// extraLabels may contain comma-separated list of `label="value"` labels, which will be added
+// to all the metrics before pushing them to pushURL.
+//
+// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments.
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+//
+// It is OK calling InitPushExt multiple times with different pushURL -
+// in this case metrics are pushed to all the provided pushURL urls.
+func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error {
+	if interval <= 0 {
+		return fmt.Errorf("interval must be positive; got %s", interval)
+	}
+	if err := validateTags(extraLabels); err != nil {
+		return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err)
+	}
+	pu, err := url.Parse(pushURL)
+	if err != nil {
+		return fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err)
+	}
+	if pu.Scheme != "http" && pu.Scheme != "https" {
+		return fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL)
+	}
+	if pu.Host == "" {
+		return fmt.Errorf("missing host in pushURL=%q", pushURL)
+	}
+	pushURLRedacted := pu.Redacted()
+	c := &http.Client{
+		Timeout: interval,
+	}
+	go func() {
+		ticker := time.NewTicker(interval)
+		var bb bytes.Buffer
+		var tmpBuf []byte
+		for range ticker.C {
+			bb.Reset()
+			writeMetrics(&bb)
+			if len(extraLabels) > 0 {
+				tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels)
+				bb.Reset()
+				bb.Write(tmpBuf)
+			}
+			resp, err := c.Post(pushURL, "text/plain", &bb)
+			if err != nil {
+				log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err)
+				continue
+			}
+			if resp.StatusCode/100 != 2 {
+				body, _ := ioutil.ReadAll(resp.Body)
+				_ = resp.Body.Close()
+				log.Printf("ERROR: metrics.push: unexpected status code in response from %q: %d; expecting 2xx; response body: %q",
+					pushURLRedacted, resp.StatusCode, body)
+				continue
+			}
+			_ = resp.Body.Close()
+		}
+	}()
+	return nil
+}
+
+func addExtraLabels(dst, src []byte, extraLabels string) []byte {
+	for len(src) > 0 {
+		var line []byte
+		n := bytes.IndexByte(src, '\n')
+		if n >= 0 {
+			line = src[:n]
+			src = src[n+1:]
+		} else {
+			line = src
+			src = nil
+		}
+		line = bytes.TrimSpace(line)
+		if len(line) == 0 {
+			// Skip empy lines
+			continue
+		}
+		if bytes.HasPrefix(line, bashBytes) {
+			// Copy comments as is
+			dst = append(dst, line...)
+			dst = append(dst, '\n')
+			continue
+		}
+		n = bytes.IndexByte(line, '{')
+		if n >= 0 {
+			dst = append(dst, line[:n+1]...)
+			dst = append(dst, extraLabels...)
+			dst = append(dst, ',')
+			dst = append(dst, line[n+1:]...)
+		} else {
+			n = bytes.LastIndexByte(line, ' ')
+			if n < 0 {
+				panic(fmt.Errorf("BUG: missing whitespace between metric name and metric value in Prometheus text exposition line %q", line))
+			}
+			dst = append(dst, line[:n]...)
+			dst = append(dst, '{')
+			dst = append(dst, extraLabels...)
+			dst = append(dst, '}')
+			dst = append(dst, line[n:]...)
+		}
+		dst = append(dst, '\n')
+	}
+	return dst
+}
+
+var bashBytes = []byte("#")

vendor/modules.txt

@@ -24,7 +24,7 @@ github.com/VictoriaMetrics/fastcache
 github.com/VictoriaMetrics/fasthttp
 github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
-# github.com/VictoriaMetrics/metrics v1.18.1
+# github.com/VictoriaMetrics/metrics v1.19.3
 ## explicit; go 1.12
 github.com/VictoriaMetrics/metrics
 # github.com/VictoriaMetrics/metricsql v0.44.1