all: add -metrics.exposeMetadata command-line flag, which can be used for adding TYPE and HELP metadata for metrics exposed at /metrics page

This may be needed for systems, which require this metadata such as Google Cloud Managed Prometheus. See https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
2024-11-21 14:44:00 +00:00 · 2023-12-19 03:19:42 +02:00 · 2023-12-19 03:19:42 +02:00 · 326a77c697
commit 326a77c697
parent bc3feebf69
28 changed files with 495 additions and 215 deletions
--- a/README.md
+++ b/README.md
@ -2722,6 +2722,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -newrelic.maxInsertRequestSize size
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@ -31,6 +31,7 @@ The sandbox cluster installation is running under the constant load generated by
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
 * FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
+* FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).

 * BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped.
 * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0).
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@ -1085,6 +1085,8 @@ Below is the output for `/path/to/vminsert -help`:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -newrelic.maxInsertRequestSize size
@ -1288,6 +1290,8 @@ Below is the output for `/path/to/vmselect -help`:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
@ -1554,6 +1558,8 @@ Below is the output for `/path/to/vmstorage -help`:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
--- a/docs/README.md
+++ b/docs/README.md
@ -2725,6 +2725,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -newrelic.maxInsertRequestSize size
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@ -2733,6 +2733,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -newrelic.maxInsertRequestSize size
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@ -1705,6 +1705,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -newrelic.maxInsertRequestSize size
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@ -1124,6 +1124,8 @@ The shortlist of configuration flags is the following:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -notifier.basicAuth.password array
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@ -881,6 +881,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
--- a/docs/vmbackup.md
+++ b/docs/vmbackup.md
@ -385,6 +385,8 @@ Run `vmbackup -help` in order to see all the available options:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -origin string
--- a/docs/vmbackupmanager.md
+++ b/docs/vmbackupmanager.md
@ -526,6 +526,8 @@ command-line flags:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
--- a/docs/vmgateway.md
+++ b/docs/vmgateway.md
@ -421,6 +421,8 @@ The shortlist of configuration flags include the following:
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
--- a/docs/vmrestore.md
+++ b/docs/vmrestore.md
@ -182,6 +182,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
+  -metrics.exposeMetadata
+     Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
  -metricsAuthKey string
     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
--- a/go.mod
+++ b/go.mod
@ -11,7 +11,7 @@ require (
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.2.0
-	github.com/VictoriaMetrics/metrics v1.26.1
+	github.com/VictoriaMetrics/metrics v1.28.2
 	github.com/VictoriaMetrics/metricsql v0.70.0
 	github.com/aws/aws-sdk-go-v2 v1.24.0
 	github.com/aws/aws-sdk-go-v2/config v1.26.1
--- a/go.sum
+++ b/go.sum
@ -63,8 +63,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT
 github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I=
 github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE=
 github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
-github.com/VictoriaMetrics/metrics v1.26.1 h1:zJmbXMED+mnirSpJeVJKxXNaFKJaX3Bj/VuV2j2uKKY=
-github.com/VictoriaMetrics/metrics v1.26.1/go.mod h1:EQ7JHshszbJYGuC1IIqkL+K73bTTTurJMQvUT5NMT5Q=
+github.com/VictoriaMetrics/metrics v1.28.2 h1:yWIq53N8G6hJI6vQ8I2NsiD4p+UsmQa/msLo3yqDfPQ=
+github.com/VictoriaMetrics/metrics v1.28.2/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
 github.com/VictoriaMetrics/metricsql v0.70.0 h1:G0k/m1yAF6pmk0dM3VT9/XI5PZ8dL7EbcLhREf4bgeI=
 github.com/VictoriaMetrics/metricsql v0.70.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
--- a/lib/appmetrics/appmetrics.go
+++ b/lib/appmetrics/appmetrics.go
@ -18,10 +18,22 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )

+var exposeMetadata = flag.Bool("metrics.exposeMetadata", false, "Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . "+
+	"The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - "+
+	"https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type")
+
+var exposeMetadataOnce sync.Once
+
+func initExposeMetadata() {
+	metrics.ExposeMetadata(*exposeMetadata)
+}
+
 var versionRe = regexp.MustCompile(`v\d+\.\d+\.\d+(?:-enterprise)?(?:-cluster)?`)

 // WritePrometheusMetrics writes all the registered metrics to w in Prometheus exposition format.
 func WritePrometheusMetrics(w io.Writer) {
+	exposeMetadataOnce.Do(initExposeMetadata)
+
 	currentTime := time.Now()
 	metricsCacheLock.Lock()
 	if currentTime.Sub(metricsCacheLastUpdateTime) > time.Second {
@ -46,22 +58,22 @@ func writePrometheusMetrics(w io.Writer) {
 	metrics.WritePrometheus(w, true)
 	metrics.WriteFDMetrics(w)

-	fmt.Fprintf(w, "vm_app_version{version=%q, short_version=%q} 1\n", buildinfo.Version,
-		versionRe.FindString(buildinfo.Version))
-	fmt.Fprintf(w, "vm_allowed_memory_bytes %d\n", memory.Allowed())
-	fmt.Fprintf(w, "vm_available_memory_bytes %d\n", memory.Allowed()+memory.Remaining())
-	fmt.Fprintf(w, "vm_available_cpu_cores %d\n", cgroup.AvailableCPUs())
-	fmt.Fprintf(w, "vm_gogc %d\n", cgroup.GetGOGC())
+	metrics.WriteGaugeUint64(w, fmt.Sprintf("vm_app_version{version=%q, short_version=%q}", buildinfo.Version, versionRe.FindString(buildinfo.Version)), 1)
+	metrics.WriteGaugeUint64(w, "vm_allowed_memory_bytes", uint64(memory.Allowed()))
+	metrics.WriteGaugeUint64(w, "vm_available_memory_bytes", uint64(memory.Allowed()+memory.Remaining()))
+	metrics.WriteGaugeUint64(w, "vm_available_cpu_cores", uint64(cgroup.AvailableCPUs()))
+	metrics.WriteGaugeUint64(w, "vm_gogc", uint64(cgroup.GetGOGC()))

 	// Export start time and uptime in seconds
-	fmt.Fprintf(w, "vm_app_start_timestamp %d\n", startTime.Unix())
-	fmt.Fprintf(w, "vm_app_uptime_seconds %d\n", int(time.Since(startTime).Seconds()))
+	metrics.WriteGaugeUint64(w, "vm_app_start_timestamp", uint64(startTime.Unix()))
+	metrics.WriteGaugeUint64(w, "vm_app_uptime_seconds", uint64(time.Since(startTime).Seconds()))

 	// Export flags as metrics.
 	isSetMap := make(map[string]bool)
 	flag.Visit(func(f *flag.Flag) {
 		isSetMap[f.Name] = true
 	})
+	metrics.WriteMetadataIfNeeded(w, "flag", "gauge")
 	flag.VisitAll(func(f *flag.Flag) {
 		lname := strings.ToLower(f.Name)
 		value := f.Value.String()
--- a/vendor/github.com/VictoriaMetrics/metrics/README.md
+++ b/vendor/github.com/VictoriaMetrics/metrics/README.md
@ -73,8 +73,11 @@ http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
 metrics.InitPush("http://victoria-metrics:8428/api/v1/import/prometheus", 10*time.Second, `instance="foobar"`, true)
 ```

-See [docs](http://godoc.org/github.com/VictoriaMetrics/metrics) for more info.
+By default, exposed metrics [do not have](https://github.com/VictoriaMetrics/metrics/issues/48#issuecomment-1620765811)
+`TYPE` or `HELP` meta information. Call [`ExposeMetadata(true)`](https://pkg.go.dev/github.com/VictoriaMetrics/metrics#ExposeMetadata)
+in order to generate `TYPE` and `HELP` meta information per each metric.

+See [docs](https://pkg.go.dev/github.com/VictoriaMetrics/metrics) for more info.

 ### Users

--- a/vendor/github.com/VictoriaMetrics/metrics/counter.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/counter.go
@ -58,6 +58,10 @@ func (c *Counter) marshalTo(prefix string, w io.Writer) {
 	fmt.Fprintf(w, "%s %d\n", prefix, v)
 }

+func (c *Counter) metricType() string {
+	return "counter"
+}
+
 // GetOrCreateCounter returns registered counter with the given name
 // or creates new counter if the registry doesn't contain counter with
 // the given name.
--- a/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go
@ -63,6 +63,10 @@ func (fc *FloatCounter) marshalTo(prefix string, w io.Writer) {
 	fmt.Fprintf(w, "%s %g\n", prefix, v)
 }

+func (fc *FloatCounter) metricType() string {
+	return "counter"
+}
+
 // GetOrCreateFloatCounter returns registered FloatCounter with the given name
 // or creates new FloatCounter if the registry doesn't contain FloatCounter with
 // the given name.
--- a/vendor/github.com/VictoriaMetrics/metrics/gauge.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/gauge.go
@ -46,6 +46,10 @@ func (g *Gauge) marshalTo(prefix string, w io.Writer) {
 	}
 }

+func (g *Gauge) metricType() string {
+	return "gauge"
+}
+
 // GetOrCreateGauge returns registered gauge with the given name
 // or creates new gauge if the registry doesn't contain gauge with
 // the given name.
--- a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go
@ -47,34 +47,34 @@ func writeGoMetrics(w io.Writer) {

 	var ms runtime.MemStats
 	runtime.ReadMemStats(&ms)
-	fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc)
-	fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc)
-	fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys)
-	fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees)
-	fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction)
-	fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys)
+	WriteGaugeUint64(w, "go_memstats_alloc_bytes", ms.Alloc)
+	WriteCounterUint64(w, "go_memstats_alloc_bytes_total", ms.TotalAlloc)
+	WriteGaugeUint64(w, "go_memstats_buck_hash_sys_bytes", ms.BuckHashSys)
+	WriteCounterUint64(w, "go_memstats_frees_total", ms.Frees)
+	WriteGaugeFloat64(w, "go_memstats_gc_cpu_fraction", ms.GCCPUFraction)
+	WriteGaugeUint64(w, "go_memstats_gc_sys_bytes", ms.GCSys)

-	fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc)
-	fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle)
-	fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse)
-	fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects)
-	fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased)
-	fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys)
-	fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9)
-	fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups)
-	fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs)
-	fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse)
-	fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys)
-	fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse)
-	fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys)
-	fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC)
-	fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys)
-	fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse)
-	fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys)
-	fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys)
+	WriteGaugeUint64(w, "go_memstats_heap_alloc_bytes", ms.HeapAlloc)
+	WriteGaugeUint64(w, "go_memstats_heap_idle_bytes", ms.HeapIdle)
+	WriteGaugeUint64(w, "go_memstats_heap_inuse_bytes", ms.HeapInuse)
+	WriteGaugeUint64(w, "go_memstats_heap_objects", ms.HeapObjects)
+	WriteGaugeUint64(w, "go_memstats_heap_released_bytes", ms.HeapReleased)
+	WriteGaugeUint64(w, "go_memstats_heap_sys_bytes", ms.HeapSys)
+	WriteGaugeFloat64(w, "go_memstats_last_gc_time_seconds", float64(ms.LastGC)/1e9)
+	WriteCounterUint64(w, "go_memstats_lookups_total", ms.Lookups)
+	WriteCounterUint64(w, "go_memstats_mallocs_total", ms.Mallocs)
+	WriteGaugeUint64(w, "go_memstats_mcache_inuse_bytes", ms.MCacheInuse)
+	WriteGaugeUint64(w, "go_memstats_mcache_sys_bytes", ms.MCacheSys)
+	WriteGaugeUint64(w, "go_memstats_mspan_inuse_bytes", ms.MSpanInuse)
+	WriteGaugeUint64(w, "go_memstats_mspan_sys_bytes", ms.MSpanSys)
+	WriteGaugeUint64(w, "go_memstats_next_gc_bytes", ms.NextGC)
+	WriteGaugeUint64(w, "go_memstats_other_sys_bytes", ms.OtherSys)
+	WriteGaugeUint64(w, "go_memstats_stack_inuse_bytes", ms.StackInuse)
+	WriteGaugeUint64(w, "go_memstats_stack_sys_bytes", ms.StackSys)
+	WriteGaugeUint64(w, "go_memstats_sys_bytes", ms.Sys)

-	fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall())
-	fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU())
+	WriteCounterUint64(w, "go_cgo_calls_count", uint64(runtime.NumCgoCall()))
+	WriteGaugeUint64(w, "go_cpu_count", uint64(runtime.NumCPU()))

 	gcPauses := histogram.NewFast()
 	for _, pauseNs := range ms.PauseNs[:] {
@ -82,20 +82,25 @@ func writeGoMetrics(w io.Writer) {
 	}
 	phis := []float64{0, 0.25, 0.5, 0.75, 1}
 	quantiles := make([]float64, 0, len(phis))
+	WriteMetadataIfNeeded(w, "go_gc_duration_seconds", "summary")
 	for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
 		fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q)
 	}
-	fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9)
-	fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC)
-	fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC)
+	fmt.Fprintf(w, "go_gc_duration_seconds_sum %g\n", float64(ms.PauseTotalNs)/1e9)
+	fmt.Fprintf(w, "go_gc_duration_seconds_count %d\n", ms.NumGC)

-	fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0))
-	fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine())
+	WriteCounterUint64(w, "go_gc_forced_count", uint64(ms.NumForcedGC))
+
+	WriteGaugeUint64(w, "go_gomaxprocs", uint64(runtime.GOMAXPROCS(0)))
+	WriteGaugeUint64(w, "go_goroutines", uint64(runtime.NumGoroutine()))
 	numThread, _ := runtime.ThreadCreateProfile(nil)
-	fmt.Fprintf(w, `go_threads %d`+"\n", numThread)
+	WriteGaugeUint64(w, "go_threads", uint64(numThread))

 	// Export build details.
+	WriteMetadataIfNeeded(w, "go_info", "gauge")
 	fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
+
+	WriteMetadataIfNeeded(w, "go_info_ext", "gauge")
 	fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
 		runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
 }
@ -117,11 +122,22 @@ func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample)
 	case runtimemetrics.KindBad:
 		panic(fmt.Errorf("BUG: unexpected runtimemetrics.KindBad for sample.Name=%q", sample.Name))
 	case runtimemetrics.KindUint64:
-		fmt.Fprintf(w, "%s %d\n", name, sample.Value.Uint64())
+		v := sample.Value.Uint64()
+		if strings.HasSuffix(name, "_total") {
+			WriteCounterUint64(w, name, v)
+		} else {
+			WriteGaugeUint64(w, name, v)
+		}
 	case runtimemetrics.KindFloat64:
-		fmt.Fprintf(w, "%s %g\n", name, sample.Value.Float64())
+		v := sample.Value.Float64()
+		if isCounterName(name) {
+			WriteCounterFloat64(w, name, v)
+		} else {
+			WriteGaugeFloat64(w, name, v)
+		}
 	case runtimemetrics.KindFloat64Histogram:
-		writeRuntimeHistogramMetric(w, name, sample.Value.Float64Histogram())
+		h := sample.Value.Float64Histogram()
+		writeRuntimeHistogramMetric(w, name, h)
 	default:
 		panic(fmt.Errorf("unexpected metric kind=%d", kind))
 	}
@ -149,6 +165,7 @@ func writeRuntimeHistogramMetric(w io.Writer, name string, h *runtimemetrics.Flo

 	totalCount := uint64(0)
 	iNext := 0.0
+	WriteMetadataIfNeeded(w, name, "histogram")
 	for i, count := range counts {
 		totalCount += count
 		if float64(i) >= iNext {
--- a/vendor/github.com/VictoriaMetrics/metrics/histogram.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/histogram.go
@ -228,3 +228,7 @@ func (h *Histogram) getSum() float64 {
 	h.mu.Unlock()
 	return sum
 }
+
+func (h *Histogram) metricType() string {
+	return "histogram"
+}
--- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go
@ -13,9 +13,12 @@
 package metrics

 import (
+	"fmt"
 	"io"
 	"sort"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"unsafe"
 )

@ -27,6 +30,7 @@ type namedMetric struct {

 type metric interface {
 	marshalTo(prefix string, w io.Writer)
+	metricType() string
 }

 var defaultSet = NewSet()
@ -241,3 +245,76 @@ func ListMetricNames() []string {
 func GetDefaultSet() *Set {
 	return defaultSet
 }
+
+// ExposeMetadata allows enabling adding TYPE and HELP metadata to the exposed metrics globally.
+//
+// It is safe to call this method multiple times. It is allowed to change it in runtime.
+// ExposeMetadata is set to false by default.
+func ExposeMetadata(v bool) {
+	n := 0
+	if v {
+		n = 1
+	}
+	atomic.StoreUint32(&exposeMetadata, uint32(n))
+}
+
+func isMetadataEnabled() bool {
+	n := atomic.LoadUint32(&exposeMetadata)
+	return n != 0
+}
+
+var exposeMetadata uint32
+
+func isCounterName(name string) bool {
+	return strings.HasSuffix(name, "_total")
+}
+
+// WriteGaugeUint64 writes gauge metric with the given name and value to w in Prometheus text exposition format.
+func WriteGaugeUint64(w io.Writer, name string, value uint64) {
+	writeMetricUint64(w, name, "gauge", value)
+}
+
+// WriteGaugeFloat64 writes gauge metric with the given name and value to w in Prometheus text exposition format.
+func WriteGaugeFloat64(w io.Writer, name string, value float64) {
+	writeMetricFloat64(w, name, "gauge", value)
+}
+
+// WriteCounterUint64 writes counter metric with the given name and value to w in Prometheus text exposition format.
+func WriteCounterUint64(w io.Writer, name string, value uint64) {
+	writeMetricUint64(w, name, "counter", value)
+}
+
+// WriteCounterFloat64 writes counter metric with the given name and value to w in Prometheus text exposition format.
+func WriteCounterFloat64(w io.Writer, name string, value float64) {
+	writeMetricFloat64(w, name, "counter", value)
+}
+
+func writeMetricUint64(w io.Writer, metricName, metricType string, value uint64) {
+	WriteMetadataIfNeeded(w, metricName, metricType)
+	fmt.Fprintf(w, "%s %d\n", metricName, value)
+}
+
+func writeMetricFloat64(w io.Writer, metricName, metricType string, value float64) {
+	WriteMetadataIfNeeded(w, metricName, metricType)
+	fmt.Fprintf(w, "%s %g\n", metricName, value)
+}
+
+// WriteMetadataIfNeeded writes HELP and TYPE metadata for the given metricName and metricType if this is globally enabled via ExposeMetadata().
+//
+// If the metadata exposition isn't enabled, then this function is no-op.
+func WriteMetadataIfNeeded(w io.Writer, metricName, metricType string) {
+	if !isMetadataEnabled() {
+		return
+	}
+	metricFamily := getMetricFamily(metricName)
+	fmt.Fprintf(w, "# HELP %s\n", metricFamily)
+	fmt.Fprintf(w, "# TYPE %s %s\n", metricFamily, metricType)
+}
+
+func getMetricFamily(metricName string) string {
+	n := strings.IndexByte(metricName, '{')
+	if n < 0 {
+		return metricName
+	}
+	return metricName[:n]
+}
--- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go
@ -74,15 +74,15 @@ func writeProcessMetrics(w io.Writer) {

 	utime := float64(p.Utime) / userHZ
 	stime := float64(p.Stime) / userHZ
-	fmt.Fprintf(w, "process_cpu_seconds_system_total %g\n", stime)
-	fmt.Fprintf(w, "process_cpu_seconds_total %g\n", utime+stime)
-	fmt.Fprintf(w, "process_cpu_seconds_user_total %g\n", utime)
-	fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt)
-	fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt)
-	fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads)
-	fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096)
-	fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds)
-	fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize)
+	WriteCounterFloat64(w, "process_cpu_seconds_system_total", stime)
+	WriteCounterFloat64(w, "process_cpu_seconds_total", utime+stime)
+	WriteCounterFloat64(w, "process_cpu_seconds_user_total", utime)
+	WriteCounterUint64(w, "process_major_pagefaults_total", uint64(p.Majflt))
+	WriteCounterUint64(w, "process_minor_pagefaults_total", uint64(p.Minflt))
+	WriteGaugeUint64(w, "process_num_threads", uint64(p.NumThreads))
+	WriteGaugeUint64(w, "process_resident_memory_bytes", uint64(p.Rss)*4096)
+	WriteGaugeUint64(w, "process_start_time_seconds", uint64(startTimeSeconds))
+	WriteGaugeUint64(w, "process_virtual_memory_bytes", uint64(p.Vsize))
 	writeProcessMemMetrics(w)
 	writeIOMetrics(w)
 }
@ -133,12 +133,12 @@ func writeIOMetrics(w io.Writer) {
 			writeBytes = getInt(s)
 		}
 	}
-	fmt.Fprintf(w, "process_io_read_bytes_total %d\n", rchar)
-	fmt.Fprintf(w, "process_io_written_bytes_total %d\n", wchar)
-	fmt.Fprintf(w, "process_io_read_syscalls_total %d\n", syscr)
-	fmt.Fprintf(w, "process_io_write_syscalls_total %d\n", syscw)
-	fmt.Fprintf(w, "process_io_storage_read_bytes_total %d\n", readBytes)
-	fmt.Fprintf(w, "process_io_storage_written_bytes_total %d\n", writeBytes)
+	WriteGaugeUint64(w, "process_io_read_bytes_total", uint64(rchar))
+	WriteGaugeUint64(w, "process_io_written_bytes_total", uint64(wchar))
+	WriteGaugeUint64(w, "process_io_read_syscalls_total", uint64(syscr))
+	WriteGaugeUint64(w, "process_io_write_syscalls_total", uint64(syscw))
+	WriteGaugeUint64(w, "process_io_storage_read_bytes_total", uint64(readBytes))
+	WriteGaugeUint64(w, "process_io_storage_written_bytes_total", uint64(writeBytes))
 }

 var startTimeSeconds = time.Now().Unix()
@ -155,8 +155,8 @@ func writeFDMetrics(w io.Writer) {
 		log.Printf("ERROR: metrics: cannot determine the limit on open file descritors: %s", err)
 		return
 	}
-	fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs)
-	fmt.Fprintf(w, "process_open_fds %d\n", totalOpenFDs)
+	WriteGaugeUint64(w, "process_max_fds", maxOpenFDs)
+	WriteGaugeUint64(w, "process_open_fds", totalOpenFDs)
 }

 func getOpenFDsCount(path string) (uint64, error) {
@ -224,11 +224,11 @@ func writeProcessMemMetrics(w io.Writer) {
 		log.Printf("ERROR: metrics: cannot determine memory status: %s", err)
 		return
 	}
-	fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak)
-	fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", ms.rssPeak)
-	fmt.Fprintf(w, "process_resident_memory_anon_bytes %d\n", ms.rssAnon)
-	fmt.Fprintf(w, "process_resident_memory_file_bytes %d\n", ms.rssFile)
-	fmt.Fprintf(w, "process_resident_memory_shared_bytes %d\n", ms.rssShmem)
+	WriteGaugeUint64(w, "process_virtual_memory_peak_bytes", ms.vmPeak)
+	WriteGaugeUint64(w, "process_resident_memory_peak_bytes", ms.rssPeak)
+	WriteGaugeUint64(w, "process_resident_memory_anon_bytes", ms.rssAnon)
+	WriteGaugeUint64(w, "process_resident_memory_file_bytes", ms.rssFile)
+	WriteGaugeUint64(w, "process_resident_memory_shared_bytes", ms.rssShmem)

 }

--- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go
@ -4,7 +4,6 @@
 package metrics

 import (
-	"fmt"
 	"io"
 	"log"
 	"syscall"
@ -55,16 +54,16 @@ func writeProcessMetrics(w io.Writer) {
 		log.Printf("ERROR: metrics: cannot read process memory information: %s", err)
 		return
 	}
-	stimeSeconds := (uint64(stime.HighDateTime)<<32 + uint64(stime.LowDateTime)) / 1e7
-	utimeSeconds := (uint64(utime.HighDateTime)<<32 + uint64(utime.LowDateTime)) / 1e7
-	fmt.Fprintf(w, "process_cpu_seconds_system_total %d\n", stimeSeconds)
-	fmt.Fprintf(w, "process_cpu_seconds_total %d\n", stimeSeconds+utimeSeconds)
-	fmt.Fprintf(w, "process_cpu_seconds_user_total %d\n", stimeSeconds)
-	fmt.Fprintf(w, "process_pagefaults_total %d\n", mc.PageFaultCount)
-	fmt.Fprintf(w, "process_start_time_seconds %d\n", startTime.Nanoseconds()/1e9)
-	fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", mc.PrivateUsage)
-	fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", mc.PeakWorkingSetSize)
-	fmt.Fprintf(w, "process_resident_memory_bytes %d\n", mc.WorkingSetSize)
+	stimeSeconds := float64(uint64(stime.HighDateTime)<<32+uint64(stime.LowDateTime)) / 1e7
+	utimeSeconds := float64(uint64(utime.HighDateTime)<<32+uint64(utime.LowDateTime)) / 1e7
+	WriteCounterFloat64(w, "process_cpu_seconds_system_total", stimeSeconds)
+	WriteCounterFloat64(w, "process_cpu_seconds_total", stimeSeconds+utimeSeconds)
+	WriteCounterFloat64(w, "process_cpu_seconds_user_total", stimeSeconds)
+	WriteCounterUint64(w, "process_pagefaults_total", uint64(mc.PageFaultCount))
+	WriteGaugeUint64(w, "process_start_time_seconds", uint64(startTime.Nanoseconds())/1e9)
+	WriteGaugeUint64(w, "process_virtual_memory_bytes", uint64(mc.PrivateUsage))
+	WriteGaugeUint64(w, "process_resident_memory_peak_bytes", uint64(mc.PeakWorkingSetSize))
+	WriteGaugeUint64(w, "process_resident_memory_bytes", uint64(mc.WorkingSetSize))
 }

 func writeFDMetrics(w io.Writer) {
@ -80,6 +79,6 @@ func writeFDMetrics(w io.Writer) {
 	}
 	// it seems to be hard-coded limit for 64-bit systems
 	// https://learn.microsoft.com/en-us/archive/blogs/markrussinovich/pushing-the-limits-of-windows-handles#maximum-number-of-handles
-	fmt.Fprintf(w, "process_max_fds %d\n", 16777216)
-	fmt.Fprintf(w, "process_open_fds %d\n", count)
+	WriteGaugeUint64(w, "process_max_fds", 16777216)
+	WriteGaugeUint64(w, "process_open_fds", uint64(count))
 }
--- a/vendor/github.com/VictoriaMetrics/metrics/push.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/push.go
@ -11,6 +11,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 	"time"

 	"compress/gzip"
@ -69,10 +70,7 @@ func InitPushWithOptions(ctx context.Context, pushURL string, interval time.Dura
 // It is OK calling InitPushProcessMetrics multiple times with different pushURL -
 // in this case metrics are pushed to all the provided pushURL urls.
 func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels string) error {
-	writeMetrics := func(w io.Writer) {
-		WriteProcessMetrics(w)
-	}
-	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
+	return InitPushExt(pushURL, interval, extraLabels, WriteProcessMetrics)
 }

 // InitPush sets up periodic push for globally registered metrics to the given pushURL with the given interval.
@ -97,6 +95,21 @@ func InitPush(pushURL string, interval time.Duration, extraLabels string, pushPr
 	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
 }

+// PushMetrics pushes globally registered metrics to pushURL.
+//
+// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL.
+//
+// opts may contain additional configuration options if non-nil.
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+func PushMetrics(ctx context.Context, pushURL string, pushProcessMetrics bool, opts *PushOptions) error {
+	writeMetrics := func(w io.Writer) {
+		WritePrometheus(w, pushProcessMetrics)
+	}
+	return PushMetricsExt(ctx, pushURL, writeMetrics, opts)
+}
+
 // InitPushWithOptions sets up periodic push for metrics from s to the given pushURL with the given interval.
 //
 // The periodic push is stopped when the ctx is canceled.
@ -112,10 +125,7 @@ func InitPush(pushURL string, interval time.Duration, extraLabels string, pushPr
 // It is OK calling InitPushWithOptions multiple times with different pushURL -
 // in this case metrics are pushed to all the provided pushURL urls.
 func (s *Set) InitPushWithOptions(ctx context.Context, pushURL string, interval time.Duration, opts *PushOptions) error {
-	writeMetrics := func(w io.Writer) {
-		s.WritePrometheus(w)
-	}
-	return InitPushExtWithOptions(ctx, pushURL, interval, writeMetrics, opts)
+	return InitPushExtWithOptions(ctx, pushURL, interval, s.WritePrometheus, opts)
 }

 // InitPush sets up periodic push for metrics from s to the given pushURL with the given interval.
@ -132,10 +142,17 @@ func (s *Set) InitPushWithOptions(ctx context.Context, pushURL string, interval
 // It is OK calling InitPush multiple times with different pushURL -
 // in this case metrics are pushed to all the provided pushURL urls.
 func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels string) error {
-	writeMetrics := func(w io.Writer) {
-		s.WritePrometheus(w)
-	}
-	return InitPushExt(pushURL, interval, extraLabels, writeMetrics)
+	return InitPushExt(pushURL, interval, extraLabels, s.WritePrometheus)
+}
+
+// PushMetrics pushes s metrics to pushURL.
+//
+// opts may contain additional configuration options if non-nil.
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+func (s *Set) PushMetrics(ctx context.Context, pushURL string, opts *PushOptions) error {
+	return PushMetricsExt(ctx, pushURL, s.WritePrometheus, opts)
 }

 // InitPushExt sets up periodic push for metrics obtained by calling writeMetrics with the given interval.
@ -179,147 +196,202 @@ func InitPushExt(pushURL string, interval time.Duration, extraLabels string, wri
 // It is OK calling InitPushExtWithOptions multiple times with different writeMetrics -
 // in this case all the metrics generated by writeMetrics callbacks are written to pushURL.
 func InitPushExtWithOptions(ctx context.Context, pushURL string, interval time.Duration, writeMetrics func(w io.Writer), opts *PushOptions) error {
-	// validate pushURL
-	pu, err := url.Parse(pushURL)
+	pc, err := newPushContext(pushURL, opts)
 	if err != nil {
-		return fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err)
-	}
-	if pu.Scheme != "http" && pu.Scheme != "https" {
-		return fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL)
-	}
-	if pu.Host == "" {
-		return fmt.Errorf("missing host in pushURL=%q", pushURL)
+		return err
 	}

 	// validate interval
 	if interval <= 0 {
 		return fmt.Errorf("interval must be positive; got %s", interval)
 	}
+	pushMetricsSet.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pc.pushURLRedacted)).Set(interval.Seconds())

-	// validate ExtraLabels
-	var extraLabels string
-	if opts != nil {
-		extraLabels = opts.ExtraLabels
-	}
-	if err := validateTags(extraLabels); err != nil {
-		return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err)
-	}
-
-	// validate Headers
-	headers := make(http.Header)
-	if opts != nil {
-		for _, h := range opts.Headers {
-			n := strings.IndexByte(h, ':')
-			if n < 0 {
-				return fmt.Errorf("missing `:` delimiter in the header %q", h)
-			}
-			name := strings.TrimSpace(h[:n])
-			value := strings.TrimSpace(h[n+1:])
-			headers.Add(name, value)
-		}
-	}
-
-	// validate DisableCompression
-	disableCompression := false
-	if opts != nil {
-		disableCompression = opts.DisableCompression
-	}
-
-	// Initialize metrics for the given pushURL
-	pushURLRedacted := pu.Redacted()
-	pushesTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted))
-	pushErrorsTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted))
-	bytesPushedTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted))
-	pushDuration := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted))
-	pushBlockSize := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted))
-	pushMetrics.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pushURLRedacted)).Set(interval.Seconds())
-
-	c := &http.Client{
-		Timeout: interval,
-	}
 	go func() {
 		ticker := time.NewTicker(interval)
-		var bb bytes.Buffer
-		var tmpBuf []byte
-		zw := gzip.NewWriter(&bb)
+		defer ticker.Stop()
 		stopCh := ctx.Done()
 		for {
 			select {
 			case <-ticker.C:
+				ctxLocal, cancel := context.WithTimeout(ctx, interval+time.Second)
+				err := pc.pushMetrics(ctxLocal, writeMetrics)
+				cancel()
+				if err != nil {
+					log.Printf("ERROR: metrics.push: %s", err)
+				}
 			case <-stopCh:
 				return
 			}
-
-			bb.Reset()
-			writeMetrics(&bb)
-			if len(extraLabels) > 0 {
-				tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels)
-				bb.Reset()
-				if _, err := bb.Write(tmpBuf); err != nil {
-					panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err))
-				}
-			}
-			if !disableCompression {
-				tmpBuf = append(tmpBuf[:0], bb.Bytes()...)
-				bb.Reset()
-				zw.Reset(&bb)
-				if _, err := zw.Write(tmpBuf); err != nil {
-					panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err))
-				}
-				if err := zw.Close(); err != nil {
-					panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err))
-				}
-			}
-			pushesTotal.Inc()
-			blockLen := bb.Len()
-			bytesPushedTotal.Add(blockLen)
-			pushBlockSize.Update(float64(blockLen))
-			req, err := http.NewRequestWithContext(ctx, "GET", pushURL, &bb)
-			if err != nil {
-				panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pushURLRedacted, err))
-			}
-
-			// Set the needed headers
-			for name, values := range headers {
-				for _, value := range values {
-					req.Header.Add(name, value)
-				}
-			}
-			req.Header.Set("Content-Type", "text/plain")
-
-			if !disableCompression {
-				req.Header.Set("Content-Encoding", "gzip")
-			}
-
-			// Perform the request
-			startTime := time.Now()
-			resp, err := c.Do(req)
-			pushDuration.UpdateDuration(startTime)
-			if err != nil {
-				if !errors.Is(err, context.Canceled) {
-					log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err)
-					pushErrorsTotal.Inc()
-				}
-				continue
-			}
-			if resp.StatusCode/100 != 2 {
-				body, _ := ioutil.ReadAll(resp.Body)
-				_ = resp.Body.Close()
-				log.Printf("ERROR: metrics.push: unexpected status code in response from %q: %d; expecting 2xx; response body: %q",
-					pushURLRedacted, resp.StatusCode, body)
-				pushErrorsTotal.Inc()
-				continue
-			}
-			_ = resp.Body.Close()
 		}
 	}()
+
 	return nil
 }

-var pushMetrics = NewSet()
+// PushMetricsExt pushes metrics generated by wirteMetrics to pushURL.
+//
+// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments.
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format
+//
+// opts may contain additional configuration options if non-nil.
+//
+// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to
+// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format
+func PushMetricsExt(ctx context.Context, pushURL string, writeMetrics func(w io.Writer), opts *PushOptions) error {
+	pc, err := newPushContext(pushURL, opts)
+	if err != nil {
+		return err
+	}
+	return pc.pushMetrics(ctx, writeMetrics)
+}
+
+type pushContext struct {
+	pushURL            *url.URL
+	pushURLRedacted    string
+	extraLabels        string
+	headers            http.Header
+	disableCompression bool
+
+	client *http.Client
+
+	pushesTotal      *Counter
+	bytesPushedTotal *Counter
+	pushBlockSize    *Histogram
+	pushDuration     *Histogram
+	pushErrors       *Counter
+}
+
+func newPushContext(pushURL string, opts *PushOptions) (*pushContext, error) {
+	if opts == nil {
+		opts = &PushOptions{}
+	}
+
+	// validate pushURL
+	pu, err := url.Parse(pushURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err)
+	}
+	if pu.Scheme != "http" && pu.Scheme != "https" {
+		return nil, fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL)
+	}
+	if pu.Host == "" {
+		return nil, fmt.Errorf("missing host in pushURL=%q", pushURL)
+	}
+
+	// validate ExtraLabels
+	extraLabels := opts.ExtraLabels
+	if err := validateTags(extraLabels); err != nil {
+		return nil, fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err)
+	}
+
+	// validate Headers
+	headers := make(http.Header)
+	for _, h := range opts.Headers {
+		n := strings.IndexByte(h, ':')
+		if n < 0 {
+			return nil, fmt.Errorf("missing `:` delimiter in the header %q", h)
+		}
+		name := strings.TrimSpace(h[:n])
+		value := strings.TrimSpace(h[n+1:])
+		headers.Add(name, value)
+	}
+
+	pushURLRedacted := pu.Redacted()
+	client := &http.Client{}
+	return &pushContext{
+		pushURL:            pu,
+		pushURLRedacted:    pushURLRedacted,
+		extraLabels:        extraLabels,
+		headers:            headers,
+		disableCompression: opts.DisableCompression,
+
+		client: client,
+
+		pushesTotal:      pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted)),
+		bytesPushedTotal: pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted)),
+		pushBlockSize:    pushMetricsSet.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted)),
+		pushDuration:     pushMetricsSet.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted)),
+		pushErrors:       pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted)),
+	}, nil
+}
+
+func (pc *pushContext) pushMetrics(ctx context.Context, writeMetrics func(w io.Writer)) error {
+	bb := getBytesBuffer()
+	defer putBytesBuffer(bb)
+
+	writeMetrics(bb)
+
+	if len(pc.extraLabels) > 0 {
+		bbTmp := getBytesBuffer()
+		bbTmp.B = append(bbTmp.B[:0], bb.B...)
+		bb.B = addExtraLabels(bb.B[:0], bbTmp.B, pc.extraLabels)
+		putBytesBuffer(bbTmp)
+	}
+	if !pc.disableCompression {
+		bbTmp := getBytesBuffer()
+		bbTmp.B = append(bbTmp.B[:0], bb.B...)
+		bb.B = bb.B[:0]
+		zw := getGzipWriter(bb)
+		if _, err := zw.Write(bbTmp.B); err != nil {
+			panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(bbTmp.B), err))
+		}
+		if err := zw.Close(); err != nil {
+			panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err))
+		}
+		putGzipWriter(zw)
+		putBytesBuffer(bbTmp)
+	}
+
+	// Update metrics
+	pc.pushesTotal.Inc()
+	blockLen := len(bb.B)
+	pc.bytesPushedTotal.Add(blockLen)
+	pc.pushBlockSize.Update(float64(blockLen))
+
+	// Prepare the request to sent to pc.pushURL
+	reqBody := bytes.NewReader(bb.B)
+	req, err := http.NewRequestWithContext(ctx, "GET", pc.pushURL.String(), reqBody)
+	if err != nil {
+		panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pc.pushURLRedacted, err))
+	}
+
+	// Set the needed headers
+	for name, values := range pc.headers {
+		for _, value := range values {
+			req.Header.Add(name, value)
+		}
+	}
+	req.Header.Set("Content-Type", "text/plain")
+	if !pc.disableCompression {
+		req.Header.Set("Content-Encoding", "gzip")
+	}
+
+	// Perform the request
+	startTime := time.Now()
+	resp, err := pc.client.Do(req)
+	pc.pushDuration.UpdateDuration(startTime)
+	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
+		pc.pushErrors.Inc()
+		return fmt.Errorf("cannot push metrics to %q: %s", pc.pushURLRedacted, err)
+	}
+	if resp.StatusCode/100 != 2 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		pc.pushErrors.Inc()
+		return fmt.Errorf("unexpected status code in response from %q: %d; expecting 2xx; response body: %q", pc.pushURLRedacted, resp.StatusCode, body)
+	}
+	_ = resp.Body.Close()
+	return nil
+}
+
+var pushMetricsSet = NewSet()

 func writePushMetrics(w io.Writer) {
-	pushMetrics.WritePrometheus(w)
+	pushMetricsSet.WritePrometheus(w)
 }

 func addExtraLabels(dst, src []byte, extraLabels string) []byte {
@ -367,3 +439,44 @@ func addExtraLabels(dst, src []byte, extraLabels string) []byte {
 }

 var bashBytes = []byte("#")
+
+func getBytesBuffer() *bytesBuffer {
+	v := bytesBufferPool.Get()
+	if v == nil {
+		return &bytesBuffer{}
+	}
+	return v.(*bytesBuffer)
+}
+
+func putBytesBuffer(bb *bytesBuffer) {
+	bb.B = bb.B[:0]
+	bytesBufferPool.Put(bb)
+}
+
+var bytesBufferPool sync.Pool
+
+type bytesBuffer struct {
+	B []byte
+}
+
+func (bb *bytesBuffer) Write(p []byte) (int, error) {
+	bb.B = append(bb.B, p...)
+	return len(p), nil
+}
+
+func getGzipWriter(w io.Writer) *gzip.Writer {
+	v := gzipWriterPool.Get()
+	if v == nil {
+		return gzip.NewWriter(w)
+	}
+	zw := v.(*gzip.Writer)
+	zw.Reset(w)
+	return zw
+}
+
+func putGzipWriter(zw *gzip.Writer) {
+	zw.Reset(io.Discard)
+	gzipWriterPool.Put(zw)
+}
+
+var gzipWriterPool sync.Pool
--- a/vendor/github.com/VictoriaMetrics/metrics/set.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/set.go
@ -47,9 +47,17 @@ func (s *Set) WritePrometheus(w io.Writer) {
 	sa := append([]*namedMetric(nil), s.a...)
 	s.mu.Unlock()

-	// Call marshalTo without the global lock, since certain metric types such as Gauge
-	// can call a callback, which, in turn, can try calling s.mu.Lock again.
+	prevMetricFamily := ""
 	for _, nm := range sa {
+		metricFamily := getMetricFamily(nm.name)
+		if metricFamily != prevMetricFamily {
+			// write meta info only once per metric family
+			metricType := nm.metric.metricType()
+			WriteMetadataIfNeeded(&bb, nm.name, metricType)
+			prevMetricFamily = metricFamily
+		}
+		// Call marshalTo without the global lock, since certain metric types such as Gauge
+		// can call a callback, which, in turn, can try calling s.mu.Lock again.
 		nm.metric.marshalTo(nm.name, &bb)
 	}
 	w.Write(bb.Bytes())
--- a/vendor/github.com/VictoriaMetrics/metrics/summary.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/summary.go
@ -119,6 +119,10 @@ func (sm *Summary) marshalTo(prefix string, w io.Writer) {
 	}
 }

+func (sm *Summary) metricType() string {
+	return "summary"
+}
+
 func splitMetricName(name string) (string, string) {
 	n := strings.IndexByte(name, '{')
 	if n < 0 {
@ -196,6 +200,10 @@ func (qv *quantileValue) marshalTo(prefix string, w io.Writer) {
 	}
 }

+func (qv *quantileValue) metricType() string {
+	return "unsupported"
+}
+
 func addTag(name, tag string) string {
 	if len(name) == 0 || name[len(name)-1] != '}' {
 		return fmt.Sprintf("%s{%s}", name, tag)
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -97,8 +97,8 @@ github.com/VictoriaMetrics/fastcache
 github.com/VictoriaMetrics/fasthttp
 github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
-# github.com/VictoriaMetrics/metrics v1.26.1
-## explicit; go 1.16
+# github.com/VictoriaMetrics/metrics v1.28.2
+## explicit; go 1.17
 github.com/VictoriaMetrics/metrics
 # github.com/VictoriaMetrics/metricsql v0.70.0
 ## explicit; go 1.13