From f96804658bac738cdf7d70e92926c8fe1e372e4d Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 07:32:11 -0700
Subject: [PATCH 01/19] app/vmctl: document that -vm-native-step-interval
 command-line option now supports `week` value

This is a follow-up for d322ee4b35ab1a2335618406fc2bccf789d1a22b

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738
---
 app/vmctl/README.md        | 4 ++--
 app/vmctl/flags.go         | 5 +++--
 app/vmctl/stepper/split.go | 4 ++--
 docs/vmctl.md              | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/app/vmctl/README.md b/app/vmctl/README.md
index c5f008995..50d11161e 100644
--- a/app/vmctl/README.md
+++ b/app/vmctl/README.md
@@ -826,7 +826,7 @@ It is possible split migration process into set of smaller batches based on time
 migrating large volumes of data as this adds indication of progress and ability to restore process from certain point 
 in case of failure.
 
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`, `minute`.
+To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `week`, `day`, `hour`, `minute`.
 Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for 
 export process.
 
@@ -836,7 +836,7 @@ Every range is being processed independently, which means that:
 so it is possible to restart process starting from failed range.
 
 It is recommended using the `month` step when migrating the data over multiple months, 
-since the migration with `day` and `hour` steps may take longer time to complete because of additional overhead.
+since the migration with `week`, `day` and `hour` steps may take longer time to complete because of additional overhead.
 
 Usage example:
 ```console
diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go
index 92c582f29..7a71d4ab0 100644
--- a/app/vmctl/flags.go
+++ b/app/vmctl/flags.go
@@ -361,8 +361,9 @@ var (
 			Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
 		},
 		&cli.StringFlag{
-			Name:  vmNativeStepInterval,
-			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
+			Name: vmNativeStepInterval,
+			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart,
+				stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
 			Value: stepper.StepMonth,
 		},
 		&cli.BoolFlag{
diff --git a/app/vmctl/stepper/split.go b/app/vmctl/stepper/split.go
index 8f890f659..104b38ad8 100644
--- a/app/vmctl/stepper/split.go
+++ b/app/vmctl/stepper/split.go
@@ -8,10 +8,10 @@ import (
 const (
 	// StepMonth represents a one month interval
 	StepMonth string = "month"
-	// StepDay represents a one day interval
-	StepDay string = "day"
 	// StepWeek represents a one week interval
 	StepWeek string = "week"
+	// StepDay represents a one day interval
+	StepDay string = "day"
 	// StepHour represents a one hour interval
 	StepHour string = "hour"
 	// StepMinute represents a one minute interval
diff --git a/docs/vmctl.md b/docs/vmctl.md
index 6015c0ba5..88e5b10a6 100644
--- a/docs/vmctl.md
+++ b/docs/vmctl.md
@@ -837,7 +837,7 @@ It is possible split migration process into set of smaller batches based on time
 migrating large volumes of data as this adds indication of progress and ability to restore process from certain point 
 in case of failure.
 
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`, `minute`.
+To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `week`, `day`, `hour`, `minute`.
 Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for 
 export process.
 
@@ -847,7 +847,7 @@ Every range is being processed independently, which means that:
 so it is possible to restart process starting from failed range.
 
 It is recommended using the `month` step when migrating the data over multiple months, 
-since the migration with `day` and `hour` steps may take longer time to complete because of additional overhead.
+since the migration with `week`, `day` and `hour` steps may take longer time to complete because of additional overhead.
 
 Usage example:
 ```console

From 1a57815769655e6b3efcc0acbf88640446c50316 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 12:25:03 -0700
Subject: [PATCH 02/19] deployment: update Docker image tags for
 VictoriaMetrics components from v1.92.1 to v1.93.0

See https://docs.victoriametrics.com/CHANGELOG.html#v1930
---
 deployment/docker/docker-compose-cluster.yml         | 12 ++++++------
 deployment/docker/docker-compose.yml                 |  6 +++---
 deployment/logs-benchmark/docker-compose.yml         |  2 +-
 .../digitialocean/one-click-droplet/RELEASE_GUIDE.md |  2 +-
 .../files/etc/update-motd.d/99-one-click             |  4 ++--
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml
index 1208c59e3..a659dd14b 100644
--- a/deployment/docker/docker-compose-cluster.yml
+++ b/deployment/docker/docker-compose-cluster.yml
@@ -2,7 +2,7 @@ version: '3.5'
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.92.1
+    image: victoriametrics/vmagent:v1.93.0
     depends_on:
       - "vminsert"
     ports:
@@ -32,7 +32,7 @@ services:
 
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.92.1-cluster
+    image: victoriametrics/vmstorage:v1.93.0-cluster
     ports:
       - 8482
       - 8400
@@ -44,7 +44,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.92.1-cluster
+    image: victoriametrics/vmstorage:v1.93.0-cluster
     ports:
       - 8482
       - 8400
@@ -56,7 +56,7 @@ services:
     restart: always
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.92.1-cluster
+    image: victoriametrics/vminsert:v1.93.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
     restart: always
   vmselect:
     container_name: vmselect
-    image: victoriametrics/vmselect:v1.92.1-cluster
+    image: victoriametrics/vmselect:v1.93.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -82,7 +82,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.92.1
+    image: victoriametrics/vmalert:v1.93.0
     depends_on:
       - "vmselect"
     ports:
diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
index d2a5ff16c..dc4b90929 100644
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.5"
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.92.1
+    image: victoriametrics/vmagent:v1.93.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
     restart: always
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.92.1
+    image: victoriametrics/victoria-metrics:v1.93.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -56,7 +56,7 @@ services:
     restart: always
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.92.1
+    image: victoriametrics/vmalert:v1.93.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"
diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml
index 9d04d23c3..416046a35 100644
--- a/deployment/logs-benchmark/docker-compose.yml
+++ b/deployment/logs-benchmark/docker-compose.yml
@@ -105,7 +105,7 @@ services:
       - '--config=/config.yml'
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.92.1
+    image: victoriametrics/victoria-metrics:v1.93.0
     ports:
       - '8428:8428'
     command:
diff --git a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
index 21040199f..41fc7cd72 100644
--- a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
+++ b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
@@ -8,7 +8,7 @@
 4. Set variables `DIGITALOCEAN_API_TOKEN` with `VM_VERSION` for `packer` environment and run make from example below:
 
 ```console
-make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.92.1"
+make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.93.0"
 ```
 
 
diff --git a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
index f07cffca9..e0765d5e2 100755
--- a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
+++ b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
@@ -19,8 +19,8 @@ On the server:
   * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
 
 ********************************************************************************
-  # This image includes 1.92.1 version of VictoriaMetrics.
-  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1
+  # This image includes 1.93.0 version of VictoriaMetrics.
+  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0
 
   # Welcome to VictoriaMetrics droplet!
 

From fae59146ad33e2a6d776acfa95bf6d58a08768ba Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 12:28:08 -0700
Subject: [PATCH 03/19] docs/CHANGELOG.md: document LTS release v1.79.14

See https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.14
---
 docs/CHANGELOG.md      |  4 ++++
 docs/CHANGELOG_2022.md | 16 ++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 71e15e5b8..573c79102 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -660,6 +660,10 @@ See changes [here](https://docs.victoriametrics.com/CHANGELOG_2022.html#v1810)
 
 See changes [here](https://docs.victoriametrics.com/CHANGELOG_2022.html#v1800)
 
+## [v1.79.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.14)
+
+See changes [here](https://docs.victoriametrics.com/CHANGELOG_2022.html#v17914)
+
 ## [v1.79.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.13)
 
 See changes [here](https://docs.victoriametrics.com/CHANGELOG_2022.html#v17913)
diff --git a/docs/CHANGELOG_2022.md b/docs/CHANGELOG_2022.md
index d38a3faf6..31604ff4b 100644
--- a/docs/CHANGELOG_2022.md
+++ b/docs/CHANGELOG_2022.md
@@ -376,6 +376,22 @@ Released at 2022-08-08
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly show date picker at `Table` tab. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2874).
 * BUGFIX: properly generate http redirects if `-http.pathPrefix` command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2918).
 
+## [v1.79.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.14)
+
+Released at 2023-08-12
+
+**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
+
+* SECURITY: upgrade Go builder from Go1.20.4 to Go1.21.0.
+* SECURITY: upgrade base docker image (Alpine) from 3.18.2 to 3.18.3. See [alpine 3.18.3 release notes](https://alpinelinux.org/posts/Alpine-3.15.10-3.16.7-3.17.5-3.18.3-released.html).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): Properly form path to static assets in WEB UI if `http.pathPrefix` set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4349).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): Properly set datasource query params. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4340). Thanks to @gsakun for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4341).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` and `/api/v1/alerts` API handlers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` for groups with present name but absent `rules`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221).
+
 ## [v1.79.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.13)
 
 Released at 2023-05-18

From 11329c3d16ff66a000e6b0e050e284dcc6a4caaa Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 14:49:08 -0700
Subject: [PATCH 04/19] docs/CHANGELOG.md: document changes in the v1.87.7 LTS
 release

---
 docs/CHANGELOG.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 573c79102..750d9cde5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -386,6 +386,33 @@ Released at 2023-02-24
 * BUGFIX: properly parse timestamps in milliseconds when [ingesting data via OpenTSDB telnet put protocol](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol). Previously timestamps in milliseconds were mistakenly multiplied by 1000. Thanks to @Droxenator for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3810).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): do not add extrapolated points outside the real points when using [interpolate()](https://docs.victoriametrics.com/MetricsQL.html#interpolate) function. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3816).
 
+## [v1.87.7](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.7)
+
+Released at 2023-08-12
+
+**v1.87.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.87.x line will be supported for at least 12 months since [v1.87.0](https://docs.victoriametrics.com/CHANGELOG.html#v1870) release**
+
+* SECURITY: upgrade Go builder from Go1.20.4 to Go1.21.0.
+* SECURITY: upgrade base docker image (Alpine) from 3.18.2 to 3.18.3. See [alpine 3.18.3 release notes](https://alpinelinux.org/posts/Alpine-3.15.10-3.16.7-3.17.5-3.18.3-released.html).
+
+* BUGFIX: vmselect: fix timestamp alignment for Prometheus querying API if time argument is less than 10m from the beginning of Unix epoch.
+* BUGFIX: vminsert: fixed decoding of label values with slash when accepting data via [pushgateway protocol](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format). This fixes Prometheus golang client compatibility. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4692).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly validate scheme for `proxy_url` field at the scrape config. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4811) for details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): close HTTP connections to [service discovery](https://docs.victoriametrics.com/sd_configs.html) servers when they are no longer needed. This should prevent from possible connection exhasution in some cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix possible panic at shutdown when [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) is enabled. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4407) for details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): Properly handle LOCAL command for proxy protocol. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3335#issuecomment-1569864108).
+* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly return error from [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) at `vmselect` when the `-search.maxSamplesPerQuery` or `-search.maxSamplesPerSeries` [limit](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#resource-usage-limits) is exceeded. Previously incomplete response could be returned without the error if `vmselect` runs with `-replicationFactor` greater than 1. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4472).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly calculate evaluation time for rules. Before, there was a low probability for discrepancy between actual time and rules evaluation time if evaluation interval was lower than the execution time for rules within the group.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): reset evaluation timestamp after modifying group interval. Before, there could have latency on rule evaluation time.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): Properly set datasource query params. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4340). Thanks to @gsakun for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4341).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): Properly form path to static assets in WEB UI if `http.pathPrefix` set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4349).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` for groups with present name but absent `rules`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): interrupt explore procedure in influx mode if vmctl found no numeric fields.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix panic in case `--remote-read-filter-time-start` flag is not set for remote-read mode. This flag is now required to use remote-read mode. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4553).
+
 ## [v1.87.6](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.6)
 
 Released at 2023-05-18

From e06d8556365dee571e0213cdc0f343ef7b202ab5 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 14:37:57 -0700
Subject: [PATCH 05/19] deployment/docker/Makefile: do not overwrite `latest`
 tag when pushing Docker images for LTS release

The `latest` tag is reserved for the latest release
---
 deployment/docker/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index b372cfc38..44e9cb4ec 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -78,7 +78,6 @@ publish-via-docker: \
 		--build-arg root_image=$(ROOT_IMAGE) \
 		--build-arg APP_NAME=$(APP_NAME) \
 		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
-		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
 		-o type=image \
 		-f app/$(APP_NAME)/multiarch/Dockerfile \
 		--push \

From 7a19b2a14ca853e87873ee60d4fadad293f93d78 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sat, 12 Aug 2023 15:30:13 -0700
Subject: [PATCH 06/19] docs/CHANGELOG.md: document that v1.93.x is a new line
 of LTS releases

---
 docs/CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 750d9cde5..8a3f0dfb2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -28,6 +28,9 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 Released at 2023-08-12
 
+**v1.93.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.93.x line will be supported for at least 12 months since [v1.93.0](https://docs.victoriametrics.com/CHANGELOG.html#v1930) release**
+
 **Update note**: starting from this release, [vmagent](https://docs.victoriametrics.com/vmagent.html) ignores timestamps provided by scrape targets by default - it associates scraped metrics with local timestamps instead. Set `honor_timestamps: true` in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) if timestamps provided by scrape targets must be used instead. This change helps removing gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) such as `container_memory_usage_bytes`. This also improves data compression and query performance over metrics collected from `cadvisor`. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697).
 
 * SECURITY: upgrade Go builder from Go1.20.6 to Go1.21.0 in order to fix [this issue](https://github.com/golang/go/issues/61460).

From a63fb21ab2db31b3c0fe11f4bc802650ac5ff3de Mon Sep 17 00:00:00 2001
From: Dmytro Kozlov <kozlovdmitriyy@gmail.com>
Date: Wed, 16 Aug 2023 14:54:51 +0200
Subject: [PATCH 07/19] app/vmctl: fix migration process if tenant have no data
 (#4799)

app/vmctl: don't interrupt migration process if tenant has no data

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Alexander Marshalov <_@marshalov.org>
---
 app/vmctl/vm_native.go | 7 ++++++-
 docs/CHANGELOG.md      | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go
index 8d94d2bbe..31bdb6b38 100644
--- a/app/vmctl/vm_native.go
+++ b/app/vmctl/vm_native.go
@@ -201,7 +201,12 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		}
 
 		if len(metrics) == 0 {
-			return fmt.Errorf("no metrics found")
+			errMsg := "no metrics found"
+			if tenantID != "" {
+				errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
+			}
+			log.Println(errMsg)
+			return nil
 		}
 		foundSeriesMsg = fmt.Sprintf("Found %d metrics to import", len(metrics))
 	}
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8a3f0dfb2..b14751bc6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## tip
 
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
+
 ## [v1.93.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0)
 
 Released at 2023-08-12

From b4529df08d3118b493c8a5916aaec7ca7d66009d Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <roman@victoriametrics.com>
Date: Wed, 16 Aug 2023 14:45:35 +0200
Subject: [PATCH 08/19] vmbackup: correctly check if specified `-dst` belongs
 to specified `-storageDataPath` (#4841)

See this issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837

Signed-off-by: hagen1778 <roman@victoriametrics.com>
---
 app/vmbackup/main.go      | 15 ++++++++++++++-
 app/vmbackup/main_test.go |  5 +++++
 docs/CHANGELOG.md         |  1 +
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go
index 06c9e475d..f42dfa73f 100644
--- a/app/vmbackup/main.go
+++ b/app/vmbackup/main.go
@@ -205,7 +205,20 @@ func hasFilepathPrefix(path, prefix string) bool {
 	if err != nil {
 		return false
 	}
-	return strings.HasPrefix(pathAbs, prefixAbs)
+	if prefixAbs == pathAbs {
+		return true
+	}
+	rel, err := filepath.Rel(prefixAbs, pathAbs)
+	if err != nil {
+		// if paths can't be related - they don't match
+		return false
+	}
+	if i := strings.Index(rel, "."); i == 0 {
+		// if path can be related only with . as first char - they still don't match
+		return false
+	}
+	// if paths are related - it is a match
+	return true
 }
 
 func newOriginFS() (common.OriginFS, error) {
diff --git a/app/vmbackup/main_test.go b/app/vmbackup/main_test.go
index f3288fb49..c1a87e51d 100644
--- a/app/vmbackup/main_test.go
+++ b/app/vmbackup/main_test.go
@@ -26,4 +26,9 @@ func TestHasFilepathPrefix(t *testing.T) {
 	f("fs://"+pwd+"/foo", pwd+"/foo/bar", false)
 	f("fs://"+pwd+"/foo/bar", pwd+"/foo", true)
 	f("fs://"+pwd+"/foo", pwd+"/bar", false)
+	f("fs:///data1", "/data", false)
+	f("fs:///data", "/data1", false)
+	f("fs:///data", "/data/foo", false)
+	f("fs:///data/foo", "/data", true)
+	f("fs:///data/foo/", "/data/", true)
 }
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b14751bc6..af043aebd 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## tip
 
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
 
 ## [v1.93.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0)

From b4c79fc60678d77b0d341a0254a075d578f04937 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 14 Aug 2023 16:14:40 +0200
Subject: [PATCH 09/19] lib/promrelabel: properly replace `:` char with `_` in
 metric names when -usePromCompatibleNaming command-line flag is set

This addresses https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071 comment from @johnseekins
---
 app/vmagent/remotewrite/relabel.go     |  4 ++--
 app/vmctl/opentsdb/parser.go           |  4 ++--
 app/vminsert/relabel/relabel.go        |  4 ++--
 docs/CHANGELOG.md                      |  1 +
 lib/promrelabel/relabel.go             | 25 ++++++++++++++++++------
 lib/promrelabel/relabel_test.go        | 22 ++++++++++++++++++---
 lib/promrelabel/relabel_timing_test.go | 27 ++++++++++++++++++++++----
 7 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index b1cfc16e6..146930f8d 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -114,9 +114,9 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 			for j := range tmpLabels {
 				label := &tmpLabels[j]
 				if label.Name == "__name__" {
-					label.Value = promrelabel.SanitizeName(label.Value)
+					label.Value = promrelabel.SanitizeMetricName(label.Value)
 				} else {
-					label.Name = promrelabel.SanitizeName(label.Name)
+					label.Name = promrelabel.SanitizeLabelName(label.Name)
 				}
 			}
 		}
diff --git a/app/vmctl/opentsdb/parser.go b/app/vmctl/opentsdb/parser.go
index 6993e8d13..6c2f209e7 100644
--- a/app/vmctl/opentsdb/parser.go
+++ b/app/vmctl/opentsdb/parser.go
@@ -180,7 +180,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 	/*
 		replace bad characters in metric name with _ per the data model
 	*/
-	finalMsg.Metric = promrelabel.SanitizeName(name)
+	finalMsg.Metric = promrelabel.SanitizeMetricName(name)
 	// replace bad characters in tag keys with _ per the data model
 	for key, value := range msg.Tags {
 		// if normalization requested, lowercase the key and value
@@ -191,7 +191,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 		/*
 			replace all explicitly bad characters with _
 		*/
-		key = promrelabel.SanitizeName(key)
+		key = promrelabel.SanitizeLabelName(key)
 		// tags that start with __ are considered custom stats for internal prometheus stuff, we should drop them
 		if !strings.HasPrefix(key, "__") {
 			finalMsg.Tags[key] = value
diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go
index fcb107c24..3409fb4c0 100644
--- a/app/vminsert/relabel/relabel.go
+++ b/app/vminsert/relabel/relabel.go
@@ -134,9 +134,9 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 		for i := range tmpLabels {
 			label := &tmpLabels[i]
 			if label.Name == "__name__" {
-				label.Value = promrelabel.SanitizeName(label.Value)
+				label.Value = promrelabel.SanitizeMetricName(label.Value)
 			} else {
-				label.Name = promrelabel.SanitizeName(label.Name)
+				label.Name = promrelabel.SanitizeLabelName(label.Name)
 			}
 		}
 	}
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index af043aebd..e2858098d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## tip
 
+* BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
 
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index c6c17b4c9..e2d5d68da 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -619,15 +619,28 @@ func fillLabelReferences(dst []byte, replacement string, labels []prompbmarshal.
 	return dst
 }
 
-// SanitizeName replaces unsupported by Prometheus chars in metric names and label names with _.
+// SanitizeLabelName replaces unsupported by Prometheus chars in label names with _.
 //
 // See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-func SanitizeName(name string) string {
-	return promSanitizer.Transform(name)
+func SanitizeLabelName(name string) string {
+	return labelNameSanitizer.Transform(name)
 }
 
-var promSanitizer = bytesutil.NewFastStringTransformer(func(s string) string {
-	return unsupportedPromChars.ReplaceAllString(s, "_")
+var labelNameSanitizer = bytesutil.NewFastStringTransformer(func(s string) string {
+	return unsupportedLabelNameChars.ReplaceAllString(s, "_")
 })
 
-var unsupportedPromChars = regexp.MustCompile(`[^a-zA-Z0-9_:]`)
+var unsupportedLabelNameChars = regexp.MustCompile(`[^a-zA-Z0-9_]`)
+
+// SanitizeMetricName replaces unsupported by Prometheus chars in metric names with _.
+//
+// See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+func SanitizeMetricName(value string) string {
+	return metricNameSanitizer.Transform(value)
+}
+
+var metricNameSanitizer = bytesutil.NewFastStringTransformer(func(s string) string {
+	return unsupportedMetricNameChars.ReplaceAllString(s, "_")
+})
+
+var unsupportedMetricNameChars = regexp.MustCompile(`[^a-zA-Z0-9_:]`)
diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go
index 987374252..df8080af4 100644
--- a/lib/promrelabel/relabel_test.go
+++ b/lib/promrelabel/relabel_test.go
@@ -9,13 +9,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-func TestSanitizeName(t *testing.T) {
+func TestSanitizeMetricName(t *testing.T) {
 	f := func(s, resultExpected string) {
 		t.Helper()
 		for i := 0; i < 5; i++ {
-			result := SanitizeName(s)
+			result := SanitizeMetricName(s)
 			if result != resultExpected {
-				t.Fatalf("unexpected result for SanitizeName(%q) at iteration %d; got %q; want %q", s, i, result, resultExpected)
+				t.Fatalf("unexpected result for SanitizeMetricName(%q) at iteration %d; got %q; want %q", s, i, result, resultExpected)
 			}
 		}
 	}
@@ -25,6 +25,22 @@ func TestSanitizeName(t *testing.T) {
 	f("foo...bar", "foo___bar")
 }
 
+func TestSanitizeLabelName(t *testing.T) {
+	f := func(s, resultExpected string) {
+		t.Helper()
+		for i := 0; i < 5; i++ {
+			result := SanitizeLabelName(s)
+			if result != resultExpected {
+				t.Fatalf("unexpected result for SanitizeLabelName(%q) at iteration %d; got %q; want %q", s, i, result, resultExpected)
+			}
+		}
+	}
+	f("", "")
+	f("a", "a")
+	f("foo.bar/baz:a", "foo_bar_baz_a")
+	f("foo...bar", "foo___bar")
+}
+
 func TestLabelsToString(t *testing.T) {
 	f := func(labels []prompbmarshal.Label, sExpected string) {
 		t.Helper()
diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go
index 31682e787..485002020 100644
--- a/lib/promrelabel/relabel_timing_test.go
+++ b/lib/promrelabel/relabel_timing_test.go
@@ -8,20 +8,39 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func BenchmarkSanitizeName(b *testing.B) {
+func BenchmarkSanitizeMetricName(b *testing.B) {
 	for _, name := range []string{"", "foo", "foo-bar-baz", "http_requests_total"} {
 		b.Run(name, func(b *testing.B) {
-			benchmarkSanitizeName(b, name)
+			benchmarkSanitizeMetricName(b, name)
 		})
 	}
 }
 
-func benchmarkSanitizeName(b *testing.B, name string) {
+func benchmarkSanitizeMetricName(b *testing.B, name string) {
 	b.ReportAllocs()
 	b.SetBytes(1)
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			sanitizedName := SanitizeName(name)
+			sanitizedName := SanitizeMetricName(name)
+			GlobalSink += len(sanitizedName)
+		}
+	})
+}
+
+func BenchmarkSanitizeLabelName(b *testing.B) {
+	for _, name := range []string{"", "foo", "foo-bar-baz", "http_requests_total"} {
+		b.Run(name, func(b *testing.B) {
+			benchmarkSanitizeLabelName(b, name)
+		})
+	}
+}
+
+func benchmarkSanitizeLabelName(b *testing.B, name string) {
+	b.ReportAllocs()
+	b.SetBytes(1)
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			sanitizedName := SanitizeLabelName(name)
 			GlobalSink += len(sanitizedName)
 		}
 	})

From c6154f8f52a48021b88dd4c0eb7a1f646ca0e6e1 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 14 Aug 2023 15:10:00 +0200
Subject: [PATCH 10/19] lib/promrelabel: stop emitting DEBUG log lines when
 parsing `if` expressions

These lines were accidentally left in the commit 62651570bb6142f94025266e9e65353f3d6c37e2

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635
---
 docs/CHANGELOG.md                | 1 +
 lib/promrelabel/if_expression.go | 3 ---
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e2858098d..473a0fb87 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## tip
 
+* BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
diff --git a/lib/promrelabel/if_expression.go b/lib/promrelabel/if_expression.go
index 78519d3e8..1f3506dfb 100644
--- a/lib/promrelabel/if_expression.go
+++ b/lib/promrelabel/if_expression.go
@@ -80,7 +80,6 @@ func (ie *IfExpression) UnmarshalYAML(f func(interface{}) error) error {
 }
 
 func (ie *IfExpression) unmarshalFromInterface(v interface{}) error {
-	logger.Infof("DEBUG: unmarshaling ifExpr from %#v", v)
 	ies := ie.ies[:0]
 	switch t := v.(type) {
 	case string:
@@ -89,7 +88,6 @@ func (ie *IfExpression) unmarshalFromInterface(v interface{}) error {
 			return fmt.Errorf("unexpected `match` option: %w", err)
 		}
 		ies = append(ies, ieLocal)
-		logger.Infof("DEBUG: unmarshaled ifExpr from %#v to %s", t, ieLocal)
 	case []interface{}:
 		for _, x := range t {
 			s, ok := x.(string)
@@ -102,7 +100,6 @@ func (ie *IfExpression) unmarshalFromInterface(v interface{}) error {
 			}
 			ies = append(ies, ieLocal)
 		}
-		logger.Infof("DEBUG: unmarshaled ifExpr from %#v to %s", t, ies)
 	default:
 		return fmt.Errorf("unexpected `match` type; got %#v; want string or an array of strings", t)
 	}

From 4b86a181059d2d900c644ff02ad7de877b530435 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 17 Aug 2023 13:57:29 +0200
Subject: [PATCH 11/19] docs/CHANGELOG.md: mention that this is v1.93.x LTS
 release line

---
 docs/CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 473a0fb87..3542d278e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -22,7 +22,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 * [How to build vmctl](https://docs.victoriametrics.com/vmctl.html#how-to-build)
 
 
-## tip
+## v1.93.x long-time support release (LTS)
 
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).

From d4123e135f070659463fff437dca75f2bd70d2d3 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 17 Aug 2023 13:26:10 +0200
Subject: [PATCH 12/19] lib/envflag: do not allow unsupported form for boolean
 command-line flags in the form `-boolFlag value`

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845
---
 docs/CHANGELOG.md      | 1 +
 lib/envflag/envflag.go | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3542d278e..55dc470d3 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## v1.93.x long-time support release (LTS)
 
+* BUGFIX: do not allow starting VictoriaMetrics components with improperly set boolean command-line flags in the form `-boolFlagName value`, since this leads to silent incomplete flags' parsing. This form should be replaced with `-boolFlagName=value`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845).
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
diff --git a/lib/envflag/envflag.go b/lib/envflag/envflag.go
index ab2b3fd9a..6603a127d 100644
--- a/lib/envflag/envflag.go
+++ b/lib/envflag/envflag.go
@@ -32,6 +32,11 @@ func ParseFlagSet(fs *flag.FlagSet, args []string) {
 		// Do not use lib/logger here, since it is uninitialized yet.
 		log.Fatalf("cannot parse flags %q: %s", args, err)
 	}
+	if fs.NArg() > 0 {
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845
+		log.Fatalf("unprocessed command-line args left: %s; the most likely reason is missing `=` between boolean flag name and value; "+
+			"see https://pkg.go.dev/flag#hdr-Command_line_flag_syntax", fs.Args())
+	}
 	if !*enable {
 		return
 	}

From 927ded6c3b82ba470c35fd4eada2f01383708b92 Mon Sep 17 00:00:00 2001
From: Alexander Marshalov <_@marshalov.org>
Date: Tue, 15 Aug 2023 13:47:48 +0200
Subject: [PATCH 13/19] fixed applying `remoteWrite.label` for pushed metrics
 (#4247) (#4824)

vmagent: properly add extra labels before sending data to remote storage

labels from `remoteWrite.label` are now added to sent metrics just before they
 are pushed to `remoteWrite.url` after all relabelings, including stream aggregation relabelings (#4247)

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247

Signed-off-by: Alexander Marshalov <_@marshalov.org>
Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>
---
 app/vmagent/remotewrite/relabel.go      | 46 ++++++++++++++++++-------
 app/vmagent/remotewrite/relabel_test.go | 33 ++++++++++++++----
 app/vmagent/remotewrite/remotewrite.go  |  9 +++--
 docs/CHANGELOG.md                       |  1 +
 docs/stream-aggregation.md              | 14 ++++++--
 5 files changed, 80 insertions(+), 23 deletions(-)

diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index 146930f8d..b79021c64 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -87,8 +87,8 @@ func initLabelsGlobal() {
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
-	if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// Nothing to change.
 		return tss
 	}
@@ -98,16 +98,6 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
-		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
-		for j := range extraLabels {
-			extraLabel := &extraLabels[j]
-			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
-			if tmp != nil {
-				tmp.Value = extraLabel.Value
-			} else {
-				labels = append(labels, *extraLabel)
-			}
-		}
 		if *usePromCompatibleNaming {
 			// Replace unsupported Prometheus chars in label names and metric names with underscores.
 			tmpLabels := labels[labelsLen:]
@@ -135,6 +125,38 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 	return tssDst
 }
 
+func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 {
+		return tss
+	}
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		for j := range extraLabels {
+			extraLabel := extraLabels[j]
+			if *usePromCompatibleNaming {
+				extraLabel.Name = promrelabel.SanitizeLabelName(extraLabel.Name)
+			}
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, extraLabel)
+			}
+		}
+		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	return tssDst
+}
+
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
diff --git a/app/vmagent/remotewrite/relabel_test.go b/app/vmagent/remotewrite/relabel_test.go
index eb3540002..441283f23 100644
--- a/app/vmagent/remotewrite/relabel_test.go
+++ b/app/vmagent/remotewrite/relabel_test.go
@@ -10,18 +10,16 @@ import (
 )
 
 func TestApplyRelabeling(t *testing.T) {
-	f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
+	f := func(pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
+		gotTss := rctx.applyRelabeling(tss, pcs)
 		if !reflect.DeepEqual(gotTss, expTss) {
 			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
 		}
 	}
 
-	f(nil, nil, "up", "up")
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
+	f(nil, "up", "up")
 
 	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
 - target_label: "foo"
@@ -32,11 +30,32 @@ func TestApplyRelabeling(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
+	f(pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
 
 	oldVal := *usePromCompatibleNaming
 	*usePromCompatibleNaming = true
-	f(nil, nil, `foo.bar`, `foo_bar`)
+	f(nil, `foo.bar`, `foo_bar`)
+	*usePromCompatibleNaming = oldVal
+}
+
+func TestAppendExtraLabels(t *testing.T) {
+	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
+		rctx := &relabelCtx{}
+		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
+		gotTss := rctx.appendExtraLabels(tss, extraLabels)
+		if !reflect.DeepEqual(gotTss, expTss) {
+			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
+		}
+	}
+
+	f(nil, "up", "up")
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
+
+	oldVal := *usePromCompatibleNaming
+	*usePromCompatibleNaming = true
+	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo_bar="baz"}`)
 	*usePromCompatibleNaming = oldVal
 }
 
diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index 29099e114..ac5681aa4 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -386,7 +386,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		if rctx != nil {
 			rowsCountBeforeRelabel := getRowsCount(tssBlock)
-			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
+			tssBlock = rctx.applyRelabeling(tssBlock, pcsGlobal)
 			rowsCountAfterRelabel := getRowsCount(tssBlock)
 			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
@@ -668,7 +668,7 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rowsCountBeforeRelabel := getRowsCount(tss)
-		tss = rctx.applyRelabeling(tss, nil, pcs)
+		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
 		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
@@ -719,6 +719,11 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 }
 
 func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+	if len(labelsGlobal) > 0 {
+		rctx := getRelabelCtx()
+		tss = rctx.appendExtraLabels(tss, labelsGlobal)
+		putRelabelCtx(rctx)
+	}
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	pss[idx].Push(tss)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 55dc470d3..ae0934ab6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -25,6 +25,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 ## v1.93.x long-time support release (LTS)
 
 * BUGFIX: do not allow starting VictoriaMetrics components with improperly set boolean command-line flags in the form `-boolFlagName value`, since this leads to silent incomplete flags' parsing. This form should be replaced with `-boolFlagName=value`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): according to [the docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics) labels from `-remoteWrite.label` cmd-line flag are now added to the sent metrics just before they are pushed to the `-remoteWrite.url` (after all relabeling, including stream aggregation relabeling). In addition, it allows adding labels for identifying vmagent instances when using streaming aggregation in vmagents [cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247) and [these docs](https://docs.victoriametrics.com/stream-aggregation.html#cluster-mode) for more details.
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index 9be338bac..eb01c9f64 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -1,7 +1,7 @@
 ---
 sort: 98
 weight: 98
-title: streaming aggregation
+title: Streaming aggregation
 menu:
   docs:
     parent: "victoriametrics"
@@ -10,7 +10,7 @@ aliases:
 - /stream-aggregation.html
 ---
 
-# streaming aggregation
+# Streaming aggregation
 
 [vmagent](https://docs.victoriametrics.com/vmagent.html) and [single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html)
 can aggregate incoming [samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) in streaming mode by time and by labels before data is written to remote storage.
@@ -673,3 +673,13 @@ support the following approaches for hot reloading stream aggregation configs fr
   ```
 
 * By sending HTTP request to `/-/reload` endpoint (e.g. `http://vmagent:8429/-/reload` or `http://victoria-metrics:8428/-/reload).
+
+## Cluster mode
+
+If you use [vmagent in cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) for streaming aggregation
+(with `-promscrape.cluster.*` parameters or with `VMAgent.spec.shardCount > 1` for [vmoperator](https://docs.victoriametrics.com/operator))
+then be careful when aggregating metrics via `by`, `without` or modifying via`*_relabel_configs` parameters, as incorrect usage may result in duplicates and data collision. For example, if more than one vmagent calculates `increase` for metric `http_requests_total` with `by: [path]` directive, then the resulting time series written to the remote database will be indistinguishable, as there is no way to tell to which `instance` the aggregation belongs. The proper fix would be to add a unique aggregation dimension: `by: [instance, path]`. With this change, aggregates between `instances` won't collide.
+
+If adding a new aggregation dimension isn't feasible (due to cardinality reduction purposes), then it is worth at least differentiating by which vmagent the aggregation was performed. You can do it with `remoteWrite.label` [parameter in vmagent](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics).
+For example, for running in docker or k8s you can use `remoteWrite.label` with `POD_NAME` or `HOSTNAME` environment variable: `remoteWrite.label='vmagent=%{HOSTNAME}'`.
+See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247#issue-1692894073) for details.

From c060c6d839e4812443af716f44c9b6bd22f8d4c9 Mon Sep 17 00:00:00 2001
From: Alexander Marshalov <_@marshalov.org>
Date: Thu, 17 Aug 2023 12:15:03 +0200
Subject: [PATCH 14/19] vmagent: fixed premature release of the context (after
 #4247 / #4824) (#4849)

Follow-up after https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a27c2f37731986f4bf6738404bb6388b1f42ffde

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247

Signed-off-by: Alexander Marshalov <_@marshalov.org>
---
 app/vmagent/remotewrite/remotewrite.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index ac5681aa4..b4f16ad3e 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -721,9 +721,10 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	if len(labelsGlobal) > 0 {
 		rctx := getRelabelCtx()
+		defer putRelabelCtx(rctx)
 		tss = rctx.appendExtraLabels(tss, labelsGlobal)
-		putRelabelCtx(rctx)
 	}
+
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	pss[idx].Push(tss)

From d688f9a744c58600a890abf42aaa64d84170f59d Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 17 Aug 2023 14:35:26 +0200
Subject: [PATCH 15/19] app/vmagent/remotewrite: follow-up after
 a27c2f37731986f4bf6738404bb6388b1f42ffde

- Fix Prometheus-compatible naming after applying the relabeling if -usePromCompatibleNaming command-line flag is set.
  This should prevent from possible Prometheus-incompatible metric names and label names generated by the relabeling.
- Do not return anything from relabelCtx.appendExtraLabels() function, since it cannot change the number of time series
  passed to it. Append labels for the passed time series in-place.
- Remove promrelabel.FinalizeLabels() call after adding extra labels to time series, since this call has been already
  made at relabelCtx.applyRelabeling(). It is user's responsibility if he passes labels with double underscore prefixes
  to -remoteWrite.label.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247
---
 app/vmagent/remotewrite/relabel.go      | 39 ++++++++++++-------------
 app/vmagent/remotewrite/relabel_test.go |  6 ++--
 app/vmagent/remotewrite/remotewrite.go  |  4 +--
 3 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index b79021c64..93b4caee7 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -98,24 +98,15 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
-		if *usePromCompatibleNaming {
-			// Replace unsupported Prometheus chars in label names and metric names with underscores.
-			tmpLabels := labels[labelsLen:]
-			for j := range tmpLabels {
-				label := &tmpLabels[j]
-				if label.Name == "__name__" {
-					label.Value = promrelabel.SanitizeMetricName(label.Value)
-				} else {
-					label.Name = promrelabel.SanitizeLabelName(label.Name)
-				}
-			}
-		}
 		labels = pcs.Apply(labels, labelsLen)
 		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
 		}
+		if *usePromCompatibleNaming {
+			fixPromCompatibleNaming(labels[labelsLen:])
+		}
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
@@ -125,11 +116,10 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 	return tssDst
 }
 
-func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) []prompbmarshal.TimeSeries {
+func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) {
 	if len(extraLabels) == 0 {
-		return tss
+		return
 	}
-	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
@@ -147,14 +137,9 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 				labels = append(labels, extraLabel)
 			}
 		}
-		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: ts.Samples,
-		})
+		ts.Labels = labels[labelsLen:]
 	}
 	rctx.labels = labels
-	return tssDst
 }
 
 type relabelCtx struct {
@@ -181,3 +166,15 @@ func putRelabelCtx(rctx *relabelCtx) {
 	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
+
+func fixPromCompatibleNaming(labels []prompbmarshal.Label) {
+	// Replace unsupported Prometheus chars in label names and metric names with underscores.
+	for i := range labels {
+		label := &labels[i]
+		if label.Name == "__name__" {
+			label.Value = promrelabel.SanitizeMetricName(label.Value)
+		} else {
+			label.Name = promrelabel.SanitizeLabelName(label.Name)
+		}
+	}
+}
diff --git a/app/vmagent/remotewrite/relabel_test.go b/app/vmagent/remotewrite/relabel_test.go
index 441283f23..c84a31610 100644
--- a/app/vmagent/remotewrite/relabel_test.go
+++ b/app/vmagent/remotewrite/relabel_test.go
@@ -42,9 +42,9 @@ func TestAppendExtraLabels(t *testing.T) {
 	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.appendExtraLabels(tss, extraLabels)
-		if !reflect.DeepEqual(gotTss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
+		rctx.appendExtraLabels(tss, extraLabels)
+		if !reflect.DeepEqual(tss, expTss) {
+			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, tss)
 		}
 	}
 
diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index b4f16ad3e..62dd1a908 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -355,7 +355,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
@@ -722,7 +722,7 @@ func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	if len(labelsGlobal) > 0 {
 		rctx := getRelabelCtx()
 		defer putRelabelCtx(rctx)
-		tss = rctx.appendExtraLabels(tss, labelsGlobal)
+		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
 	pss := rwctx.pss

From 382721a3accc0a6098ae5d261684d5d80a6364e3 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 17 Aug 2023 15:18:22 +0200
Subject: [PATCH 16/19] docs/stream-aggregation.md: clarify the usage of
 `-remoteWrite.label` after the fix at
 a27c2f37731986f4bf6738404bb6388b1f42ffde

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247
---
 README.md                             |  3 ++-
 docs/CHANGELOG.md                     |  2 +-
 docs/README.md                        |  3 ++-
 docs/Single-server-VictoriaMetrics.md |  3 ++-
 docs/stream-aggregation.md            | 12 ++++++++----
 5 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index f792d1473..aebd03b8f 100644
--- a/README.md
+++ b/README.md
@@ -173,7 +173,8 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in `yaml` configuration files (such as `-promscrape.config`)
+and in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
 This expansion is performed by VictoriaMetrics itself.
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ae0934ab6..74b545616 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -25,7 +25,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 ## v1.93.x long-time support release (LTS)
 
 * BUGFIX: do not allow starting VictoriaMetrics components with improperly set boolean command-line flags in the form `-boolFlagName value`, since this leads to silent incomplete flags' parsing. This form should be replaced with `-boolFlagName=value`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): according to [the docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics) labels from `-remoteWrite.label` cmd-line flag are now added to the sent metrics just before they are pushed to the `-remoteWrite.url` (after all relabeling, including stream aggregation relabeling). In addition, it allows adding labels for identifying vmagent instances when using streaming aggregation in vmagents [cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247) and [these docs](https://docs.victoriametrics.com/stream-aggregation.html#cluster-mode) for more details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly set labels from `-remoteWrite.label` command-line flag just before sending samples to the configured `-remoteWrite.url` according to [these docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics). Previously these labels were incorrectly set before [the relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) configured via `-remoteWrite.urlRelabelConfigs` and [the stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) configured via `-remoteWrite.streamAggr.config`, so these labels could be lost or incorrectly transformed before sending the samples to remote storage. The fix allows using `-remoteWrite.label` for identifying `vmagent` instances in [cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247) and [these docs](https://docs.victoriametrics.com/stream-aggregation.html#cluster-mode) for more details.
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
diff --git a/docs/README.md b/docs/README.md
index 448d59f56..34412c343 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -176,7 +176,8 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in `yaml` configuration files (such as `-promscrape.config`)
+and in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
 This expansion is performed by VictoriaMetrics itself.
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index b3e54edf7..d26ed865d 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -184,7 +184,8 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in `yaml` configuration files (such as `-promscrape.config`)
+and in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
 This expansion is performed by VictoriaMetrics itself.
diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index eb01c9f64..161d6011f 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -678,8 +678,12 @@ support the following approaches for hot reloading stream aggregation configs fr
 
 If you use [vmagent in cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) for streaming aggregation
 (with `-promscrape.cluster.*` parameters or with `VMAgent.spec.shardCount > 1` for [vmoperator](https://docs.victoriametrics.com/operator))
-then be careful when aggregating metrics via `by`, `without` or modifying via`*_relabel_configs` parameters, as incorrect usage may result in duplicates and data collision. For example, if more than one vmagent calculates `increase` for metric `http_requests_total` with `by: [path]` directive, then the resulting time series written to the remote database will be indistinguishable, as there is no way to tell to which `instance` the aggregation belongs. The proper fix would be to add a unique aggregation dimension: `by: [instance, path]`. With this change, aggregates between `instances` won't collide.
+then be careful when aggregating metrics via `by`, `without` or modifying via `*_relabel_configs` parameters, since incorrect usage
+may result in duplicates and data collision. For example, if more than one `vmagent` instance calculates `increase` for metric `http_requests_total`
+with `by: [path]` directive, then all the `vmagent` instances will aggregate samples to the same set of time series with different `path` labels.
+The proper fix would be to add an unique [`-remoteWrite.label`]https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics() per each `vmagent`,
+so every `vmagent` aggregates data into distinct set of time series. These time series then can be aggregated later as needed during querying.
 
-If adding a new aggregation dimension isn't feasible (due to cardinality reduction purposes), then it is worth at least differentiating by which vmagent the aggregation was performed. You can do it with `remoteWrite.label` [parameter in vmagent](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics).
-For example, for running in docker or k8s you can use `remoteWrite.label` with `POD_NAME` or `HOSTNAME` environment variable: `remoteWrite.label='vmagent=%{HOSTNAME}'`.
-See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247#issue-1692894073) for details.
+For example, if `vmagent` instances run in Docker or Kubernetes, then you can refer `POD_NAME` or `HOSTNAME` environment variables
+as an unique label value per each `vmagent`: `-remoteWrite.label='vmagent=%{HOSTNAME}` . See [these docs](https://docs.victoriametrics.com/#environment-variables)
+on how to refer environment variables in VictoriaMetrics components.

From a2c901423b451001d1a502e4a068bc01ef5033b8 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 17 Aug 2023 15:24:44 +0200
Subject: [PATCH 17/19] docs/stream-aggregation.md: typo fix after
 54f522ac257ea82db2ca18d0e458921f8ec5a510

---
 docs/stream-aggregation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index 161d6011f..80756a3f0 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -681,7 +681,7 @@ If you use [vmagent in cluster mode](https://docs.victoriametrics.com/vmagent.ht
 then be careful when aggregating metrics via `by`, `without` or modifying via `*_relabel_configs` parameters, since incorrect usage
 may result in duplicates and data collision. For example, if more than one `vmagent` instance calculates `increase` for metric `http_requests_total`
 with `by: [path]` directive, then all the `vmagent` instances will aggregate samples to the same set of time series with different `path` labels.
-The proper fix would be to add an unique [`-remoteWrite.label`]https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics() per each `vmagent`,
+The proper fix would be to add an unique [`-remoteWrite.label`](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics) per each `vmagent`,
 so every `vmagent` aggregates data into distinct set of time series. These time series then can be aggregated later as needed during querying.
 
 For example, if `vmagent` instances run in Docker or Kubernetes, then you can refer `POD_NAME` or `HOSTNAME` environment variables

From 35263983a6d080c76761c64d645ce299ec3b1daa Mon Sep 17 00:00:00 2001
From: Nikolay <nik@victoriametrics.com>
Date: Wed, 23 Aug 2023 13:22:53 +0200
Subject: [PATCH 18/19] lib/storage: properly caclucate nextRotationTimestamp
 (#4874)

cause of typo unix millis was used instead of unix for current timestamp
calculation
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873

(cherry picked from commit c5aac34b68f6c55cb465feb712c575b681e2a6bc)
---
 docs/CHANGELOG.md      | 1 +
 lib/storage/storage.go | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 74b545616..c2dab4c58 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -57,6 +57,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
 * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. 
 
+* BUGFIX: [storage](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): return human readable error if opentelemetry has json encoding. Follow-up after [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2570).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly validate scheme for `proxy_url` field at the scrape config. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4811) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816).
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index 5ef0a8290..e61c0f0c0 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -757,7 +757,7 @@ func (s *Storage) mustRotateIndexDB(currentTime time.Time) {
 	idbNew := mustOpenIndexDB(idbNewPath, s, &s.isReadOnly)
 
 	// Update nextRotationTimestamp
-	nextRotationTimestamp := currentTime.UnixMilli() + s.retentionMsecs/1000
+	nextRotationTimestamp := currentTime.Unix() + s.retentionMsecs/1000
 	atomic.StoreInt64(&s.nextRotationTimestamp, nextRotationTimestamp)
 
 	// Set idbNext to idbNew

From c8c20b7f7aa8f383dbc826af820e950a9b96806d Mon Sep 17 00:00:00 2001
From: Dmytro Kozlov <kozlovdmitriyy@gmail.com>
Date: Fri, 18 Aug 2023 08:55:42 +0200
Subject: [PATCH 19/19] docs: cut 1.93.1-lts in changelog

Signed-off-by: hagen1778 <roman@victoriametrics.com>
---
 docs/CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c2dab4c58..89d1d76d3 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,9 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## v1.93.x long-time support release (LTS)
 
+## v1.93.1 long-time support release (LTS)
+
+* BUGFIX: [storage](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: do not allow starting VictoriaMetrics components with improperly set boolean command-line flags in the form `-boolFlagName value`, since this leads to silent incomplete flags' parsing. This form should be replaced with `-boolFlagName=value`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly set labels from `-remoteWrite.label` command-line flag just before sending samples to the configured `-remoteWrite.url` according to [these docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics). Previously these labels were incorrectly set before [the relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) configured via `-remoteWrite.urlRelabelConfigs` and [the stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) configured via `-remoteWrite.streamAggr.config`, so these labels could be lost or incorrectly transformed before sending the samples to remote storage. The fix allows using `-remoteWrite.label` for identifying `vmagent` instances in [cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247) and [these docs](https://docs.victoriametrics.com/stream-aggregation.html#cluster-mode) for more details.
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).