Merge branch 'master' into 7127_vmagent_k8s_discovery

2025-02-09 15:27:11 +00:00 · 2024-11-21 00:19:21 +01:00 · 2024-11-21 00:19:21 +01:00 · 822ca028b8
commit 822ca028b8
parent f77929edce 61532930e6
84 changed files with 1722 additions and 733 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -88,6 +88,35 @@ jobs:
        run: make ${{ matrix.scenario}}

      - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
        with:
          file: ./coverage.txt
+
+  integration-test:
+    name: integration-test
+    needs: [lint, test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+
+      - name: Run integration tests
+        run: make integration-test
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@ -510,7 +510,13 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusNoContent)
+		statusCode := http.StatusNoContent
+		if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
+			// Return 200 status code for pushgateway requests.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
+			statusCode = http.StatusOK
+		}
+		w.WriteHeader(statusCode)
 		return true
 	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {
--- a/app/vmalert/remotewrite/client.go
+++ b/app/vmalert/remotewrite/client.go
@ -27,7 +27,7 @@ var defaultConcurrency = cgroup.AvailableCPUs() * 2

 const (
 	defaultMaxBatchSize  = 1e4
-	defaultMaxQueueSize  = 1e6
+	defaultMaxQueueSize  = 1e5
 	defaultFlushInterval = 2 * time.Second
 	defaultWriteTimeout  = 30 * time.Second
 )
--- a/app/vmselect/vmui/asset-manifest.json
+++ b/app/vmselect/vmui/asset-manifest.json
@ -1,13 +1,13 @@
 {
  "files": {
    "main.css": "./static/css/main.d781989c.css",
-    "main.js": "./static/js/main.7ec4e6eb.js",
+    "main.js": "./static/js/main.a7037969.js",
    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
    "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
    "index.html": "./index.html"
  },
  "entrypoints": [
    "static/css/main.d781989c.css",
-    "static/js/main.7ec4e6eb.js"
+    "static/js/main.a7037969.js"
  ]
 }
--- a/app/vmselect/vmui/index.html
+++ b/app/vmselect/vmui/index.html
@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.7ec4e6eb.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.a7037969.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
--- a/app/vmselect/vmui/static/js/main.a7037969.js
+++ b/app/vmselect/vmui/static/js/main.a7037969.js
--- a/app/vmselect/vmui/static/js/main.a7037969.js.LICENSE.txt
+++ b/app/vmselect/vmui/static/js/main.a7037969.js.LICENSE.txt
--- a/apptest/client.go
+++ b/apptest/client.go
@ -128,3 +128,31 @@ func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
 	t.Fatalf("metic not found: %s", metricName)
 	return 0
 }
+
+// GetMetricsByPrefix retrieves the values of all metrics that start with given
+// prefix.
+func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
+	t.Helper()
+
+	values := []float64{}
+
+	metrics := app.cli.Get(t, app.metricsURL, http.StatusOK)
+	for _, metric := range strings.Split(metrics, "\n") {
+		if !strings.HasPrefix(metric, prefix) {
+			continue
+		}
+
+		parts := strings.Split(metric, " ")
+		if len(parts) < 2 {
+			t.Fatalf("unexpected record format: got %q, want metric name and value separated by a space", metric)
+		}
+
+		value, err := strconv.ParseFloat(parts[len(parts)-1], 64)
+		if err != nil {
+			t.Fatalf("could not parse metric value %s: %v", metric, err)
+		}
+
+		values = append(values, value)
+	}
+	return values
+}
--- a/apptest/model.go
+++ b/apptest/model.go
@ -3,7 +3,9 @@ package apptest
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 )
@ -20,6 +22,20 @@ type PrometheusWriter interface {
 	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
 }

+// StorageFlusher defines a method that forces the flushing of data inserted
+// into the storage, so it becomes available for searching immediately.
+type StorageFlusher interface {
+	ForceFlush(t *testing.T)
+}
+
+// PrometheusWriteQuerier encompasses the methods for writing, flushing and
+// querying the data.
+type PrometheusWriteQuerier interface {
+	PrometheusWriter
+	PrometheusQuerier
+	StorageFlusher
+}
+
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
 	Tenant  string
@ -119,3 +135,19 @@ func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1Se
 	}
 	return res
 }
+
+// Sort sorts the response data.
+func (r *PrometheusAPIV1SeriesResponse) Sort() {
+	str := func(m map[string]string) string {
+		s := []string{}
+		for k, v := range m {
+			s = append(s, k+v)
+		}
+		slices.Sort(s)
+		return strings.Join(s, "")
+	}
+
+	slices.SortFunc(r.Data, func(a, b map[string]string) int {
+		return strings.Compare(str(a), str(b))
+	})
+}
--- a/apptest/testcase.go
+++ b/apptest/testcase.go
@ -1,9 +1,12 @@
 package apptest

 import (
+	"fmt"
 	"testing"
+	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/google/go-cmp/cmp"
 )

 // TestCase holds the state and defines clean-up procedure common for all test
@ -103,6 +106,124 @@ func (tc *TestCase) MustStartVminsert(instance string, flags []string) *Vminsert
 	return app
 }

+type vmcluster struct {
+	*Vminsert
+	*Vmselect
+	vmstorages []*Vmstorage
+}
+
+func (c *vmcluster) ForceFlush(t *testing.T) {
+	for _, s := range c.vmstorages {
+		s.ForceFlush(t)
+	}
+}
+
+// MustStartCluster is a typical cluster configuration.
+//
+// The cluster consists of two vmstorages, one vminsert and one vmselect, no
+// data replication.
+//
+// Such configuration is suitable for tests that don't verify the
+// cluster-specific behavior (such as sharding, replication, or multilevel
+// vmselect) but instead just need a typical cluster configuration to verify
+// some business logic (such as API surface, or MetricsQL). Such cluster
+// tests usually come paired with corresponding vmsingle tests.
+func (tc *TestCase) MustStartCluster() PrometheusWriteQuerier {
+	tc.t.Helper()
+
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
+		"-retentionPeriod=100y",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	})
+
+	return &vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
+}
+
 func (tc *TestCase) addApp(app Stopper) {
 	tc.startedApps = append(tc.startedApps, app)
 }
+
+// AssertOptions hold the assertion params, such as got and wanted values as
+// well as the message that should be included into the assertion error message
+// in case of failure.
+//
+// In VictoriaMetrics (especially the cluster version) the inserted data does
+// not become visible for querying right away. Therefore, the first comparisons
+// may fail. AssertOptions allow to configure how many times the actual result
+// must be retrieved and compared with the expected one and for long to wait
+// between the retries. If these two params (`Retries` and `Period`) are not
+// set, the default values will be used.
+//
+// If it is known that the data is available, then the retry functionality can
+// be disabled by setting the `DoNotRetry` field.
+//
+// AssertOptions are used by the TestCase.Assert() method, and this method uses
+// cmp.Diff() from go-cmp package for comparing got and wanted values.
+// AssertOptions, therefore, allows to pass cmp.Options to cmp.Diff() via
+// `CmpOpts` field.
+//
+// Finally the `FailNow` field controls whether the assertion should fail using
+// `testing.T.Errorf()` or `testing.T.Fatalf()`.
+type AssertOptions struct {
+	Msg        string
+	Got        func() any
+	Want       any
+	CmpOpts    []cmp.Option
+	DoNotRetry bool
+	Retries    int
+	Period     time.Duration
+	FailNow    bool
+}
+
+// Assert compares the actual result with the expected one possibly multiple
+// times in order to account for the fact that the inserted data does not become
+// available for querying right away (especially in cluster version of
+// VictoriaMetrics).
+func (tc *TestCase) Assert(opts *AssertOptions) {
+	tc.t.Helper()
+
+	const (
+		defaultRetries = 20
+		defaultPeriod  = 100 * time.Millisecond
+	)
+
+	if opts.DoNotRetry {
+		opts.Retries = 1
+		opts.Period = 0
+	} else {
+		if opts.Retries <= 0 {
+			opts.Retries = defaultRetries
+		}
+		if opts.Period <= 0 {
+			opts.Period = defaultPeriod
+		}
+	}
+
+	var diff string
+
+	for range opts.Retries {
+		diff = cmp.Diff(opts.Want, opts.Got(), opts.CmpOpts...)
+		if diff == "" {
+			return
+		}
+		time.Sleep(opts.Period)
+	}
+
+	msg := fmt.Sprintf("%s (-want, +got):\n%s", opts.Msg, diff)
+
+	if opts.FailNow {
+		tc.t.Fatal(msg)
+	} else {
+		tc.t.Error(msg)
+	}
+}
--- a/apptest/tests/key_concepts_test.go
+++ b/apptest/tests/key_concepts_test.go
@ -2,7 +2,6 @@ package tests

 import (
 	"testing"
-	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/google/go-cmp/cmp"
@ -29,67 +28,41 @@ var docData = []string{
 }

 // TestSingleKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+// for vm-single.
 func TestSingleKeyConceptsQuery(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()

-	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+	sut := tc.MustStartVmsingle("vmsingle", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage",
 		"-retentionPeriod=100y",
 	})

-	opts := apptest.QueryOpts{Timeout: "5s"}
-
-	// Insert example data from documentation.
-	vmsingle.PrometheusAPIV1ImportPrometheus(t, docData, opts)
-	vmsingle.ForceFlush(t)
-
-	testInstantQuery(t, vmsingle, opts)
-	testRangeQuery(t, vmsingle, opts)
-	testRangeQueryIsEquivalentToManyInstantQueries(t, vmsingle, opts)
+	testKeyConceptsQueryData(t, sut)
 }

-// TestClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
-func TestClusterKeyConceptsQuery(t *testing.T) {
+// TestClusterKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+// for vm-cluster.
+func TestClusterKeyConceptsQueryData(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()

-	// Set up the following cluster configuration:
-	//
-	// - two vmstorage instances
-	// - vminsert points to the two vmstorages, its replication setting
-	//   is off which means it will only shard the incoming data across the two
-	//   vmstorages.
-	// - vmselect points to the two vmstorages and is expected to query both
-	//   vmstorages and build the full result out of the two partial results.
+	sut := tc.MustStartCluster()

-	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
-		"-retentionPeriod=100y",
-	})
-	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
-		"-retentionPeriod=100y",
-	})
-	vminsert := tc.MustStartVminsert("vminsert", []string{
-		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	})
-	vmselect := tc.MustStartVmselect("vmselect", []string{
-		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	})
+	testKeyConceptsQueryData(t, sut)
+}

+// testClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+func testKeyConceptsQueryData(t *testing.T, sut apptest.PrometheusWriteQuerier) {
 	opts := apptest.QueryOpts{Timeout: "5s", Tenant: "0"}

 	// Insert example data from documentation.
-	vminsert.PrometheusAPIV1ImportPrometheus(t, docData, opts)
-	time.Sleep(2 * time.Second)
+	sut.PrometheusAPIV1ImportPrometheus(t, docData, opts)
+	sut.ForceFlush(t)

-	vmstorage1.ForceFlush(t)
-	vmstorage2.ForceFlush(t)
-
-	testInstantQuery(t, vmselect, opts)
-	testRangeQuery(t, vmselect, opts)
-	testRangeQueryIsEquivalentToManyInstantQueries(t, vmselect, opts)
+	testInstantQuery(t, sut, opts)
+	testRangeQuery(t, sut, opts)
+	testRangeQueryIsEquivalentToManyInstantQueries(t, sut, opts)
 }

 // testInstantQuery verifies the statements made in the `Instant query` section
--- a/apptest/tests/multilevel_test.go
+++ b/apptest/tests/multilevel_test.go
@ -4,7 +4,6 @@ import (
 	"fmt"
 	"math/rand/v2"
 	"testing"
-	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
@ -33,30 +32,41 @@ func TestClusterMultilevelSelect(t *testing.T) {
 		"-storageNode=" + vmselectL1.ClusternativeListenAddr(),
 	})

-	// Insert 1000 unique time series.Wait for 2 seconds to let vmstorage
-	// flush pending items so they become searchable.
+	// Insert 1000 unique time series.

 	const numMetrics = 1000
 	records := make([]string, numMetrics)
+	want := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+		Data:      make([]map[string]string, numMetrics),
+	}
 	for i := range numMetrics {
-		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
+		name := fmt.Sprintf("metric_%d", i)
+		records[i] = fmt.Sprintf("%s %d", name, rand.IntN(1000))
+		want.Data[i] = map[string]string{"__name__": name}
 	}
-	vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{Tenant: "0"})
-	time.Sleep(2 * time.Second)
+	want.Sort()
+	qopts := apptest.QueryOpts{Tenant: "0"}
+	vminsert.PrometheusAPIV1ImportPrometheus(t, records, qopts)
+	vmstorage.ForceFlush(t)

-	// Retrieve all time series and verify that vmselect (L1) serves the complete
-	// set of time series.
+	// Retrieve all time series and verify that both vmselect (L1) and
+	// vmselect (L2) serve the complete set of time series.

-	seriesL1 := vmselectL1.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
-	if got, want := len(seriesL1.Data), numMetrics; got != want {
-		t.Fatalf("unexpected level-1 series count: got %d, want %d", got, want)
-	}
-
-	// Retrieve all time series and verify that vmselect (L2) serves the complete
-	// set of time series.
-
-	seriesL2 := vmselectL2.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
-	if got, want := len(seriesL2.Data), numMetrics; got != want {
-		t.Fatalf("unexpected level-2 series count: got %d, want %d", got, want)
+	got := func(app *apptest.Vmselect) any {
+		res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, qopts)
+		res.Sort()
+		return res
 	}
+	tc.Assert(&apptest.AssertOptions{
+		Msg:  "unexpected level-1 series count",
+		Got:  func() any { return got(vmselectL1) },
+		Want: want,
+	})
+	tc.Assert(&apptest.AssertOptions{
+		Msg:  "unexpected level-2 series count",
+		Got:  func() any { return got(vmselectL2) },
+		Want: want,
+	})
 }
--- a/apptest/tests/sharding_test.go
+++ b/apptest/tests/sharding_test.go
@ -4,7 +4,6 @@ import (
 	"fmt"
 	"math/rand/v2"
 	"testing"
-	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
@ -35,20 +34,28 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
 	})

-	// Insert 1000 unique time series and verify the that inserted data has been
-	// indeed sharded by checking various metrics exposed by vminsert and
-	// vmstorage.
-	// Also wait for 2 seconds to let vminsert and vmstorage servers to update
-	// the values of the metrics they expose and to let vmstorages flush pending
-	// items so they become searchable.
+	// Insert 1000 unique time series.

 	const numMetrics = 1000
 	records := make([]string, numMetrics)
-	for i := range numMetrics {
-		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
+	want := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+		Data:      make([]map[string]string, numMetrics),
 	}
-	vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{Tenant: "0"})
-	time.Sleep(2 * time.Second)
+	for i := range numMetrics {
+		name := fmt.Sprintf("metric_%d", i)
+		records[i] = fmt.Sprintf("%s %d", name, rand.IntN(1000))
+		want.Data[i] = map[string]string{"__name__": name}
+	}
+	want.Sort()
+	qopts := apptest.QueryOpts{Tenant: "0"}
+	vminsert.PrometheusAPIV1ImportPrometheus(t, records, qopts)
+	vmstorage1.ForceFlush(t)
+	vmstorage2.ForceFlush(t)
+
+	// Verify that inserted data has been indeed sharded by checking metrics
+	// exposed by vmstorage.

 	numMetrics1 := vmstorage1.GetIntMetric(t, "vm_vminsert_metrics_read_total")
 	if numMetrics1 == 0 {
@ -63,16 +70,15 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 	}

 	// Retrieve all time series and verify that vmselect serves the complete set
-	//of time series.
+	// of time series.

-	series := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
-	if got, want := series.Status, "success"; got != want {
-		t.Fatalf("unexpected /ap1/v1/series response status: got %s, want %s", got, want)
-	}
-	if got, want := series.IsPartial, false; got != want {
-		t.Fatalf("unexpected /ap1/v1/series response isPartial value: got %t, want %t", got, want)
-	}
-	if got, want := len(series.Data), numMetrics; got != want {
-		t.Fatalf("unexpected /ap1/v1/series response series count: got %d, want %d", got, want)
-	}
+	tc.Assert(&apptest.AssertOptions{
+		Msg: "unexpected /api/v1/series response",
+		Got: func() any {
+			res := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, qopts)
+			res.Sort()
+			return res
+		},
+		Want: want,
+	})
 }
--- a/apptest/vminsert.go
+++ b/apptest/vminsert.go
@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 )

 // Vminsert holds the state of a vminsert app and provides vminsert-specific
@ -55,10 +56,47 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	t.Helper()

 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.Tenant)
+	wantRowsSentCount := app.rpcRowsSentTotal(t) + len(records)
 	app.cli.Post(t, url, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
+	app.waitUntilSent(t, wantRowsSentCount)
 }

 // String returns the string representation of the vminsert app state.
 func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
 }
+
+// waitUntilSent waits until vminsert sends buffered data to vmstorage.
+//
+// Waiting is implemented a retrieving the value of `vm_rpc_rows_sent_total`
+// metric and checking whether it is equal or greater than the wanted value.
+// If it is, then the data has been sent to vmstorage.
+//
+// Unreliable if the records are inserted concurrently.
+func (app *Vminsert) waitUntilSent(t *testing.T, wantRowsSentCount int) {
+	t.Helper()
+
+	const (
+		retries = 20
+		period  = 100 * time.Millisecond
+	)
+
+	for range retries {
+		if app.rpcRowsSentTotal(t) >= wantRowsSentCount {
+			return
+		}
+		time.Sleep(period)
+	}
+	t.Fatalf("timed out while waiting for inserted rows to be sent to vmstorage")
+}
+
+// rpcRowsSentTotal retrieves the values of all vminsert
+// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
+// returns their integer sum.
+func (app *Vminsert) rpcRowsSentTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range app.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
+		total += v
+	}
+	return int(total)
+}
--- a/deployment/docker/README.md
+++ b/deployment/docker/README.md
@ -167,6 +167,8 @@ The list of alerting rules is the following:
  alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component;
 * [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml):
    alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/);
+* [alerts-vmanomaly.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmanomaly.yml):
+    alerting rules related to [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/);

 Please, also see [how to monitor](https://docs.victoriametrics.com/single-server-victoriametrics/#monitoring) 
 VictoriaMetrics installations.
--- a/deployment/docker/docker-compose-cluster.yml
+++ b/deployment/docker/docker-compose-cluster.yml
@ -4,7 +4,7 @@ services:
  #  And forward them to --remoteWrite.url
  vmagent:
    container_name: vmagent
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    depends_on:
      - "vminsert"
    ports:
@ -39,7 +39,7 @@ services:
  # where N is number of vmstorages (2 in this case).
  vmstorage-1:
    container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.106.0-cluster
+    image: victoriametrics/vmstorage:v1.106.1-cluster
    ports:
      - 8482
      - 8400
@ -51,7 +51,7 @@ services:
    restart: always
  vmstorage-2:
    container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.106.0-cluster
+    image: victoriametrics/vmstorage:v1.106.1-cluster
    ports:
      - 8482
      - 8400
@ -66,7 +66,7 @@ services:
  # pre-process them and distributes across configured vmstorage shards.
  vminsert:
    container_name: vminsert
-    image: victoriametrics/vminsert:v1.106.0-cluster
+    image: victoriametrics/vminsert:v1.106.1-cluster
    depends_on:
      - "vmstorage-1"
      - "vmstorage-2"
@ -81,7 +81,7 @@ services:
  # vmselect collects results from configured `--storageNode` shards.
  vmselect-1:
    container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.106.0-cluster
+    image: victoriametrics/vmselect:v1.106.1-cluster
    depends_on:
      - "vmstorage-1"
      - "vmstorage-2"
@ -94,7 +94,7 @@ services:
    restart: always
  vmselect-2:
    container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.106.0-cluster
+    image: victoriametrics/vmselect:v1.106.1-cluster
    depends_on:
      - "vmstorage-1"
      - "vmstorage-2"
@ -112,7 +112,7 @@ services:
  # It can be used as an authentication proxy.
  vmauth:
    container_name: vmauth
-    image: victoriametrics/vmauth:v1.106.0
+    image: victoriametrics/vmauth:v1.106.1
    depends_on:
      - "vmselect-1"
      - "vmselect-2"
@ -127,7 +127,7 @@ services:
  # vmalert executes alerting and recording rules
  vmalert:
    container_name: vmalert
-    image: victoriametrics/vmalert:v1.106.0
+    image: victoriametrics/vmalert:v1.106.1
    depends_on:
      - "vmauth"
    ports:
--- a/deployment/docker/docker-compose-victorialogs.yml
+++ b/deployment/docker/docker-compose-victorialogs.yml
@ -16,7 +16,7 @@ services:
      - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
      - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
    environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.6.2/victorialogs-datasource-v0.6.2.zip;victorialogs-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.8.0/victorialogs-datasource-v0.8.0.zip;victorialogs-datasource"
      - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource"
    networks:
      - vm_net
@ -55,7 +55,7 @@ services:
  # scraping, storing metrics and serve read requests.
  victoriametrics:
    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - 8428:8428
    volumes:
@ -74,7 +74,7 @@ services:
  # depending on the requested path.
  vmauth:
    container_name: vmauth
-    image: victoriametrics/vmauth:v1.106.0
+    image: victoriametrics/vmauth:v1.106.1
    depends_on:
      - "victoriametrics"
      - "victorialogs"
@ -91,7 +91,7 @@ services:
  # vmalert executes alerting and recording rules according to given rule type.
  vmalert:
    container_name: vmalert
-    image: victoriametrics/vmalert:v1.106.0
+    image: victoriametrics/vmalert:v1.106.1
    depends_on:
      - "vmauth"
      - "alertmanager"
@ -104,6 +104,8 @@ services:
      - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
      - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
      - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      # vlogs rule
+      - ./rules/vlogs-example-alerts.yml:/etc/alerts/vlogs-example-alerts.yml
    command:
      - "--datasource.url=http://vmauth:8427/"
      - "--remoteRead.url=http://victoriametrics:8428/"
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@ -4,7 +4,7 @@ services:
  #  And forward them to --remoteWrite.url
  vmagent:
    container_name: vmagent
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    depends_on:
      - "victoriametrics"
    ports:
@ -22,7 +22,7 @@ services:
  # storing metrics and serve read requests.
  victoriametrics:
    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - 8428:8428
      - 8089:8089
@ -65,7 +65,7 @@ services:
  # vmalert executes alerting and recording rules
  vmalert:
    container_name: vmalert
-    image: victoriametrics/vmalert:v1.106.0
+    image: victoriametrics/vmalert:v1.106.1
    depends_on:
      - "victoriametrics"
      - "alertmanager"
--- a/deployment/docker/rules/alerts-vmanomaly.yml
+++ b/deployment/docker/rules/alerts-vmanomaly.yml
@ -0,0 +1,121 @@
+# This file provides a recommended list of alerts to monitor the health of VictoriaMetrics Anomaly Detection (vmanomaly).
+# Note: The alerts below are general recommendations and may require customization,
+# including threshold adjustments, to suit the specifics of your setup.
+
+groups:
+    # Note - Adjust the `job` filter to match your specific setup.
+    # By default, the `job` label for vmanomaly in push-based self-monitoring mode is set to `vmanomaly`.
+    # However, this can be overridden using additional labels. For further details, refer to the example here:
+    # https://docs.victoriametrics.com/anomaly-detection/components/monitoring/?highlight=extra_labels#monitoring-section-config-example
+  - name: vmanomaly-health
+    rules:
+      - alert: TooManyRestarts
+        expr: changes(process_start_time_seconds{job=~".*vmanomaly.*"}[15m]) > 2
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
+          description: |
+            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
+            It might be crashlooping. Please check the logs for more details.
+            Additionally, refer to the "r:errors" value in the "Instance Overview" section of the self-monitoring Grafana dashboard.
+
+      # works if you use Prometheus scraping (pull model only)
+      - alert: ServiceDown
+        expr: up{job=~".*vmanomaly.*"} == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
+
+      - alert: ProcessNearFDLimits
+        expr: (process_max_fds{job=~".*vmanomaly.*"} - process_open_fds{job=~".*vmanomaly.*"}) < 100
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
+          description: |
+            Exhausting OS file descriptors limit can cause severe degradation of the process.
+            Consider to increase the limit as fast as possible.
+
+      - alert: TooHighCPUUsage
+        expr: > 
+          sum(rate(process_cpu_seconds_total{job=~".*vmanomaly.*"}[5m])) by (job, instance) /
+          sum(vmanomaly_cpu_cores_available{job=~".*vmanomaly.*"}[5m]) by (job, instance) > 0.9
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
+          description: >
+            Too high CPU usage may be a sign of insufficient resources and make process unstable.
+            Consider to either increase available CPU resources or decrease the load on the process.
+
+      - alert: TooHighMemoryUsage
+        expr: (min_over_time(process_resident_memory_bytes[10m]) / vmanomaly_available_memory_bytes) > 0.85
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "It is more than 85% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: |
+            Too high memory usage may result into multiple issues such as OOMs or degraded performance.
+            E.g. it can be caused by high churn rate in your input data.
+            Consider to either increase available memory or decrease the load on the process.
+
+  - name: vmanomaly-issues
+    rules:
+      - alert: ServiceErrorsDetected
+        expr: sum(increase(vmanomaly_model_run_errors_total{job=~".*vmanomaly.*"}[5m])) by (job, instance, stage) > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Model Run Errors in \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") stage: {{ $labels.stage }} during the last 5m"
+          description: >
+            Errors in the service may indicate a problem with the service itself or its dependencies.
+            Investigate the logs for more details.
+      - alert: SkippedModelRunsDetected
+        expr: sum(increase(vmanomaly_model_runs_skipped_total{job=~".*vmanomaly.*"}[5m])) by (job, instance, stage) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Skipped Model Runs in \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") stage: {{ $labels.stage }} during the last 5m"
+          description: >
+            Skipped model runs may indicate issues like:
+              1. No new or valid data is available for the current run.
+              2. The presence of new time series that do not have a trained model yet.
+              3. No new (or valid) datapoints produced during inference.
+            Investigate the logs for more details.
+      - alert: HighReadErrorRate
+        expr: >
+          (
+            sum(increase(vmanomaly_reader_responses_total{job=~".*vmanomaly.*", code=~"2.."}[5m])) by (job, instance, url) /
+            sum(increase(vmanomaly_reader_responses_total{job=~".*vmanomaly.*"}[5m])) by (job, instance, url)
+          ) < 0.95
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High error rate in read requests for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for url: {{ $labels.url }} during the last 5m"
+          description: >
+            Reading errors may indicate issues with the input data source, server-side constraint violations, security or network issues. 
+            Investigate the logs for more details.
+      - alert: HighWriteErrorRate
+        expr: >
+          (
+            sum(increase(vmanomaly_writer_responses_total{job=~".*vmanomaly.*", code=~"2.."}[5m])) by (job, instance, url) /
+            sum(increase(vmanomaly_writer_responses_total{job=~".*vmanomaly.*"}[5m])) by (job, instance, url)
+          ) < 0.95
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High error rate in write requests for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for url: {{ $labels.url }} during the last 5m"
+          description: >
+            Writing errors may indicate issues with the destination source, server-side constraint violations, security, or network issues. 
+            Investigate the logs for more details.
--- a/deployment/docker/rules/vlogs-example-alerts.yml
+++ b/deployment/docker/rules/vlogs-example-alerts.yml
@ -1,5 +1,5 @@
 groups:
-  - name: TestGroup
+  - name: log-rules
    type: vlogs
    interval: 1m
    rules:
--- a/deployment/docker/victorialogs/compose-base.yml
+++ b/deployment/docker/victorialogs/compose-base.yml
@ -18,7 +18,7 @@ services:
      retries: 10

  dd-logs:
-    image: docker.io/victoriametrics/vmauth:v1.106.0
+    image: docker.io/victoriametrics/vmauth:v1.106.1
    restart: on-failure
    volumes:
      - ./:/etc/vmauth
--- a/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml
+++ b/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml
@ -1,7 +1,7 @@
 services:
  vmagent:
    container_name: vmagent
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    depends_on:
      - "victoriametrics"
    ports:
@ -18,7 +18,7 @@ services:

  victoriametrics:
    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - 8428:8428
    volumes:
@ -50,7 +50,7 @@ services:

  vmalert:
    container_name: vmalert
-    image: victoriametrics/vmalert:v1.106.0
+    image: victoriametrics/vmalert:v1.106.1
    depends_on:
      - "victoriametrics"
    ports:
@ -72,7 +72,7 @@ services:
    restart: always
  vmanomaly:
    container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.3
+    image: victoriametrics/vmanomaly:v1.18.4
    depends_on:
      - "victoriametrics"
    ports:
--- a/deployment/logs-benchmark/docker-compose.yml
+++ b/deployment/logs-benchmark/docker-compose.yml
@ -46,7 +46,7 @@ services:
      - "--config=/config.yml"

  vmsingle:
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - "8428:8428"
    command:
--- a/docs/Articles.md
+++ b/docs/Articles.md
@ -92,7 +92,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 ### Announcements

 * [Open-source strategy at VictoriaMetrics](https://www.youtube.com/watch?v=-DbbIZzFHIY)
-* [Open-sourcing VictoriaMetrics](https://blog.usejournal.com/open-sourcing-victoriametrics-f31e34485c2b)
+* [Open-sourcing VictoriaMetrics](https://valyala.medium.com/open-sourcing-victoriametrics-f31e34485c2b)
 * [VictoriaMetrics — creating the best remote storage for Prometheus](https://faun.pub/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac)
 * [Anomaly Detection in VictoriaMetrics](https://victoriametrics.medium.com/anomaly-detection-in-victoriametrics-9528538786a7)

@ -119,6 +119,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 * [Improving histogram usability for Prometheus and Grafana](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
 * [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
 * [VictoriaMetrics: PromQL compliance](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e)
+* [How do open source solutions for logs work: Elasticsearch, Loki and VictoriaLogs](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f)

 ### Tutorials, guides and how-to articles

--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@ -95,7 +95,7 @@ We are open to third-party pull requests provided they follow [KISS design princ

 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and debugged by wider audience.

-Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/) has no the following "features" popular in distributed computing world:
+Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/) has none of the following "features" popular in distributed computing world:

 - Fragile gossip protocols. See [failed attempt in Thanos](https://github.com/improbable-eng/thanos/blob/030bc345c12c446962225221795f4973848caab5/docs/proposals/completed/201809_gossip-removal.md).
 - Hard-to-understand-and-implement-properly [Paxos protocols](https://www.quora.com/In-distributed-systems-what-is-a-simple-explanation-of-the-Paxos-algorithm).
--- a/docs/VictoriaLogs/FAQ.md
+++ b/docs/VictoriaLogs/FAQ.md
@ -34,7 +34,7 @@ VictoriaLogs is optimized specifically for logs. So it provides the following fe
 - Easy to setup and operate. There is no need in tuning configuration for optimal performance or in creating any indexes for various log types.
  Just run VictoriaLogs on the most suitable hardware, ingest logs into it via [supported data ingestion protocols](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
  and get the best available performance out of the box.
- Up to 30x less RAM usage than Elasticsearch for the same workload.
+- Up to 30x less RAM usage than Elasticsearch for the same workload. See [this article](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f) for details.
 - Up to 15x less disk space usage than Elasticsearch for the same amounts of stored logs.
 - Ability to work efficiently with hundreds of terabytes of logs on a single node.
 - Easy to use query language optimized for typical log analysis tasks - [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
--- a/docs/VictoriaLogs/README.md
+++ b/docs/VictoriaLogs/README.md
@ -15,7 +15,7 @@ VictoriaLogs provides the following features:
 - VictoriaLogs' capacity and performance scales linearly with the available resources (CPU, RAM, disk IO, disk space).
  It runs smoothly on Raspberry PI and on servers with hundreds of CPU cores and terabytes of RAM.
 - It can handle up to 30x bigger data volumes than Elasticsearch and Grafana Loki when running on the same hardware.
-  See [these docs](#benchmarks).
+  See [these docs](#benchmarks) and [this article](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f) for details.
 - It provides fast full-text search out of the box for [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
  with high cardinality (e.g. high number of unique values) such as `trace_id`, `user_id` and `ip`.
 - It supports multitenancy - see [these docs](#multitenancy).
--- a/docs/VictoriaLogs/data-ingestion/README.md
+++ b/docs/VictoriaLogs/data-ingestion/README.md
@ -317,7 +317,7 @@ Here is the list of log collectors and their ingestion formats supported by Vict
 | [Filebeat](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No | No | No | No | No |
 | [Fluentbit](https://docs.victoriametrics.com/victorialogs/data-ingestion/fluentbit/) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/syslog) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/opentelemetry) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/datadog) |
 | [Logstash](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/)   | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-syslog.html) | [Yes](https://github.com/paulgrav/logstash-output-opentelemetry) | No | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-datadog.html) |
-| [Vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/http/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | No | [Yes](https://vector.dev/docs/reference/configuration/sources/opentelemetry/) | No | [Yes](https://vector.dev/docs/reference/configuration/sinks/datadog_logs/) |
+| [Vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/http/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | No | No | No | [Yes](https://vector.dev/docs/reference/configuration/sinks/datadog_logs/) |
 | [Promtail](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/)   | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | No | No | No | No |
 | [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter) | No | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/lokiexporter) | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/syslogexporter) | [Yes](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter) | No | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter) |
 | [Telegraf](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/http) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/loki) | [Yes](https://github.com/influxdata/telegraf/blob/master/plugins/outputs/syslog) | Yes | No | No |
--- a/docs/VictoriaLogs/victorialogs-datasource.md
+++ b/docs/VictoriaLogs/victorialogs-datasource.md
@ -18,6 +18,7 @@ The VictoriaLogs datasource plugin allows you to query and visualize
 [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) data in [Grafana](https://grafana.com).

 * [Installation](#installation)
+* [Getting started development](#getting-started-development)
 * [How to make new release](#how-to-make-new-release)
 * [Notes](#notes)
 * [License](#license)
@ -79,7 +80,7 @@ Please find the example of provisioning Grafana instance with VictoriaLogs datas
       grafana:
         image: grafana/grafana:11.0.0
         environment:
-         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.7.0/victorialogs-datasource-v0.7.0.zip;victorialogs-datasource
+         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.8.0/victorialogs-datasource-v0.8.0.zip;victorialogs-datasource
         - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource
         ports:
         - 3000:3000/tcp
@ -107,7 +108,7 @@ Option 1. Using Grafana provisioning:

 ``` yaml
 env:
-  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.7.0/victorialogs-datasource-v0.7.0.zip;victorialogs-datasource"
+  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.8.0/victorialogs-datasource-v0.8.0.zip;victorialogs-datasource"
  GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "victorialogs-datasource"
 ```

@ -115,7 +116,7 @@ Option 2. Using Grafana plugins section in `values.yaml`:

 ``` yaml
 plugins:
-  - https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.7.0/victorialogs-datasource-v0.7.0.zip;victorialogs-datasource
+  - https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.8.0/victorialogs-datasource-v0.8.0.zip;victorialogs-datasource
 ```

 Option 3. Using init container:
@ -158,7 +159,7 @@ sidecar:

 See more about chart settings [here](https://github.com/grafana/helm-charts/blob/541d97051de87a309362e02d08741ffc868cfcd6/charts/grafana/values.yaml)

-Option 4 would be to build custom Grafana image with plugin based on same installation instructions.
+Option 4. would be to build custom Grafana image with plugin based on same installation instructions.

 #### Grafana operator

--- a/docs/anomaly-detection/CHANGELOG.md
+++ b/docs/anomaly-detection/CHANGELOG.md
@ -11,6 +11,11 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.

+## v1.18.4
+Released: 2024-11-18
+
+- IMPROVEMENT: Introduced [self-monitoring guide](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/) for `vmanomaly`. Added metrics for total RAM `vmanomaly_available_memory_bytes` and the number of logical CPU cores `vmanomaly_cpu_cores_available` to the [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#metrics-generated-by-vmanomaly).
+
 ## v1.18.3
 Released: 2024-11-14

--- a/docs/anomaly-detection/FAQ.md
+++ b/docs/anomaly-detection/FAQ.md
@ -159,7 +159,7 @@ services:
  # ...
  vmanomaly:
    container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.3
+    image: victoriametrics/vmanomaly:v1.18.4
    # ...
    ports:
      - "8490:8490"
@ -361,3 +361,9 @@ reader:
      - '{region="region_name_2"}'
      # ...
 ```
+
+## Monitoring vmanomaly
+
+`vmanomaly` includes self-monitoring features that allow you to track its health, performance, and detect arising issues. Metrics related to resource usage, model runs, errors, and I/O operations are visualized using a Grafana Dashboard and are complemented by alerting rules that notify you of critical conditions. These monitoring tools help ensure stability and efficient troubleshooting of the service.
+
+For detailed instructions on setting up self-monitoring, dashboards, and alerting rules, refer to the [self-monitoring documentation](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/).
--- a/docs/anomaly-detection/Overview.md
+++ b/docs/anomaly-detection/Overview.md
@ -229,7 +229,7 @@ This will expose metrics at `http://0.0.0.0:8080/metrics` page.
 To use *vmanomaly* you need to pull docker image:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.3
+docker pull victoriametrics/vmanomaly:v1.18.4
 ```

 > Note: please check what is latest release in [CHANGELOG](https://docs.victoriametrics.com/anomaly-detection/changelog/)
@ -239,7 +239,7 @@ docker pull victoriametrics/vmanomaly:v1.18.3
 You can put a tag on it for your convenience:

 ```sh
-docker image tag victoriametrics/vmanomaly:v1.18.3 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.18.4 vmanomaly
 ```
 Here is an example of how to run *vmanomaly* docker container with [license file](#licensing):

--- a/docs/anomaly-detection/QuickStart.md
+++ b/docs/anomaly-detection/QuickStart.md
@ -58,13 +58,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.3
+docker pull victoriametrics/vmanomaly:v1.18.4
 ```

 2. (Optional step) tag the `vmanomaly` Docker image:

 ```sh
-docker image tag victoriametrics/vmanomaly:v1.18.3 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.18.4 vmanomaly
 ```

 3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@ -98,7 +98,7 @@ docker run -it --user 1000:1000 \
 services:
  # ...
  vmanomaly:
-    image: victoriametrics/vmanomaly:v1.18.3
+    image: victoriametrics/vmanomaly:v1.18.4
    volumes:
        $YOUR_LICENSE_FILE_PATH:/license
        $YOUR_CONFIG_FILE_PATH:/config.yml
--- a/docs/anomaly-detection/README.md
+++ b/docs/anomaly-detection/README.md
@ -1,17 +1,19 @@
-In the dynamic and complex world of system monitoring, VictoriaMetrics Anomaly Detection, being a part of our [Enterprise offering](https://victoriametrics.com/products/enterprise/), stands as a pivotal tool for achieving advanced observability. It empowers SREs and DevOps teams by automating the intricate task of identifying abnormal behavior in time-series data. It goes beyond traditional threshold-based alerting, utilizing machine learning techniques to not only detect anomalies but also minimize false positives, thus reducing alert fatigue. By providing simplified alerting mechanisms atop of [unified anomaly scores](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output), it enables teams to spot and address potential issues faster, ensuring system reliability and operational efficiency.
+In the dynamic and complex world of system monitoring, [VictoriaMetrics Anomaly Detection](https://victoriametrics.com/products/enterprise/anomaly-detection/) (or shortly, `vmanomaly`), being a part of our [Enterprise offering](https://victoriametrics.com/products/enterprise/), stands as a pivotal tool for achieving advanced observability. It empowers SREs and DevOps teams by automating the identification of abnormal behavior in time-series data. It goes beyond traditional threshold-based alerting, utilizing machine learning techniques to not only detect anomalies but also minimize false positives, thus reducing alert fatigue. By providing simplified alerting mechanisms atop of [unified anomaly scores](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output), it enables teams to spot and address potential issues faster, ensuring system reliability and operational efficiency.

 ## Practical Guides and Installation
-Begin your VictoriaMetrics Anomaly Detection journey with ease using our guides and installation instructions:

- **Quickstart**: Check out how to get `vmanomaly` up and running [here](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
- **Overview**: Find out how `vmanomaly` service operates [here](https://docs.victoriametrics.com/anomaly-detection/overview/)
- **Integration**: Integrate anomaly detection into your observability ecosystem. Get started [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
- **Anomaly Detection Presets**: Enable anomaly detection on predefined set of indicators, that require frequently changing static thresholds for alerting. Find more information [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
+Get started with VictoriaMetrics Anomaly Detection efficiently by following our guides and installation options:

- **Installation Options**: Select the method that aligns with your technical requirements:
-    - **Docker Installation**: Suitable for containerized environments. See [Docker guide](https://docs.victoriametrics.com/anomaly-detection/overview/#run-vmanomaly-docker-container).
-    - **Helm Chart Installation**: Appropriate for those using Kubernetes. See our [Helm charts](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-anomaly).
+- **Quickstart**: Learn how to quickly set up `vmanomaly` by following the [Quickstart Guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
+- **Overview**: Understand the architecture and operation of the `vmanomaly` service [here](https://docs.victoriametrics.com/anomaly-detection/overview/).
+- **Integration**: Integrate anomaly detection into your existing observability stack. Find detailed steps [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
+- **Anomaly Detection Presets**: Enable anomaly detection on predefined sets of metrics that require frequent static threshold changes for alerting. Learn more [here](https://docs.victoriametrics.com/anomaly-detection/presets/).

+- **Installation Options**: Choose the installation method that best fits your infrastructure:
+    - **Docker Installation**: Ideal for containerized environments. Follow the [Docker Installation Guide](https://docs.victoriametrics.com/anomaly-detection/overview/#run-vmanomaly-docker-container).
+    - **Helm Chart Installation**: Recommended for Kubernetes deployments. See our [Helm charts](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-anomaly).
+
+- **Self-Monitoring**: Ensure `vmanomaly` is functioning optimally with built-in self-monitoring capabilities. Use the provided Grafana dashboards and alerting rules to track service health and operational metrics. Find the complete docs [here](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/).

 > **Note**: starting from [v1.5.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v150) `vmanomaly` requires a [license key](https://docs.victoriametrics.com/anomaly-detection/overview/#licensing) to run. You can obtain a trial license key [**here**](https://victoriametrics.com/products/enterprise/trial/).

@ -32,6 +34,9 @@ Enhance your knowledge with our handbook on Anomaly Detection & Root Cause Analy
    - [Techniques and Models for Anomaly Detection](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-3/)
 * Follow the [`#anomaly-detection`](https://victoriametrics.com/blog/tags/anomaly-detection/) tag in our blog

+## Product Updates
+Stay up-to-date with the latest improvements and features in VictoriaMetrics Anomaly Detection, and the rest of our products on our [blog](https://victoriametrics.com/blog/tags/product-updates/).
+
 ## Frequently Asked Questions (FAQ)
 Got questions about VictoriaMetrics Anomaly Detection? Chances are, we've got the answers ready for you. 

--- a/docs/anomaly-detection/Self-monitoring.md
+++ b/docs/anomaly-detection/Self-monitoring.md
@ -0,0 +1,147 @@
+---
+weight: 5
+title: Self-monitoring
+menu:
+  docs:
+    identifier: "vmanomaly-self-monitoring"
+    parent: "anomaly-detection"
+    weight: 5
+aliases:
+- /anomaly-detection/self-monitoring.html
+---
+
+## What is Self-Monitoring
+
+Self-monitoring refers to the ability of the service to track and report its own health, operational performance, and any potential issues in real time. This process enables proactive maintenance and problem identification, making it easier for administrators and users to maintain system stability and performance.
+
+VictoriaMetrics Anomaly Detection (`vmanomaly`) supports self-monitoring by generating metrics related to different operational aspects of the service, including model execution, reader behavior, writer behavior, and overall service health. These metrics provide insights into system bottlenecks, errors, or unusual behavior, and allow for data-driven decisions on tuning performance and reliability.
+
+Self-monitoring metrics are available in both the [push](https://docs.victoriametrics.com/keyconcepts/#push-model) and [pull](https://docs.victoriametrics.com/keyconcepts/#pull-model) models, providing flexibility to fit into different monitoring environments. By specifying relevant parameters in the `monitoring` section of the configuration, `vmanomaly` components can seamlessly integrate with VictoriaMetrics, Prometheus or other monitoring solutions, enabling centralized visibility into both service and anomaly detection outcomes.
+
+> **Note**: For the detailed overview of self-monitoring metrics that are produced by `vmanomaly` and how to enable their tracking for push/pull models, please refer to [monitoring](https://docs.victoriametrics.com/anomaly-detection/components/monitoring) section docs.
+
+The self-monitoring assets of `vmanomaly` include Grafana dashboard and accompanying alerting rules.
+
+## Grafana Dashboard
+
+> **Note**: Recent revision of Grafana dashboard is designed to work with metrics produced by `vmanomaly` version [v1.18.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1184) or higher.
+
+
+### Overview
+
+To visualize and interact with the self-monitoring metrics, `vmanomaly` provides a [Grafana Dashboard](https://grafana.com/grafana/dashboards/22337/). It offers an overview of service health (per job, instance) and performance metrics from different operational stages, including model behavior, reader, and writer components.
+
+The Grafana Dashboard is helpful for:
+
+- Monitoring common metrics such as uptime, RAM, and CPU usage.
+- Tracking key metrics like skipped model runs, request errors, and stage timings across `vmanomaly` components.
+- Identifying service-specific bottlenecks or performance issues at a glance.
+
+### Navigating the Dashboard
+
+> **Note**: Use the top-level filters to refine metrics by job, instance, or specific components for more focused monitoring. The time range filter, along with `job` and `instance` filters, is applied across all components. All other filters apply to all dashboard sections except "Instance Overview." Hover over the (i) icon for detailed filter descriptions.
+
+<img src="vmanomaly-dashboard-1-filters.webp" alt="vmanomaly-dashboard-1-filters" width="800px"/>
+
+The Grafana Dashboard for `vmanomaly` is organized into various panels that offer insights into different components and their operational metrics. The main sections are as follows:
+
+### Instance Overview
+
+<img src="vmanomaly-dashboard-2-instance-overview.webp" alt="vmanomaly-dashboard-2-instance-overview" width="800px"/>
+
+This panel provides general information about the state at individual `instance` level, including metrics such as uptime, restarts, errors, license expiration, and overall status. It serves as a critical starting point for assessing the health of the anomaly detection service. If any issues are identified with a particular instance — such as a low success rate, a high number of skipped or erroneous runs, or increased resource consumption — you can drill down further by using the dashboard filter `instance={{instance}}` for more detailed analysis.
+
+**Healthy scenario**:
+- **I/O: Reads, Writes, Total**: These should all be close to 100%, indicating successful data handling.
+- **Acceptance Rate (`r:acc.rate`)**: Should be close to 100%, meaning there is no invalid data (e.g., `Inf` or `NaN`).
+- **Resource Usage (CPU, RAM)**: CPU and RAM usage should be within expected limits.
+- **Errors (`r:errors`)**: Should be zero, indicating no major faults in operations.
+- **Skipped Runs (`r:skipped`)**: Should be close to zero, reflecting minimal disruptions in data processing.
+
+### Global Statistics
+
+#### Models
+This global panel holds statistics related to models, filtered by the dashboard settings, including:
+- The number of unique models in use and the entities they interact with (e.g., queries, schedulers, instances).
+- Counts of successful, skipped, or erroneous model runs.
+- Average timings for different model stages.
+
+<img src="vmanomaly-dashboard-3-global-panel-models.webp" alt="vmanomaly-dashboard-3-global-panel-models" width="800px"/>
+
+**Healthy scenario**:
+- **Data Acceptance**: Should be consistently high, ideally close to 100%. This indicates that the system is successfully processing the majority of incoming data without issues (e.g., no NaNs or Inf values).
+- **Erroneous Runs**: There should be zero erroneous runs. Any errors suggest potential issues with the service or uncaught corner cases that need immediate attention.
+- **Skipped Runs**: Should be minimal, ideally none. A significant number of skipped runs may indicate missing data, configuration problems, high churn rate (resulting in no models trained yet for new time series), or other underlying issues.
+- **Problematic Runs Percentage**: This should be close to 0%. A higher percentage warrants further investigation into potential bottlenecks or misconfigurations.
+- **Timings**: Processing time sparklines should remain stable over time without significant peaks.
+
+#### I/O
+This global panel holds statistics related to I/O operations and data processing, filtered by the dashboard settings.
+
+<img src="vmanomaly-dashboard-3-global-panel-io.webp" alt="vmanomaly-dashboard-3-global-io" width="800px"/>
+
+**Healthy scenario**:
+- **I/O success, %**: Should be close to 100%.
+- **Timeseries graphs**: Should appear stable over time, without significant spikes or drops.
+
+#### Latency
+This global panel holds latency statistics (reads, writes, response processing by stages), filtered by the dashboard settings.
+
+<img src="vmanomaly-dashboard-3-global-panel-latency.webp" alt="vmanomaly-dashboard-3-global-latency" width="800px"/>
+
+**Healthy scenario**:
+- **Timeseries graphs**: Should appear stable over time, without significant spikes or drops.
+- **Histograms**: Should not have right-hand buckets significantly higher than all the other buckets, indicating the absence of consecutive latency spikes.
+
+#### Resource
+
+This global panel holds resource utilization (CPU, RAM, File Descriptors) on both an overall and per-`instance` level, filtered by the dashboard settings.
+
+<img src="vmanomaly-dashboard-3-global-panel-resources.webp" alt="vmanomaly-dashboard-3-global-resources" width="800px"/>
+
+**Healthy scenario**:
+- **Timeseries graphs**: Should appear stable over time, without significant spikes or drops. An absence of upward trends (e.g., trends in RAM usage may indicate a [high churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) in your input data).
+
+### Model Statistics
+
+These panels contain repeated blocks for each unique `model_alias` (a distinct entity defined in the `models` [configuration section](https://docs.victoriametrics.com/anomaly-detection/components/models)), filtered according to the current dashboard settings. They provide information on the number of unique entities (such as queries, schedulers, and instances) that a particular `model_alias` interacts with, as well as the count of active model instances available for inferring new data.
+
+<img src="vmanomaly-dashboard-4-model-sections.webp" alt="vmanomaly-dashboard-4-model-sections" width="800px"/>
+
+**Healthy scenario**:
+- **Erroneous Runs**: There should be zero erroneous runs. Any errors suggest potential issues with the service or uncaught corner cases that need immediate attention.
+- **Skipped Runs**: Should be minimal, ideally none. A significant number of skipped runs may indicate missing data, configuration problems, high churn rate (resulting in no models trained yet for new time series), or other underlying issues.
+- **Timeseries graphs**: Should remain stable over time, without significant spikes or drops. An absence of upward trends (e.g., in RAM usage) could indicate normal operation without excessive [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate).
+
+---
+
+## Alerting Rules
+
+### Overview
+
+Alerting rules for `vmanomaly` are a critical part of ensuring that any severe issues are promptly detected, so the users can be timely notified of problems such as service downtime, excessive error rates, or insufficient system resources.
+
+The alerting rules are provided in a YAML file called [`alerts-vmanomaly.yml`](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/rules/alerts-vmanomaly.yml).
+
+### Using the Alerting Rules
+
+These alerting rules complements the [dashboard](#grafana-dashboard) to monitor the health of `vmanomaly`. Each alert has annotations to help understand the issue and guide troubleshooting efforts. Below are the key alerts included, grouped into 2 sections:
+
+<img src="firing-alerts-groups.webp" alt="firing-alerts-groups" width="800px"/>
+
+`vmanomaly-health` alerting group:
+- **`TooManyRestarts`**: Triggers if an instance restarts more than twice within 15 minutes, suggesting the process might be crashlooping and needs investigation.
+- **`ServiceDown`**: Alerts if an instance is down for more than 5 minutes, indicating a service outage.
+- **`ProcessNearFDLimits`**: Alerts when the number of available file descriptors falls below 100, which could lead to severe degradation if the limit is exhausted.
+- **`TooHighCPUUsage`**: Alerts when CPU usage exceeds 90% for a continuous 5-minute period, indicating possible resource exhaustion and the need to adjust resource allocation or load.
+- **`TooHighMemoryUsage`**: Alerts when RAM usage exceeds 85% for a continuous 5-minute period and the need to adjust resource allocation or load.
+
+<img src="firing-alerts-example-too-many-restarts.webp" alt="firing-alerts-example-too-many-restarts" width="800px"/>
+
+`vmanomaly-issues` alerting group:
+- **`ServiceErrorsDetected`**: Alerts if model run errors are detected, indicating problems with the anomaly detection service or its dependencies.
+- **`SkippedModelRunsDetected`**: Alerts if model runs are skipped, potentially due to no new valid data, absence of trained ML models for new time series, or invalid data points (like `Inf` or `NaN`).
+- **`HighReadErrorRate`**: Alerts when the error rate for read operations exceeds 5% in a 5-minute window, suggesting issues with the data source, server constraints, or network.
+- **`HighWriteErrorRate`**: Alerts when the error rate for write operations exceeds 5% in a 5-minute window, indicating issues with data writing, potential server-side violations, or network problems.
+
+<img src="firing-alerts-example-skipped-runs.webp" alt="firing-alerts-example-skipped-runs" width="800px"/>
--- a/docs/anomaly-detection/components/models.md
+++ b/docs/anomaly-detection/components/models.md
@ -993,7 +993,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.3
+docker pull victoriametrics/vmanomaly:v1.18.4
 ```

 Now we can run the docker container putting as volumes both config and model file:
@ -1007,7 +1007,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.18.3 /config.yaml \
+victoriametrics/vmanomaly:v1.18.4 /config.yaml \
 --licenseFile=/license
 ```

--- a/docs/anomaly-detection/components/monitoring.md
+++ b/docs/anomaly-detection/components/monitoring.md
@ -266,6 +266,22 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
            <td>Gauge</td>
            <td>vmanomaly UI version information, contained in `version` label. Added in [v1.17.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1172)</td>
        </tr>
+        <tr>
+            <td>
+
+`vmanomaly_available_memory_bytes`
+            </td>
+            <td>Gauge</td>
+            <td>Virtual memory size in bytes, available to the process. Added in [v1.18.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1184)</td>
+        </tr>
+        <tr>
+            <td>
+
+`vmanomaly_cpu_cores_available`
+            </td>
+            <td>Gauge</td>
+            <td>Number of (logical) CPU cores available to the process. Added in [v1.18.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1184)</td>
+        </tr>
    </tbody>
 </table>

--- a/docs/anomaly-detection/firing-alerts-example-skipped-runs.webp
+++ b/docs/anomaly-detection/firing-alerts-example-skipped-runs.webp
--- a/docs/anomaly-detection/firing-alerts-example-too-many-restarts.webp
+++ b/docs/anomaly-detection/firing-alerts-example-too-many-restarts.webp
--- a/docs/anomaly-detection/firing-alerts-groups.webp
+++ b/docs/anomaly-detection/firing-alerts-groups.webp
--- a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
+++ b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
@ -2,9 +2,9 @@

 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.106.0)
-  -  [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.106.0)
-  -  [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.106.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.106.1)
+  -  [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.106.1)
+  -  [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.106.1)
 - [Grafana](https://grafana.com/) (v.10.2.1)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@ -313,7 +313,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
  vmagent:
    container_name: vmagent
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    depends_on:
      - "victoriametrics"
    ports:
@ -330,7 +330,7 @@ services:

  victoriametrics:
    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - 8428:8428
    volumes:
@ -363,7 +363,7 @@ services:

  vmalert:
    container_name: vmalert
-    image: victoriametrics/vmalert:v1.106.0
+    image: victoriametrics/vmalert:v1.106.1
    depends_on:
      - "victoriametrics"
    ports:
@ -385,7 +385,7 @@ services:
    restart: always
  vmanomaly:
    container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.3
+    image: victoriametrics/vmanomaly:v1.18.4
    depends_on:
      - "victoriametrics"
    ports:
--- a/docs/anomaly-detection/vmanomaly-dashboard-1-filters.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-1-filters.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-2-instance-overview.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-2-instance-overview.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-io.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-io.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-latency.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-latency.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-models.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-models.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-resources.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-3-global-panel-resources.webp
--- a/docs/anomaly-detection/vmanomaly-dashboard-4-model-sections.webp
+++ b/docs/anomaly-detection/vmanomaly-dashboard-4-model-sections.webp
--- a/docs/changelog/CHANGELOG.md
+++ b/docs/changelog/CHANGELOG.md
@ -18,16 +18,62 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).

 ## tip

+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert): revert the default value of `-remoteWrite.maxQueueSize` from `1_000_000` to `100_000`. It was bumped in [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0), which increases memory usage and is not needed for most setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7471).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent): Properly return `200 OK` HTTP status code when importing data via [Pushgateway protocol](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format) using [multitenant URL format](https://docs.victoriametrics.com/cluster-victoriametrics/#url-format). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7571).
+
+## [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
+Released at 2024-11-15
+
 * SECURITY: upgrade Go builder from Go1.23.1 to Go1.23.3. See the list of issues addressed in [Go1.23.2](https://github.com/golang/go/issues?q=milestone%3AGo1.23.2+label%3ACherryPickApproved) and [Go1.23.3](https://github.com/golang/go/issues?q=milestone%3AGo1.23.3+label%3ACherryPickApproved).

-* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fixed unauthorized routing behavior inconsistency. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7543) for details.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): reduces memory and CPU usage for high volume of triggered alerting rules. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6894) for details.
+
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068).
 * BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details.
-* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fixed unauthorized routing behavior inconsistency. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7543) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly inherit [`drop_src_path_prefix_parts`](https://docs.victoriametrics.com/vmauth/#dropping-request-path-prefix), [`load_balancing_policy`](https://docs.victoriametrics.com/vmauth/#high-availability), [`retry_status_codes`](https://docs.victoriametrics.com/vmauth/#load-balancing) and [`discover_backend_ips`](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) options by `url_map` entries if `url_prefix` option isn't set at the [user config level](https://docs.victoriametrics.com/vmauth/#auth-config). These options were inherited only when the `url_prefix` option was set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7519).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
+* BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): add `file` label filter to vmalert dashboard panels. Previously, metrics from groups with the same name but different rule files could be mixed in the results.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Optimize resources usage for configured [downsampling](https://docs.victoriametrics.com/#downsampling) with time-series filter. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7440) for details.
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Properly return query results for search requests after index rotation. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7417) for details.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Properly handle [multitenant](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels) query request errors and correctly perform search for available tenants. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7549) for details.
+* BUGFIX: `vmagent`, `vminsert` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix a performance issue with tenant metrics counters across large numbers of tenants. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7482) for details.
+
+## [v1.102.7](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.7)
+
+Released at 2024-11-15
+
+**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
+
+* SECURITY: upgrade Go builder from Go1.23.1 to Go1.23.3. See the list of issues addressed in [Go1.23.2](https://github.com/golang/go/issues?q=milestone%3AGo1.23.2+label%3ACherryPickApproved) and [Go1.23.3](https://github.com/golang/go/issues?q=milestone%3AGo1.23.3+label%3ACherryPickApproved).
+
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068).
+* BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fixed unauthorized routing behavior inconsistency. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7543) for details.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly inherit [`drop_src_path_prefix_parts`](https://docs.victoriametrics.com/vmauth/#dropping-request-path-prefix), [`load_balancing_policy`](https://docs.victoriametrics.com/vmauth/#high-availability), [`retry_status_codes`](https://docs.victoriametrics.com/vmauth/#load-balancing) and [`discover_backend_ips`](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) options by `url_map` entries if `url_prefix` option isn't set at the [user config level](https://docs.victoriametrics.com/vmauth/#auth-config). These options were inherited only when the `url_prefix` option was set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7519).
 * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): add `file` label filter to vmalert dashboard panels. Previously, metrics from groups with the same name but different rule files could be mixed in the results.

+## [v1.97.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.12)
+
+Released at 2024-11-15
+
+**v1.97.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.97.x line will be supported for at least 12 months since [v1.97.0](https://docs.victoriametrics.com/CHANGELOG.html#v1970) release**
+
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068).
+* BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fixed unauthorized routing behavior inconsistency. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7543) for details.
+
 ## [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0)

 Released at 2024-11-04
@ -426,7 +472,7 @@ Released at 2024-04-26
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): auto-suggestion triggers at any cursor position in the query input. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5864).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): update error messages on the Query page for enhanced clarity. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6177).

-* BUGFIX: [downsampling](https://docs.victoriametrics.com/#downsampling): skip unnecessary index lookups if downsampling wasn't set for ENT versions of VictoriaMetrics. Before, users of VictoriaMetrics ENT could have experience elevated CPU usage even if no downsampling was configured. The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000).  
+* BUGFIX: [downsampling](https://docs.victoriametrics.com/#downsampling): skip unnecessary index lookups if downsampling wasn't set for ENT versions of VictoriaMetrics. Before, users of VictoriaMetrics ENT could have experience elevated CPU usage even if no downsampling was configured. The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000).
 * BUGFIX: [downsampling](https://docs.victoriametrics.com/#downsampling): properly populate downsampling metadata for data parts created by VictoriaMetrics ENT versions lower than v1.100.0. The bug could trigger the downsampling actions for parts that were downsampled already. This bug doesn't have any negative effect apart from spending extra CPU resources on the repeated downsampling. The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): avoid blocking `/api/v1/rules`, `/api/v1/alerts`, `/metrics` APIs when alerting rule uses template functions `query`, which could takes a while to execute. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079).
--- a/docs/enterprise.md
+++ b/docs/enterprise.md
@ -82,7 +82,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).

 Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.106.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.106.1-enterprise.tar.gz`.

 In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
@ -100,8 +100,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):

 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.0/victoria-metrics-linux-amd64-v1.106.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.106.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.1/victoria-metrics-linux-amd64-v1.106.1-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.106.1-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```

@ -116,7 +116,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).

 Docker images for VictoriaMetrics Enterprise are available [at VictoriaMetrics DockerHub](https://hub.docker.com/u/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.106.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.106.1-enterprise`.

 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
 flag as described [here](#binary-releases).
@ -126,13 +126,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:

 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.106.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.106.1-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```

 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:

 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.106.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.106.1-enterprise -licenseFile=/path/to/vm-license
 ```

 Example docker-compose configuration:
@ -141,7 +141,7 @@ version: "3.5"
 services:
  victoriametrics:
    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    ports:
      - 8428:8428
    volumes:
@ -173,7 +173,7 @@ is used to provide key in plain-text:
 ```yaml
 server:
  image:
-    tag: v1.106.0-enterprise
+    tag: v1.106.1-enterprise

 license:
  key: {BASE64_ENCODED_LICENSE_KEY}
@ -184,7 +184,7 @@ In order to provide key via existing secret, the following values file is used:
 ```yaml
 server:
  image:
-    tag: v1.106.0-enterprise
+    tag: v1.106.1-enterprise

 license:
  secret:
@ -233,7 +233,7 @@ spec:
  license:
    key: {BASE64_ENCODED_LICENSE_KEY}
  image:
-    tag: v1.106.0-enterprise
+    tag: v1.106.1-enterprise
 ```

 In order to provide key via existing secret, the following custom resource is used:
@ -250,7 +250,7 @@ spec:
      name: vm-license
      key: license
  image:
-    tag: v1.106.0-enterprise
+    tag: v1.106.1-enterprise
 ```

 Example secret with license key:
--- a/docs/guides/grafana-vmgateway-openid-configuration/README.md
+++ b/docs/guides/grafana-vmgateway-openid-configuration/README.md
@ -236,27 +236,27 @@ services:
      - grafana_data:/var/lib/grafana/

  vmsingle:
-    image: victoriametrics/victoria-metrics:v1.106.0
+    image: victoriametrics/victoria-metrics:v1.106.1
    command:
      - -httpListenAddr=0.0.0.0:8429

  vmstorage:
-    image: victoriametrics/vmstorage:v1.106.0-cluster
+    image: victoriametrics/vmstorage:v1.106.1-cluster

  vminsert:
-    image: victoriametrics/vminsert:v1.106.0-cluster
+    image: victoriametrics/vminsert:v1.106.1-cluster
    command:
      - -storageNode=vmstorage:8400
      - -httpListenAddr=0.0.0.0:8480

  vmselect:
-    image: victoriametrics/vmselect:v1.106.0-cluster
+    image: victoriametrics/vmselect:v1.106.1-cluster
    command:
      - -storageNode=vmstorage:8401
      - -httpListenAddr=0.0.0.0:8481

  vmagent:
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    volumes:
      - ./scrape.yaml:/etc/vmagent/config.yaml
    command:
@ -265,7 +265,7 @@ services:
      - -remoteWrite.url=http://vmsingle:8429/api/v1/write

  vmgateway-cluster:
-    image: victoriametrics/vmgateway:v1.106.0-enterprise
+    image: victoriametrics/vmgateway:v1.106.1-enterprise
    ports:
      - 8431:8431
    volumes:
@ -281,7 +281,7 @@ services:
      - -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration

  vmgateway-single:
-    image: victoriametrics/vmgateway:v1.106.0-enterprise
+    image: victoriametrics/vmgateway:v1.106.1-enterprise
    ports:
      - 8432:8431
    volumes:
@ -393,7 +393,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use

 ```yaml
  vmagent:
-    image: victoriametrics/vmagent:v1.106.0
+    image: victoriametrics/vmagent:v1.106.1
    volumes:
      - ./scrape.yaml:/etc/vmagent/config.yaml
      - ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret
--- a/docs/helm/victoria-logs-single/CHANGELOG.md
+++ b/docs/helm/victoria-logs-single/CHANGELOG.md
@ -2,6 +2,7 @@

 - Removed redundant `VECTOR_SELF_NODE_NAME` env variable from vector values. See [this issue](https://github.com/VictoriaMetrics/helm-charts/issues/1727).
 - Added Vector dashboard. See [this issue](https://github.com/VictoriaMetrics/helm-charts/issues/1721).
+- updated common dependency 0.0.23 -> 0.0.28

 ## 0.8.1

--- a/docs/helm/victoria-metrics-agent/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-agent/CHANGELOG.md
@ -2,6 +2,16 @@

 - TODO

+## 0.14.8
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.23 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
 ## 0.14.7

 **Release date:** 2024-11-12
--- a/docs/helm/victoria-metrics-agent/README.md
+++ b/docs/helm/victoria-metrics-agent/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.14.7](https://img.shields.io/badge/Version-0.14.7-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.14.8](https://img.shields.io/badge/Version-0.14.8-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-agent)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

--- a/docs/helm/victoria-metrics-alert/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-alert/CHANGELOG.md
@ -2,6 +2,17 @@

 - TODO

+## 0.12.6
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated context for serviceaccount name
+- updated common dependency 0.0.23 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
 ## 0.12.5

 **Release date:** 2024-11-12
--- a/docs/helm/victoria-metrics-alert/README.md
+++ b/docs/helm/victoria-metrics-alert/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.5](https://img.shields.io/badge/Version-0.12.5-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.6](https://img.shields.io/badge/Version-0.12.6-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-alert)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

@ -1051,6 +1051,8 @@ username: ""
 <code class="language-yaml">envflag.enable: "true"
 envflag.prefix: VM_
 loggerFormat: json
+rule:
+    - /config/alert-rules.yaml
 </code>
 </pre>
 </td>
@ -1087,7 +1089,7 @@ loggerFormat: json
 </code>
 </pre>
 </td>
-      <td><p>Extra Volume Mounts for the container</p>
+      <td><p>Extra Volume Mounts for the container. Expects a lice of <a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#volumemount-v1-core" target="_blank">volume mounts</a></p>
 </td>
    </tr>
    <tr>
--- a/docs/helm/victoria-metrics-anomaly/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-anomaly/CHANGELOG.md
@ -2,6 +2,35 @@

 - TODO

+## 1.6.7
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.18.4](https://img.shields.io/static/v1?label=AppVersion&message=v1.18.4&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- Upgraded ['vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) to [1.18.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1184).
+- updated common dependency 0.0.23 -> 0.0.28
+
+## 1.6.6
+
+**Release date:** 2024-11-14
+
+![AppVersion: v1.18.3](https://img.shields.io/static/v1?label=AppVersion&message=v1.18.3&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- Upgraded ['vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) to [1.18.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1183). This is a patch release that fixes a service crash during parallelized data processing with [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader).
+
+
+## 1.6.5
+
+**Release date:** 2024-11-13
+
+![AppVersion: v1.18.2](https://img.shields.io/static/v1?label=AppVersion&message=v1.18.2&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- Upgraded [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) to [1.18.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1182)
+
 ## 1.6.4

 **Release date:** 2024-11-12
--- a/docs/helm/victoria-metrics-anomaly/README.md
+++ b/docs/helm/victoria-metrics-anomaly/README.md
@ -1,4 +1,4 @@
-![Version: 1.6.4](https://img.shields.io/badge/Version-1.6.4-informational?style=flat-square)
+![Version: 1.6.7](https://img.shields.io/badge/Version-1.6.7-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-anomaly)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/helm-charts/blob/master/LICENSE)
--- a/docs/helm/victoria-metrics-auth/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-auth/CHANGELOG.md
@ -1,7 +1,27 @@
 ## Next release

+- TODO
+
+## 0.7.7
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.26 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
+## 0.7.6
+
+**Release date:** 2024-11-14
+
+![AppVersion: v1.106.0](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.0&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
 - set default DNS domain to `cluster.local.`
- updated common dependency 0.0.19 -> 0.0.23
+- updated common dependency 0.0.19 -> 0.0.26
+- added init containers

 ## 0.7.5

--- a/docs/helm/victoria-metrics-auth/README.md
+++ b/docs/helm/victoria-metrics-auth/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.7.5](https://img.shields.io/badge/Version-0.7.5-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.7.7](https://img.shields.io/badge/Version-0.7.7-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-auth)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

@ -544,6 +544,17 @@ loggerFormat: json
 </pre>
 </td>
      <td><p>Array of TLS objects</p>
+</td>
+    </tr>
+    <tr>
+      <td>initContainers</td>
+      <td>list</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">[]
+</code>
+</pre>
+</td>
+      <td><p>Init containers for vmauth</p>
 </td>
    </tr>
    <tr>
--- a/docs/helm/victoria-metrics-cluster/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-cluster/CHANGELOG.md
@ -2,6 +2,26 @@

 - TODO

+## 0.14.12
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.25 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
+## 0.14.11
+
+**Release date:** 2024-11-14
+
+![AppVersion: v1.106.0](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.0&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated context for serviceaccount name
+- updated common dependency 0.0.23 -> 0.0.25
+
 ## 0.14.10

 **Release date:** 2024-11-08
--- a/docs/helm/victoria-metrics-cluster/README.md
+++ b/docs/helm/victoria-metrics-cluster/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.14.10](https://img.shields.io/badge/Version-0.14.10-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.14.12](https://img.shields.io/badge/Version-0.14.12-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-cluster)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

--- a/docs/helm/victoria-metrics-common/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-common/CHANGELOG.md
@ -2,6 +2,55 @@

 ## Next release

+- TODO
+
+## 0.0.29
+
+**Release date:** 2024-11-19
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- Allow lookup in context root for `vm.url`, `vm.host` templates
+
+## 0.0.28
+
+**Release date:** 2024-11-14
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- Allow lookup in context root for `vm.url`, `vm.host` templates
+
+## 0.0.27
+
+**Release date:** 2024-11-14
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- fail fullname templates if data for appKey is not found
+- find by appKey in Values and context root
+
+## 0.0.26
+
+**Release date:** 2024-11-14
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- do not append key only if it's passed to a template
+
+## 0.0.25
+
+**Release date:** 2024-11-12
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- fixed adding suffix for `vm.plain.fullname`
+
+## 0.0.24
+
+**Release date:** 2024-11-12
+
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
 - Disabled impact of `<component>.name` on resource name to avoid confusion
 - Fixed `vm.app.name` template for appCtx that contains slice

--- a/docs/helm/victoria-metrics-common/README.md
+++ b/docs/helm/victoria-metrics-common/README.md
@ -0,0 +1,44 @@
+![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![Version: 0.0.29](https://img.shields.io/badge/Version-0.0.29-informational?style=flat-square)
+[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-common)
+
+Victoria Metrics Common - contains shared templates for all Victoria Metrics helm charts
+
+## Documentation of Helm Chart
+
+Install ``helm-docs`` following the instructions on this [tutorial](https://docs.victoriametrics.com/helm/requirements/).
+
+Generate docs with ``helm-docs`` command.
+
+```bash
+cd charts/victoria-metrics-common
+
+helm-docs
+```
+
+The markdown generation is entirely go template driven. The tool parses metadata from charts and generates a number of sub-templates that can be referenced in a template file (by default ``README.md.gotmpl``). If no template file is provided, the tool has a default internal template that will generate a reasonably formatted README.
+
+## Parameters
+
+The following table lists the template functions of the chart and description.
+
+<table class="helm-vars">
+  <thead>
+    <th class="helm-vars-key">Key</th>
+    <th class="helm-vars-type">Type</th>
+    <th class="helm-vars-default">Default</th>
+    <th class="helm-vars-description">Description</th>
+  </thead>
+  <tbody>
+    <tr>
+      <td>unitTest</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">false
+</code>
+</pre>
+</td>
+      <td></td>
+    </tr>
+  </tbody>
+</table>
+
--- a/docs/helm/victoria-metrics-distributed/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-distributed/CHANGELOG.md
@ -1,17 +1,20 @@
 ## Next release

- `vmauthIngestGlobal` was changed to `write.global.vmauth`
- `vmauthQueryGlobal` was changed to `read.global.vmauth`
- `availabilityZones[*].allowIngest` was changed to `availabilityZones[*].write.allow`
- `availabilityZones[*].allowRead` was changed to `availabilityZones[*].read.allow`
- `availabilityZones[*].nodeSelector` was moved to `availabilityZones[*].common.spec.nodeSelector`
- `availabilityZones[*].extraAffinity` was moved to `availabilityZones[*].common.spec.affinity`
- `availabilityZones[*].topologySpreadConstraints` was moved to `availabilityZones[*].common.spec.topologySpreadConstraints`
- `availabilityZones[*].vmauthIngest` was moved to `availabilityZones[*].write.vmauth`
- `availabilityZones[*].vmauthQueryPerZone` was moved to `availabilityZones[*].read.perZone.vmauth`
- `availabilityZones[*].vmauthCrossAZQuery` was moved to `availabilityZones[*].read.crossZone.vmauth`
+- TODO
+
+## 0.5.0
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+**Update note**: This release contains breaking changes. please follow [upgrade guide](../#upgrade-to-050)
+
 - set default DNS domain to `cluster.local.`
- updated common dependency 0.0.19 -> 0.0.23
+- added `.Values.zoneTpl` to define a default configuration for each `.Values.availabilityZones`
+- updated common dependency 0.0.19 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)

 ## 0.4.2

--- a/docs/helm/victoria-metrics-distributed/README.md
+++ b/docs/helm/victoria-metrics-distributed/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.4.2](https://img.shields.io/badge/Version-0.4.2-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-distributed)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

@ -23,15 +23,15 @@ The default setup is as shown below:
 For write:
 1. extra-vmagent(optional): scrapes external targets and all the components installed by this chart, sends data to global write entrypoint.
 2. vmauth-global-write: global write entrypoint, proxies requests to one of the zone `vmagent` with `least_loaded` policy.
-3. vmagent(per-zone): remote writes data to availability zones that enabled `.Values.availabilityZones.allowIngest`, and [buffer data on disk](https://docs.victoriametrics.com/vmagent/#calculating-disk-space-for-persistence-queue) when zone is unavailable to ingest.
+3. vmagent(per-zone): remote writes data to availability zones that enabled `.Values.availabilityZones[*].write.allow`, and [buffer data on disk](https://docs.victoriametrics.com/vmagent/#calculating-disk-space-for-persistence-queue) when zone is unavailable to ingest.
 4. vmauth-write-balancer(per-zone): proxies requests to vminsert instances inside it's zone with `least_loaded` policy.
 5. vmcluster(per-zone): processes write requests and stores data.

 For read:
 1. vmcluster(per-zone): processes query requests and returns results.
 2. vmauth-read-balancer(per-zone): proxies requests to vmselect instances inside it's zone with `least_loaded` policy.
-3. vmauth-read-proxy(per-zone): uses all the `vmauth-read-balancer` as servers if zone has `.Values.availabilityZones.allowQuery` enabled, always prefer "local" `vmauth-read-balancer` to reduce cross-zone traffic with `first_available` policy.
-4. vmauth-global-read: global query entrypoint, proxies requests to one of the zone `vnauth-read-proxy` with `first_available` policy.
+3. vmauth-read-proxy(per-zone): uses all the `vmauth-read-balancer` as servers if zone has `.Values.availabilityZones[*].read.allow` enabled, always prefer "local" `vmauth-read-balancer` to reduce cross-zone traffic with `first_available` policy.
+4. vmauth-global-read: global query entrypoint, proxies requests to one of the zone `vmauth-read-proxy` with `first_available` policy.
 5. grafana(optional): uses `vmauth-global-read` as default datasource.

 >Note:
@ -53,7 +53,7 @@ Optionally, you can push data to any of the per-zone vmagents, and they will rep
 ### How to query data?

 The chart provides `vmauth-global-read` as global read entrypoint, it picks the first available zone (see [first_available](https://docs.victoriametrics.com/vmauth/#high-availability) policy) as it's preferred datasource and switches automatically to next zone if first one is unavailable, check [vmauth `first_available`](https://docs.victoriametrics.com/vmauth/#high-availability) for more details.
-If you have services like [vmalert](https://docs.victoriametrics.com/vmalert) or Grafana deployed in each zone, then configure them to use local `vmauth-read-proxy`. Per-zone `vmauth-read-proxy` always prefers "local" vmcluster for querying and reduces cross-zone traffic. 
+If you have services like [vmalert](https://docs.victoriametrics.com/vmalert) or Grafana deployed in each zone, then configure them to use local `vmauth-read-proxy`. Per-zone `vmauth-read-proxy` always prefers "local" vmcluster for querying and reduces cross-zone traffic.

 You can also pick other proxies like kubernetes service which supports [Topology Aware Routing](https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/) as global read entrypoint.

@ -65,7 +65,7 @@ If availability zone `zone-eu-1` is experiencing an outage, `vmauth-global-write
 3. `vmagent` on `zone-us-1` fails to send data to `zone-eu-1.vmauth-write-balancer`, starts to buffer data on disk(unless `-remoteWrite.disableOnDiskQueue` is specified, which is not recommended for this topology);
 To keep data completeness for all the availability zones, make sure you have enough disk space on vmagent for buffer, see [this doc](https://docs.victoriametrics.com/vmagent/#calculating-disk-space-for-persistence-queue) for size recommendation.

-And to avoid getting incomplete responses from `zone-eu-1` which gets recovered from outage, check vmagent on `zone-us-1` to see if persistent queue has been drained. If not, remove `zone-eu-1` from serving query by setting `.Values.availabilityZones.{zone-eu-1}.allowQuery=false` and change it back after confirm all data are restored.
+And to avoid getting incomplete responses from `zone-eu-1` which gets recovered from outage, check vmagent on `zone-us-1` to see if persistent queue has been drained. If not, remove `zone-eu-1` from serving query by setting `.Values.availabilityZones.{zone-eu-1}.read.allow=false` and change it back after confirm all data are restored.

 ### How to use [multitenancy](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy)?

@ -196,13 +196,79 @@ helm history vmd -n NAMESPACE

 In order to serving query and ingestion while upgrading components version or changing configurations, it's recommended to perform maintenance on availability zone one by one.
 First, performing update on availability zone `zone-eu-1`:
-1. remove `zone-eu-1` from serving query by setting `.Values.availabilityZones.{zone-eu-1}.allowQuery=false`;
+1. remove `zone-eu-1` from serving query by setting `.Values.availabilityZones.{zone-eu-1}.read.allow=false`;
 2. run `helm upgrade vm-dis -n NAMESPACE` with updated configurations for `zone-eu-1` in `values.yaml`;
 3. wait for all the components on zone `zone-eu-1` running;
-4. wait `zone-us-1` vmagent persistent queue for `zone-eu-1` been drained, add `zone-eu-1` back to serving query by setting `.Values.availabilityZones.{zone-eu-1}.allowQuery=true`.
+4. wait `zone-us-1` vmagent persistent queue for `zone-eu-1` been drained, add `zone-eu-1` back to serving query by setting `.Values.availabilityZones.{zone-eu-1}.read.allow=true`.

 Then, perform update on availability zone `zone-us-1` with the same steps1~4.

+### Upgrade to 0.5.0
+
+This release was refactored, names of the parameters was changed:
+
+- `vmauthIngestGlobal` was changed to `write.global.vmauth`
+- `vmauthQueryGlobal` was changed to `read.global.vmauth`
+- `availabilityZones[*].allowIngest` was changed to `availabilityZones[*].write.allow`
+- `availabilityZones[*].allowRead` was changed to `availabilityZones[*].read.allow`
+- `availabilityZones[*].nodeSelector` was moved to `availabilityZones[*].common.spec.nodeSelector`
+- `availabilityZones[*].extraAffinity` was moved to `availabilityZones[*].common.spec.affinity`
+- `availabilityZones[*].topologySpreadConstraints` was moved to `availabilityZones[*].common.spec.topologySpreadConstraints`
+- `availabilityZones[*].vmauthIngest` was moved to `availabilityZones[*].write.vmauth`
+- `availabilityZones[*].vmauthQueryPerZone` was moved to `availabilityZones[*].read.perZone.vmauth`
+- `availabilityZones[*].vmauthCrossAZQuery` was moved to `availabilityZones[*].read.crossZone.vmauth`
+
+Example:
+
+If before an upgrade you had given below configuration
+
+```yaml
+vmauthIngestGlobal:
+  spec:
+    extraArgs:
+      discoverBackendIPs: "true"
+vmauthQueryGlobal:
+  spec:
+    extraArgs:
+      discoverBackendIPs: "true"
+availabilityZones:
+  - name: zone-eu-1
+    vmauthIngest:
+      spec:
+        extraArgs:
+          discoverBackendIPs: "true"
+    vmcluster:
+      spec:
+        retentionPeriod: "14"
+```
+
+after upgrade it will look like this:
+
+```yaml
+write:
+  global:
+    vmauth:
+      spec:
+        extraArgs:
+          discoverBackendIPs: "true"
+read:
+  global:
+    vmauth:
+      spec:
+        extraArgs:
+          discoverBackendIPs: "true"
+availabilityZones:
+  - name: zone-eu-1
+    write:
+      vmauth:
+        spec:
+          extraArgs:
+            discoverBackendIPs: "true"
+    vmcluster:
+      spec:
+        retentionPeriod: "14"
+```
+
 ## How to uninstall

 Remove application with command.
@ -240,20 +306,15 @@ Change the values according to the need of the environment in ``victoria-metrics
  </thead>
  <tbody>
    <tr>
-      <td>availabilityZones[0].common.spec</td>
-      <td>object</td>
+      <td>availabilityZones</td>
+      <td>list</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">affinity: {}
-nodeSelector:
-    topology.kubernetes.io/zone: zone-eu-1
-topologySpreadConstraints:
-    - maxSkew: 1
-      topologyKey: kubernetes.io/hostname
-      whenUnsatisfiable: ScheduleAnyway
+<code class="language-yaml">- name: zone-eu-1
+- name: zone-us-1
 </code>
 </pre>
 </td>
-      <td><p>Common for <a href="https://docs.victoriametrics.com/operator/api/#vmagentspec" target="_blank">VMAgent</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmauthspec" target="_blank">VMAuth</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmclusterspec" target="_blank">VMCluster</a> spec params, like nodeSelector, affinity, topologySpreadConstraint, etc</p>
+      <td><p>Config for all availability zones. Each element represents custom zone config, which overrides a default one from <code>zoneTpl</code></p>
 </td>
    </tr>
    <tr>
@ -265,240 +326,6 @@ topologySpreadConstraints:
 </pre>
 </td>
      <td><p>Availability zone name</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.allow</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Allow data query from this zone through global query endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.crossZone.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create a vmauth with all the zone with <code>allow: true</code> as query backends</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.crossZone.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.crossZone.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.perZone.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create vmauth as a local read endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.perZone.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].read.perZone.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">extraArgs:
-    discoverBackendIPs: "true"
-port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmagent.annotations</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">{}
-</code>
-</pre>
-</td>
-      <td><p>VMAgent remote write proxy annotations</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmagent.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create VMAgent remote write proxy</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmagent.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmagent object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmagent.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">port: "8429"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAgent CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmagentspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmcluster.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create VMCluster</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmcluster.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmcluster, by default is <zoneName></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].vmcluster.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">replicationFactor: 2
-retentionPeriod: "14"
-vminsert:
-    extraArgs: {}
-    port: "8480"
-    replicaCount: 2
-    resources: {}
-vmselect:
-    extraArgs: {}
-    port: "8481"
-    replicaCount: 2
-    resources: {}
-vmstorage:
-    replicaCount: 2
-    resources: {}
-    storageDataPath: /vm-data
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMCluster CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmclusterspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].write.allow</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Allow data ingestion to this zone</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].write.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create vmauth as a local write endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].write.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[0].write.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">extraArgs:
-    discoverBackendIPs: "true"
-port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].common.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">affinity: {}
-nodeSelector:
-    topology.kubernetes.io/zone: zone-us-1
-topologySpreadConstraints:
-    - maxSkew: 1
-      topologyKey: kubernetes.io/hostname
-      whenUnsatisfiable: ScheduleAnyway
-</code>
-</pre>
-</td>
-      <td><p>Common for <a href="https://docs.victoriametrics.com/operator/api/#vmagentspec" target="_blank">VMAgent</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmauthspec" target="_blank">VMAuth</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmclusterspec" target="_blank">VMCluster</a> spec params, like nodeSelector, affinity, topologySpreadConstraint, etc</p>
 </td>
    </tr>
    <tr>
@ -513,238 +340,21 @@ topologySpreadConstraints:
 </td>
    </tr>
    <tr>
-      <td>availabilityZones[1].read.allow</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Allow data query from this zone through global query endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.crossZone.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create a vmauth with all the zone with <code>allow: true</code> as query backends</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.crossZone.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.crossZone.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.perZone.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create vmauth as a local read endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.perZone.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].read.perZone.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">extraArgs:
-    discoverBackendIPs: "true"
-port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmagent.annotations</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">{}
-</code>
-</pre>
-</td>
-      <td><p>VMAgent remote write proxy annotations</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmagent.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create VMAgent remote write proxy</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmagent.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmagent object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmagent.spec</td>
+      <td>common.vmagent.spec</td>
      <td>object</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
 <code class="language-yaml">port: "8429"
 </code>
 </pre>
 </td>
-      <td><p>Spec for VMAgent CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmagentspec" target="_blank">here</a></p>
+      <td><p>Common VMAgent spec, which can be overridden by each VMAgent configuration. Available parameters can be found <a href="https://docs.victoriametrics.com/operator/api/index.html#vmagentspec" target="_blank">here</a></p>
 </td>
    </tr>
    <tr>
-      <td>availabilityZones[1].vmcluster.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create VMCluster</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmcluster.name</td>
+      <td>common.vmauth.spec.port</td>
      <td>string</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmcluster, by default is <zoneName></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].vmcluster.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">replicationFactor: 2
-retentionPeriod: "14"
-vminsert:
-    extraArgs: {}
-    port: "8480"
-    replicaCount: 2
-    resources: {}
-vmselect:
-    extraArgs: {}
-    port: "8481"
-    replicaCount: 2
-    resources: {}
-vmstorage:
-    replicaCount: 2
-    resources: {}
-    storageDataPath: /vm-data
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMCluster CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmclusterspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].write.allow</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Allow data ingestion to this zone</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].write.vmauth.enabled</td>
-      <td>bool</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">true
-</code>
-</pre>
-</td>
-      <td><p>Create vmauth as a local write endpoint</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].write.vmauth.name</td>
-      <td>string</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
-</code>
-</pre>
-</td>
-      <td><p>Override the name of the vmauth object</p>
-</td>
-    </tr>
-    <tr>
-      <td>availabilityZones[1].write.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">extraArgs:
-    discoverBackendIPs: "true"
-port: "8427"
-</code>
-</pre>
-</td>
-      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>common.vmagent.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">{}
-</code>
-</pre>
-</td>
-      <td><p>Common VMAgent spec, which can be overriden by each VMAgent configuration. Available parameters can be found <a href="https://docs.victoriametrics.com/operator/api/index.html#vmagentspec" target="_blank">here</a></p>
-</td>
-    </tr>
-    <tr>
-      <td>common.vmauth.spec</td>
-      <td>object</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">{}
+<code class="language-yaml">"8427"
 </code>
 </pre>
 </td>
@ -755,14 +365,17 @@ port: "8427"
      <td>object</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
 <code class="language-yaml">vminsert:
+    port: "8480"
    serviceSpec:
        spec:
            clusterIP: None
            type: ClusterIP
+vmselect:
+    port: "8481"
 </code>
 </pre>
 </td>
-      <td><p>Common VMCluster spec, which can be overriden by each VMCluster configuration. Available parameters can be found <a href="https://docs.victoriametrics.com/operator/api/index.html#vmclusterspec" target="_blank">here</a></p>
+      <td><p>Common VMCluster spec, which can be overridden by each VMCluster configuration. Available parameters can be found <a href="https://docs.victoriametrics.com/operator/api/index.html#vmclusterspec" target="_blank">here</a></p>
 </td>
    </tr>
    <tr>
@ -777,17 +390,18 @@ port: "8427"
 </td>
    </tr>
    <tr>
-      <td>extraVMAgent</td>
+      <td>extra</td>
      <td>object</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">enabled: true
-name: test-vmagent
-spec:
-    selectAllByDefault: true
+<code class="language-yaml">vmagent:
+    enabled: true
+    name: test-vmagent
+    spec:
+        selectAllByDefault: true
 </code>
 </pre>
 </td>
-      <td><p>Set up an extra vmagent to scrape all the scrape objects by default, and write data to above vmauth-global-ingest endpoint.</p>
+      <td><p>Set up an extra vmagent to scrape all the scrape objects by default, and write data to above write-global endpoint.</p>
 </td>
    </tr>
    <tr>
@ -850,7 +464,7 @@ spec:
      <td>read.global.vmauth.name</td>
      <td>string</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
+<code class="language-yaml">vmauth-global-read-{{ .fullname }}
 </code>
 </pre>
 </td>
@ -861,7 +475,7 @@ spec:
      <td>read.global.vmauth.spec</td>
      <td>object</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">port: "8427"
+<code class="language-yaml">{}
 </code>
 </pre>
 </td>
@ -908,7 +522,7 @@ vmsingle:
      <td>write.global.vmauth.name</td>
      <td>string</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="">
-<code class="language-yaml">""
+<code class="language-yaml">vmauth-global-write-{{ .fullname }}
 </code>
 </pre>
 </td>
@ -919,7 +533,301 @@ vmsingle:
      <td>write.global.vmauth.spec</td>
      <td>object</td>
      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">port: "8427"
+<code class="language-yaml">{}
+</code>
+</pre>
+</td>
+      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">common:
+    spec:
+        affinity: {}
+        nodeSelector:
+            topology.kubernetes.io/zone: '{{ (.zone).name }}'
+        topologySpreadConstraints:
+            - maxSkew: 1
+              topologyKey: kubernetes.io/hostname
+              whenUnsatisfiable: ScheduleAnyway
+read:
+    allow: true
+    crossZone:
+        vmauth:
+            enabled: true
+            name: vmauth-read-proxy-{{ (.zone).name }}
+            spec: {}
+    perZone:
+        vmauth:
+            enabled: true
+            name: vmauth-read-balancer-{{ (.zone).name }}
+            spec:
+                extraArgs:
+                    discoverBackendIPs: "true"
+vmagent:
+    annotations: {}
+    enabled: true
+    name: vmagent-{{ (.zone).name }}
+    spec: {}
+vmcluster:
+    enabled: true
+    name: vmcluster-{{ (.zone).name }}
+    spec:
+        replicationFactor: 2
+        retentionPeriod: "14"
+        vminsert:
+            extraArgs: {}
+            replicaCount: 2
+            resources: {}
+        vmselect:
+            extraArgs: {}
+            replicaCount: 2
+            resources: {}
+        vmstorage:
+            replicaCount: 2
+            resources: {}
+            storageDataPath: /vm-data
+write:
+    allow: true
+    vmauth:
+        enabled: true
+        name: vmauth-write-balancer-{{ (.zone).name }}
+        spec:
+            extraArgs:
+                discoverBackendIPs: "true"
+</code>
+</pre>
+</td>
+      <td><p>Default config for each availability zone components, including vmagent, vmcluster, vmauth etc. Defines a template for each availability zone, which can be overridden for each availability zone at <code>availabilityZones[*]</code></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.common.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">affinity: {}
+nodeSelector:
+    topology.kubernetes.io/zone: '{{ (.zone).name }}'
+topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: ScheduleAnyway
+</code>
+</pre>
+</td>
+      <td><p>Common for <a href="https://docs.victoriametrics.com/operator/api/#vmagentspec" target="_blank">VMAgent</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmauthspec" target="_blank">VMAuth</a>, <a href="https://docs.victoriametrics.com/operator/api/#vmclusterspec" target="_blank">VMCluster</a> spec params, like nodeSelector, affinity, topologySpreadConstraint, etc</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.allow</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Allow data query from this zone through global query endpoint</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.crossZone.vmauth.enabled</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Create a vmauth with all the zone with <code>allow: true</code> as query backends</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.crossZone.vmauth.name</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">vmauth-read-proxy-{{ (.zone).name }}
+</code>
+</pre>
+</td>
+      <td><p>Override the name of the vmauth object</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.crossZone.vmauth.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">{}
+</code>
+</pre>
+</td>
+      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.perZone.vmauth.enabled</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Create vmauth as a local read endpoint</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.perZone.vmauth.name</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">vmauth-read-balancer-{{ (.zone).name }}
+</code>
+</pre>
+</td>
+      <td><p>Override the name of the vmauth object</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.read.perZone.vmauth.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">extraArgs:
+    discoverBackendIPs: "true"
+</code>
+</pre>
+</td>
+      <td><p>Spec for VMAuth CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmagent.annotations</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">{}
+</code>
+</pre>
+</td>
+      <td><p>VMAgent remote write proxy annotations</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmagent.enabled</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Create VMAgent remote write proxy</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmagent.name</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">vmagent-{{ (.zone).name }}
+</code>
+</pre>
+</td>
+      <td><p>Override the name of the vmagent object</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmagent.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">{}
+</code>
+</pre>
+</td>
+      <td><p>Spec for VMAgent CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmagentspec" target="_blank">here</a></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmcluster.enabled</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Create VMCluster</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmcluster.name</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">vmcluster-{{ (.zone).name }}
+</code>
+</pre>
+</td>
+      <td><p>Override the name of the vmcluster, by default is <zoneName></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.vmcluster.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">replicationFactor: 2
+retentionPeriod: "14"
+vminsert:
+    extraArgs: {}
+    replicaCount: 2
+    resources: {}
+vmselect:
+    extraArgs: {}
+    replicaCount: 2
+    resources: {}
+vmstorage:
+    replicaCount: 2
+    resources: {}
+    storageDataPath: /vm-data
+</code>
+</pre>
+</td>
+      <td><p>Spec for VMCluster CRD, see <a href="https://docs.victoriametrics.com/operator/api#vmclusterspec" target="_blank">here</a></p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.write.allow</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Allow data ingestion to this zone</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.write.vmauth.enabled</td>
+      <td>bool</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">true
+</code>
+</pre>
+</td>
+      <td><p>Create vmauth as a local write endpoint</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.write.vmauth.name</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">vmauth-write-balancer-{{ (.zone).name }}
+</code>
+</pre>
+</td>
+      <td><p>Override the name of the vmauth object</p>
+</td>
+    </tr>
+    <tr>
+      <td>zoneTpl.write.vmauth.spec</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">extraArgs:
+    discoverBackendIPs: "true"
 </code>
 </pre>
 </td>
--- a/docs/helm/victoria-metrics-gateway/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-gateway/CHANGELOG.md
@ -1,8 +1,28 @@
 ## Next release

+- TODO
+
+## 0.5.7
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.26 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
+
+## 0.5.6
+
+**Release date:** 2024-11-14
+
+![AppVersion: v1.106.0](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.0&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
 - set default DNS domain to `cluster.local.`
- updated common dependency 0.0.19 -> 0.0.23
+- updated common dependency 0.0.19 -> 0.0.26
 - added template for configmap name
+- added init containers

 ## 0.5.5

--- a/docs/helm/victoria-metrics-gateway/README.md
+++ b/docs/helm/victoria-metrics-gateway/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.5.5](https://img.shields.io/badge/Version-0.5.5-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.5.7](https://img.shields.io/badge/Version-0.5.7-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-gateway)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)

@ -557,6 +557,17 @@ loggerFormat: json
 </pre>
 </td>
      <td><p>Array of TLS objects</p>
+</td>
+    </tr>
+    <tr>
+      <td>initContainers</td>
+      <td>list</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
+<code class="language-yaml">[]
+</code>
+</pre>
+</td>
+      <td><p>Init containers for vmgateway</p>
 </td>
    </tr>
    <tr>
--- a/docs/helm/victoria-metrics-k8s-stack/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-k8s-stack/CHANGELOG.md
@ -1,6 +1,16 @@
 ## Next release

- updated common dependency 0.0.21 -> 0.0.23
+- TODO
+
+## 0.28.4
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.21 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)

 ## 0.28.3

--- a/docs/helm/victoria-metrics-k8s-stack/README.md
+++ b/docs/helm/victoria-metrics-k8s-stack/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.28.3](https://img.shields.io/badge/Version-0.28.3-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.28.4](https://img.shields.io/badge/Version-0.28.4-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-k8s-stack)

 Kubernetes monitoring on VictoriaMetrics stack. Includes VictoriaMetrics Operator, Grafana dashboards, ServiceScrapes and VMRules
--- a/docs/helm/victoria-metrics-operator/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-operator/CHANGELOG.md
@ -1,9 +1,20 @@
 ## Next release

+- TODO
+
+## 0.38.0
+
+**Release date:** 2024-11-18
+
+![AppVersion: v0.49.1](https://img.shields.io/static/v1?label=AppVersion&message=v0.49.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
 - fix Deployment/StatefulSets when `serviceAccount.name` is empty and `serviceAccount.create: false`. See [this issue](https://github.com/VictoriaMetrics/helm-charts/issues/1683).
 - set default DNS domain to `cluster.local.`
- updated common dependency 0.0.19 -> 0.0.23
+- updated common dependency 0.0.19 -> 0.0.28
 - added back `crds.enabled: false` option, which disables CRD creation, but due to limitation of dependencies condition it allows to disable only in combination with `crds.plain: false`
+- disabled cleanup, while `crds.enabled: false`. See [this issue](https://github.com/VictoriaMetrics/helm-charts/issues/1563).
+- updates operator to [v0.49.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.49.1) version

 ## 0.37.0

--- a/docs/helm/victoria-metrics-operator/README.md
+++ b/docs/helm/victoria-metrics-operator/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.38.0](https://img.shields.io/badge/Version-0.38.0-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-operator)

 Victoria Metrics Operator
--- a/docs/helm/victoria-metrics-operator/charts/crds/README.md
+++ b/docs/helm/victoria-metrics-operator/charts/crds/README.md
@ -0,0 +1,17 @@
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.0.0](https://img.shields.io/badge/Version-0.0.0-informational?style=flat-square)
+
+A subchart stores victoriametrics operator CRDs.
+
+## Documentation of Helm Chart
+
+Install ``helm-docs`` following the instructions on this [tutorial](https://docs.victoriametrics.com/helm/requirements/).
+
+Generate docs with ``helm-docs`` command.
+
+```bash
+cd charts/crds
+
+helm-docs
+```
+
+The markdown generation is entirely go template driven. The tool parses metadata from charts and generates a number of sub-templates that can be referenced in a template file (by default ``README.md.gotmpl``). If no template file is provided, the tool has a default internal template that will generate a reasonably formatted README.
--- a/docs/helm/victoria-metrics-single/CHANGELOG.md
+++ b/docs/helm/victoria-metrics-single/CHANGELOG.md
@ -1,6 +1,16 @@
 ## Next release

- updated common dependency 0.0.20 -> 0.0.23
+- TODO
+
+## 0.12.7
+
+**Release date:** 2024-11-18
+
+![AppVersion: v1.106.1](https://img.shields.io/static/v1?label=AppVersion&message=v1.106.1&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+- updated common dependency 0.0.20 -> 0.0.28
+- bump version of VM components to [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)

 ## 0.12.6

--- a/docs/helm/victoria-metrics-single/README.md
+++ b/docs/helm/victoria-metrics-single/README.md
@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.6](https://img.shields.io/badge/Version-0.12.6-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.7](https://img.shields.io/badge/Version-0.12.7-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-single)

 Victoria Metrics Single version - high-performance, cost-effective and scalable TSDB, long-term remote storage for Prometheus
--- a/docs/operator/CHANGELOG.md
+++ b/docs/operator/CHANGELOG.md
@ -13,7 +13,10 @@ aliases:

 ## tip

- TODO
+- [vmoperator](https://docs.victoriametrics.com/operator/): add missing `container` labels to the metrics discovered with `VMServiceScrape` for `endpointslices` discovery role.
+- [vmoperator](https://docs.victoriametrics.com/operator/): bump default version of VictoriaMetrics components to [1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1).
+- [vmoperator](https://docs.victoriametrics.com/operator/): add new variable `VM_VMSERVICESCRAPEDEFAULT_ENFORCEENDPOINTSLICES` to use `endpointslices` instead of `endpoints` as discovery role for VMServiceScrape when generate scrape config for VMAgent.
+- [vmoperator](https://docs.victoriametrics.com/operator/): adds new flag `loggerJSONFields` to the operator logger configuration. It allows to change json encoder fields. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1157) for details.

 ## [v0.49.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.49.1) - 11 Nov 2024

@ -31,7 +34,7 @@ aliases:
 - [vmalertmanager](https://docs.victoriametrics.com/operator/resources/vmalertmanager): properly trigger reload when `ConfigMap` provided via `.spec.configMap` are changed.
 - [operator](https://docs.victoriametrics.com/operator/): fixed operator reconcile on storage size change
 - [operator](https://docs.victoriametrics.com/operator/): fixed converting AlertmanagerConfig to VMAlertmanagerConfig
- [vmoperator](https://docs.victoriametrics.com/operator/): bump default version of VictoriaMetrics components to [1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.6).
+- [vmoperator](https://docs.victoriametrics.com/operator/): bump default version of VictoriaMetrics components to [1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0).

 ## [v0.48.4](https://github.com/VictoriaMetrics/operator/releases/tag/v0.48.4) - 15 Oct 2024

--- a/docs/operator/api.md
+++ b/docs/operator/api.md
@ -3897,8 +3897,8 @@ _Appears in:_
 | `serviceAccountName` | ServiceAccountName is the name of the ServiceAccount to use to run the pods | _string_ | false |
 | `serviceScrapeSpec` | ServiceScrapeSpec that will be added to vmsingle VMServiceScrape spec | _[VMServiceScrapeSpec](#vmservicescrapespec)_ | false |
 | `serviceSpec` | ServiceSpec that will be added to vmsingle service spec | _[AdditionalServiceSpec](#additionalservicespec)_ | false |
-| `storage` | Storage is the definition of how storage will be used by the VMSingle<br />by default it`s empty dir | _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#persistentvolumeclaimspec-v1-core)_ | false |
-| `storageDataPath` | StorageDataPath disables spec.storage option and overrides arg for victoria-metrics binary --storageDataPath,<br />its users responsibility to mount proper device into given path. | _string_ | false |
+| `storage` | Storage is the definition of how storage will be used by the VMSingle<br />by default it`s empty dir<br />this option is ignored if storageDataPath is set | _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#persistentvolumeclaimspec-v1-core)_ | false |
+| `storageDataPath` | StorageDataPath disables spec.storage option and overrides arg for victoria-metrics binary --storageDataPath,<br />its users responsibility to mount proper device into given path.<br />It requires to provide spec.volumes and spec.volumeMounts with at least 1 value | _string_ | false |
 | `storageMetadata` | StorageMeta defines annotations and labels attached to PVC for given vmsingle CR | _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | false |
 | `streamAggrConfig` | StreamAggrConfig defines stream aggregation configuration for VMSingle | _[StreamAggrConfig](#streamaggrconfig)_ | true |
 | `terminationGracePeriodSeconds` | TerminationGracePeriodSeconds period for container graceful termination | _integer_ | false |
--- a/docs/operator/vars.md
+++ b/docs/operator/vars.md
@ -10,7 +10,7 @@ aliases:
  - /operator/vars/index.html
 ---
 <!-- this doc autogenerated - don't edit it manually -->
- updated at Tue Nov  5 17:35:31 UTC 2024
+ updated at Tue Nov 19 08:23:27 UTC 2024


 | variable name | variable default value | variable required | variable description |
@ -31,7 +31,7 @@ aliases:
 | VM_VLOGSDEFAULT_CONFIGRELOADERCPU | - | false | ignored |
 | VM_VLOGSDEFAULT_CONFIGRELOADERMEMORY | - | false | ignored |
 | VM_VMALERTDEFAULT_IMAGE | victoriametrics/vmalert | false | - |
-| VM_VMALERTDEFAULT_VERSION | v1.106.0 | false | - |
+| VM_VMALERTDEFAULT_VERSION | v1.106.1 | false | - |
 | VM_VMALERTDEFAULT_CONFIGRELOADIMAGE | jimmidyson/configmap-reload:v0.3.0 | false | - |
 | VM_VMALERTDEFAULT_PORT | 8080 | false | - |
 | VM_VMALERTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -41,8 +41,9 @@ aliases:
 | VM_VMALERTDEFAULT_RESOURCE_REQUEST_CPU | 50m | false | - |
 | VM_VMALERTDEFAULT_CONFIGRELOADERCPU | 100m | false | - |
 | VM_VMALERTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
+| VM_VMSERVICESCRAPEDEFAULT_ENFORCEENDPOINTSLICES | false | false | Use endpointslices instead of endpoints as discovery role for vmservicescrape when generate scrape config for vmagent. |
 | VM_VMAGENTDEFAULT_IMAGE | victoriametrics/vmagent | false | - |
-| VM_VMAGENTDEFAULT_VERSION | v1.106.0 | false | - |
+| VM_VMAGENTDEFAULT_VERSION | v1.106.1 | false | - |
 | VM_VMAGENTDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.68.0 | false | - |
 | VM_VMAGENTDEFAULT_PORT | 8429 | false | - |
 | VM_VMAGENTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -53,7 +54,7 @@ aliases:
 | VM_VMAGENTDEFAULT_CONFIGRELOADERCPU | 100m | false | - |
 | VM_VMAGENTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
 | VM_VMSINGLEDEFAULT_IMAGE | victoriametrics/victoria-metrics | false | - |
-| VM_VMSINGLEDEFAULT_VERSION | v1.106.0 | false | - |
+| VM_VMSINGLEDEFAULT_VERSION | v1.106.1 | false | - |
 | VM_VMSINGLEDEFAULT_CONFIGRELOADIMAGE | - | false | ignored |
 | VM_VMSINGLEDEFAULT_PORT | 8429 | false | - |
 | VM_VMSINGLEDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -65,14 +66,14 @@ aliases:
 | VM_VMSINGLEDEFAULT_CONFIGRELOADERMEMORY | - | false | ignored |
 | VM_VMCLUSTERDEFAULT_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_IMAGE | victoriametrics/vmselect | false | - |
-| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.106.0-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.106.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_PORT | 8481 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_MEM | 1000Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_CPU | 100m | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_IMAGE | victoriametrics/vmstorage | false | - |
-| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.106.0-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.106.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMINSERTPORT | 8400 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMSELECTPORT | 8401 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_PORT | 8482 | false | - |
@ -81,7 +82,7 @@ aliases:
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_CPU | 250m | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_IMAGE | victoriametrics/vminsert | false | - |
-| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.106.0-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.106.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_PORT | 8480 | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
@ -100,7 +101,7 @@ aliases:
 | VM_VMALERTMANAGER_RESOURCE_REQUEST_CPU | 30m | false | - |
 | VM_DISABLESELFSERVICESCRAPECREATION | false | false | - |
 | VM_VMBACKUP_IMAGE | victoriametrics/vmbackupmanager | false | - |
-| VM_VMBACKUP_VERSION | v1.106.0-enterprise | false | - |
+| VM_VMBACKUP_VERSION | v1.106.1-enterprise | false | - |
 | VM_VMBACKUP_PORT | 8300 | false | - |
 | VM_VMBACKUP_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMBACKUP_RESOURCE_LIMIT_MEM | 500Mi | false | - |
@ -108,7 +109,7 @@ aliases:
 | VM_VMBACKUP_RESOURCE_REQUEST_MEM | 200Mi | false | - |
 | VM_VMBACKUP_RESOURCE_REQUEST_CPU | 150m | false | - |
 | VM_VMAUTHDEFAULT_IMAGE | victoriametrics/vmauth | false | - |
-| VM_VMAUTHDEFAULT_VERSION | v1.106.0 | false | - |
+| VM_VMAUTHDEFAULT_VERSION | v1.106.1 | false | - |
 | VM_VMAUTHDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.68.0 | false | - |
 | VM_VMAUTHDEFAULT_PORT | 8427 | false | - |
 | VM_VMAUTHDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -136,4 +137,4 @@ aliases:
 | VM_PODWAITREADYINTERVALCHECK | 5s | false | Defines poll interval for pods ready check at statefulset rollout update |
 | VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. |
 | VM_ENABLESTRICTSECURITY | false | false | EnableStrictSecurity will add default `securityContext` to pods and containers created by operator Default PodSecurityContext include: 1. RunAsNonRoot: true 2. RunAsUser/RunAsGroup/FSGroup: 65534 '65534' refers to 'nobody' in all the used default images like alpine, busybox. If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext. 3. FSGroupChangePolicy: &onRootMismatch If KubeVersion>=1.20, use `FSGroupChangePolicy="onRootMismatch"` to skip the recursive permission change when the root of the volume already has the correct permissions 4. SeccompProfile:      type: RuntimeDefault Use `RuntimeDefault` seccomp profile by default, which is defined by the container runtime, instead of using the Unconfined (seccomp disabled) mode. Default container SecurityContext include: 1. AllowPrivilegeEscalation: false 2. ReadOnlyRootFilesystem: true 3. Capabilities:      drop:        - all turn off `EnableStrictSecurity` by default, see https://github.com/VictoriaMetrics/operator/issues/749 for details |
-[envconfig-sum]: 1633bf4709b7f1602ed6f44ebb3f2fa2
+[envconfig-sum]: 4b951a49a01d16512392a23bf68385f2
--- a/docs/scrape_config_examples.md
+++ b/docs/scrape_config_examples.md
@ -30,8 +30,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:

 ```
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.0/victoria-metrics-linux-amd64-v1.106.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.106.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.1/victoria-metrics-linux-amd64-v1.106.1.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.106.1.tar.gz
 ```

 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@ -146,8 +146,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco

 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.0/victoria-metrics-linux-amd64-v1.106.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.106.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.106.1/victoria-metrics-linux-amd64-v1.106.1.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.106.1.tar.gz

 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@ -157,8 +157,8 @@ name: <string>
 # up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric).
 [ concurrency: <integer> | default = 1 ]

-# Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
-# By default, "prometheus" type is used.
+# Optional type for expressions inside rules to override the `-rule.defaultRuleType(default is "prometheus")` cmd-line flag.
+# Supported values: "graphite", "prometheus" and "vlogs"(check https://docs.victoriametrics.com/victorialogs/vmalert/ for details).
 [ type: <string> ]

 # Optional
@ -1384,7 +1384,7 @@ The shortlist of configuration flags is the following:
  -remoteWrite.maxBatchSize int
     Defines max number of timeseries to be flushed at once (default 10000)
  -remoteWrite.maxQueueSize int
-     Defines the max number of pending datapoints to remote write endpoint (default 1000000)
+     Defines the max number of pending datapoints to remote write endpoint (default 100000)
  -remoteWrite.oauth2.clientID string
     Optional OAuth2 clientID to use for -remoteWrite.url
  -remoteWrite.oauth2.clientSecret string
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@ -91,13 +91,17 @@ type indexDB struct {
 	// The db must be automatically recovered after that.
 	missingMetricNamesForMetricID atomic.Uint64

-	// minMissingTimestamp is the minimum timestamp, which is missing in the given indexDB.
+	// minMissingTimestampByKey holds the minimum timestamps by index search key,
+	// which is missing in the given indexDB.
+	// Key must be formed with marshalCommonPrefix function.
 	//
 	// This field is used at containsTimeRange() function only for the previous indexDB,
 	// since this indexDB is readonly.
 	// This field cannot be used for the current indexDB, since it may receive data
 	// with bigger timestamps at any time.
-	minMissingTimestamp atomic.Int64
+	minMissingTimestampByKey map[string]int64
+	// protects minMissingTimestampByKey
+	minMissingTimestampByKeyLock sync.RWMutex

 	// generation identifies the index generation ID
 	// and is used for syncing items from different indexDBs
@ -162,6 +166,7 @@ func mustOpenIndexDB(path string, s *Storage, isReadOnly *atomic.Bool) *indexDB
 		tb:         tb,
 		name:       name,

+		minMissingTimestampByKey:   make(map[string]int64),
 		tagFiltersToMetricIDsCache: workingsetcache.New(tagFiltersCacheSize),
 		s:                          s,
 		loopsPerDateTagFilterCache: workingsetcache.New(mem / 128),
@ -1945,25 +1950,36 @@ func (is *indexSearch) containsTimeRange(tr TimeRange) bool {
 		// This means that it may contain data for the given tr with probability close to 100%.
 		return true
 	}
-
 	// The db corresponds to the previous indexDB, which is readonly.
 	// So it is safe caching the minimum timestamp, which isn't covered by the db.
-	minMissingTimestamp := db.minMissingTimestamp.Load()
-	if minMissingTimestamp != 0 && tr.MinTimestamp >= minMissingTimestamp {
+
+	// use common prefix as a key for minMissingTimestamp
+	// it's needed to properly track timestamps for cluster version
+	// which uses tenant labels for the index search
+	kb := &is.kb
+	kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixDateToMetricID)
+	key := kb.B
+
+	db.minMissingTimestampByKeyLock.RLock()
+	minMissingTimestamp, ok := db.minMissingTimestampByKey[string(key)]
+	db.minMissingTimestampByKeyLock.RUnlock()
+
+	if ok && tr.MinTimestamp >= minMissingTimestamp {
 		return false
 	}
-
-	if is.containsTimeRangeSlow(tr) {
+	if is.containsTimeRangeSlowForPrefixBuf(kb, tr) {
 		return true
 	}

-	db.minMissingTimestamp.CompareAndSwap(minMissingTimestamp, tr.MinTimestamp)
+	db.minMissingTimestampByKeyLock.Lock()
+	db.minMissingTimestampByKey[string(key)] = tr.MinTimestamp
+	db.minMissingTimestampByKeyLock.Unlock()
+
 	return false
 }

-func (is *indexSearch) containsTimeRangeSlow(tr TimeRange) bool {
+func (is *indexSearch) containsTimeRangeSlowForPrefixBuf(prefixBuf *bytesutil.ByteBuffer, tr TimeRange) bool {
 	ts := &is.ts
-	kb := &is.kb

 	// Verify whether the tr.MinTimestamp is included into `ts` or is smaller than the minimum date stored in `ts`.
 	// Do not check whether tr.MaxTimestamp is included into `ts` or is bigger than the max date stored in `ts` for performance reasons.
@ -1972,13 +1988,12 @@ func (is *indexSearch) containsTimeRangeSlow(tr TimeRange) bool {
 	// The main practical case allows skipping searching in prev indexdb (`ts`) when `tr`
 	// is located above the max date stored there.
 	minDate := uint64(tr.MinTimestamp) / msecPerDay
-	kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixDateToMetricID)
-	prefix := kb.B
-	kb.B = encoding.MarshalUint64(kb.B, minDate)
-	ts.Seek(kb.B)
+	prefix := prefixBuf.B
+	prefixBuf.B = encoding.MarshalUint64(prefixBuf.B, minDate)
+	ts.Seek(prefixBuf.B)
 	if !ts.NextItem() {
 		if err := ts.Error(); err != nil {
-			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %w", minDate, kb.B, err)
+			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %w", minDate, prefixBuf.B, err)
 		}
 		return false
 	}
--- a/lib/storage/index_db_test.go
+++ b/lib/storage/index_db_test.go
@ -2101,3 +2101,130 @@ func stopTestStorage(s *Storage) {
 	s.tsidCache.Stop()
 	fs.MustRemoveDirAtomic(s.cachePath)
 }
+
+func TestSearchContainsTimeRange(t *testing.T) {
+	path := t.Name()
+	os.RemoveAll(path)
+	s := MustOpenStorage(path, retentionMax, 0, 0)
+	db := s.idb()
+
+	is := db.getIndexSearch(noDeadline)
+
+	// Create a bunch of per-day time series
+	const (
+		days                = 6
+		tenant2IngestionDay = 8
+		metricsPerDay       = 1000
+	)
+	rotationDay := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC)
+	rotationMillis := uint64(rotationDay.UnixMilli())
+	rotationDate := rotationMillis / msecPerDay
+	var metricNameBuf []byte
+	perDayMetricIDs := make(map[uint64]*uint64set.Set)
+	labelNames := []string{
+		"__name__", "constant", "day", "UniqueId", "some_unique_id",
+	}
+
+	sort.Strings(labelNames)
+
+	newMN := func(name string, day, metric int) MetricName {
+		var mn MetricName
+		mn.MetricGroup = []byte(name)
+		mn.AddTag(
+			"constant",
+			"const",
+		)
+		mn.AddTag(
+			"day",
+			fmt.Sprintf("%v", day),
+		)
+		mn.AddTag(
+			"UniqueId",
+			fmt.Sprintf("%v", metric),
+		)
+		mn.AddTag(
+			"some_unique_id",
+			fmt.Sprintf("%v", day),
+		)
+		mn.sortTags()
+		return mn
+	}
+
+	// ingest metrics for tenant 0:0
+	for day := 0; day < days; day++ {
+		date := rotationDate - uint64(day)
+
+		var metricIDs uint64set.Set
+		for metric := range metricsPerDay {
+			mn := newMN("testMetric", day, metric)
+			metricNameBuf = mn.Marshal(metricNameBuf[:0])
+			var genTSID generationTSID
+			if !is.getTSIDByMetricName(&genTSID, metricNameBuf, date) {
+				generateTSID(&genTSID.TSID, &mn)
+				createAllIndexesForMetricName(is, &mn, &genTSID.TSID, date)
+			}
+			metricIDs.Add(genTSID.TSID.MetricID)
+		}
+
+		perDayMetricIDs[date] = &metricIDs
+	}
+	db.putIndexSearch(is)
+
+	// Flush index to disk, so it becomes visible for search
+	s.DebugFlush()
+
+	is2 := db.getIndexSearch(noDeadline)
+
+	// Check that all the metrics are found for all the days.
+	for date := rotationDate - days + 1; date <= rotationDate; date++ {
+
+		metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if !perDayMetricIDs[date].Equal(metricIDs) {
+			t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
+		}
+	}
+
+	db.putIndexSearch(is2)
+
+	// rotate indexdb
+	s.mustRotateIndexDB(rotationDay)
+	db = s.idb()
+
+	// perform search for 0:0 tenant
+	// results of previous search requests shouldn't affect it
+
+	isExt := db.extDB.getIndexSearch(noDeadline)
+	// search for range that covers prev indexDB for dates before ingestion
+	tr := TimeRange{
+		MinTimestamp: int64(rotationMillis - msecPerDay*(days)),
+		MaxTimestamp: int64(rotationMillis),
+	}
+	if !isExt.containsTimeRange(tr) {
+		t.Fatalf("expected to have given time range at prev IndexDB")
+	}
+
+	// search for range not exist at prev indexDB
+	tr = TimeRange{
+		MinTimestamp: int64(rotationMillis + msecPerDay*(days+4)),
+		MaxTimestamp: int64(rotationMillis + msecPerDay*(days+2)),
+	}
+	if isExt.containsTimeRange(tr) {
+		t.Fatalf("not expected to have given time range at prev IndexDB")
+	}
+	key := isExt.marshalCommonPrefix(nil, nsPrefixDateToMetricID)
+
+	db.extDB.minMissingTimestampByKeyLock.Lock()
+	minMissingTimetamp := db.extDB.minMissingTimestampByKey[string(key)]
+	db.extDB.minMissingTimestampByKeyLock.Unlock()
+
+	if minMissingTimetamp != tr.MinTimestamp {
+		t.Fatalf("unexpected minMissingTimestamp for 0:0 tenant got %d, want %d", minMissingTimetamp, tr.MinTimestamp)
+	}
+
+	db.extDB.putIndexSearch(isExt)
+	s.MustClose()
+	fs.MustRemoveAll(path)
+}
--- a/lib/tenantmetrics/counter_map.go
+++ b/lib/tenantmetrics/counter_map.go
@ -2,6 +2,7 @@ package tenantmetrics

 import (
 	"fmt"
+	"sync"
 	"sync/atomic"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
@ -19,17 +20,16 @@ type TenantID struct {
 type CounterMap struct {
 	metric string

-	// do not use atomic.Pointer, since the stored map there is already a pointer type.
-	m atomic.Value
+	m sync.Map
+	// mt holds value for multi-tenant metrics.
+	mt atomic.Value
 }

 // NewCounterMap creates new CounterMap for the given metric.
 func NewCounterMap(metric string) *CounterMap {
-	cm := &CounterMap{
+	return &CounterMap{
 		metric: metric,
 	}
-	cm.m.Store(make(map[TenantID]*metrics.Counter))
-	return cm
 }

 // Get returns counter for the given at
@ -38,7 +38,7 @@ func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter {
 		AccountID: at.AccountID,
 		ProjectID: at.ProjectID,
 	}
-	return cm.GetByTenant(key)
+	return cm.GetByTenant(&key)
 }

 // MultiAdd adds multiple values grouped by auth.Token
@ -49,22 +49,25 @@ func (cm *CounterMap) MultiAdd(perTenantValues map[auth.Token]int) {
 }

 // GetByTenant returns counter for the given key.
-func (cm *CounterMap) GetByTenant(key TenantID) *metrics.Counter {
-	m := cm.m.Load().(map[TenantID]*metrics.Counter)
-	if c := m[key]; c != nil {
-		// Fast path - the counter for k already exists.
-		return c
+func (cm *CounterMap) GetByTenant(key *TenantID) *metrics.Counter {
+	if key == nil {
+		mtm := cm.mt.Load()
+		if mtm == nil {
+			mtc := metrics.GetOrCreateCounter(createMetricNameMultitenant(cm.metric))
+			cm.mt.Store(mtc)
+			return mtc
+		}
+		return mtm.(*metrics.Counter)
 	}

-	// Slow path - create missing counter for k and re-create m.
-	newM := make(map[TenantID]*metrics.Counter, len(m)+1)
-	for k, c := range m {
-		newM[k] = c
+	if counter, ok := cm.m.Load(*key); ok {
+		return counter.(*metrics.Counter)
 	}
-	metricName := createMetricName(cm.metric, key)
+
+	// Slow path - create missing counter for k.
+	metricName := createMetricName(cm.metric, *key)
 	c := metrics.GetOrCreateCounter(metricName)
-	newM[key] = c
-	cm.m.Store(newM)
+	cm.m.Store(*key, c)
 	return c
 }

@ -79,3 +82,15 @@ func createMetricName(metric string, key TenantID) string {
 	// Metric with labels.
 	return fmt.Sprintf(`%s,accountID="%d",projectID="%d"}`, metric[:len(metric)-1], key.AccountID, key.ProjectID)
 }
+
+func createMetricNameMultitenant(metric string) string {
+	if len(metric) == 0 {
+		logger.Panicf("BUG: metric cannot be empty")
+	}
+	if metric[len(metric)-1] != '}' {
+		// Metric without labels.
+		return fmt.Sprintf(`%s{accountID="multitenant",projectID="multitenant"}`, metric)
+	}
+	// Metric with labels.
+	return fmt.Sprintf(`%s,accountID="multitenant",projectID="multitenant"}`, metric[:len(metric)-1])
+}
--- a/lib/tenantmetrics/counter_map_timing_test.go
+++ b/lib/tenantmetrics/counter_map_timing_test.go
@ -0,0 +1,46 @@
+package tenantmetrics
+
+import (
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+)
+
+func BenchmarkCounterMapGrowth(b *testing.B) {
+	f := func(b *testing.B, numTenants uint32, nProcs int) {
+		b.Helper()
+
+		for i := 0; i < b.N; i++ {
+			cm := NewCounterMap("foobar")
+			var wg sync.WaitGroup
+			for range nProcs {
+				wg.Add(1)
+				go func() {
+					for i := range numTenants {
+						cm.Get(&auth.Token{AccountID: i, ProjectID: i}).Inc()
+					}
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+		}
+	}
+
+	b.Run("n=100,nProcs=GOMAXPROCS", func(b *testing.B) {
+		f(b, 100, runtime.GOMAXPROCS(0))
+	})
+
+	b.Run("n=100,nProcs=2", func(b *testing.B) {
+		f(b, 100, 2)
+	})
+
+	b.Run("n=1000,nProcs=2", func(b *testing.B) {
+		f(b, 1000, 2)
+	})
+
+	b.Run("n=10000,nProcs=2", func(b *testing.B) {
+		f(b, 10000, 2)
+	})
+}