From 9c70c1f21f5fc51cbf7055d1190682fb5bb75db9 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Thu, 18 Feb 2021 22:07:29 +0200
Subject: [PATCH 01/32] app/vmselect/promql: reduce the probability of
 `duplicate time series` errors when querying Kubernetes metrics

---
 app/vmselect/promql/binary_op.go | 9 +++++++--
 docs/CHANGELOG.md                | 2 ++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/vmselect/promql/binary_op.go b/app/vmselect/promql/binary_op.go
index 7d827607a..d258d6330 100644
--- a/app/vmselect/promql/binary_op.go
+++ b/app/vmselect/promql/binary_op.go
@@ -252,16 +252,21 @@ func mergeNonOverlappingTimeseries(dst, src *timeseries) bool {
 	// Verify whether the time series can be merged.
 	srcValues := src.Values
 	dstValues := dst.Values
+	overlaps := 0
 	_ = dstValues[len(srcValues)-1]
 	for i, v := range srcValues {
 		if math.IsNaN(v) {
 			continue
 		}
 		if !math.IsNaN(dstValues[i]) {
-			return false
+			overlaps++
 		}
 	}
-
+	// Allow up to two overlapping datapoints, which can appear due to staleness algorithm,
+	// which can add a few datapoints in the end of time series.
+	if overlaps > 2 {
+		return false
+	}
 	// Time series can be merged. Merge them.
 	for i, v := range srcValues {
 		if math.IsNaN(v) {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 361d7d61f..4780edae1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 # tip
 
+* BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
+
 
 # [v1.54.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.1)
 

From f26162ec99e797a3acd7ac7d33ce68448baa117c Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Thu, 18 Feb 2021 23:51:29 +0200
Subject: [PATCH 02/32] lib/promscrape: add scrape_align_interval config option
 into scrape config

This option allows aligning scrapes to a particular intervals.
---
 app/vmagent/README.md         | 10 +++++++++
 docs/CHANGELOG.md             |  2 ++
 docs/vmagent.md               | 10 +++++++++
 lib/promscrape/config.go      | 10 ++++++---
 lib/promscrape/config_test.go | 14 +++++++------
 lib/promscrape/scrapework.go  | 38 ++++++++++++++++++++++-------------
 6 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index bf40e17ba..866862761 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -298,6 +298,16 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
   the url may contain sensitive information such as auth tokens or passwords.
   Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
 
+* If scrapes must be aligned in time (for instance, if they must be performed at the beginning of every hour), then set `scrape_align_interval` option
+  in the corresponding scrape config. For example, the following config aligns hourly scrapes to the nearest 10 minutes:
+
+  ```yml
+  scrape_configs:
+  - job: foo
+    scrape_interval: 1h
+    scrape_align_interval: 10m
+  ```
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
   or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
   See available options below if you prefer fixing the root cause of the error:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4780edae1..17674a6e9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 # tip
 
+* FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details.
+
 * BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
 
 
diff --git a/docs/vmagent.md b/docs/vmagent.md
index bf40e17ba..866862761 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -298,6 +298,16 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
   the url may contain sensitive information such as auth tokens or passwords.
   Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
 
+* If scrapes must be aligned in time (for instance, if they must be performed at the beginning of every hour), then set `scrape_align_interval` option
+  in the corresponding scrape config. For example, the following config aligns hourly scrapes to the nearest 10 minutes:
+
+  ```yml
+  scrape_configs:
+  - job: foo
+    scrape_interval: 1h
+    scrape_align_interval: 10m
+  ```
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
   or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
   See available options below if you prefer fixing the root cause of the error:
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 31e1ce858..aade32faa 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -89,9 +89,10 @@ type ScrapeConfig struct {
 	SampleLimit          int                         `yaml:"sample_limit,omitempty"`
 
 	// These options are supported only by lib/promscrape.
-	DisableCompression bool `yaml:"disable_compression,omitempty"`
-	DisableKeepAlive   bool `yaml:"disable_keepalive,omitempty"`
-	StreamParse        bool `yaml:"stream_parse,omitempty"`
+	DisableCompression  bool          `yaml:"disable_compression,omitempty"`
+	DisableKeepAlive    bool          `yaml:"disable_keepalive,omitempty"`
+	StreamParse         bool          `yaml:"stream_parse,omitempty"`
+	ScrapeAlignInterval time.Duration `yaml:"scrape_align_interval,omitempty"`
 
 	// This is set in loadConfig
 	swc *scrapeWorkConfig
@@ -508,6 +509,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		disableCompression:   sc.DisableCompression,
 		disableKeepAlive:     sc.DisableKeepAlive,
 		streamParse:          sc.StreamParse,
+		scrapeAlignInterval:  sc.ScrapeAlignInterval,
 	}
 	return swc, nil
 }
@@ -530,6 +532,7 @@ type scrapeWorkConfig struct {
 	disableCompression   bool
 	disableKeepAlive     bool
 	streamParse          bool
+	scrapeAlignInterval  time.Duration
 }
 
 func appendKubernetesScrapeWork(dst []*ScrapeWork, sdc *kubernetes.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
@@ -761,6 +764,7 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 		DisableCompression:   swc.disableCompression,
 		DisableKeepAlive:     swc.disableKeepAlive,
 		StreamParse:          swc.streamParse,
+		ScrapeAlignInterval:  swc.scrapeAlignInterval,
 
 		jobNameOriginal: swc.jobName,
 	})
diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go
index 9982efb1a..a1dbc5f57 100644
--- a/lib/promscrape/config_test.go
+++ b/lib/promscrape/config_test.go
@@ -1275,6 +1275,7 @@ scrape_configs:
     disable_keepalive: true
     disable_compression: true
     stream_parse: true
+    scrape_align_interval: 1s
     static_configs:
       - targets:
         - 192.168.1.2  # SNMP device.
@@ -1323,12 +1324,13 @@ scrape_configs:
 					Value: "snmp",
 				},
 			},
-			AuthConfig:         &promauth.Config{},
-			SampleLimit:        100,
-			DisableKeepAlive:   true,
-			DisableCompression: true,
-			StreamParse:        true,
-			jobNameOriginal:    "snmp",
+			AuthConfig:          &promauth.Config{},
+			SampleLimit:         100,
+			DisableKeepAlive:    true,
+			DisableCompression:  true,
+			StreamParse:         true,
+			ScrapeAlignInterval: time.Second,
+			jobNameOriginal:     "snmp",
 		},
 	})
 	f(`
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 585529e01..82a064094 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -90,6 +90,9 @@ type ScrapeWork struct {
 	// Whether to parse target responses in a streaming manner.
 	StreamParse bool
 
+	// The interval for aligning the first scrape.
+	ScrapeAlignInterval time.Duration
+
 	// The original 'job_name'
 	jobNameOriginal string
 }
@@ -100,9 +103,9 @@ type ScrapeWork struct {
 func (sw *ScrapeWork) key() string {
 	// Do not take into account OriginalLabels.
 	key := fmt.Sprintf("ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, Labels=%s, "+
-		"AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v",
+		"AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, ScrapeAlignInterval=%s",
 		sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.LabelsString(),
-		sw.AuthConfig.String(), sw.metricRelabelConfigsString(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse)
+		sw.AuthConfig.String(), sw.metricRelabelConfigsString(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.ScrapeAlignInterval)
 	return key
 }
 
@@ -180,20 +183,27 @@ type scrapeWork struct {
 }
 
 func (sw *scrapeWork) run(stopCh <-chan struct{}) {
-	// Calculate start time for the first scrape from ScrapeURL and labels.
-	// This should spread load when scraping many targets with different
-	// scrape urls and labels.
-	// This also makes consistent scrape times across restarts
-	// for a target with the same ScrapeURL and labels.
 	scrapeInterval := sw.Config.ScrapeInterval
-	key := fmt.Sprintf("ScrapeURL=%s, Labels=%s", sw.Config.ScrapeURL, sw.Config.LabelsString())
-	h := uint32(xxhash.Sum64([]byte(key)))
-	randSleep := uint64(float64(scrapeInterval) * (float64(h) / (1 << 32)))
-	sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval)
-	if randSleep < sleepOffset {
-		randSleep += uint64(scrapeInterval)
+	var randSleep uint64
+	if sw.Config.ScrapeAlignInterval <= 0 {
+		// Calculate start time for the first scrape from ScrapeURL and labels.
+		// This should spread load when scraping many targets with different
+		// scrape urls and labels.
+		// This also makes consistent scrape times across restarts
+		// for a target with the same ScrapeURL and labels.
+		key := fmt.Sprintf("ScrapeURL=%s, Labels=%s", sw.Config.ScrapeURL, sw.Config.LabelsString())
+		h := uint32(xxhash.Sum64([]byte(key)))
+		randSleep := uint64(float64(scrapeInterval) * (float64(h) / (1 << 32)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(scrapeInterval)
+		}
+		randSleep -= sleepOffset
+	} else {
+		d := uint64(sw.Config.ScrapeAlignInterval)
+		randSleep = d - uint64(time.Now().UnixNano())%d
+		randSleep %= uint64(scrapeInterval)
 	}
-	randSleep -= sleepOffset
 	timer := timerpool.Get(time.Duration(randSleep))
 	var timestamp int64
 	var ticker *time.Ticker

From 49e36e8d9d82107ddfaeec7a245ef124c72c6157 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Thu, 18 Feb 2021 23:57:23 +0200
Subject: [PATCH 03/32] app/vmagent: fix scrape config example for
 scrape_align_interval option

---
 app/vmagent/README.md | 2 +-
 docs/vmagent.md       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 866862761..145491fcb 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -303,7 +303,7 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
 
   ```yml
   scrape_configs:
-  - job: foo
+  - job_name: foo
     scrape_interval: 1h
     scrape_align_interval: 10m
   ```
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 866862761..145491fcb 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -303,7 +303,7 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
 
   ```yml
   scrape_configs:
-  - job: foo
+  - job_name: foo
     scrape_interval: 1h
     scrape_align_interval: 10m
   ```

From c2678754e46880645f7f48ac1f5df1b3d5429750 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 19 Feb 2021 00:31:07 +0200
Subject: [PATCH 04/32] app/vmagent: properly perform graceful shutdown, which
 was broken in the commit 1d1ba889fe61b2ce55216e616428839261e8d07c

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
---
 app/vmagent/remotewrite/remotewrite.go |  1 +
 docs/CHANGELOG.md                      |  1 +
 lib/persistentqueue/fastqueue.go       | 16 ++++++++++++----
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index 337772560..824ea9ad7 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -227,6 +227,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	}
 	rwctx.idx = 0
 	rwctx.pss = nil
+	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
 	rwctx.fq.MustClose()
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 17674a6e9..94403e514 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 * FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details.
 
+* BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
 * BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
 
 
diff --git a/lib/persistentqueue/fastqueue.go b/lib/persistentqueue/fastqueue.go
index 5a3dd33e2..5c25933f6 100644
--- a/lib/persistentqueue/fastqueue.go
+++ b/lib/persistentqueue/fastqueue.go
@@ -52,16 +52,24 @@ func MustOpenFastQueue(path, name string, maxInmemoryBlocks, maxPendingBytes int
 	return fq
 }
 
-// MustClose unblocks all the readers.
-//
-// It is expected no new writers during and after the call.
-func (fq *FastQueue) MustClose() {
+// UnblockAllReaders unblocks all the readers.
+func (fq *FastQueue) UnblockAllReaders() {
 	fq.mu.Lock()
 	defer fq.mu.Unlock()
 
 	// Unblock blocked readers
 	fq.mustStop = true
 	fq.cond.Broadcast()
+}
+
+// MustClose unblocks all the readers.
+//
+// It is expected no new writers during and after the call.
+func (fq *FastQueue) MustClose() {
+	fq.UnblockAllReaders()
+
+	fq.mu.Lock()
+	defer fq.mu.Unlock()
 
 	// flush blocks from fq.ch to fq.pq, so they can be persisted
 	fq.flushInmemoryBlocksToFileLocked()

From 2cfb376945ddd166814503d501fb543c86cd6942 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 19 Feb 2021 00:33:37 +0200
Subject: [PATCH 05/32] lib/promscrape: typo fix after the commit
 f26162ec99e797a3acd7ac7d33ce68448baa117c

---
 lib/promscrape/scrapework.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 82a064094..adff00c2c 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -193,7 +193,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}) {
 		// for a target with the same ScrapeURL and labels.
 		key := fmt.Sprintf("ScrapeURL=%s, Labels=%s", sw.Config.ScrapeURL, sw.Config.LabelsString())
 		h := uint32(xxhash.Sum64([]byte(key)))
-		randSleep := uint64(float64(scrapeInterval) * (float64(h) / (1 << 32)))
+		randSleep = uint64(float64(scrapeInterval) * (float64(h) / (1 << 32)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval)
 		if randSleep < sleepOffset {
 			randSleep += uint64(scrapeInterval)

From cb311bb156ef74159f6c4e1fa29c0da3db7ca1f6 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 21:18:49 +0200
Subject: [PATCH 06/32] lib/{mergeset,storage}: do not use pools for indexBlock
 and inmemoryBlock during their caching, since this results in higher memory
 usage in production without any performance gains

---
 lib/mergeset/block_header.go |  5 +++++
 lib/mergeset/part.go         | 40 +++++-------------------------------
 lib/mergeset/part_search.go  |  2 +-
 lib/storage/part.go          | 22 --------------------
 lib/storage/part_search.go   |  2 +-
 5 files changed, 12 insertions(+), 59 deletions(-)

diff --git a/lib/mergeset/block_header.go b/lib/mergeset/block_header.go
index 5404c2af6..49e35aa09 100644
--- a/lib/mergeset/block_header.go
+++ b/lib/mergeset/block_header.go
@@ -3,6 +3,7 @@ package mergeset
 import (
 	"fmt"
 	"sort"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -34,6 +35,10 @@ type blockHeader struct {
 	lensBlockSize uint32
 }
 
+func (bh *blockHeader) SizeBytes() int {
+	return int(unsafe.Sizeof(*bh)) + cap(bh.commonPrefix) + cap(bh.firstItem)
+}
+
 func (bh *blockHeader) Reset() {
 	bh.commonPrefix = bh.commonPrefix[:0]
 	bh.firstItem = bh.firstItem[:0]
diff --git a/lib/mergeset/part.go b/lib/mergeset/part.go
index 2b01d94be..7da60ceb3 100644
--- a/lib/mergeset/part.go
+++ b/lib/mergeset/part.go
@@ -138,24 +138,14 @@ type indexBlock struct {
 }
 
 func (idxb *indexBlock) SizeBytes() int {
-	return cap(idxb.bhs) * int(unsafe.Sizeof(blockHeader{}))
-}
-
-func getIndexBlock() *indexBlock {
-	v := indexBlockPool.Get()
-	if v == nil {
-		return &indexBlock{}
+	bhs := idxb.bhs[:cap(idxb.bhs)]
+	n := int(unsafe.Sizeof(*idxb))
+	for i := range bhs {
+		n += bhs[i].SizeBytes()
 	}
-	return v.(*indexBlock)
+	return n
 }
 
-func putIndexBlock(idxb *indexBlock) {
-	idxb.bhs = idxb.bhs[:0]
-	indexBlockPool.Put(idxb)
-}
-
-var indexBlockPool sync.Pool
-
 type indexBlockCache struct {
 	// Atomically updated counters must go first in the struct, so they are properly
 	// aligned to 8 bytes on 32-bit architectures.
@@ -194,12 +184,6 @@ func newIndexBlockCache() *indexBlockCache {
 func (idxbc *indexBlockCache) MustClose() {
 	close(idxbc.cleanerStopCh)
 	idxbc.cleanerWG.Wait()
-
-	// It is safe returning idxbc.m to pool, since the MustClose can be called
-	// when the idxbc entries are no longer accessed by concurrent goroutines.
-	for _, idxbe := range idxbc.m {
-		putIndexBlock(idxbe.idxb)
-	}
 	idxbc.m = nil
 }
 
@@ -223,8 +207,6 @@ func (idxbc *indexBlockCache) cleanByTimeout() {
 	for k, idxbe := range idxbc.m {
 		// Delete items accessed more than two minutes ago.
 		if currentTime-atomic.LoadUint64(&idxbe.lastAccessTime) > 2*60 {
-			// do not call putIndexBlock(ibxbc.m[k]), since it
-			// may be used by concurrent goroutines.
 			delete(idxbc.m, k)
 		}
 	}
@@ -257,8 +239,6 @@ func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) {
 		// Remove 10% of items from the cache.
 		overflow = int(float64(len(idxbc.m)) * 0.1)
 		for k := range idxbc.m {
-			// do not call putIndexBlock(ibxbc.m[k]), since it
-			// may be used by concurrent goroutines.
 			delete(idxbc.m, k)
 			overflow--
 			if overflow == 0 {
@@ -348,12 +328,6 @@ func newInmemoryBlockCache() *inmemoryBlockCache {
 func (ibc *inmemoryBlockCache) MustClose() {
 	close(ibc.cleanerStopCh)
 	ibc.cleanerWG.Wait()
-
-	// It is safe returning ibc.m entries to pool, since the MustClose can be called
-	// only if no other goroutines access ibc entries.
-	for _, ibe := range ibc.m {
-		putInmemoryBlock(ibe.ib)
-	}
 	ibc.m = nil
 }
 
@@ -377,8 +351,6 @@ func (ibc *inmemoryBlockCache) cleanByTimeout() {
 	for k, ibe := range ibc.m {
 		// Delete items accessed more than a two minutes ago.
 		if currentTime-atomic.LoadUint64(&ibe.lastAccessTime) > 2*60 {
-			// do not call putInmemoryBlock(ibc.m[k]), since it
-			// may be used by concurrent goroutines.
 			delete(ibc.m, k)
 		}
 	}
@@ -412,8 +384,6 @@ func (ibc *inmemoryBlockCache) Put(k inmemoryBlockCacheKey, ib *inmemoryBlock) {
 		// Remove 10% of items from the cache.
 		overflow = int(float64(len(ibc.m)) * 0.1)
 		for k := range ibc.m {
-			// do not call putInmemoryBlock(ib), since the ib
-			// may be used by concurrent goroutines.
 			delete(ibc.m, k)
 			overflow--
 			if overflow == 0 {
diff --git a/lib/mergeset/part_search.go b/lib/mergeset/part_search.go
index 0c7b17855..ea4b5ca25 100644
--- a/lib/mergeset/part_search.go
+++ b/lib/mergeset/part_search.go
@@ -279,7 +279,7 @@ func (ps *partSearch) readIndexBlock(mr *metaindexRow) (*indexBlock, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot decompress index block: %w", err)
 	}
-	idxb := getIndexBlock()
+	idxb := &indexBlock{}
 	idxb.bhs, err = unmarshalBlockHeaders(idxb.bhs[:0], ps.indexBuf, int(mr.blockHeadersCount))
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal block headers from index block (offset=%d, size=%d): %w", mr.indexBlockOffset, mr.indexBlockSize, err)
diff --git a/lib/storage/part.go b/lib/storage/part.go
index 52b0ec41a..378cdc869 100644
--- a/lib/storage/part.go
+++ b/lib/storage/part.go
@@ -145,21 +145,6 @@ func (idxb *indexBlock) SizeBytes() int {
 	return cap(idxb.bhs) * int(unsafe.Sizeof(blockHeader{}))
 }
 
-func getIndexBlock() *indexBlock {
-	v := indexBlockPool.Get()
-	if v == nil {
-		return &indexBlock{}
-	}
-	return v.(*indexBlock)
-}
-
-func putIndexBlock(ib *indexBlock) {
-	ib.bhs = ib.bhs[:0]
-	indexBlockPool.Put(ib)
-}
-
-var indexBlockPool sync.Pool
-
 type indexBlockCache struct {
 	// Put atomic counters to the top of struct in order to align them to 8 bytes on 32-bit architectures.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
@@ -198,12 +183,6 @@ func newIndexBlockCache() *indexBlockCache {
 func (ibc *indexBlockCache) MustClose(isBig bool) {
 	close(ibc.cleanerStopCh)
 	ibc.cleanerWG.Wait()
-
-	// It is safe returning ibc.m itemst to the pool, since Reset must
-	// be called only when no other goroutines access ibc entries.
-	for _, ibe := range ibc.m {
-		putIndexBlock(ibe.ib)
-	}
 	ibc.m = nil
 }
 
@@ -259,7 +238,6 @@ func (ibc *indexBlockCache) Put(k uint64, ib *indexBlock) {
 		// Remove 10% of items from the cache.
 		overflow = int(float64(len(ibc.m)) * 0.1)
 		for k := range ibc.m {
-			// Do not call putIndexBlock on ibc.m entries, since they may be used by concurrent goroutines.
 			delete(ibc.m, k)
 			overflow--
 			if overflow == 0 {
diff --git a/lib/storage/part_search.go b/lib/storage/part_search.go
index 705595b53..5acc087a4 100644
--- a/lib/storage/part_search.go
+++ b/lib/storage/part_search.go
@@ -218,7 +218,7 @@ func (ps *partSearch) readIndexBlock(mr *metaindexRow) (*indexBlock, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot decompress index block: %w", err)
 	}
-	ib := getIndexBlock()
+	ib := &indexBlock{}
 	ib.bhs, err = unmarshalBlockHeaders(ib.bhs[:0], ps.indexBuf, int(mr.BlockHeadersCount))
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal index block: %w", err)

From 48656dcc383cdd86ad6468dc8a5fa8e834a0fdfd Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 21:25:58 +0200
Subject: [PATCH 07/32] lib/{mergeset,storage}: allow merging smaller number of
 small parts

While this may increase CPU and disk IO usage needed for background merge,
this also recudes CPU usage during queries in production. This is because
such queries tend to read recently added data and it is better to have lower number
of parts for such data in order to reduce CPU usage.

This partially reverts ebf8da3730b0823f4b3804fede125f607c340f5b
---
 lib/mergeset/table.go         | 6 ------
 lib/storage/partition.go      | 6 ------
 lib/storage/partition_test.go | 6 ++++--
 3 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go
index d66639253..ac57e975d 100644
--- a/lib/mergeset/table.go
+++ b/lib/mergeset/table.go
@@ -1333,12 +1333,6 @@ func appendPartsToMerge(dst, src []*partWrapper, maxPartsToMerge int, maxItems u
 			for _, pw := range a {
 				itemsSum += pw.p.ph.itemsCount
 			}
-			if itemsSum < 1e6 && len(a) < maxPartsToMerge {
-				// Do not merge parts with too small number of items if the number of source parts
-				// isn't equal to maxPartsToMerge. This should reduce CPU usage and disk IO usage
-				// for small parts merge.
-				continue
-			}
 			if itemsSum > maxItems {
 				// There is no sense in checking the remaining bigger parts.
 				break
diff --git a/lib/storage/partition.go b/lib/storage/partition.go
index ff2ca08ab..3d9518c55 100644
--- a/lib/storage/partition.go
+++ b/lib/storage/partition.go
@@ -1469,12 +1469,6 @@ func appendPartsToMerge(dst, src []*partWrapper, maxPartsToMerge int, maxRows ui
 				continue
 			}
 			rowsCount := getRowsCount(a)
-			if rowsCount < 1e6 && len(a) < maxPartsToMerge {
-				// Do not merge parts with too small number of rows if the number of source parts
-				// isn't equal to maxPartsToMerge. This should reduce CPU usage and disk IO usage
-				// for small parts merge.
-				continue
-			}
 			if rowsCount > maxRows {
 				// There is no need in verifying remaining parts with higher number of rows
 				needFreeSpace = true
diff --git a/lib/storage/partition_test.go b/lib/storage/partition_test.go
index 0deebbadd..60c3a84d2 100644
--- a/lib/storage/partition_test.go
+++ b/lib/storage/partition_test.go
@@ -26,9 +26,11 @@ func TestAppendPartsToMerge(t *testing.T) {
 	testAppendPartsToMerge(t, 2, []uint64{4, 2, 4}, []uint64{4, 4})
 	testAppendPartsToMerge(t, 2, []uint64{1, 3, 7, 2}, nil)
 	testAppendPartsToMerge(t, 3, []uint64{1, 3, 7, 2}, []uint64{1, 2, 3})
-	testAppendPartsToMerge(t, 4, []uint64{1, 3, 7, 2}, nil)
+	testAppendPartsToMerge(t, 4, []uint64{1, 3, 7, 2}, []uint64{1, 2, 3})
+	testAppendPartsToMerge(t, 5, []uint64{1, 3, 7, 2}, nil)
 	testAppendPartsToMerge(t, 4, []uint64{1e6, 3e6, 7e6, 2e6}, []uint64{1e6, 2e6, 3e6})
-	testAppendPartsToMerge(t, 4, []uint64{2, 3, 7, 2}, []uint64{2, 2, 3, 7})
+	testAppendPartsToMerge(t, 4, []uint64{2, 3, 7, 2}, []uint64{2, 2, 3})
+	testAppendPartsToMerge(t, 5, []uint64{2, 3, 7, 2}, nil)
 	testAppendPartsToMerge(t, 3, []uint64{11, 1, 10, 100, 10}, []uint64{10, 10, 11})
 }
 

From 388cdb1980442ffc291fa631063fb811134cb6a7 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 21:32:52 +0200
Subject: [PATCH 08/32] lib/storage: do not re-calculate stats for heavy tag
 filters

This should reduce the number of slow queries when stats for heavy tag filters was recalculated.
---
 lib/storage/index_db.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index e01efa14c..3ba6af5ed 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -2797,9 +2797,12 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
 		tf := &tfs.tfs[i]
 		loopsCount, lastQueryTimestamp := is.getLoopsCountAndTimestampForDateFilter(date, tf)
 		origLoopsCount := loopsCount
-		if currentTime > lastQueryTimestamp+60*60 {
-			// Reset loopsCount to 0 every hour for collecting updated stats for the tf.
-			loopsCount = 0
+		if currentTime > lastQueryTimestamp+3*3600 {
+			// Update stats once per 3 hours only for relatively fast tag filters.
+			// There is no need in spending CPU resources on updating stats for slow tag filters.
+			if loopsCount <= 10e6 {
+				loopsCount = 0
+			}
 		}
 		if loopsCount == 0 {
 			// Prevent from possible thundering herd issue when heavy tf is executed from multiple concurrent queries

From 636c55b526837d392dbb210c86837647d65f731e Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 22:06:45 +0200
Subject: [PATCH 09/32] lib/mergeset: reduce memory usage for inmemoryBlock by
 using more compact items representation

This also should reduce CPU time spent by GC, since inmemoryBlock.items don't have pointers now,
so GC doesn't need visiting them.
---
 lib/mergeset/block_stream_reader.go      |  10 +-
 lib/mergeset/block_stream_reader_test.go |   6 +-
 lib/mergeset/encoding.go                 | 187 +++++++++++++++--------
 lib/mergeset/encoding_test.go            |  26 ++--
 lib/mergeset/inmemory_part.go            |   4 +-
 lib/mergeset/merge.go                    |  38 +++--
 lib/mergeset/merge_test.go               |   8 +-
 lib/mergeset/part_search.go              |  29 ++--
 lib/mergeset/table_search_timing_test.go |   4 +-
 lib/mergeset/table_test.go               |   2 +-
 lib/storage/block_header_test.go         |   2 +-
 lib/storage/index_db.go                  |  52 ++++---
 lib/storage/index_db_test.go             |  30 ++--
 13 files changed, 248 insertions(+), 150 deletions(-)

diff --git a/lib/mergeset/block_stream_reader.go b/lib/mergeset/block_stream_reader.go
index a89710887..06a28b238 100644
--- a/lib/mergeset/block_stream_reader.go
+++ b/lib/mergeset/block_stream_reader.go
@@ -195,7 +195,8 @@ func (bsr *blockStreamReader) Next() bool {
 		if err := bsr.readNextBHS(); err != nil {
 			if err == io.EOF {
 				// Check the last item.
-				lastItem := bsr.Block.items[len(bsr.Block.items)-1]
+				b := &bsr.Block
+				lastItem := b.items[len(b.items)-1].Bytes(b.data)
 				if string(bsr.ph.lastItem) != string(lastItem) {
 					err = fmt.Errorf("unexpected last item; got %X; want %X", lastItem, bsr.ph.lastItem)
 				}
@@ -240,12 +241,13 @@ func (bsr *blockStreamReader) Next() bool {
 	}
 	if !bsr.firstItemChecked {
 		bsr.firstItemChecked = true
-		if string(bsr.ph.firstItem) != string(bsr.Block.items[0]) {
-			bsr.err = fmt.Errorf("unexpected first item; got %X; want %X", bsr.Block.items[0], bsr.ph.firstItem)
+		b := &bsr.Block
+		firstItem := b.items[0].Bytes(b.data)
+		if string(bsr.ph.firstItem) != string(firstItem) {
+			bsr.err = fmt.Errorf("unexpected first item; got %X; want %X", firstItem, bsr.ph.firstItem)
 			return false
 		}
 	}
-
 	return true
 }
 
diff --git a/lib/mergeset/block_stream_reader_test.go b/lib/mergeset/block_stream_reader_test.go
index 056b05cec..c9175549d 100644
--- a/lib/mergeset/block_stream_reader_test.go
+++ b/lib/mergeset/block_stream_reader_test.go
@@ -44,8 +44,10 @@ func testBlockStreamReaderRead(ip *inmemoryPart, items []string) error {
 	bsr := newTestBlockStreamReader(ip)
 	i := 0
 	for bsr.Next() {
-		for _, item := range bsr.Block.items {
-			if string(item) != items[i] {
+		data := bsr.Block.data
+		for _, it := range bsr.Block.items {
+			item := it.String(data)
+			if item != items[i] {
 				return fmt.Errorf("unexpected item[%d]; got %q; want %q", i, item, items[i])
 			}
 			i++
diff --git a/lib/mergeset/encoding.go b/lib/mergeset/encoding.go
index 647b0a674..8cb57082c 100644
--- a/lib/mergeset/encoding.go
+++ b/lib/mergeset/encoding.go
@@ -3,6 +3,7 @@ package mergeset
 import (
 	"fmt"
 	"os"
+	"reflect"
 	"sort"
 	"strings"
 	"sync"
@@ -13,35 +14,62 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-type byteSliceSorter [][]byte
+// Item represents a single item for storing in a mergeset.
+type Item struct {
+	// Start is start offset for the item in data.
+	Start uint32
 
-func (s byteSliceSorter) Len() int { return len(s) }
-func (s byteSliceSorter) Less(i, j int) bool {
-	return string(s[i]) < string(s[j])
+	// End is end offset for the item in data.
+	End uint32
 }
-func (s byteSliceSorter) Swap(i, j int) {
-	s[i], s[j] = s[j], s[i]
+
+// Bytes returns bytes representation of it obtained from data.
+//
+// The returned bytes representation belongs to data.
+func (it Item) Bytes(data []byte) []byte {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	sh.Cap = int(it.End - it.Start)
+	sh.Len = int(it.End - it.Start)
+	sh.Data += uintptr(it.Start)
+	return data
+}
+
+// String returns string represetnation of it obtained from data.
+//
+// The returned string representation belongs to data.
+func (it Item) String(data []byte) string {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	sh.Data += uintptr(it.Start)
+	sh.Len = int(it.End - it.Start)
+	return *(*string)(unsafe.Pointer(sh))
+}
+
+func (ib *inmemoryBlock) Len() int { return len(ib.items) }
+
+func (ib *inmemoryBlock) Less(i, j int) bool {
+	data := ib.data
+	items := ib.items
+	return string(items[i].Bytes(data)) < string(items[j].Bytes(data))
+}
+
+func (ib *inmemoryBlock) Swap(i, j int) {
+	items := ib.items
+	items[i], items[j] = items[j], items[i]
 }
 
 type inmemoryBlock struct {
 	commonPrefix []byte
 	data         []byte
-	items        byteSliceSorter
+	items        []Item
 }
 
 func (ib *inmemoryBlock) SizeBytes() int {
-	return int(unsafe.Sizeof(*ib)) + cap(ib.commonPrefix) + cap(ib.data) + cap(ib.items)*int(unsafe.Sizeof([]byte{}))
+	return int(unsafe.Sizeof(*ib)) + cap(ib.commonPrefix) + cap(ib.data) + cap(ib.items)*int(unsafe.Sizeof(Item{}))
 }
 
 func (ib *inmemoryBlock) Reset() {
 	ib.commonPrefix = ib.commonPrefix[:0]
 	ib.data = ib.data[:0]
-
-	items := ib.items
-	for i := range items {
-		// Remove reference to by slice, so GC could free the byte slice.
-		items[i] = nil
-	}
 	ib.items = ib.items[:0]
 }
 
@@ -50,12 +78,14 @@ func (ib *inmemoryBlock) updateCommonPrefix() {
 	if len(ib.items) == 0 {
 		return
 	}
-	cp := ib.items[0]
+	items := ib.items
+	data := ib.data
+	cp := items[0].Bytes(data)
 	if len(cp) == 0 {
 		return
 	}
-	for _, item := range ib.items[1:] {
-		cpLen := commonPrefixLen(cp, item)
+	for _, it := range items[1:] {
+		cpLen := commonPrefixLen(cp, it.Bytes(data))
 		if cpLen == 0 {
 			return
 		}
@@ -82,15 +112,21 @@ func commonPrefixLen(a, b []byte) int {
 //
 // false is returned if x isn't added to ib due to block size contraints.
 func (ib *inmemoryBlock) Add(x []byte) bool {
-	if len(x)+len(ib.data) > maxInmemoryBlockSize {
+	data := ib.data
+	if len(x)+len(data) > maxInmemoryBlockSize {
 		return false
 	}
-	if cap(ib.data) < maxInmemoryBlockSize {
-		dataLen := len(ib.data)
-		ib.data = bytesutil.Resize(ib.data, maxInmemoryBlockSize)[:dataLen]
+	if cap(data) < maxInmemoryBlockSize {
+		dataLen := len(data)
+		data = bytesutil.Resize(data, maxInmemoryBlockSize)[:dataLen]
 	}
-	ib.data = append(ib.data, x...)
-	ib.items = append(ib.items, ib.data[len(ib.data)-len(x):])
+	dataLen := len(data)
+	data = append(data, x...)
+	ib.items = append(ib.items, Item{
+		Start: uint32(dataLen),
+		End:   uint32(len(data)),
+	})
+	ib.data = data
 	return true
 }
 
@@ -100,16 +136,21 @@ func (ib *inmemoryBlock) Add(x []byte) bool {
 const maxInmemoryBlockSize = 64 * 1024
 
 func (ib *inmemoryBlock) sort() {
-	// Use sort.Sort instead of sort.Slice in order to eliminate memory allocation.
-	sort.Sort(&ib.items)
+	sort.Sort(ib)
+	data := ib.data
+	items := ib.items
 	bb := bbPool.Get()
-	b := bytesutil.Resize(bb.B, len(ib.data))
+	b := bytesutil.Resize(bb.B, len(data))
 	b = b[:0]
-	for i, item := range ib.items {
-		b = append(b, item...)
-		ib.items[i] = b[len(b)-len(item):]
+	for i, it := range items {
+		bLen := len(b)
+		b = append(b, it.String(data)...)
+		items[i] = Item{
+			Start: uint32(bLen),
+			End:   uint32(len(b)),
+		}
 	}
-	bb.B, ib.data = ib.data, b
+	bb.B, ib.data = data, b
 	bbPool.Put(bb)
 }
 
@@ -140,7 +181,7 @@ func checkMarshalType(mt marshalType) error {
 
 func (ib *inmemoryBlock) isSorted() bool {
 	// Use sort.IsSorted instead of sort.SliceIsSorted in order to eliminate memory allocation.
-	return sort.IsSorted(&ib.items)
+	return sort.IsSorted(ib)
 }
 
 // MarshalUnsortedData marshals unsorted items from ib to sb.
@@ -179,9 +220,11 @@ func (ib *inmemoryBlock) MarshalSortedData(sb *storageBlock, firstItemDst, commo
 
 func (ib *inmemoryBlock) debugItemsString() string {
 	var sb strings.Builder
-	var prevItem []byte
-	for i, item := range ib.items {
-		if string(item) < string(prevItem) {
+	var prevItem string
+	data := ib.data
+	for i, it := range ib.items {
+		item := it.String(data)
+		if item < prevItem {
 			fmt.Fprintf(&sb, "!!! the next item is smaller than the previous item !!!\n")
 		}
 		fmt.Fprintf(&sb, "%05d %X\n", i, item)
@@ -201,7 +244,9 @@ func (ib *inmemoryBlock) marshalData(sb *storageBlock, firstItemDst, commonPrefi
 		logger.Panicf("BUG: the number of items in the block must be smaller than %d; got %d items", uint64(1<<32), len(ib.items))
 	}
 
-	firstItemDst = append(firstItemDst, ib.items[0]...)
+	data := ib.data
+	firstItem := ib.items[0].Bytes(data)
+	firstItemDst = append(firstItemDst, firstItem...)
 	commonPrefixDst = append(commonPrefixDst, ib.commonPrefix...)
 
 	if len(ib.data)-len(ib.commonPrefix)*len(ib.items) < 64 || len(ib.items) < 2 {
@@ -221,10 +266,11 @@ func (ib *inmemoryBlock) marshalData(sb *storageBlock, firstItemDst, commonPrefi
 	defer encoding.PutUint64s(xs)
 
 	cpLen := len(ib.commonPrefix)
-	prevItem := ib.items[0][cpLen:]
+	prevItem := firstItem[cpLen:]
 	prevPrefixLen := uint64(0)
-	for i, item := range ib.items[1:] {
-		item := item[cpLen:]
+	for i, it := range ib.items[1:] {
+		it.Start += uint32(cpLen)
+		item := it.Bytes(data)
 		prefixLen := uint64(commonPrefixLen(prevItem, item))
 		bItems = append(bItems, item[prefixLen:]...)
 		xLen := prefixLen ^ prevPrefixLen
@@ -240,9 +286,9 @@ func (ib *inmemoryBlock) marshalData(sb *storageBlock, firstItemDst, commonPrefi
 	bbPool.Put(bbItems)
 
 	// Marshal lens data.
-	prevItemLen := uint64(len(ib.items[0]) - cpLen)
-	for i, item := range ib.items[1:] {
-		itemLen := uint64(len(item) - cpLen)
+	prevItemLen := uint64(len(firstItem) - cpLen)
+	for i, it := range ib.items[1:] {
+		itemLen := uint64(int(it.End-it.Start) - cpLen)
 		xLen := itemLen ^ prevItemLen
 		prevItemLen = itemLen
 
@@ -346,11 +392,15 @@ func (ib *inmemoryBlock) UnmarshalData(sb *storageBlock, firstItem, commonPrefix
 	}
 	data := bytesutil.Resize(ib.data, maxInmemoryBlockSize)
 	if n := int(itemsCount) - cap(ib.items); n > 0 {
-		ib.items = append(ib.items[:cap(ib.items)], make([][]byte, n)...)
+		ib.items = append(ib.items[:cap(ib.items)], make([]Item, n)...)
 	}
 	ib.items = ib.items[:itemsCount]
 	data = append(data[:0], firstItem...)
-	ib.items[0] = data
+	items := ib.items
+	items[0] = Item{
+		Start: 0,
+		End:   uint32(len(data)),
+	}
 	prevItem := data[len(commonPrefix):]
 	b := bb.B
 	for i := 1; i < int(itemsCount); i++ {
@@ -363,17 +413,19 @@ func (ib *inmemoryBlock) UnmarshalData(sb *storageBlock, firstItem, commonPrefix
 		if uint64(len(b)) < suffixLen {
 			return fmt.Errorf("not enough data for decoding item from itemsData; want %d bytes; remained %d bytes", suffixLen, len(b))
 		}
-		data = append(data, commonPrefix...)
-
 		if prefixLen > uint64(len(prevItem)) {
 			return fmt.Errorf("prefixLen cannot exceed %d; got %d", len(prevItem), prefixLen)
 		}
+		dataLen := len(data)
+		data = append(data, commonPrefix...)
 		data = append(data, prevItem[:prefixLen]...)
 		data = append(data, b[:suffixLen]...)
-		item := data[len(data)-int(itemLen)-len(commonPrefix):]
-		ib.items[i] = item
+		items[i] = Item{
+			Start: uint32(dataLen),
+			End:   uint32(len(data)),
+		}
 		b = b[suffixLen:]
-		prevItem = item[len(commonPrefix):]
+		prevItem = data[len(data)-int(itemLen):]
 	}
 	if len(b) > 0 {
 		return fmt.Errorf("unexpected tail left after itemsData with len %d: %q", len(b), b)
@@ -381,30 +433,33 @@ func (ib *inmemoryBlock) UnmarshalData(sb *storageBlock, firstItem, commonPrefix
 	if uint64(len(data)) != dataLen {
 		return fmt.Errorf("unexpected data len; got %d; want %d", len(data), dataLen)
 	}
+	ib.data = data
 	if !ib.isSorted() {
 		return fmt.Errorf("decoded data block contains unsorted items; items:\n%s", ib.debugItemsString())
 	}
-	ib.data = data
 	return nil
 }
 
 var bbPool bytesutil.ByteBufferPool
 
 func (ib *inmemoryBlock) marshalDataPlain(sb *storageBlock) {
+	data := ib.data
+
 	// Marshal items data.
 	// There is no need in marshaling the first item, since it is returned
 	// to the caller in marshalData.
 	cpLen := len(ib.commonPrefix)
 	b := sb.itemsData[:0]
-	for _, item := range ib.items[1:] {
-		b = append(b, item[cpLen:]...)
+	for _, it := range ib.items[1:] {
+		it.Start += uint32(cpLen)
+		b = append(b, it.String(data)...)
 	}
 	sb.itemsData = b
 
 	// Marshal length data.
 	b = sb.lensData[:0]
-	for _, item := range ib.items[1:] {
-		b = encoding.MarshalUint64(b, uint64(len(item)-cpLen))
+	for _, it := range ib.items[1:] {
+		b = encoding.MarshalUint64(b, uint64(int(it.End-it.Start)-cpLen))
 	}
 	sb.lensData = b
 }
@@ -431,26 +486,34 @@ func (ib *inmemoryBlock) unmarshalDataPlain(sb *storageBlock, firstItem []byte,
 	}
 
 	// Unmarshal items data.
-	ib.data = bytesutil.Resize(ib.data, len(firstItem)+len(sb.itemsData)+len(commonPrefix)*int(itemsCount))
-	ib.data = append(ib.data[:0], firstItem...)
-	ib.items = append(ib.items[:0], ib.data)
-
+	data := ib.data
+	items := ib.items
+	data = bytesutil.Resize(data, len(firstItem)+len(sb.itemsData)+len(commonPrefix)*int(itemsCount))
+	data = append(data[:0], firstItem...)
+	items = append(items[:0], Item{
+		Start: 0,
+		End:   uint32(len(data)),
+	})
 	b = sb.itemsData
 	for i := 1; i < int(itemsCount); i++ {
 		itemLen := lb.lens[i]
 		if uint64(len(b)) < itemLen {
 			return fmt.Errorf("not enough data for decoding item from itemsData; want %d bytes; remained %d bytes", itemLen, len(b))
 		}
-		ib.data = append(ib.data, commonPrefix...)
-		ib.data = append(ib.data, b[:itemLen]...)
-		item := ib.data[len(ib.data)-int(itemLen)-len(commonPrefix):]
-		ib.items = append(ib.items, item)
+		dataLen := len(data)
+		data = append(data, commonPrefix...)
+		data = append(data, b[:itemLen]...)
+		items = append(items, Item{
+			Start: uint32(dataLen),
+			End:   uint32(len(data)),
+		})
 		b = b[itemLen:]
 	}
+	ib.data = data
+	ib.items = items
 	if len(b) > 0 {
 		return fmt.Errorf("unexpected tail left after itemsData with len %d: %q", len(b), b)
 	}
-
 	return nil
 }
 
diff --git a/lib/mergeset/encoding_test.go b/lib/mergeset/encoding_test.go
index 549a45d65..993c4b6ef 100644
--- a/lib/mergeset/encoding_test.go
+++ b/lib/mergeset/encoding_test.go
@@ -37,8 +37,10 @@ func TestInmemoryBlockAdd(t *testing.T) {
 		if len(ib.data) != totalLen {
 			t.Fatalf("unexpected ib.data len; got %d; want %d", len(ib.data), totalLen)
 		}
-		for j, item := range ib.items {
-			if items[j] != string(item) {
+		data := ib.data
+		for j, it := range ib.items {
+			item := it.String(data)
+			if items[j] != item {
 				t.Fatalf("unexpected item at index %d out of %d, loop %d\ngot\n%X\nwant\n%X", j, len(items), i, item, items[j])
 			}
 		}
@@ -75,8 +77,10 @@ func TestInmemoryBlockSort(t *testing.T) {
 		if len(ib.data) != totalLen {
 			t.Fatalf("unexpected ib.data len; got %d; want %d", len(ib.data), totalLen)
 		}
-		for j, item := range ib.items {
-			if items[j] != string(item) {
+		data := ib.data
+		for j, it := range ib.items {
+			item := it.String(data)
+			if items[j] != item {
 				t.Fatalf("unexpected item at index %d out of %d, loop %d\ngot\n%X\nwant\n%X", j, len(items), i, item, items[j])
 			}
 		}
@@ -122,8 +126,9 @@ func TestInmemoryBlockMarshalUnmarshal(t *testing.T) {
 		if int(itemsLen) != len(ib.items) {
 			t.Fatalf("unexpected number of items marshaled; got %d; want %d", itemsLen, len(ib.items))
 		}
-		if string(firstItem) != string(ib.items[0]) {
-			t.Fatalf("unexpected the first item\ngot\n%q\nwant\n%q", firstItem, ib.items[0])
+		firstItemExpected := ib.items[0].String(ib.data)
+		if string(firstItem) != firstItemExpected {
+			t.Fatalf("unexpected the first item\ngot\n%q\nwant\n%q", firstItem, firstItemExpected)
 		}
 		if err := checkMarshalType(mt); err != nil {
 			t.Fatalf("invalid mt: %s", err)
@@ -143,12 +148,15 @@ func TestInmemoryBlockMarshalUnmarshal(t *testing.T) {
 			t.Fatalf("unexpected ib.data len; got %d; want %d", len(ib2.data), totalLen)
 		}
 		for j := range items {
-			if len(items[j]) != len(ib2.items[j]) {
+			it2 := ib2.items[j]
+			item2 := it2.String(ib2.data)
+			if len(items[j]) != len(item2) {
 				t.Fatalf("items length mismatch at index %d out of %d, loop %d\ngot\n(len=%d) %X\nwant\n(len=%d) %X",
-					j, len(items), i, len(ib2.items[j]), ib2.items[j], len(items[j]), items[j])
+					j, len(items), i, len(item2), item2, len(items[j]), items[j])
 			}
 		}
-		for j, item := range ib2.items {
+		for j, it := range ib2.items {
+			item := it.String(ib2.data)
 			if items[j] != string(item) {
 				t.Fatalf("unexpected item at index %d out of %d, loop %d\ngot\n(len=%d) %X\nwant\n(len=%d) %X",
 					j, len(items), i, len(item), item, len(items[j]), items[j])
diff --git a/lib/mergeset/inmemory_part.go b/lib/mergeset/inmemory_part.go
index 8297ec5f1..e2f4f02bd 100644
--- a/lib/mergeset/inmemory_part.go
+++ b/lib/mergeset/inmemory_part.go
@@ -56,8 +56,8 @@ func (ip *inmemoryPart) Init(ib *inmemoryBlock) {
 
 	ip.ph.itemsCount = uint64(len(ib.items))
 	ip.ph.blocksCount = 1
-	ip.ph.firstItem = append(ip.ph.firstItem[:0], ib.items[0]...)
-	ip.ph.lastItem = append(ip.ph.lastItem[:0], ib.items[len(ib.items)-1]...)
+	ip.ph.firstItem = append(ip.ph.firstItem[:0], ib.items[0].String(ib.data)...)
+	ip.ph.lastItem = append(ip.ph.lastItem[:0], ib.items[len(ib.items)-1].String(ib.data)...)
 
 	fs.MustWriteData(&ip.itemsData, ip.sb.itemsData)
 	ip.bh.itemsBlockOffset = 0
diff --git a/lib/mergeset/merge.go b/lib/mergeset/merge.go
index 276354e8d..7f2fa5ccf 100644
--- a/lib/mergeset/merge.go
+++ b/lib/mergeset/merge.go
@@ -16,7 +16,7 @@ import (
 //
 // The callback must return sorted items. The first and the last item must be unchanged.
 // The callback can re-use data and items for storing the result.
-type PrepareBlockCallback func(data []byte, items [][]byte) ([]byte, [][]byte)
+type PrepareBlockCallback func(data []byte, items []Item) ([]byte, []Item)
 
 // mergeBlockStreams merges bsrs and writes result to bsw.
 //
@@ -122,8 +122,10 @@ again:
 		nextItem = bsm.bsrHeap[0].bh.firstItem
 		hasNextItem = true
 	}
+	items := bsr.Block.items
+	data := bsr.Block.data
 	for bsr.blockItemIdx < len(bsr.Block.items) {
-		item := bsr.Block.items[bsr.blockItemIdx]
+		item := items[bsr.blockItemIdx].Bytes(data)
 		if hasNextItem && string(item) > string(nextItem) {
 			break
 		}
@@ -148,32 +150,36 @@ again:
 
 	// The next item in the bsr.Block exceeds nextItem.
 	// Adjust bsr.bh.firstItem and return bsr to heap.
-	bsr.bh.firstItem = append(bsr.bh.firstItem[:0], bsr.Block.items[bsr.blockItemIdx]...)
+	bsr.bh.firstItem = append(bsr.bh.firstItem[:0], bsr.Block.items[bsr.blockItemIdx].String(bsr.Block.data)...)
 	heap.Push(&bsm.bsrHeap, bsr)
 	goto again
 }
 
 func (bsm *blockStreamMerger) flushIB(bsw *blockStreamWriter, ph *partHeader, itemsMerged *uint64) {
-	if len(bsm.ib.items) == 0 {
+	items := bsm.ib.items
+	data := bsm.ib.data
+	if len(items) == 0 {
 		// Nothing to flush.
 		return
 	}
-	atomic.AddUint64(itemsMerged, uint64(len(bsm.ib.items)))
+	atomic.AddUint64(itemsMerged, uint64(len(items)))
 	if bsm.prepareBlock != nil {
-		bsm.firstItem = append(bsm.firstItem[:0], bsm.ib.items[0]...)
-		bsm.lastItem = append(bsm.lastItem[:0], bsm.ib.items[len(bsm.ib.items)-1]...)
-		bsm.ib.data, bsm.ib.items = bsm.prepareBlock(bsm.ib.data, bsm.ib.items)
-		if len(bsm.ib.items) == 0 {
+		bsm.firstItem = append(bsm.firstItem[:0], items[0].String(data)...)
+		bsm.lastItem = append(bsm.lastItem[:0], items[len(items)-1].String(data)...)
+		data, items = bsm.prepareBlock(data, items)
+		bsm.ib.data = data
+		bsm.ib.items = items
+		if len(items) == 0 {
 			// Nothing to flush
 			return
 		}
 		// Consistency checks after prepareBlock call.
-		firstItem := bsm.ib.items[0]
-		if string(firstItem) != string(bsm.firstItem) {
+		firstItem := items[0].String(data)
+		if firstItem != string(bsm.firstItem) {
 			logger.Panicf("BUG: prepareBlock must return first item equal to the original first item;\ngot\n%X\nwant\n%X", firstItem, bsm.firstItem)
 		}
-		lastItem := bsm.ib.items[len(bsm.ib.items)-1]
-		if string(lastItem) != string(bsm.lastItem) {
+		lastItem := items[len(items)-1].String(data)
+		if lastItem != string(bsm.lastItem) {
 			logger.Panicf("BUG: prepareBlock must return last item equal to the original last item;\ngot\n%X\nwant\n%X", lastItem, bsm.lastItem)
 		}
 		// Verify whether the bsm.ib.items are sorted only in tests, since this
@@ -182,12 +188,12 @@ func (bsm *blockStreamMerger) flushIB(bsw *blockStreamWriter, ph *partHeader, it
 			logger.Panicf("BUG: prepareBlock must return sorted items;\ngot\n%s", bsm.ib.debugItemsString())
 		}
 	}
-	ph.itemsCount += uint64(len(bsm.ib.items))
+	ph.itemsCount += uint64(len(items))
 	if !bsm.phFirstItemCaught {
-		ph.firstItem = append(ph.firstItem[:0], bsm.ib.items[0]...)
+		ph.firstItem = append(ph.firstItem[:0], items[0].String(data)...)
 		bsm.phFirstItemCaught = true
 	}
-	ph.lastItem = append(ph.lastItem[:0], bsm.ib.items[len(bsm.ib.items)-1]...)
+	ph.lastItem = append(ph.lastItem[:0], items[len(items)-1].String(data)...)
 	bsw.WriteBlock(&bsm.ib)
 	bsm.ib.Reset()
 	ph.blocksCount++
diff --git a/lib/mergeset/merge_test.go b/lib/mergeset/merge_test.go
index 8d34ce421..f042bae0f 100644
--- a/lib/mergeset/merge_test.go
+++ b/lib/mergeset/merge_test.go
@@ -157,10 +157,12 @@ func testCheckItems(dstIP *inmemoryPart, items []string) error {
 		if bh.itemsCount <= 0 {
 			return fmt.Errorf("unexpected empty block")
 		}
-		if string(bh.firstItem) != string(dstBsr.Block.items[0]) {
-			return fmt.Errorf("unexpected blockHeader.firstItem; got %q; want %q", bh.firstItem, dstBsr.Block.items[0])
+		item := dstBsr.Block.items[0].Bytes(dstBsr.Block.data)
+		if string(bh.firstItem) != string(item) {
+			return fmt.Errorf("unexpected blockHeader.firstItem; got %q; want %q", bh.firstItem, item)
 		}
-		for _, item := range dstBsr.Block.items {
+		for _, it := range dstBsr.Block.items {
+			item := it.Bytes(dstBsr.Block.data)
 			dstItems = append(dstItems, string(item))
 		}
 	}
diff --git a/lib/mergeset/part_search.go b/lib/mergeset/part_search.go
index ea4b5ca25..8671561b6 100644
--- a/lib/mergeset/part_search.go
+++ b/lib/mergeset/part_search.go
@@ -142,14 +142,17 @@ func (ps *partSearch) Seek(k []byte) {
 
 	// Locate the first item to scan in the block.
 	items := ps.ib.items
+	data := ps.ib.data
 	cpLen := commonPrefixLen(ps.ib.commonPrefix, k)
 	if cpLen > 0 {
 		keySuffix := k[cpLen:]
 		ps.ibItemIdx = sort.Search(len(items), func(i int) bool {
-			return string(keySuffix) <= string(items[i][cpLen:])
+			it := items[i]
+			it.Start += uint32(cpLen)
+			return string(keySuffix) <= it.String(data)
 		})
 	} else {
-		ps.ibItemIdx = binarySearchKey(items, k)
+		ps.ibItemIdx = binarySearchKey(data, items, k)
 	}
 	if ps.ibItemIdx < len(items) {
 		// The item has been found.
@@ -168,13 +171,14 @@ func (ps *partSearch) tryFastSeek(k []byte) bool {
 	if ps.ib == nil {
 		return false
 	}
+	data := ps.ib.data
 	items := ps.ib.items
 	idx := ps.ibItemIdx
 	if idx >= len(items) {
 		// The ib is exhausted.
 		return false
 	}
-	if string(k) > string(items[len(items)-1]) {
+	if string(k) > items[len(items)-1].String(data) {
 		// The item is located in next blocks.
 		return false
 	}
@@ -183,8 +187,8 @@ func (ps *partSearch) tryFastSeek(k []byte) bool {
 	if idx > 0 {
 		idx--
 	}
-	if string(k) < string(items[idx]) {
-		if string(k) < string(items[0]) {
+	if string(k) < items[idx].String(data) {
+		if string(k) < items[0].String(data) {
 			// The item is located in previous blocks.
 			return false
 		}
@@ -192,7 +196,7 @@ func (ps *partSearch) tryFastSeek(k []byte) bool {
 	}
 
 	// The item is located in the current block
-	ps.ibItemIdx = idx + binarySearchKey(items[idx:], k)
+	ps.ibItemIdx = idx + binarySearchKey(data, items[idx:], k)
 	return true
 }
 
@@ -204,10 +208,11 @@ func (ps *partSearch) NextItem() bool {
 		return false
 	}
 
-	if ps.ibItemIdx < len(ps.ib.items) {
+	items := ps.ib.items
+	if ps.ibItemIdx < len(items) {
 		// Fast path - the current block contains more items.
 		// Proceed to the next item.
-		ps.Item = ps.ib.items[ps.ibItemIdx]
+		ps.Item = items[ps.ibItemIdx].Bytes(ps.ib.data)
 		ps.ibItemIdx++
 		return true
 	}
@@ -219,7 +224,7 @@ func (ps *partSearch) NextItem() bool {
 	}
 
 	// Invariant: len(ps.ib.items) > 0 after nextBlock.
-	ps.Item = ps.ib.items[0]
+	ps.Item = ps.ib.items[0].Bytes(ps.ib.data)
 	ps.ibItemIdx++
 	return true
 }
@@ -319,11 +324,11 @@ func (ps *partSearch) readInmemoryBlock(bh *blockHeader) (*inmemoryBlock, error)
 	return ib, nil
 }
 
-func binarySearchKey(items [][]byte, key []byte) int {
+func binarySearchKey(data []byte, items []Item, key []byte) int {
 	if len(items) == 0 {
 		return 0
 	}
-	if string(key) <= string(items[0]) {
+	if string(key) <= items[0].String(data) {
 		// Fast path - the item is the first.
 		return 0
 	}
@@ -335,7 +340,7 @@ func binarySearchKey(items [][]byte, key []byte) int {
 	i, j := uint(0), n
 	for i < j {
 		h := uint(i+j) >> 1
-		if h >= 0 && h < uint(len(items)) && string(key) > string(items[h]) {
+		if h >= 0 && h < uint(len(items)) && string(key) > items[h].String(data) {
 			i = h + 1
 		} else {
 			j = h
diff --git a/lib/mergeset/table_search_timing_test.go b/lib/mergeset/table_search_timing_test.go
index f8d6e3515..add04e46e 100644
--- a/lib/mergeset/table_search_timing_test.go
+++ b/lib/mergeset/table_search_timing_test.go
@@ -46,7 +46,7 @@ func benchmarkTableSearch(b *testing.B, itemsCount int) {
 	b.Run("sequential-keys-exact", func(b *testing.B) {
 		benchmarkTableSearchKeys(b, tb, keys, 0)
 	})
-	b.Run("sequential-keys-without-siffux", func(b *testing.B) {
+	b.Run("sequential-keys-without-suffix", func(b *testing.B) {
 		benchmarkTableSearchKeys(b, tb, keys, 4)
 	})
 
@@ -57,7 +57,7 @@ func benchmarkTableSearch(b *testing.B, itemsCount int) {
 	b.Run("random-keys-exact", func(b *testing.B) {
 		benchmarkTableSearchKeys(b, tb, randKeys, 0)
 	})
-	b.Run("random-keys-without-siffux", func(b *testing.B) {
+	b.Run("random-keys-without-suffix", func(b *testing.B) {
 		benchmarkTableSearchKeys(b, tb, randKeys, 4)
 	})
 }
diff --git a/lib/mergeset/table_test.go b/lib/mergeset/table_test.go
index aa43fe9e1..f61d2e635 100644
--- a/lib/mergeset/table_test.go
+++ b/lib/mergeset/table_test.go
@@ -218,7 +218,7 @@ func TestTableAddItemsConcurrent(t *testing.T) {
 		atomic.AddUint64(&flushes, 1)
 	}
 	var itemsMerged uint64
-	prepareBlock := func(data []byte, items [][]byte) ([]byte, [][]byte) {
+	prepareBlock := func(data []byte, items []Item) ([]byte, []Item) {
 		atomic.AddUint64(&itemsMerged, uint64(len(items)))
 		return data, items
 	}
diff --git a/lib/storage/block_header_test.go b/lib/storage/block_header_test.go
index cffcfcb7e..aaa597914 100644
--- a/lib/storage/block_header_test.go
+++ b/lib/storage/block_header_test.go
@@ -76,6 +76,6 @@ func testBlockHeaderMarshalUnmarshal(t *testing.T, bh *blockHeader) {
 		t.Fatalf("unexpected tail after unmarshaling bh=%+v; got\n%x; want\n%x", bh, tail, suffix)
 	}
 	if !reflect.DeepEqual(bh, &bh2) {
-		t.Fatalf("unexpected bh unmarshaled after adding siffux; got\n%+v; want\n%+v", &bh2, bh)
+		t.Fatalf("unexpected bh unmarshaled after adding suffix; got\n%+v; want\n%+v", &bh2, bh)
 	}
 }
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 3ba6af5ed..a7b8521f1 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -3461,24 +3461,24 @@ func (mp *tagToMetricIDsRowParser) IsDeletedTag(dmis *uint64set.Set) bool {
 	return true
 }
 
-func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) {
+func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
 	data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
 	data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)
 	return data, items
 }
 
-func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte) ([]byte, [][]byte) {
+func mergeTagToMetricIDsRowsInternal(data []byte, items []mergeset.Item, nsPrefix byte) ([]byte, []mergeset.Item) {
 	// Perform quick checks whether items contain rows starting from nsPrefix
 	// based on the fact that items are sorted.
 	if len(items) <= 2 {
 		// The first and the last row must remain unchanged.
 		return data, items
 	}
-	firstItem := items[0]
+	firstItem := items[0].Bytes(data)
 	if len(firstItem) > 0 && firstItem[0] > nsPrefix {
 		return data, items
 	}
-	lastItem := items[len(items)-1]
+	lastItem := items[len(items)-1].Bytes(data)
 	if len(lastItem) > 0 && lastItem[0] < nsPrefix {
 		return data, items
 	}
@@ -3491,14 +3491,18 @@ func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte)
 	mpPrev := &tmm.mpPrev
 	dstData := data[:0]
 	dstItems := items[:0]
-	for i, item := range items {
+	for i, it := range items {
+		item := it.Bytes(data)
 		if len(item) == 0 || item[0] != nsPrefix || i == 0 || i == len(items)-1 {
 			// Write rows not starting with nsPrefix as-is.
 			// Additionally write the first and the last row as-is in order to preserve
 			// sort order for adjancent blocks.
 			dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
 			dstData = append(dstData, item...)
-			dstItems = append(dstItems, dstData[len(dstData)-len(item):])
+			dstItems = append(dstItems, mergeset.Item{
+				Start: uint32(len(dstData) - len(item)),
+				End:   uint32(len(dstData)),
+			})
 			continue
 		}
 		if err := mp.Init(item, nsPrefix); err != nil {
@@ -3507,7 +3511,10 @@ func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte)
 		if mp.MetricIDsLen() >= maxMetricIDsPerRow {
 			dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
 			dstData = append(dstData, item...)
-			dstItems = append(dstItems, dstData[len(dstData)-len(item):])
+			dstItems = append(dstItems, mergeset.Item{
+				Start: uint32(len(dstData) - len(item)),
+				End:   uint32(len(dstData)),
+			})
 			continue
 		}
 		if !mp.EqualPrefix(mpPrev) {
@@ -3523,7 +3530,7 @@ func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte)
 	if len(tmm.pendingMetricIDs) > 0 {
 		logger.Panicf("BUG: tmm.pendingMetricIDs must be empty at this point; got %d items: %d", len(tmm.pendingMetricIDs), tmm.pendingMetricIDs)
 	}
-	if !checkItemsSorted(dstItems) {
+	if !checkItemsSorted(dstData, dstItems) {
 		// Items could become unsorted if initial items contain duplicate metricIDs:
 		//
 		//   item1: 1, 1, 5
@@ -3541,15 +3548,8 @@ func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte)
 		// into the same new time series from multiple concurrent goroutines.
 		atomic.AddUint64(&indexBlocksWithMetricIDsIncorrectOrder, 1)
 		dstData = append(dstData[:0], tmm.dataCopy...)
-		dstItems = dstItems[:0]
-		// tmm.itemsCopy can point to overwritten data, so it must be updated
-		// to point to real data from tmm.dataCopy.
-		buf := dstData
-		for _, item := range tmm.itemsCopy {
-			dstItems = append(dstItems, buf[:len(item)])
-			buf = buf[len(item):]
-		}
-		if !checkItemsSorted(dstItems) {
+		dstItems = append(dstItems[:0], tmm.itemsCopy...)
+		if !checkItemsSorted(dstData, dstItems) {
 			logger.Panicf("BUG: the original items weren't sorted; items=%q", dstItems)
 		}
 	}
@@ -3561,13 +3561,14 @@ func mergeTagToMetricIDsRowsInternal(data []byte, items [][]byte, nsPrefix byte)
 var indexBlocksWithMetricIDsIncorrectOrder uint64
 var indexBlocksWithMetricIDsProcessed uint64
 
-func checkItemsSorted(items [][]byte) bool {
+func checkItemsSorted(data []byte, items []mergeset.Item) bool {
 	if len(items) == 0 {
 		return true
 	}
-	prevItem := items[0]
-	for _, currItem := range items[1:] {
-		if string(prevItem) > string(currItem) {
+	prevItem := items[0].String(data)
+	for _, it := range items[1:] {
+		currItem := it.String(data)
+		if prevItem > currItem {
 			return false
 		}
 		prevItem = currItem
@@ -3595,7 +3596,7 @@ type tagToMetricIDsRowsMerger struct {
 	mp               tagToMetricIDsRowParser
 	mpPrev           tagToMetricIDsRowParser
 
-	itemsCopy [][]byte
+	itemsCopy []mergeset.Item
 	dataCopy  []byte
 }
 
@@ -3608,7 +3609,7 @@ func (tmm *tagToMetricIDsRowsMerger) Reset() {
 	tmm.dataCopy = tmm.dataCopy[:0]
 }
 
-func (tmm *tagToMetricIDsRowsMerger) flushPendingMetricIDs(dstData []byte, dstItems [][]byte, mp *tagToMetricIDsRowParser) ([]byte, [][]byte) {
+func (tmm *tagToMetricIDsRowsMerger) flushPendingMetricIDs(dstData []byte, dstItems []mergeset.Item, mp *tagToMetricIDsRowParser) ([]byte, []mergeset.Item) {
 	if len(tmm.pendingMetricIDs) == 0 {
 		// Nothing to flush
 		return dstData, dstItems
@@ -3623,7 +3624,10 @@ func (tmm *tagToMetricIDsRowsMerger) flushPendingMetricIDs(dstData []byte, dstIt
 	for _, metricID := range tmm.pendingMetricIDs {
 		dstData = encoding.MarshalUint64(dstData, metricID)
 	}
-	dstItems = append(dstItems, dstData[dstDataLen:])
+	dstItems = append(dstItems, mergeset.Item{
+		Start: uint32(dstDataLen),
+		End:   uint32(len(dstData)),
+	})
 	tmm.pendingMetricIDs = tmm.pendingMetricIDs[:0]
 	return dstData, dstItems
 }
diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go
index c77d99e02..25fbe858c 100644
--- a/lib/storage/index_db_test.go
+++ b/lib/storage/index_db_test.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 )
@@ -36,33 +37,38 @@ func TestMergeTagToMetricIDsRows(t *testing.T) {
 	f := func(items []string, expectedItems []string) {
 		t.Helper()
 		var data []byte
-		var itemsB [][]byte
+		var itemsB []mergeset.Item
 		for _, item := range items {
 			data = append(data, item...)
-			itemsB = append(itemsB, data[len(data)-len(item):])
+			itemsB = append(itemsB, mergeset.Item{
+				Start: uint32(len(data) - len(item)),
+				End:   uint32(len(data)),
+			})
 		}
-		if !checkItemsSorted(itemsB) {
+		if !checkItemsSorted(data, itemsB) {
 			t.Fatalf("source items aren't sorted; items:\n%q", itemsB)
 		}
 		resultData, resultItemsB := mergeTagToMetricIDsRows(data, itemsB)
 		if len(resultItemsB) != len(expectedItems) {
 			t.Fatalf("unexpected len(resultItemsB); got %d; want %d", len(resultItemsB), len(expectedItems))
 		}
-		if !checkItemsSorted(resultItemsB) {
+		if !checkItemsSorted(resultData, resultItemsB) {
 			t.Fatalf("result items aren't sorted; items:\n%q", resultItemsB)
 		}
-		for i, item := range resultItemsB {
-			if !bytes.HasPrefix(resultData, item) {
-				t.Fatalf("unexpected prefix for resultData #%d;\ngot\n%X\nwant\n%X", i, resultData, item)
+		buf := resultData
+		for i, it := range resultItemsB {
+			item := it.Bytes(resultData)
+			if !bytes.HasPrefix(buf, item) {
+				t.Fatalf("unexpected prefix for resultData #%d;\ngot\n%X\nwant\n%X", i, buf, item)
 			}
-			resultData = resultData[len(item):]
+			buf = buf[len(item):]
 		}
-		if len(resultData) != 0 {
-			t.Fatalf("unexpected tail left in resultData: %X", resultData)
+		if len(buf) != 0 {
+			t.Fatalf("unexpected tail left in resultData: %X", buf)
 		}
 		var resultItems []string
-		for _, item := range resultItemsB {
-			resultItems = append(resultItems, string(item))
+		for _, it := range resultItemsB {
+			resultItems = append(resultItems, string(it.Bytes(resultData)))
 		}
 		if !reflect.DeepEqual(expectedItems, resultItems) {
 			t.Fatalf("unexpected items;\ngot\n%X\nwant\n%X", resultItems, expectedItems)

From 901e12024d0d37d29925f4dcc0acd644232e3ef2 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 22:50:03 +0200
Subject: [PATCH 10/32] vendor: update github.com/VictoriaMetrics/metrics from
 v1.14.0 to v1.15.0

The v1.15.0 exports the following additional metrics:

    process_io_read_bytes_total - the number of bytes read via io syscalls such as read and pread
    process_io_written_bytes_total - the number of bytes written via io syscalls such as write and pwrite
    process_io_read_syscalls_total - the number of read syscalls such as read and pread
    process_io_write_syscalls_total - the number of write syscalls such as write and pwrite
    process_io_storage_read_bytes_total - the number of bytes read from storage layer
    process_io_storage_written_bytes_total - the number of bytes written to storage layer

These metrics can be used for monitoring process io
---
 docs/CHANGELOG.md                             |  8 +++
 go.mod                                        |  2 +-
 go.sum                                        |  4 +-
 .../VictoriaMetrics/metrics/README.md         |  2 +-
 .../metrics/process_metrics_linux.go          | 54 +++++++++++++++++--
 vendor/modules.txt                            |  2 +-
 6 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 94403e514..d19346a89 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -3,6 +3,14 @@
 # tip
 
 * FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details.
+* FEATURE: expose io-related metrics at `/metrics` page for every VictoriaMetrics component:
+  * `process_io_read_bytes_total` - the number of bytes read via io syscalls such as read and pread
+  * `process_io_written_bytes_total` - the number of bytes written via io syscalls such as write and pwrite
+  * `process_io_read_syscalls_total` - the number of read syscalls such as read and pread
+  * `process_io_write_syscalls_total` - the number of write syscalls such as write and pwrite
+  * `process_io_storage_read_bytes_total` - the number of bytes read from storage layer
+  * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
+
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
 * BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
diff --git a/go.mod b/go.mod
index 857a1b9f7..a6fcfe489 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.12
-	github.com/VictoriaMetrics/metrics v1.14.0
+	github.com/VictoriaMetrics/metrics v1.15.0
 	github.com/VictoriaMetrics/metricsql v0.10.1
 	github.com/aws/aws-sdk-go v1.37.12
 	github.com/cespare/xxhash/v2 v2.1.1
diff --git a/go.sum b/go.sum
index 5f0da9295..9de56d296 100644
--- a/go.sum
+++ b/go.sum
@@ -85,8 +85,8 @@ github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6Ro
 github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE=
 github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU=
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metrics v1.14.0 h1:yvyEVo7cPN2Hv+Hrm1zPTA1f/squmEZTq6xtPH/8F64=
-github.com/VictoriaMetrics/metrics v1.14.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
+github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js=
+github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metricsql v0.10.1 h1:wLl/YbMmBGFPyLKMfqNLC333iygibosSM5iSvlH2B4A=
 github.com/VictoriaMetrics/metricsql v0.10.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
diff --git a/vendor/github.com/VictoriaMetrics/metrics/README.md b/vendor/github.com/VictoriaMetrics/metrics/README.md
index 4f1283abb..c76406cc2 100644
--- a/vendor/github.com/VictoriaMetrics/metrics/README.md
+++ b/vendor/github.com/VictoriaMetrics/metrics/README.md
@@ -93,7 +93,7 @@ exposed from your application.
 #### How to implement [CounterVec](https://godoc.org/github.com/prometheus/client_golang/prometheus#CounterVec) in `metrics`?
 
 Just use [GetOrCreateCounter](http://godoc.org/github.com/VictoriaMetrics/metrics#GetOrCreateCounter)
-instead of `CounterVec.With`. See [this example](https://godoc.org/github.com/VictoriaMetrics/metrics#example-Counter--Vec) for details.
+instead of `CounterVec.With`. See [this example](https://pkg.go.dev/github.com/VictoriaMetrics/metrics#example-Counter-Vec) for details.
 
 
 #### Why [Histogram](http://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) buckets contain `vmrange` labels instead of `le` labels like in Prometheus histograms?
diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go
index f04a50393..895ca72dd 100644
--- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go
@@ -12,8 +12,6 @@ import (
 	"time"
 )
 
-const statFilepath = "/proc/self/stat"
-
 // See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 .
 const userHZ = 100
 
@@ -44,6 +42,7 @@ type procStat struct {
 }
 
 func writeProcessMetrics(w io.Writer) {
+	statFilepath := "/proc/self/stat"
 	data, err := ioutil.ReadFile(statFilepath)
 	if err != nil {
 		log.Printf("ERROR: cannot open %s: %s", statFilepath, err)
@@ -68,7 +67,8 @@ func writeProcessMetrics(w io.Writer) {
 	}
 
 	// It is expensive obtaining `process_open_fds` when big number of file descriptors is opened,
-	// don't do it here.
+	// so don't do it here.
+	// See writeFDMetrics instead.
 
 	utime := float64(p.Utime) / userHZ
 	stime := float64(p.Stime) / userHZ
@@ -81,6 +81,54 @@ func writeProcessMetrics(w io.Writer) {
 	fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096)
 	fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds)
 	fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize)
+
+	writeIOMetrics(w)
+}
+
+func writeIOMetrics(w io.Writer) {
+	ioFilepath := "/proc/self/io"
+	data, err := ioutil.ReadFile(ioFilepath)
+	if err != nil {
+		log.Printf("ERROR: cannot open %q: %s", ioFilepath, err)
+	}
+	getInt := func(s string) int64 {
+		n := strings.IndexByte(s, ' ')
+		if n < 0 {
+			log.Printf("ERROR: cannot find whitespace in %q at %q", s, ioFilepath)
+			return 0
+		}
+		v, err := strconv.ParseInt(s[n+1:], 10, 64)
+		if err != nil {
+			log.Printf("ERROR: cannot parse %q at %q: %s", s, ioFilepath, err)
+			return 0
+		}
+		return v
+	}
+	var rchar, wchar, syscr, syscw, readBytes, writeBytes int64
+	lines := strings.Split(string(data), "\n")
+	for _, s := range lines {
+		s = strings.TrimSpace(s)
+		switch {
+		case strings.HasPrefix(s, "rchar: "):
+			rchar = getInt(s)
+		case strings.HasPrefix(s, "wchar: "):
+			wchar = getInt(s)
+		case strings.HasPrefix(s, "syscr: "):
+			syscr = getInt(s)
+		case strings.HasPrefix(s, "syscw: "):
+			syscw = getInt(s)
+		case strings.HasPrefix(s, "read_bytes: "):
+			readBytes = getInt(s)
+		case strings.HasPrefix(s, "write_bytes: "):
+			writeBytes = getInt(s)
+		}
+	}
+	fmt.Fprintf(w, "process_io_read_bytes_total %d\n", rchar)
+	fmt.Fprintf(w, "process_io_written_bytes_total %d\n", wchar)
+	fmt.Fprintf(w, "process_io_read_syscalls_total %d\n", syscr)
+	fmt.Fprintf(w, "process_io_write_syscalls_total %d\n", syscw)
+	fmt.Fprintf(w, "process_io_storage_read_bytes_total %d\n", readBytes)
+	fmt.Fprintf(w, "process_io_storage_written_bytes_total %d\n", writeBytes)
 }
 
 var startTimeSeconds = time.Now().Unix()
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 242c38ca3..61b2281b4 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -14,7 +14,7 @@ github.com/VictoriaMetrics/fastcache
 github.com/VictoriaMetrics/fasthttp
 github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
-# github.com/VictoriaMetrics/metrics v1.14.0
+# github.com/VictoriaMetrics/metrics v1.15.0
 github.com/VictoriaMetrics/metrics
 # github.com/VictoriaMetrics/metricsql v0.10.1
 github.com/VictoriaMetrics/metricsql

From ff5bbc4b889f5c10bb792bcc40f509413b85e5f3 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 21 Feb 2021 23:21:17 +0200
Subject: [PATCH 11/32] lib/promscrape: export
 vm_promscrape_target_relabel_duration_seconds metric

---
 docs/CHANGELOG.md        | 1 +
 lib/promscrape/config.go | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d19346a89..6ca294ee4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -10,6 +10,7 @@
   * `process_io_write_syscalls_total` - the number of write syscalls such as write and pwrite
   * `process_io_storage_read_bytes_total` - the number of bytes read from storage layer
   * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
+* FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
 
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index aade32faa..6c1e2f502 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
+	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 )
 
@@ -608,6 +609,7 @@ func appendGCEScrapeWork(dst []*ScrapeWork, sdc *gce.SDConfig, swc *scrapeWorkCo
 }
 
 func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, targetLabels []map[string]string, sectionName string) []*ScrapeWork {
+	startTime := time.Now()
 	for _, metaLabels := range targetLabels {
 		target := metaLabels["__address__"]
 		var err error
@@ -617,6 +619,7 @@ func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, t
 			continue
 		}
 	}
+	metrics.GetOrCreateHistogram(fmt.Sprintf("vm_promscrape_target_relabel_duration_seconds{type=%q}", sectionName)).UpdateDuration(startTime)
 	return dst
 }
 

From dd1e53b11922675d24ade46f39b6189af44bbb26 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Mon, 22 Feb 2021 00:50:57 +0200
Subject: [PATCH 12/32] lib/promrelabel: optimize relabeling performance for
 common cases

---
 docs/CHANGELOG.md                      |   1 +
 lib/promrelabel/relabel.go             |  88 +++++++++--
 lib/promrelabel/relabel_test.go        | 181 ++++++++++++++++++++++-
 lib/promrelabel/relabel_timing_test.go | 197 ++++++++++++++++++++++++-
 4 files changed, 446 insertions(+), 21 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6ca294ee4..cb42435f6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -11,6 +11,7 @@
   * `process_io_storage_read_bytes_total` - the number of bytes read from storage layer
   * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
 * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
+* FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
 
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index 0a7a02646..95cd0d924 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -115,12 +115,23 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "replace":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		if len(bb.B) == 0 && prc.Regex == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInReplacement && !prc.hasCaptureGroupInTargetLabel {
-			// Fast path for the following rule that just sets label value:
-			// - target_label: foobar
-			//   replacement: something-here
-			relabelBufPool.Put(bb)
-			return setLabelValue(labels, labelsOffset, prc.TargetLabel, prc.Replacement)
+		if prc.Regex == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInTargetLabel {
+			if prc.Replacement == "$1" {
+				// Fast path for the rule that copies source label values to destination:
+				// - source_labels: [...]
+				//   target_label: foobar
+				valueStr := string(bb.B)
+				relabelBufPool.Put(bb)
+				return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
+			}
+			if !prc.hasCaptureGroupInReplacement {
+				// Fast path for the rule that sets label value:
+				// - target_label: foobar
+				//   replacement: something-here
+				relabelBufPool.Put(bb)
+				labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, prc.Replacement)
+				return labels
+			}
 		}
 		match := prc.Regex.FindSubmatchIndex(bb.B)
 		if match == nil {
@@ -139,6 +150,13 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "replace_all":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
+		if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
+			// Fast path - string replacement without regexp.
+			sourceStr := string(bb.B)
+			relabelBufPool.Put(bb)
+			valueStr := strings.ReplaceAll(sourceStr, prefix, prc.Replacement)
+			return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
+		}
 		if !prc.Regex.Match(bb.B) {
 			// Fast path - nothing to replace.
 			relabelBufPool.Put(bb)
@@ -175,7 +193,14 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "keep":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		keep := prc.Regex.Match(bb.B)
+		keep := false
+		if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+			// Fast path - simple string match
+			keep = prefix == string(bb.B)
+		} else {
+			// Slow path - match by regexp
+			keep = prc.Regex.Match(bb.B)
+		}
 		relabelBufPool.Put(bb)
 		if !keep {
 			return labels[:labelsOffset]
@@ -184,7 +209,14 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "drop":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		drop := prc.Regex.Match(bb.B)
+		drop := false
+		if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+			// Fast path - simple string match
+			drop = prefix == string(bb.B)
+		} else {
+			// Slow path - match by regexp
+			drop = prc.Regex.Match(bb.B)
+		}
 		relabelBufPool.Put(bb)
 		if drop {
 			return labels[:labelsOffset]
@@ -214,16 +246,28 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "labelmap_all":
 		for i := range src {
 			label := &src[i]
-			if !prc.Regex.MatchString(label.Name) {
-				continue
+			if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
+				// Fast path - replace without regexp
+				label.Name = strings.ReplaceAll(label.Name, prefix, prc.Replacement)
+			} else {
+				// Slow path - replace with regexp.
+				if !prc.Regex.MatchString(label.Name) {
+					continue
+				}
+				label.Name = prc.Regex.ReplaceAllString(label.Name, prc.Replacement)
 			}
-			label.Name = prc.Regex.ReplaceAllString(label.Name, prc.Replacement)
 		}
 		return labels
 	case "labeldrop":
 		keepSrc := true
 		for i := range src {
-			if prc.Regex.MatchString(src[i].Name) {
+			label := &src[i]
+			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+				if prefix == label.Name {
+					keepSrc = false
+					break
+				}
+			} else if prc.Regex.MatchString(label.Name) {
 				keepSrc = false
 				break
 			}
@@ -234,7 +278,11 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		dst := labels[:labelsOffset]
 		for i := range src {
 			label := &src[i]
-			if !prc.Regex.MatchString(label.Name) {
+			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+				if prefix != label.Name {
+					dst = append(dst, *label)
+				}
+			} else if !prc.Regex.MatchString(label.Name) {
 				dst = append(dst, *label)
 			}
 		}
@@ -242,7 +290,13 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "labelkeep":
 		keepSrc := true
 		for i := range src {
-			if !prc.Regex.MatchString(src[i].Name) {
+			label := &src[i]
+			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+				if prefix != label.Name {
+					keepSrc = false
+					break
+				}
+			} else if !prc.Regex.MatchString(src[i].Name) {
 				keepSrc = false
 				break
 			}
@@ -253,7 +307,11 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		dst := labels[:labelsOffset]
 		for i := range src {
 			label := &src[i]
-			if prc.Regex.MatchString(label.Name) {
+			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
+				if prefix == label.Name {
+					dst = append(dst, *label)
+				}
+			} else if prc.Regex.MatchString(label.Name) {
 				dst = append(dst, *label)
 			}
 		}
diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go
index a57e8bee6..6185f029b 100644
--- a/lib/promrelabel/relabel_test.go
+++ b/lib/promrelabel/relabel_test.go
@@ -232,6 +232,28 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace_all-hit", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "replace_all",
+				SourceLabels: []string{"xxx"},
+				Separator:    ";",
+				TargetLabel:  "xxx",
+				Regex:        regexp.MustCompile("-"),
+				Replacement:  ".",
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "a-b-c",
+			},
+		}, false, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "a.b.c",
+			},
+		})
+	})
+	t.Run("replace_all-regex-hit", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{
 				Action:                       "replace_all",
@@ -265,11 +287,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				hasCaptureGroupInReplacement: true,
 			},
 			{
-				Action:       "replace",
-				SourceLabels: []string{"bar"},
-				TargetLabel:  "zar",
-				Regex:        defaultRegexForRelabelConfig,
-				Replacement:  "b-$1",
+				Action:                       "replace",
+				SourceLabels:                 []string{"bar"},
+				TargetLabel:                  "zar",
+				Regex:                        defaultRegexForRelabelConfig,
+				Replacement:                  "b-$1",
+				hasCaptureGroupInReplacement: true,
 			},
 		}, []prompbmarshal.Label{
 			{
@@ -444,6 +467,25 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("keep-hit", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "keep",
+				SourceLabels: []string{"foo"},
+				Regex:        regexp.MustCompile("yyy"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+		}, false, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("keep-hit-regexp", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{
 				Action:       "keep",
@@ -489,6 +531,20 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("drop-hit", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "drop",
+				SourceLabels: []string{"foo"},
+				Regex:        regexp.MustCompile("yyy"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+		}, true, []prompbmarshal.Label{})
+	})
+	t.Run("drop-hit-regexp", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{
 				Action:       "drop",
@@ -608,7 +664,80 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		})
 	})
+	t.Run("labelmap_all-regexp", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:      "labelmap_all",
+				Regex:       regexp.MustCompile(`ba(.)`),
+				Replacement: "${1}ss",
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "foo.bar.baz",
+				Value: "yyy",
+			},
+			{
+				Name:  "foozar",
+				Value: "aaa",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "foo.rss.zss",
+				Value: "yyy",
+			},
+			{
+				Name:  "foozar",
+				Value: "aaa",
+			},
+		})
+	})
 	t.Run("labeldrop", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action: "labeldrop",
+				Regex:  regexp.MustCompile("dropme"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "aaa",
+				Value: "bbb",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "aaa",
+				Value: "bbb",
+			},
+		})
+		f([]ParsedRelabelConfig{
+			{
+				Action: "labeldrop",
+				Regex:  regexp.MustCompile("dropme"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+			{
+				Name:  "dropme",
+				Value: "aaa",
+			},
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+		}, false, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("labeldrop-regexp", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{
 				Action: "labeldrop",
@@ -655,6 +784,48 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labelkeep", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action: "labelkeep",
+				Regex:  regexp.MustCompile("keepme"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "keepme",
+				Value: "aaa",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "keepme",
+				Value: "aaa",
+			},
+		})
+		f([]ParsedRelabelConfig{
+			{
+				Action: "labelkeep",
+				Regex:  regexp.MustCompile("keepme"),
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "keepme",
+				Value: "aaa",
+			},
+			{
+				Name:  "aaaa",
+				Value: "awef",
+			},
+			{
+				Name:  "keepme-aaa",
+				Value: "234",
+			},
+		}, false, []prompbmarshal.Label{
+			{
+				Name:  "keepme",
+				Value: "aaa",
+			},
+		})
+	})
+	t.Run("labelkeep-regexp", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{
 				Action: "labelkeep",
diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go
index c16a3a113..fb0a0207c 100644
--- a/lib/promrelabel/relabel_timing_test.go
+++ b/lib/promrelabel/relabel_timing_test.go
@@ -183,7 +183,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
-	b.Run("replace-match", func(b *testing.B) {
+	b.Run("replace-match-regex", func(b *testing.B) {
 		prcs := []ParsedRelabelConfig{
 			{
 				Action:       "replace",
@@ -272,6 +272,37 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("drop-match", func(b *testing.B) {
+		prcs := []ParsedRelabelConfig{
+			{
+				Action:       "drop",
+				SourceLabels: []string{"id"},
+				Regex:        regexp.MustCompile("yes"),
+			},
+		}
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "yes",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				if len(labels) != 0 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
+				}
+			}
+		})
+	})
+	b.Run("drop-match-regexp", func(b *testing.B) {
 		prcs := []ParsedRelabelConfig{
 			{
 				Action:       "drop",
@@ -334,6 +365,49 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("keep-match", func(b *testing.B) {
+		prcs := []ParsedRelabelConfig{
+			{
+				Action:       "keep",
+				SourceLabels: []string{"id"},
+				Regex:        regexp.MustCompile("yes"),
+			},
+		}
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "yes",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				if len(labels) != len(labelsOrig) {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
+				}
+				if labels[0].Name != "__name__" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
+				}
+				if labels[0].Value != "metric" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "metric"))
+				}
+				if labels[1].Name != "id" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[1].Name, "id"))
+				}
+				if labels[1].Value != "yes" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "yes"))
+				}
+			}
+		})
+	})
+	b.Run("keep-match-regexp", func(b *testing.B) {
 		prcs := []ParsedRelabelConfig{
 			{
 				Action:       "keep",
@@ -454,6 +528,42 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
+	b.Run("labeldrop-match-regexp", func(b *testing.B) {
+		prcs := []ParsedRelabelConfig{
+			{
+				Action: "labeldrop",
+				Regex:  regexp.MustCompile("id.*"),
+			},
+		}
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				if len(labels) != 1 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
+				}
+				if labels[0].Name != "__name__" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
+				}
+				if labels[0].Value != "metric" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "metric"))
+				}
+			}
+		})
+	})
 	b.Run("labelkeep-mismatch", func(b *testing.B) {
 		prcs := []ParsedRelabelConfig{
 			{
@@ -520,6 +630,91 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
+	b.Run("labelkeep-match-regexp", func(b *testing.B) {
+		prcs := []ParsedRelabelConfig{
+			{
+				Action: "labelkeep",
+				Regex:  regexp.MustCompile("id.*"),
+			},
+		}
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				if len(labels) != 1 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
+				}
+				if labels[0].Name != "id" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "id"))
+				}
+				if labels[0].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
+				}
+			}
+		})
+	})
+	b.Run("labelmap", func(b *testing.B) {
+		prcs := []ParsedRelabelConfig{
+			{
+				Action:      "labelmap",
+				Regex:       regexp.MustCompile("a(.*)"),
+				Replacement: "$1",
+			},
+		}
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "aabc",
+				Value: "foobar-random-string-here",
+			},
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				if len(labels) != 3 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 3, labels))
+				}
+				if labels[0].Name != "aabc" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "aabc"))
+				}
+				if labels[0].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
+				}
+				if labels[1].Name != "abc" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[1].Name, "abc"))
+				}
+				if labels[1].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "foobar-random-string-here"))
+				}
+				if labels[2].Name != "foo" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[2].Name, "foo"))
+				}
+				if labels[2].Value != "bar" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[2].Value, "bar"))
+				}
+			}
+		})
+	})
 	b.Run("hashmod", func(b *testing.B) {
 		prcs := []ParsedRelabelConfig{
 			{

From d136081040b4a0564dee952d2e568ccd39dc133b Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Mon, 22 Feb 2021 16:33:55 +0200
Subject: [PATCH 13/32] lib/promrelabel: add more optimizations for relabeling
 for common cases

---
 app/vmagent/remotewrite/relabel.go     |  12 +-
 app/vmagent/remotewrite/remotewrite.go |  12 +-
 app/vminsert/relabel/relabel.go        |  26 +-
 lib/promrelabel/config.go              | 100 ++--
 lib/promrelabel/config_test.go         |  43 +-
 lib/promrelabel/relabel.go             | 185 +++----
 lib/promrelabel/relabel_test.go        | 646 ++++++++++++-------------
 lib/promrelabel/relabel_timing_test.go | 350 +++++++-------
 lib/promscrape/config.go               |  12 +-
 lib/promscrape/config_test.go          |  20 +-
 lib/promscrape/scrapework.go           |  15 +-
 lib/promscrape/scrapework_test.go      |  55 ++-
 12 files changed, 751 insertions(+), 725 deletions(-)

diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index 875694173..ad7c967f6 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -41,7 +41,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), len(*remoteWriteURLs))
 	}
-	rcs.perURL = make([][]promrelabel.ParsedRelabelConfig, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.
@@ -57,8 +57,8 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 }
 
 type relabelConfigs struct {
-	global []promrelabel.ParsedRelabelConfig
-	perURL [][]promrelabel.ParsedRelabelConfig
+	global *promrelabel.ParsedConfigs
+	perURL []*promrelabel.ParsedConfigs
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.
@@ -79,8 +79,8 @@ func initLabelsGlobal() {
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
-	if len(extraLabels) == 0 && len(prcs) == 0 {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && pcs.Len() == 0 {
 		// Nothing to change.
 		return tss
 	}
@@ -100,7 +100,7 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 				labels = append(labels, *extraLabel)
 			}
 		}
-		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
+		labels = pcs.Apply(labels, labelsLen, true)
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index 824ea9ad7..4e5ef6751 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -142,8 +142,8 @@ func Stop() {
 func Push(wr *prompbmarshal.WriteRequest) {
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
-	prcsGlobal := rcs.global
-	if len(prcsGlobal) > 0 || len(labelsGlobal) > 0 {
+	pcsGlobal := rcs.global
+	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
@@ -167,7 +167,7 @@ func Push(wr *prompbmarshal.WriteRequest) {
 		}
 		if rctx != nil {
 			tssBlockLen := len(tssBlock)
-			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, prcsGlobal)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
 			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
 		}
 		for _, rwctx := range rwctxs {
@@ -240,8 +240,8 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
-	prcs := rcs.perURL[rwctx.idx]
-	if len(prcs) > 0 {
+	pcs := rcs.perURL[rwctx.idx]
+	if pcs.Len() > 0 {
 		rctx = getRelabelCtx()
 		// Make a copy of tss before applying relabeling in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
@@ -250,7 +250,7 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		tssLen := len(tss)
-		tss = rctx.applyRelabeling(tss, nil, prcs)
+		tss = rctx.applyRelabeling(tss, nil, pcs)
 		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
 	}
 	pss := rwctx.pss
diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go
index 11b9cc774..b9a2e62e4 100644
--- a/app/vminsert/relabel/relabel.go
+++ b/app/vminsert/relabel/relabel.go
@@ -19,11 +19,11 @@ var relabelConfig = flag.String("relabelConfig", "", "Optional path to a file wi
 
 // Init must be called after flag.Parse and before using the relabel package.
 func Init() {
-	prcs, err := loadRelabelConfig()
+	pcs, err := loadRelabelConfig()
 	if err != nil {
 		logger.Fatalf("cannot load relabelConfig: %s", err)
 	}
-	prcsGlobal.Store(&prcs)
+	pcsGlobal.Store(pcs)
 	if len(*relabelConfig) == 0 {
 		return
 	}
@@ -31,34 +31,34 @@ func Init() {
 	go func() {
 		for range sighupCh {
 			logger.Infof("received SIGHUP; reloading -relabelConfig=%q...", *relabelConfig)
-			prcs, err := loadRelabelConfig()
+			pcs, err := loadRelabelConfig()
 			if err != nil {
 				logger.Errorf("cannot load the updated relabelConfig: %s; preserving the previous config", err)
 				continue
 			}
-			prcsGlobal.Store(&prcs)
+			pcsGlobal.Store(pcs)
 			logger.Infof("successfully reloaded -relabelConfig=%q", *relabelConfig)
 		}
 	}()
 }
 
-var prcsGlobal atomic.Value
+var pcsGlobal atomic.Value
 
-func loadRelabelConfig() ([]promrelabel.ParsedRelabelConfig, error) {
+func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) {
 	if len(*relabelConfig) == 0 {
 		return nil, nil
 	}
-	prcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig)
+	pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error when reading -relabelConfig=%q: %w", *relabelConfig, err)
 	}
-	return prcs, nil
+	return pcs, nil
 }
 
 // HasRelabeling returns true if there is global relabeling.
 func HasRelabeling() bool {
-	prcs := prcsGlobal.Load().(*[]promrelabel.ParsedRelabelConfig)
-	return len(*prcs) > 0
+	pcs := pcsGlobal.Load().(*promrelabel.ParsedConfigs)
+	return pcs.Len() > 0
 }
 
 // Ctx holds relabeling context.
@@ -77,8 +77,8 @@ func (ctx *Ctx) Reset() {
 //
 // The returned labels are valid until the next call to ApplyRelabeling.
 func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
-	prcs := prcsGlobal.Load().(*[]promrelabel.ParsedRelabelConfig)
-	if len(*prcs) == 0 {
+	pcs := pcsGlobal.Load().(*promrelabel.ParsedConfigs)
+	if pcs.Len() == 0 {
 		// There are no relabeling rules.
 		return labels
 	}
@@ -97,7 +97,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 	}
 
 	// Apply relabeling
-	tmpLabels = promrelabel.ApplyRelabelConfigs(tmpLabels, 0, *prcs, true)
+	tmpLabels = pcs.Apply(tmpLabels, 0, true)
 	ctx.tmpLabels = tmpLabels
 	if len(tmpLabels) == 0 {
 		metricsDropped.Inc()
diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go
index abd9419a4..29d624174 100644
--- a/lib/promrelabel/config.go
+++ b/lib/promrelabel/config.go
@@ -23,38 +23,78 @@ type RelabelConfig struct {
 	Action       string   `yaml:"action,omitempty"`
 }
 
+// ParsedConfigs represents parsed relabel configs.
+type ParsedConfigs struct {
+	prcs []*parsedRelabelConfig
+}
+
+// Len returns the number of relabel configs in pcs.
+func (pcs *ParsedConfigs) Len() int {
+	if pcs == nil {
+		return 0
+	}
+	return len(pcs.prcs)
+}
+
+// String returns human-readabale representation for pcs.
+func (pcs *ParsedConfigs) String() string {
+	if pcs == nil {
+		return ""
+	}
+	var sb strings.Builder
+	for _, prc := range pcs.prcs {
+		fmt.Fprintf(&sb, "%s", prc.String())
+	}
+	return sb.String()
+}
+
 // LoadRelabelConfigs loads relabel configs from the given path.
-func LoadRelabelConfigs(path string) ([]ParsedRelabelConfig, error) {
+func LoadRelabelConfigs(path string) (*ParsedConfigs, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err)
 	}
 	data = envtemplate.Replace(data)
-	var rcs []RelabelConfig
-	if err := yaml.UnmarshalStrict(data, &rcs); err != nil {
+	pcs, err := ParseRelabelConfigsData(data)
+	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err)
 	}
-	return ParseRelabelConfigs(nil, rcs)
+	return pcs, nil
+}
+
+// ParseRelabelConfigsData parses relabel configs from the given data.
+func ParseRelabelConfigsData(data []byte) (*ParsedConfigs, error) {
+	var rcs []RelabelConfig
+	if err := yaml.UnmarshalStrict(data, &rcs); err != nil {
+		return nil, err
+	}
+	return ParseRelabelConfigs(rcs)
 }
 
 // ParseRelabelConfigs parses rcs to dst.
-func ParseRelabelConfigs(dst []ParsedRelabelConfig, rcs []RelabelConfig) ([]ParsedRelabelConfig, error) {
+func ParseRelabelConfigs(rcs []RelabelConfig) (*ParsedConfigs, error) {
 	if len(rcs) == 0 {
-		return dst, nil
+		return nil, nil
 	}
+	prcs := make([]*parsedRelabelConfig, len(rcs))
 	for i := range rcs {
-		var err error
-		dst, err = parseRelabelConfig(dst, &rcs[i])
+		prc, err := parseRelabelConfig(&rcs[i])
 		if err != nil {
-			return dst, fmt.Errorf("error when parsing `relabel_config` #%d: %w", i+1, err)
+			return nil, fmt.Errorf("error when parsing `relabel_config` #%d: %w", i+1, err)
 		}
+		prcs[i] = prc
 	}
-	return dst, nil
+	return &ParsedConfigs{
+		prcs: prcs,
+	}, nil
 }
 
-var defaultRegexForRelabelConfig = regexp.MustCompile("^(.*)$")
+var (
+	defaultOriginalRegexForRelabelConfig = regexp.MustCompile(".*")
+	defaultRegexForRelabelConfig         = regexp.MustCompile("^(.*)$")
+)
 
-func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedRelabelConfig, error) {
+func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 	sourceLabels := rc.SourceLabels
 	separator := ";"
 	if rc.Separator != nil {
@@ -62,6 +102,7 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR
 	}
 	targetLabel := rc.TargetLabel
 	regexCompiled := defaultRegexForRelabelConfig
+	regexOriginalCompiled := defaultOriginalRegexForRelabelConfig
 	if rc.Regex != nil {
 		regex := *rc.Regex
 		if rc.Action != "replace_all" && rc.Action != "labelmap_all" {
@@ -69,9 +110,14 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR
 		}
 		re, err := regexp.Compile(regex)
 		if err != nil {
-			return dst, fmt.Errorf("cannot parse `regex` %q: %w", regex, err)
+			return nil, fmt.Errorf("cannot parse `regex` %q: %w", regex, err)
 		}
 		regexCompiled = re
+		reOriginal, err := regexp.Compile(*rc.Regex)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse `regex` %q: %w", *rc.Regex, err)
+		}
+		regexOriginalCompiled = reOriginal
 	}
 	modulus := rc.Modulus
 	replacement := "$1"
@@ -85,49 +131,49 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR
 	switch action {
 	case "replace":
 		if targetLabel == "" {
-			return dst, fmt.Errorf("missing `target_label` for `action=replace`")
+			return nil, fmt.Errorf("missing `target_label` for `action=replace`")
 		}
 	case "replace_all":
 		if len(sourceLabels) == 0 {
-			return dst, fmt.Errorf("missing `source_labels` for `action=replace_all`")
+			return nil, fmt.Errorf("missing `source_labels` for `action=replace_all`")
 		}
 		if targetLabel == "" {
-			return dst, fmt.Errorf("missing `target_label` for `action=replace_all`")
+			return nil, fmt.Errorf("missing `target_label` for `action=replace_all`")
 		}
 	case "keep_if_equal":
 		if len(sourceLabels) < 2 {
-			return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=keep_if_equal`; got %q", sourceLabels)
+			return nil, fmt.Errorf("`source_labels` must contain at least two entries for `action=keep_if_equal`; got %q", sourceLabels)
 		}
 	case "drop_if_equal":
 		if len(sourceLabels) < 2 {
-			return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=drop_if_equal`; got %q", sourceLabels)
+			return nil, fmt.Errorf("`source_labels` must contain at least two entries for `action=drop_if_equal`; got %q", sourceLabels)
 		}
 	case "keep":
 		if len(sourceLabels) == 0 {
-			return dst, fmt.Errorf("missing `source_labels` for `action=keep`")
+			return nil, fmt.Errorf("missing `source_labels` for `action=keep`")
 		}
 	case "drop":
 		if len(sourceLabels) == 0 {
-			return dst, fmt.Errorf("missing `source_labels` for `action=drop`")
+			return nil, fmt.Errorf("missing `source_labels` for `action=drop`")
 		}
 	case "hashmod":
 		if len(sourceLabels) == 0 {
-			return dst, fmt.Errorf("missing `source_labels` for `action=hashmod`")
+			return nil, fmt.Errorf("missing `source_labels` for `action=hashmod`")
 		}
 		if targetLabel == "" {
-			return dst, fmt.Errorf("missing `target_label` for `action=hashmod`")
+			return nil, fmt.Errorf("missing `target_label` for `action=hashmod`")
 		}
 		if modulus < 1 {
-			return dst, fmt.Errorf("unexpected `modulus` for `action=hashmod`: %d; must be greater than 0", modulus)
+			return nil, fmt.Errorf("unexpected `modulus` for `action=hashmod`: %d; must be greater than 0", modulus)
 		}
 	case "labelmap":
 	case "labelmap_all":
 	case "labeldrop":
 	case "labelkeep":
 	default:
-		return dst, fmt.Errorf("unknown `action` %q", action)
+		return nil, fmt.Errorf("unknown `action` %q", action)
 	}
-	dst = append(dst, ParsedRelabelConfig{
+	return &parsedRelabelConfig{
 		SourceLabels: sourceLabels,
 		Separator:    separator,
 		TargetLabel:  targetLabel,
@@ -136,8 +182,8 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR
 		Replacement:  replacement,
 		Action:       action,
 
+		regexOriginal:                regexOriginalCompiled,
 		hasCaptureGroupInTargetLabel: strings.Contains(targetLabel, "$"),
 		hasCaptureGroupInReplacement: strings.Contains(replacement, "$"),
-	})
-	return dst, nil
+	}, nil
 }
diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go
index 3a1ae1477..f514d4d9a 100644
--- a/lib/promrelabel/config_test.go
+++ b/lib/promrelabel/config_test.go
@@ -7,12 +7,12 @@ import (
 
 func TestLoadRelabelConfigsSuccess(t *testing.T) {
 	path := "testdata/relabel_configs_valid.yml"
-	prcs, err := LoadRelabelConfigs(path)
+	pcs, err := LoadRelabelConfigs(path)
 	if err != nil {
 		t.Fatalf("cannot load relabel configs from %q: %s", path, err)
 	}
-	if len(prcs) != 9 {
-		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, len(prcs), 9)
+	if n := pcs.Len(); n != 9 {
+		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, n, 9)
 	}
 }
 
@@ -23,7 +23,7 @@ func TestLoadRelabelConfigsFailure(t *testing.T) {
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if len(rcs) != 0 {
+		if rcs.Len() != 0 {
 			t.Fatalf("unexpected non-empty rcs: %#v", rcs)
 		}
 	}
@@ -36,14 +36,14 @@ func TestLoadRelabelConfigsFailure(t *testing.T) {
 }
 
 func TestParseRelabelConfigsSuccess(t *testing.T) {
-	f := func(rcs []RelabelConfig, prcsExpected []ParsedRelabelConfig) {
+	f := func(rcs []RelabelConfig, pcsExpected *ParsedConfigs) {
 		t.Helper()
-		prcs, err := ParseRelabelConfigs(nil, rcs)
+		pcs, err := ParseRelabelConfigs(rcs)
 		if err != nil {
 			t.Fatalf("unexected error: %s", err)
 		}
-		if !reflect.DeepEqual(prcs, prcsExpected) {
-			t.Fatalf("unexpected prcs; got\n%#v\nwant\n%#v", prcs, prcsExpected)
+		if !reflect.DeepEqual(pcs, pcsExpected) {
+			t.Fatalf("unexpected pcs; got\n%#v\nwant\n%#v", pcs, pcsExpected)
 		}
 	}
 	f(nil, nil)
@@ -52,16 +52,19 @@ func TestParseRelabelConfigsSuccess(t *testing.T) {
 			SourceLabels: []string{"foo", "bar"},
 			TargetLabel:  "xxx",
 		},
-	}, []ParsedRelabelConfig{
-		{
-			SourceLabels: []string{"foo", "bar"},
-			Separator:    ";",
-			TargetLabel:  "xxx",
-			Regex:        defaultRegexForRelabelConfig,
-			Replacement:  "$1",
-			Action:       "replace",
+	}, &ParsedConfigs{
+		prcs: []*parsedRelabelConfig{
+			{
+				SourceLabels: []string{"foo", "bar"},
+				Separator:    ";",
+				TargetLabel:  "xxx",
+				Regex:        defaultRegexForRelabelConfig,
+				Replacement:  "$1",
+				Action:       "replace",
 
-			hasCaptureGroupInReplacement: true,
+				regexOriginal:                defaultOriginalRegexForRelabelConfig,
+				hasCaptureGroupInReplacement: true,
+			},
 		},
 	})
 }
@@ -69,12 +72,12 @@ func TestParseRelabelConfigsSuccess(t *testing.T) {
 func TestParseRelabelConfigsFailure(t *testing.T) {
 	f := func(rcs []RelabelConfig) {
 		t.Helper()
-		prcs, err := ParseRelabelConfigs(nil, rcs)
+		pcs, err := ParseRelabelConfigs(rcs)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if len(prcs) > 0 {
-			t.Fatalf("unexpected non-empty prcs: %#v", prcs)
+		if pcs.Len() > 0 {
+			t.Fatalf("unexpected non-empty pcs: %#v", pcs)
 		}
 	}
 	t.Run("invalid-regex", func(t *testing.T) {
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index 95cd0d924..a161e55cc 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -12,10 +12,10 @@ import (
 	xxhash "github.com/cespare/xxhash/v2"
 )
 
-// ParsedRelabelConfig contains parsed `relabel_config`.
+// parsedRelabelConfig contains parsed `relabel_config`.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
-type ParsedRelabelConfig struct {
+type parsedRelabelConfig struct {
 	SourceLabels []string
 	Separator    string
 	TargetLabel  string
@@ -24,29 +24,32 @@ type ParsedRelabelConfig struct {
 	Replacement  string
 	Action       string
 
+	regexOriginal                *regexp.Regexp
 	hasCaptureGroupInTargetLabel bool
 	hasCaptureGroupInReplacement bool
 }
 
 // String returns human-readable representation for prc.
-func (prc *ParsedRelabelConfig) String() string {
+func (prc *parsedRelabelConfig) String() string {
 	return fmt.Sprintf("SourceLabels=%s, Separator=%s, TargetLabel=%s, Regex=%s, Modulus=%d, Replacement=%s, Action=%s",
 		prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.Regex.String(), prc.Modulus, prc.Replacement, prc.Action)
 }
 
-// ApplyRelabelConfigs applies prcs to labels starting from the labelsOffset.
+// Apply applies pcs to labels starting from the labelsOffset.
 //
 // If isFinalize is set, then FinalizeLabels is called on the labels[labelsOffset:].
 //
 // The returned labels at labels[labelsOffset:] are sorted.
-func ApplyRelabelConfigs(labels []prompbmarshal.Label, labelsOffset int, prcs []ParsedRelabelConfig, isFinalize bool) []prompbmarshal.Label {
-	for i := range prcs {
-		tmp := applyRelabelConfig(labels, labelsOffset, &prcs[i])
-		if len(tmp) == labelsOffset {
-			// All the labels have been removed.
-			return tmp
+func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool) []prompbmarshal.Label {
+	if pcs != nil {
+		for _, prc := range pcs.prcs {
+			tmp := prc.apply(labels, labelsOffset)
+			if len(tmp) == labelsOffset {
+				// All the labels have been removed.
+				return tmp
+			}
+			labels = tmp
 		}
-		labels = tmp
 	}
 	labels = removeEmptyLabels(labels, labelsOffset)
 	if isFinalize {
@@ -106,10 +109,10 @@ func FinalizeLabels(dst, src []prompbmarshal.Label) []prompbmarshal.Label {
 	return dst
 }
 
-// applyRelabelConfig applies relabeling according to prc.
+// apply applies relabeling according to prc.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
-func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *ParsedRelabelConfig) []prompbmarshal.Label {
+func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label {
 	src := labels[labelsOffset:]
 	switch prc.Action {
 	case "replace":
@@ -150,22 +153,13 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "replace_all":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
-			// Fast path - string replacement without regexp.
-			sourceStr := string(bb.B)
-			relabelBufPool.Put(bb)
-			valueStr := strings.ReplaceAll(sourceStr, prefix, prc.Replacement)
-			return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
-		}
-		if !prc.Regex.Match(bb.B) {
-			// Fast path - nothing to replace.
-			relabelBufPool.Put(bb)
-			return labels
-		}
-		sourceStr := string(bb.B) // Make a copy of bb, since it can be returned from ReplaceAllString
+		sourceStr := string(bb.B)
 		relabelBufPool.Put(bb)
-		valueStr := prc.Regex.ReplaceAllString(sourceStr, prc.Replacement)
-		return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
+		valueStr, ok := prc.replaceStringSubmatches(sourceStr, prc.Replacement, prc.hasCaptureGroupInReplacement)
+		if ok {
+			labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
+		}
+		return labels
 	case "keep_if_equal":
 		// Keep the entry if all the label values in source_labels are equal.
 		// For example:
@@ -193,14 +187,7 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "keep":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		keep := false
-		if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-			// Fast path - simple string match
-			keep = prefix == string(bb.B)
-		} else {
-			// Slow path - match by regexp
-			keep = prc.Regex.Match(bb.B)
-		}
+		keep := prc.matchString(bytesutil.ToUnsafeString(bb.B))
 		relabelBufPool.Put(bb)
 		if !keep {
 			return labels[:labelsOffset]
@@ -209,14 +196,7 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "drop":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		drop := false
-		if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-			// Fast path - simple string match
-			drop = prefix == string(bb.B)
-		} else {
-			// Slow path - match by regexp
-			drop = prc.Regex.Match(bb.B)
-		}
+		drop := prc.matchString(bytesutil.ToUnsafeString(bb.B))
 		relabelBufPool.Put(bb)
 		if drop {
 			return labels[:labelsOffset]
@@ -232,42 +212,23 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	case "labelmap":
 		for i := range src {
 			label := &src[i]
-			match := prc.Regex.FindStringSubmatchIndex(label.Name)
-			if match == nil {
-				continue
+			labelName, ok := prc.replaceFullString(label.Name, prc.Replacement, prc.hasCaptureGroupInReplacement)
+			if ok {
+				labels = setLabelValue(labels, labelsOffset, labelName, label.Value)
 			}
-			value := relabelBufPool.Get()
-			value.B = prc.Regex.ExpandString(value.B[:0], prc.Replacement, label.Name, match)
-			labelName := string(value.B)
-			relabelBufPool.Put(value)
-			labels = setLabelValue(labels, labelsOffset, labelName, label.Value)
 		}
 		return labels
 	case "labelmap_all":
 		for i := range src {
 			label := &src[i]
-			if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
-				// Fast path - replace without regexp
-				label.Name = strings.ReplaceAll(label.Name, prefix, prc.Replacement)
-			} else {
-				// Slow path - replace with regexp.
-				if !prc.Regex.MatchString(label.Name) {
-					continue
-				}
-				label.Name = prc.Regex.ReplaceAllString(label.Name, prc.Replacement)
-			}
+			label.Name, _ = prc.replaceStringSubmatches(label.Name, prc.Replacement, prc.hasCaptureGroupInReplacement)
 		}
 		return labels
 	case "labeldrop":
 		keepSrc := true
 		for i := range src {
 			label := &src[i]
-			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-				if prefix == label.Name {
-					keepSrc = false
-					break
-				}
-			} else if prc.Regex.MatchString(label.Name) {
+			if prc.matchString(label.Name) {
 				keepSrc = false
 				break
 			}
@@ -278,11 +239,7 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		dst := labels[:labelsOffset]
 		for i := range src {
 			label := &src[i]
-			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-				if prefix != label.Name {
-					dst = append(dst, *label)
-				}
-			} else if !prc.Regex.MatchString(label.Name) {
+			if !prc.matchString(label.Name) {
 				dst = append(dst, *label)
 			}
 		}
@@ -291,12 +248,7 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		keepSrc := true
 		for i := range src {
 			label := &src[i]
-			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-				if prefix != label.Name {
-					keepSrc = false
-					break
-				}
-			} else if !prc.Regex.MatchString(src[i].Name) {
+			if !prc.matchString(label.Name) {
 				keepSrc = false
 				break
 			}
@@ -307,11 +259,7 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		dst := labels[:labelsOffset]
 		for i := range src {
 			label := &src[i]
-			if prefix, complete := prc.Regex.LiteralPrefix(); complete {
-				if prefix == label.Name {
-					dst = append(dst, *label)
-				}
-			} else if prc.Regex.MatchString(label.Name) {
+			if prc.matchString(label.Name) {
 				dst = append(dst, *label)
 			}
 		}
@@ -322,7 +270,74 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 	}
 }
 
-func (prc *ParsedRelabelConfig) expandCaptureGroups(template, source string, match []int) string {
+func (prc *parsedRelabelConfig) replaceFullString(s, replacement string, hasCaptureGroupInReplacement bool) (string, bool) {
+	prefix, complete := prc.regexOriginal.LiteralPrefix()
+	if complete && !hasCaptureGroupInReplacement {
+		if s == prefix {
+			return replacement, true
+		}
+		return s, false
+	}
+	if !strings.HasPrefix(s, prefix) {
+		return s, false
+	}
+	if replacement == "$1" {
+		// Fast path for commonly used rule for deleting label prefixes such as:
+		//
+		// - action: labelmap
+		//   regex: __meta_kubernetes_node_label_(.+)
+		//
+		reStr := prc.regexOriginal.String()
+		if strings.HasPrefix(reStr, prefix) {
+			suffix := s[len(prefix):]
+			reSuffix := reStr[len(prefix):]
+			switch reSuffix {
+			case "(.*)":
+				return suffix, true
+			case "(.+)":
+				if len(suffix) > 0 {
+					return suffix, true
+				}
+				return s, false
+			}
+		}
+	}
+	// Slow path - regexp processing
+	match := prc.Regex.FindStringSubmatchIndex(s)
+	if match == nil {
+		return s, false
+	}
+	bb := relabelBufPool.Get()
+	bb.B = prc.Regex.ExpandString(bb.B[:0], replacement, s, match)
+	result := string(bb.B)
+	relabelBufPool.Put(bb)
+	return result, true
+}
+
+func (prc *parsedRelabelConfig) replaceStringSubmatches(s, replacement string, hasCaptureGroupInReplacement bool) (string, bool) {
+	re := prc.regexOriginal
+	prefix, complete := re.LiteralPrefix()
+	if complete && !hasCaptureGroupInReplacement {
+		if !strings.Contains(s, prefix) {
+			return s, false
+		}
+		return strings.ReplaceAll(s, prefix, replacement), true
+	}
+	if !re.MatchString(s) {
+		return s, false
+	}
+	return re.ReplaceAllString(s, replacement), true
+}
+
+func (prc *parsedRelabelConfig) matchString(s string) bool {
+	prefix, complete := prc.regexOriginal.LiteralPrefix()
+	if complete {
+		return prefix == s
+	}
+	return strings.HasPrefix(s, prefix) && prc.Regex.MatchString(s)
+}
+
+func (prc *parsedRelabelConfig) expandCaptureGroups(template, source string, match []int) string {
 	bb := relabelBufPool.Get()
 	bb.B = prc.Regex.ExpandString(bb.B[:0], template, source, match)
 	s := string(bb.B)
diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go
index 6185f029b..cc06887e7 100644
--- a/lib/promrelabel/relabel_test.go
+++ b/lib/promrelabel/relabel_test.go
@@ -2,24 +2,27 @@ package promrelabel
 
 import (
 	"reflect"
-	"regexp"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestApplyRelabelConfigs(t *testing.T) {
-	f := func(prcs []ParsedRelabelConfig, labels []prompbmarshal.Label, isFinalize bool, resultExpected []prompbmarshal.Label) {
+	f := func(config string, labels []prompbmarshal.Label, isFinalize bool, resultExpected []prompbmarshal.Label) {
 		t.Helper()
-		result := ApplyRelabelConfigs(labels, 0, prcs, isFinalize)
+		pcs, err := ParseRelabelConfigsData([]byte(config))
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", config, err)
+		}
+		result := pcs.Apply(labels, 0, isFinalize)
 		if !reflect.DeepEqual(result, resultExpected) {
 			t.Fatalf("unexpected result; got\n%v\nwant\n%v", result, resultExpected)
 		}
 	}
 	t.Run("empty_relabel_configs", func(t *testing.T) {
-		f(nil, nil, false, nil)
-		f(nil, nil, true, nil)
-		f(nil, []prompbmarshal.Label{
+		f("", nil, false, nil)
+		f("", nil, true, nil)
+		f("", []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "bar",
@@ -30,7 +33,7 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "bar",
 			},
 		})
-		f(nil, []prompbmarshal.Label{
+		f("", []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "bar",
@@ -55,35 +58,20 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, nil, false, []prompbmarshal.Label{})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, nil, false, []prompbmarshal.Label{})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  target_label: bar
+`, nil, false, []prompbmarshal.Label{})
+		f(`
+- action: replace
+  source_labels: ["foo"]
+  target_label: bar
+`, nil, false, []prompbmarshal.Label{})
+		f(`
+- action: replace
+  source_labels: ["foo"]
+  target_label: "bar"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -94,16 +82,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "yyy",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        regexp.MustCompile(".+"),
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  source_labels: ["foo"]
+  target_label: "bar"
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -116,17 +100,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"xxx", "foo"},
-				Separator:                    ";",
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "a-$1-b",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  source_labels: ["xxx", "foo"]
+  target_label: "bar"
+  replacement: "a-$1-b"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -143,18 +122,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-hit-target-label-with-capture-group", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"xxx", "foo"},
-				Separator:                    ";",
-				TargetLabel:                  "bar-$1",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "a-$1-b",
-				hasCaptureGroupInTargetLabel: true,
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  source_labels: ["xxx", "foo"]
+  target_label: "bar-$1"
+  replacement: "a-$1-b"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -171,35 +144,21 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace_all-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace_all",
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, nil, false, []prompbmarshal.Label{})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace_all",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, nil, false, []prompbmarshal.Label{})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace_all",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace_all
+  source_labels: [foo]
+  target_label: "bar"
+`, nil, false, []prompbmarshal.Label{})
+		f(`
+- action: replace_all
+  source_labels: ["foo"]
+  target_label: "bar"
+`, nil, false, []prompbmarshal.Label{})
+		f(`
+- action: replace_all
+  source_labels: ["foo"]
+  target_label: "bar"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -210,16 +169,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "yyy",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace_all",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "bar",
-				Regex:                        regexp.MustCompile(".+"),
-				Replacement:                  "$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace_all
+  source_labels: ["foo"]
+  target_label: "bar"
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -232,16 +187,13 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace_all-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "replace_all",
-				SourceLabels: []string{"xxx"},
-				Separator:    ";",
-				TargetLabel:  "xxx",
-				Regex:        regexp.MustCompile("-"),
-				Replacement:  ".",
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace_all
+  source_labels: ["xxx"]
+  target_label: "xxx"
+  regex: "-"
+  replacement: "."
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "a-b-c",
@@ -254,17 +206,13 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace_all-regex-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace_all",
-				SourceLabels:                 []string{"xxx", "foo"},
-				Separator:                    ";",
-				TargetLabel:                  "xxx",
-				Regex:                        regexp.MustCompile("(;)"),
-				Replacement:                  "-$1-",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace_all
+  source_labels: ["xxx", "foo"]
+  target_label: "xxx"
+  regex: "(;)"
+  replacement: "-$1-"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "y;y",
@@ -277,24 +225,16 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-add-multi-labels", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"xxx"},
-				TargetLabel:                  "bar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "a-$1",
-				hasCaptureGroupInReplacement: true,
-			},
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"bar"},
-				TargetLabel:                  "zar",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "b-$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  source_labels: ["xxx"]
+  target_label: "bar"
+  replacement: "a-$1"
+- action: replace
+  source_labels: ["bar"]
+  target_label: "zar"
+  replacement: "b-$1"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -323,16 +263,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-self", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:                       "replace",
-				SourceLabels:                 []string{"foo"},
-				TargetLabel:                  "foo",
-				Regex:                        defaultRegexForRelabelConfig,
-				Replacement:                  "a-$1",
-				hasCaptureGroupInReplacement: true,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: replace
+  source_labels: ["foo"]
+  target_label: "foo"
+  replacement: "a-$1"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "aaxx",
@@ -345,14 +281,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("replace-missing-source", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:      "replace",
-				TargetLabel: "foo",
-				Regex:       defaultRegexForRelabelConfig,
-				Replacement: "foobar",
-			},
-		}, []prompbmarshal.Label{}, true, []prompbmarshal.Label{
+		f(`
+- action: replace
+  target_label: foo
+  replacement: "foobar"
+`, []prompbmarshal.Label{}, true, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "foobar",
@@ -360,18 +293,14 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("keep_if_equal-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep_if_equal",
-				SourceLabels: []string{"foo", "bar"},
-			},
-		}, nil, true, nil)
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep_if_equal",
-				SourceLabels: []string{"xxx", "bar"},
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: keep_if_equal
+  source_labels: ["foo", "bar"]
+`, nil, true, nil)
+		f(`
+- action: keep_if_equal
+  source_labels: ["xxx", "bar"]
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -379,12 +308,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("keep_if_equal-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep_if_equal",
-				SourceLabels: []string{"xxx", "bar"},
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: keep_if_equal
+  source_labels: ["xxx", "bar"]
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -405,18 +332,14 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("drop_if_equal-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop_if_equal",
-				SourceLabels: []string{"foo", "bar"},
-			},
-		}, nil, true, nil)
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop_if_equal",
-				SourceLabels: []string{"xxx", "bar"},
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: drop_if_equal
+  source_labels: ["foo", "bar"]
+`, nil, true, nil)
+		f(`
+- action: drop_if_equal
+  source_labels: ["xxx", "bar"]
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -429,12 +352,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("drop_if_equal-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop_if_equal",
-				SourceLabels: []string{"xxx", "bar"},
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: drop_if_equal
+  source_labels: [xxx, bar]
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -446,20 +367,16 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("keep-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, nil, true, nil)
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: keep
+  source_labels: [foo]
+  regex: ".+"
+`, nil, true, nil)
+		f(`
+- action: keep
+  source_labels: [foo]
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -467,13 +384,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("keep-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile("yyy"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: keep
+  source_labels: [foo]
+  regex: "yyy"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -486,13 +401,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("keep-hit-regexp", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: keep
+  source_labels: ["foo"]
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -505,20 +418,16 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("drop-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, nil, false, nil)
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: drop
+  source_labels: [foo]
+  regex: ".+"
+`, nil, false, nil)
+		f(`
+- action: drop
+  source_labels: [foo]
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -531,13 +440,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("drop-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile("yyy"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: drop
+  source_labels: [foo]
+  regex: yyy
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -545,13 +452,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("drop-hit-regexp", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"foo"},
-				Regex:        regexp.MustCompile(".+"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: drop
+  source_labels: [foo]
+  regex: ".+"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -559,14 +464,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		}, true, []prompbmarshal.Label{})
 	})
 	t.Run("hashmod-miss", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "hashmod",
-				SourceLabels: []string{"foo"},
-				TargetLabel:  "aaa",
-				Modulus:      123,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: hashmod
+  source_labels: [foo]
+  target_label: aaa
+  modulus: 123
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -583,14 +486,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("hashmod-hit", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:       "hashmod",
-				SourceLabels: []string{"foo"},
-				TargetLabel:  "aaa",
-				Modulus:      123,
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: hashmod
+  source_labels: [foo]
+  target_label: aaa
+  modulus: 123
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -606,14 +507,97 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		})
 	})
-	t.Run("labelmap", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
+	t.Run("labelmap-copy-label", func(t *testing.T) {
+		f(`
+- action: labelmap
+  regex: "foo"
+  replacement: "bar"
+`, []prompbmarshal.Label{
 			{
-				Action:      "labelmap",
-				Regex:       regexp.MustCompile("foo(.+)"),
-				Replacement: "$1-x",
+				Name:  "foo",
+				Value: "yyy",
 			},
-		}, []prompbmarshal.Label{
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "bar",
+				Value: "yyy",
+			},
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+		})
+	})
+	t.Run("labelmap-remove-prefix-dot-star", func(t *testing.T) {
+		f(`
+- action: labelmap
+  regex: "foo(.*)"
+`, []prompbmarshal.Label{
+			{
+				Name:  "xoo",
+				Value: "yyy",
+			},
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "bar",
+				Value: "aaa",
+			},
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+			{
+				Name:  "xoo",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("labelmap-remove-prefix-dot-plus", func(t *testing.T) {
+		f(`
+- action: labelmap
+  regex: "foo(.+)"
+`, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "bar",
+				Value: "aaa",
+			},
+			{
+				Name:  "foo",
+				Value: "yyy",
+			},
+			{
+				Name:  "foobar",
+				Value: "aaa",
+			},
+		})
+	})
+	t.Run("labelmap-regex", func(t *testing.T) {
+		f(`
+- action: labelmap
+  regex: "foo(.+)"
+  replacement: "$1-x"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo",
 				Value: "yyy",
@@ -638,13 +622,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labelmap_all", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:      "labelmap_all",
-				Regex:       regexp.MustCompile(`\.`),
-				Replacement: "-",
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelmap_all
+  regex: "\\."
+  replacement: "-"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo.bar.baz",
 				Value: "yyy",
@@ -665,13 +647,11 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labelmap_all-regexp", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action:      "labelmap_all",
-				Regex:       regexp.MustCompile(`ba(.)`),
-				Replacement: "${1}ss",
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelmap_all
+  regex: "ba(.)"
+  replacement: "${1}ss"
+`, []prompbmarshal.Label{
 			{
 				Name:  "foo.bar.baz",
 				Value: "yyy",
@@ -692,12 +672,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labeldrop", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("dropme"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labeldrop
+  regex: dropme
+`, []prompbmarshal.Label{
 			{
 				Name:  "aaa",
 				Value: "bbb",
@@ -708,12 +686,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "bbb",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("dropme"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labeldrop
+  regex: dropme
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -738,12 +714,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labeldrop-regexp", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("dropme.*"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labeldrop
+  regex: "dropme.*"
+`, []prompbmarshal.Label{
 			{
 				Name:  "aaa",
 				Value: "bbb",
@@ -754,12 +728,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "bbb",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("dropme.*"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labeldrop
+  regex: "dropme.*"
+`, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
 				Value: "yyy",
@@ -784,12 +756,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labelkeep", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("keepme"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelkeep
+  regex: "keepme"
+`, []prompbmarshal.Label{
 			{
 				Name:  "keepme",
 				Value: "aaa",
@@ -800,12 +770,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "aaa",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("keepme"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelkeep
+  regex: keepme
+`, []prompbmarshal.Label{
 			{
 				Name:  "keepme",
 				Value: "aaa",
@@ -826,12 +794,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 	})
 	t.Run("labelkeep-regexp", func(t *testing.T) {
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("keepme.*"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelkeep
+  regex: "keepme.*"
+`, []prompbmarshal.Label{
 			{
 				Name:  "keepme",
 				Value: "aaa",
@@ -842,12 +808,10 @@ func TestApplyRelabelConfigs(t *testing.T) {
 				Value: "aaa",
 			},
 		})
-		f([]ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("keepme.*"),
-			},
-		}, []prompbmarshal.Label{
+		f(`
+- action: labelkeep
+  regex: "keepme.*"
+`, []prompbmarshal.Label{
 			{
 				Name:  "keepme",
 				Value: "aaa",
diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go
index fb0a0207c..4904bb0ef 100644
--- a/lib/promrelabel/relabel_timing_test.go
+++ b/lib/promrelabel/relabel_timing_test.go
@@ -2,7 +2,6 @@ package promrelabel
 
 import (
 	"fmt"
-	"regexp"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -10,15 +9,11 @@ import (
 
 func BenchmarkApplyRelabelConfigs(b *testing.B) {
 	b.Run("replace-label-copy", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "replace",
-				SourceLabels: []string{"id"},
-				TargetLabel:  "__name__",
-				Regex:        defaultRegexForRelabelConfig,
-				Replacement:  "$1",
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: replace
+  source_labels: [id]
+  target_label: __name__
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -35,7 +30,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -55,14 +50,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("replace-set-label", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:      "replace",
-				TargetLabel: "__name__",
-				Regex:       defaultRegexForRelabelConfig,
-				Replacement: "foobar",
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: replace
+  target_label: __name__
+  replacement: foobar
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -79,7 +71,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -99,14 +91,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("replace-add-label", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:      "replace",
-				TargetLabel: "aaa",
-				Regex:       defaultRegexForRelabelConfig,
-				Replacement: "foobar",
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: replace
+  target_label: aaa
+  replacement: foobar
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -119,7 +108,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 2 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 2, labels))
 				}
@@ -139,15 +128,12 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("replace-mismatch", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "replace",
-				SourceLabels: []string{"non-existing-label"},
-				TargetLabel:  "id",
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-				Replacement:  "$1",
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: replace
+  source_labels: ["non-existing-label"]
+  target_label: id
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -164,7 +150,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -184,15 +170,12 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("replace-match-regex", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "replace",
-				SourceLabels: []string{"id"},
-				TargetLabel:  "id",
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-				Replacement:  "$1",
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: replace
+  source_labels: [id]
+  target_label: id
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -209,7 +192,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -229,13 +212,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("drop-mismatch", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"non-existing-label"},
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: drop
+  source_labels: ["non-existing-label"]
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -252,7 +233,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -272,13 +253,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("drop-match", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"id"},
-				Regex:        regexp.MustCompile("yes"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: drop
+  source_labels: [id]
+  regex: yes
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -295,7 +274,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 0 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
 				}
@@ -303,13 +282,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("drop-match-regexp", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"id"},
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: drop
+  source_labels: [id]
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -326,7 +303,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 0 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
 				}
@@ -334,13 +311,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("keep-mismatch", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"non-existing-label"},
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: keep
+  source_labels: ["non-existing-label"]
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -357,7 +332,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 0 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
 				}
@@ -365,13 +340,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("keep-match", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"id"},
-				Regex:        regexp.MustCompile("yes"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: keep
+  source_labels: [id]
+  regex: yes
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -388,7 +361,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -408,13 +381,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("keep-match-regexp", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "keep",
-				SourceLabels: []string{"id"},
-				Regex:        regexp.MustCompile("(foobar)-.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: keep
+  source_labels: [id]
+  regex: "(foobar)-.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -431,7 +402,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -451,12 +422,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labeldrop-mismatch", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("non-existing-label"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labeldrop
+  regex: "non-existing-label"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -473,7 +442,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -493,12 +462,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labeldrop-match", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("id"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labeldrop
+  regex: id
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -515,7 +482,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 1 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
 				}
@@ -529,12 +496,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labeldrop-match-regexp", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("id.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labeldrop
+  regex: "id.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -551,7 +516,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 1 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
 				}
@@ -565,12 +530,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labelkeep-mismatch", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("non-existing-label"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labelkeep
+  regex: "non-existing-label"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -587,7 +550,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 0 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
 				}
@@ -595,12 +558,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labelkeep-match", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("id"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labelkeep
+  regex: id
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -617,7 +578,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 1 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
 				}
@@ -631,12 +592,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 	b.Run("labelkeep-match-regexp", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action: "labelkeep",
-				Regex:  regexp.MustCompile("id.*"),
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: labelkeep
+  regex: "id.*"
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -653,7 +612,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != 1 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
 				}
@@ -666,19 +625,12 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
-	b.Run("labelmap", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:      "labelmap",
-				Regex:       regexp.MustCompile("a(.*)"),
-				Replacement: "$1",
-			},
-		}
+	b.Run("labelmap-mismatch", func(b *testing.B) {
+		pcs := mustParseRelabelConfigs(`
+- action: labelmap
+  regex: "a(.*)"
+`)
 		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "aabc",
-				Value: "foobar-random-string-here",
-			},
 			{
 				Name:  "foo",
 				Value: "bar",
@@ -690,8 +642,38 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
-				if len(labels) != 3 {
+				labels = pcs.Apply(labels, 0, true)
+				if len(labels) != 1 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 3, labels))
+				}
+				if labels[0].Name != "foo" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "foo"))
+				}
+				if labels[0].Value != "bar" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "bar"))
+				}
+			}
+		})
+	})
+	b.Run("labelmap-match-remove-prefix", func(b *testing.B) {
+		pcs := mustParseRelabelConfigs(`
+- action: labelmap
+  regex: "a(.*)"
+`)
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "aabc",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = pcs.Apply(labels, 0, true)
+				if len(labels) != 2 {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 3, labels))
 				}
 				if labels[0].Name != "aabc" {
@@ -706,24 +688,52 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 				if labels[1].Value != "foobar-random-string-here" {
 					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "foobar-random-string-here"))
 				}
-				if labels[2].Name != "foo" {
-					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[2].Name, "foo"))
+			}
+		})
+	})
+	b.Run("labelmap-match-regexp", func(b *testing.B) {
+		pcs := mustParseRelabelConfigs(`
+- action: labelmap
+  regex: "(.*)bc"
+`)
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "aabc",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = pcs.Apply(labels, 0, true)
+				if len(labels) != 2 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 3, labels))
 				}
-				if labels[2].Value != "bar" {
-					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[2].Value, "bar"))
+				if labels[0].Name != "aa" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "aa"))
+				}
+				if labels[0].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
+				}
+				if labels[1].Name != "aabc" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[1].Name, "aabc"))
+				}
+				if labels[1].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "foobar-random-string-here"))
 				}
 			}
 		})
 	})
 	b.Run("hashmod", func(b *testing.B) {
-		prcs := []ParsedRelabelConfig{
-			{
-				Action:       "hashmod",
-				SourceLabels: []string{"id"},
-				TargetLabel:  "id",
-				Modulus:      23,
-			},
-		}
+		pcs := mustParseRelabelConfigs(`
+- action: hashmod
+  source_labels: [id]
+  target_label: id
+  modulus: 23
+`)
 		labelsOrig := []prompbmarshal.Label{
 			{
 				Name:  "__name__",
@@ -740,7 +750,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			var labels []prompbmarshal.Label
 			for pb.Next() {
 				labels = append(labels[:0], labelsOrig...)
-				labels = ApplyRelabelConfigs(labels, 0, prcs, true)
+				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
 					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
 				}
@@ -760,3 +770,11 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 		})
 	})
 }
+
+func mustParseRelabelConfigs(config string) *ParsedConfigs {
+	pcs, err := ParseRelabelConfigsData([]byte(config))
+	if err != nil {
+		panic(fmt.Errorf("unexpected error: %w", err))
+	}
+	return pcs
+}
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 6c1e2f502..4f97506b6 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -482,13 +482,11 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config for `job_name` %q: %w", jobName, err)
 	}
-	var relabelConfigs []promrelabel.ParsedRelabelConfig
-	relabelConfigs, err = promrelabel.ParseRelabelConfigs(relabelConfigs[:0], sc.RelabelConfigs)
+	relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
-	var metricRelabelConfigs []promrelabel.ParsedRelabelConfig
-	metricRelabelConfigs, err = promrelabel.ParseRelabelConfigs(metricRelabelConfigs[:0], sc.MetricRelabelConfigs)
+	metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
@@ -527,8 +525,8 @@ type scrapeWorkConfig struct {
 	honorLabels          bool
 	honorTimestamps      bool
 	externalLabels       map[string]string
-	relabelConfigs       []promrelabel.ParsedRelabelConfig
-	metricRelabelConfigs []promrelabel.ParsedRelabelConfig
+	relabelConfigs       *promrelabel.ParsedConfigs
+	metricRelabelConfigs *promrelabel.ParsedConfigs
 	sampleLimit          int
 	disableCompression   bool
 	disableKeepAlive     bool
@@ -695,7 +693,7 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 		// Reduce memory usage by interning all the strings in originalLabels.
 		internLabelStrings(originalLabels)
 	}
-	labels = promrelabel.ApplyRelabelConfigs(labels, 0, swc.relabelConfigs, false)
+	labels = swc.relabelConfigs.Apply(labels, 0, false)
 	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
 	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
 	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go
index a1dbc5f57..f2273ccc4 100644
--- a/lib/promscrape/config_test.go
+++ b/lib/promscrape/config_test.go
@@ -4,13 +4,11 @@ import (
 	"crypto/tls"
 	"fmt"
 	"reflect"
-	"regexp"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestLoadStaticConfigs(t *testing.T) {
@@ -1034,13 +1032,6 @@ scrape_configs:
 		},
 	})
 
-	prcs, err := promrelabel.ParseRelabelConfigs(nil, []promrelabel.RelabelConfig{{
-		SourceLabels: []string{"foo"},
-		TargetLabel:  "abc",
-	}})
-	if err != nil {
-		t.Fatalf("unexpected error when parsing relabel configs: %s", err)
-	}
 	f(`
 scrape_configs:
 - job_name: foo
@@ -1076,9 +1067,12 @@ scrape_configs:
 					Value: "foo",
 				},
 			},
-			AuthConfig:           &promauth.Config{},
-			MetricRelabelConfigs: prcs,
-			jobNameOriginal:      "foo",
+			AuthConfig: &promauth.Config{},
+			MetricRelabelConfigs: mustParseRelabelConfigs(`
+- source_labels: [foo]
+  target_label: abc
+`),
+			jobNameOriginal: "foo",
 		},
 	})
 	f(`
@@ -1374,8 +1368,6 @@ scrape_configs:
 	})
 }
 
-var defaultRegexForRelabelConfig = regexp.MustCompile("^(.*)$")
-
 func equalStaticConfigForScrapeWorks(a, b []*ScrapeWork) bool {
 	if len(a) != len(b) {
 		return false
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index adff00c2c..313734d6b 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -6,7 +6,6 @@ import (
 	"math"
 	"math/bits"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -76,7 +75,7 @@ type ScrapeWork struct {
 	ProxyURL proxy.URL
 
 	// Optional `metric_relabel_configs`.
-	MetricRelabelConfigs []promrelabel.ParsedRelabelConfig
+	MetricRelabelConfigs *promrelabel.ParsedConfigs
 
 	// The maximum number of metrics to scrape after relabeling.
 	SampleLimit int
@@ -105,18 +104,10 @@ func (sw *ScrapeWork) key() string {
 	key := fmt.Sprintf("ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, Labels=%s, "+
 		"AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, ScrapeAlignInterval=%s",
 		sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.LabelsString(),
-		sw.AuthConfig.String(), sw.metricRelabelConfigsString(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.ScrapeAlignInterval)
+		sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.ScrapeAlignInterval)
 	return key
 }
 
-func (sw *ScrapeWork) metricRelabelConfigsString() string {
-	var sb strings.Builder
-	for _, prc := range sw.MetricRelabelConfigs {
-		fmt.Fprintf(&sb, "%s", prc.String())
-	}
-	return sb.String()
-}
-
 // Job returns job for the ScrapeWork
 func (sw *ScrapeWork) Job() string {
 	return promrelabel.GetLabelValueByName(sw.Labels, "job")
@@ -503,7 +494,7 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim
 	labelsLen := len(wc.labels)
 	wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels)
 	if needRelabel {
-		wc.labels = promrelabel.ApplyRelabelConfigs(wc.labels, labelsLen, sw.Config.MetricRelabelConfigs, true)
+		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true)
 	} else {
 		wc.labels = promrelabel.FinalizeLabels(wc.labels[:labelsLen], wc.labels[labelsLen:])
 		promrelabel.SortLabels(wc.labels[labelsLen:])
diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go
index 3e9fb9577..74b6f67e8 100644
--- a/lib/promscrape/scrapework_test.go
+++ b/lib/promscrape/scrapework_test.go
@@ -2,7 +2,6 @@ package promscrape
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
 	"testing"
 
@@ -102,7 +101,8 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) {
 		sw.PushData = func(wr *prompbmarshal.WriteRequest) {
 			pushDataCalls++
 			if len(wr.Timeseries) > len(timeseriesExpected) {
-				pushDataErr = fmt.Errorf("too many time series obtained; got %d; want %d", len(wr.Timeseries), len(timeseriesExpected))
+				pushDataErr = fmt.Errorf("too many time series obtained; got %d; want %d\ngot\n%+v\nwant\n%+v",
+					len(wr.Timeseries), len(timeseriesExpected), wr.Timeseries, timeseriesExpected)
 				return
 			}
 			tsExpected := timeseriesExpected[:len(wr.Timeseries)]
@@ -271,20 +271,14 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) {
 				Value: "foo.com",
 			},
 		},
-		MetricRelabelConfigs: []promrelabel.ParsedRelabelConfig{
-			{
-				SourceLabels: []string{"__address__", "job"},
-				Separator:    "/",
-				TargetLabel:  "instance",
-				Regex:        defaultRegexForRelabelConfig,
-				Replacement:  "$1",
-				Action:       "replace",
-			},
-			{
-				Action: "labeldrop",
-				Regex:  regexp.MustCompile("^c$"),
-			},
-		},
+		MetricRelabelConfigs: mustParseRelabelConfigs(`
+- action: replace
+  source_labels: ["__address__", "job"]
+  separator: "/"
+  target_label: "instance"
+- action: labeldrop
+  regex: c
+`),
 	}, `
 		foo{bar="baz",job="xx",instance="foo.com/xx"} 34.44 123
 		bar{a="b",job="xx",instance="foo.com/xx"} -3e4 123
@@ -311,18 +305,15 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) {
 				Value: "foo.com",
 			},
 		},
-		MetricRelabelConfigs: []promrelabel.ParsedRelabelConfig{
-			{
-				Action:       "drop",
-				SourceLabels: []string{"a", "c"},
-				Regex:        regexp.MustCompile("^bd$"),
-			},
-			{
-				Action:       "drop",
-				SourceLabels: []string{"__name__"},
-				Regex:        regexp.MustCompile("^(dropme|up)$"),
-			},
-		},
+		MetricRelabelConfigs: mustParseRelabelConfigs(`
+- action: drop
+  separator: ""
+  source_labels: [a, c]
+  regex: "^bd$"
+- action: drop
+  source_labels: [__name__]
+  regex: "dropme|up"
+`),
 	}, `
 		foo{bar="baz",job="xx",instance="foo.com"} 34.44 123
 		up{job="xx",instance="foo.com"} 1 123
@@ -440,3 +431,11 @@ func timeseriesToString(ts *prompbmarshal.TimeSeries) string {
 	fmt.Fprintf(&sb, "%g %d", s.Value, s.Timestamp)
 	return sb.String()
 }
+
+func mustParseRelabelConfigs(config string) *promrelabel.ParsedConfigs {
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config))
+	if err != nil {
+		panic(fmt.Errorf("cannot parse %q: %w", config, err))
+	}
+	return pcs
+}

From 2549469d5d022a06ca57162bb0e773a337839a01 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Mon, 22 Feb 2021 18:29:23 +0200
Subject: [PATCH 14/32] app/vmselect/graphite: support `Storage-Step` header
 value and `storage_step` query arg at /render API

---
 docs/Single-server-VictoriaMetrics.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 54526ccdb..7aba95a44 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -605,7 +605,8 @@ and it is easier to use when migrating from Graphite to VictoriaMetrics.
 ### Graphite Render API usage
 
 [VictoriaMetrics Enterprise](https://victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
-at `/render` endpoint. This subset is required for [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
+at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
+It supports `Storage-Step` http request header, which must be set to a step between data points stored in VictoriaMetrics when configuring Graphite datasource in Grafana.
 
 
 ### Graphite Metrics API usage

From fa03e0d21031c8de455eece2f4af792e28058239 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Mon, 22 Feb 2021 19:12:12 +0200
Subject: [PATCH 15/32] app/vmselect/promql: add `increase_pure()` function to
 MetricsQL

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962
---
 app/vmselect/promql/exec_test.go              | 11 +++++++++
 app/vmselect/promql/rollup.go                 | 23 +++++++++++++++++++
 app/vmselect/promql/rollup_test.go            |  1 +
 docs/CHANGELOG.md                             |  1 +
 docs/MetricsQL.md                             |  1 +
 go.mod                                        |  2 +-
 go.sum                                        |  4 ++--
 .../VictoriaMetrics/metricsql/rollup.go       |  1 +
 vendor/modules.txt                            |  2 +-
 9 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 6c3cd65df..8ed198353 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -5252,6 +5252,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`increase_pure(time())`, func(t *testing.T) {
+		t.Parallel()
+		q := `increase_pure(time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increase(time())`, func(t *testing.T) {
 		t.Parallel()
 		q := `increase(time())`
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 3c62fbdc7..c7f973a82 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -53,6 +53,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"distinct_over_time":    newRollupFuncOneArg(rollupDistinct),
 	"increases_over_time":   newRollupFuncOneArg(rollupIncreases),
 	"decreases_over_time":   newRollupFuncOneArg(rollupDecreases),
+	"increase_pure":         newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
 	"integrate":             newRollupFuncOneArg(rollupIntegrate),
 	"ideriv":                newRollupFuncOneArg(rollupIderiv),
 	"lifetime":              newRollupFuncOneArg(rollupLifetime),
@@ -123,6 +124,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"distinct_over_time":  rollupDistinct,
 	"increases_over_time": rollupIncreases,
 	"decreases_over_time": rollupDecreases,
+	"increase_pure":       rollupIncreasePure,
 	"integrate":           rollupIntegrate,
 	"ideriv":              rollupIderiv,
 	"lifetime":            rollupLifetime,
@@ -160,6 +162,7 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{
 	"distinct_over_time":  true,
 	"increases_over_time": true,
 	"decreases_over_time": true,
+	"increase_pure":       true,
 	"integrate":           true,
 	"ascent_over_time":    true,
 	"descent_over_time":   true,
@@ -172,6 +175,7 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 	"rate":            true,
 	"rollup_rate":     true,
 	"rollup_increase": true,
+	"increase_pure":   true,
 }
 
 var rollupFuncsKeepMetricGroup = map[string]bool{
@@ -1323,6 +1327,25 @@ func rollupStdvar(rfa *rollupFuncArg) float64 {
 	return q / count
 }
 
+func rollupIncreasePure(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	prevValue := rfa.prevValue
+	if math.IsNaN(prevValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		// Assume the counter starts from 0.
+		prevValue = 0
+	}
+	if len(values) == 0 {
+		// Assume the counter didsn't change since prevValue.
+		return 0
+	}
+	return values[len(values)-1] - prevValue
+}
+
 func rollupDelta(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go
index 2820b5b7f..4b5eb540f 100644
--- a/app/vmselect/promql/rollup_test.go
+++ b/app/vmselect/promql/rollup_test.go
@@ -476,6 +476,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("ideriv", 0)
 	f("decreases_over_time", 5)
 	f("increases_over_time", 5)
+	f("increase_pure", 398)
 	f("ascent_over_time", 142)
 	f("descent_over_time", 231)
 	f("zscore_over_time", -0.4254336383156416)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index cb42435f6..a8ddf8c3e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -12,6 +12,7 @@
   * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
 * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
 * FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
+* FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
 
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index 65d75612c..15bf2638c 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -72,6 +72,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
 - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
 - `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`.
+- `increase_pure(m[d])` - for calculating increase of `m` over `d` without edge-case handling compared to `increase(m[d])`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
 - `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`.
 - `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range.
 - `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details.
diff --git a/go.mod b/go.mod
index a6fcfe489..fe007c0dd 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.12
 	github.com/VictoriaMetrics/metrics v1.15.0
-	github.com/VictoriaMetrics/metricsql v0.10.1
+	github.com/VictoriaMetrics/metricsql v0.11.0
 	github.com/aws/aws-sdk-go v1.37.12
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/cheggaaa/pb/v3 v3.0.6
diff --git a/go.sum b/go.sum
index 9de56d296..343c504a0 100644
--- a/go.sum
+++ b/go.sum
@@ -87,8 +87,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js=
 github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.10.1 h1:wLl/YbMmBGFPyLKMfqNLC333iygibosSM5iSvlH2B4A=
-github.com/VictoriaMetrics/metricsql v0.10.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.11.0 h1:85zbY8NxWNALppctOAkWfLDC7dDFcvTEn5IHMKwOGag=
+github.com/VictoriaMetrics/metricsql v0.11.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
index 80cfe58f9..085ee96c7 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
@@ -38,6 +38,7 @@ var rollupFuncs = map[string]bool{
 	"distinct_over_time":    true,
 	"increases_over_time":   true,
 	"decreases_over_time":   true,
+	"increase_pure":         true,
 	"integrate":             true,
 	"ideriv":                true,
 	"lifetime":              true,
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 61b2281b4..fa5b55f16 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.15.0
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.10.1
+# github.com/VictoriaMetrics/metricsql v0.11.0
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/VividCortex/ewma v1.1.1

From 15d61c48791bf1a43e99c09ce7703b5ee0abf8e5 Mon Sep 17 00:00:00 2001
From: faceair <git@faceair.me>
Date: Tue, 23 Feb 2021 03:46:56 +0800
Subject: [PATCH 16/32] lib/storage: correct tagfilter match cost (#1079)

---
 lib/storage/index_db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index a7b8521f1..6ff77048f 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -2458,7 +2458,7 @@ func (is *indexSearch) getMetricIDsForTagFilterSlow(tf *tagFilter, filter *uint6
 		}
 		// Slow path: need tf.matchSuffix call.
 		ok, err := tf.matchSuffix(suffix)
-		loopsCount += reMatchCost
+		loopsCount += tf.matchCost
 		if err != nil {
 			return loopsCount, fmt.Errorf("error when matching %s against suffix %q: %w", tf, suffix, err)
 		}

From 2c44178645bbaf7bc3543f65b95f3c9ded3393c6 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 23 Feb 2021 15:47:19 +0200
Subject: [PATCH 17/32] lib/storage: consistency renaming:
 durationsPerDateTagFilterCache -> loopsPerDateTagFilterCache

---
 lib/storage/index_db.go | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 6ff77048f..2aea6a621 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -104,9 +104,9 @@ type indexDB struct {
 	// matching low number of metrics.
 	uselessTagFiltersCache *workingsetcache.Cache
 
-	// Cache for (date, tagFilter) -> filterDuration, which is used for reducing
+	// Cache for (date, tagFilter) -> loopsCount, which is used for reducing
 	// the amount of work when matching a set of filters.
-	durationsPerDateTagFilterCache *workingsetcache.Cache
+	loopsPerDateTagFilterCache *workingsetcache.Cache
 
 	indexSearchPool sync.Pool
 
@@ -150,12 +150,12 @@ func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *working
 		tb:       tb,
 		name:     name,
 
-		tagCache:                       workingsetcache.New(mem/32, time.Hour),
-		metricIDCache:                  metricIDCache,
-		metricNameCache:                metricNameCache,
-		tsidCache:                      tsidCache,
-		uselessTagFiltersCache:         workingsetcache.New(mem/128, time.Hour),
-		durationsPerDateTagFilterCache: workingsetcache.New(mem/128, time.Hour),
+		tagCache:                   workingsetcache.New(mem/32, time.Hour),
+		metricIDCache:              metricIDCache,
+		metricNameCache:            metricNameCache,
+		tsidCache:                  tsidCache,
+		uselessTagFiltersCache:     workingsetcache.New(mem/128, 3*time.Hour),
+		loopsPerDateTagFilterCache: workingsetcache.New(mem/128, 3*time.Hour),
 
 		minTimestampForCompositeIndex: minTimestampForCompositeIndex,
 	}
@@ -320,14 +320,14 @@ func (db *indexDB) decRef() {
 	// Free space occupied by caches owned by db.
 	db.tagCache.Stop()
 	db.uselessTagFiltersCache.Stop()
-	db.durationsPerDateTagFilterCache.Stop()
+	db.loopsPerDateTagFilterCache.Stop()
 
 	db.tagCache = nil
 	db.metricIDCache = nil
 	db.metricNameCache = nil
 	db.tsidCache = nil
 	db.uselessTagFiltersCache = nil
-	db.durationsPerDateTagFilterCache = nil
+	db.loopsPerDateTagFilterCache = nil
 
 	if atomic.LoadUint64(&db.mustDrop) == 0 {
 		return
@@ -3105,7 +3105,7 @@ func (is *indexSearch) getLoopsCountAndTimestampForDateFilter(date uint64, tf *t
 	is.kb.B = appendDateTagFilterCacheKey(is.kb.B[:0], date, tf)
 	kb := kbPool.Get()
 	defer kbPool.Put(kb)
-	kb.B = is.db.durationsPerDateTagFilterCache.Get(kb.B[:0], is.kb.B)
+	kb.B = is.db.loopsPerDateTagFilterCache.Get(kb.B[:0], is.kb.B)
 	if len(kb.B) != 16 {
 		return 0, 0
 	}
@@ -3125,7 +3125,7 @@ func (is *indexSearch) storeLoopsCountForDateFilter(date uint64, tf *tagFilter,
 	kb := kbPool.Get()
 	kb.B = encoding.MarshalUint64(kb.B[:0], loopsCount)
 	kb.B = encoding.MarshalUint64(kb.B, currentTimestamp)
-	is.db.durationsPerDateTagFilterCache.Set(is.kb.B, kb.B)
+	is.db.loopsPerDateTagFilterCache.Set(is.kb.B, kb.B)
 	kbPool.Put(kb)
 }
 

From f4135b0d149035a7019a7cee37fe8643129e0e05 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 24 Feb 2021 00:40:16 +0200
Subject: [PATCH 18/32] app/vmselect/promql: properly calculate
 `histogram_quantile() over zero buckets and only a single non-zero
 `le="+Inf"` bucket like Prometheus does

---
 app/vmselect/promql/exec_test.go | 20 ++++++++++++++++++++
 app/vmselect/promql/transform.go |  8 +-------
 docs/CHANGELOG.md                |  1 +
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 8ed198353..f12fbbda0 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -2757,6 +2757,26 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0.6, label_set(100, "le", "+Inf"))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0.6, (
+			label_set(100, "le", "+Inf"),
+			label_set(0, "le", "42"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{42, 42, 42, 42, 42, 42},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "le", "200"))`
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index 6d1b01cbd..20961e409 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -649,14 +649,9 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	m := groupLeTimeseries(tss)
 
 	// Calculate quantile for each group in m
-
 	lastNonInf := func(i int, xss []leTimeseries) float64 {
 		for len(xss) > 0 {
 			xsLast := xss[len(xss)-1]
-			v := xsLast.ts.Values[i]
-			if v == 0 {
-				return nan
-			}
 			if !math.IsInf(xsLast.le, 0) {
 				return xsLast.le
 			}
@@ -700,8 +695,7 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 				continue
 			}
 			if math.IsInf(le, 0) {
-				vv := lastNonInf(i, xss)
-				return vv, vv, inf
+				break
 			}
 			if v == vPrev {
 				return lePrev, lePrev, v
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a8ddf8c3e..663f6cbed 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -17,6 +17,7 @@
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
 * BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
+* BUGFIX: properly calculate `histogram_quantile()` over time series with only a single non-zero bucket with `{le="+Inf"}`. Previously `NaN` was returned, now the value for the last bucket before `{le="+Inf"}` is returned like Prometheus does.
 
 
 # [v1.54.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.1)

From f050e3f4926e2dad131675debedd7784facc88ee Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 24 Feb 2021 11:48:30 +0200
Subject: [PATCH 19/32] docs/CHANGELOG.md: mention about a bugfix from
 4805b80977b7992d3f49e9be71726bc0c7690363

---
 docs/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 663f6cbed..192a75ff5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -18,6 +18,7 @@
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065
 * BUGFIX: reduce the probability of `duplicate time series` errors when querying Kubernetes metrics.
 * BUGFIX: properly calculate `histogram_quantile()` over time series with only a single non-zero bucket with `{le="+Inf"}`. Previously `NaN` was returned, now the value for the last bucket before `{le="+Inf"}` is returned like Prometheus does.
+* BUGFIX: vmselect: do not cache partial query results on timeout when receiving data from `vmstorage` nodes. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1085
 
 
 # [v1.54.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.1)

From 2cd23362f5d311656d180326378c84f5f87d7098 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 24 Feb 2021 17:24:18 +0200
Subject: [PATCH 20/32] README.md: sync with
 docs/Single-server-Victoria-Metrics.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 54526ccdb..7aba95a44 100644
--- a/README.md
+++ b/README.md
@@ -605,7 +605,8 @@ and it is easier to use when migrating from Graphite to VictoriaMetrics.
 ### Graphite Render API usage
 
 [VictoriaMetrics Enterprise](https://victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
-at `/render` endpoint. This subset is required for [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
+at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
+It supports `Storage-Step` http request header, which must be set to a step between data points stored in VictoriaMetrics when configuring Graphite datasource in Grafana.
 
 
 ### Graphite Metrics API usage

From 98854e5f2b80dcc8e50bc16271ceef27ec24c284 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 24 Feb 2021 17:24:54 +0200
Subject: [PATCH 21/32] app/vmselect: add `sign(q)` and `clamp(q, min, max)`
 functions, which will be added in the upcoming Prometheus release

See https://twitter.com/roidelapluie/status/1363428376162295811

The `last_over_time(m[d])` function already exists in MetricsQL.
---
 app/vmselect/promql/exec_test.go              | 26 +++++++++-
 app/vmselect/promql/transform.go              | 47 +++++++++++++++++++
 docs/CHANGELOG.md                             |  1 +
 go.mod                                        |  2 +-
 go.sum                                        |  4 +-
 .../VictoriaMetrics/metricsql/transform.go    |  2 +
 vendor/modules.txt                            |  2 +-
 7 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index f12fbbda0..0bc3efc56 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -673,6 +673,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("clamp(time(), 1400, 1800)", func(t *testing.T) {
+		t.Parallel()
+		q := `clamp(time(), 1400, 1800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1400, 1400, 1400, 1600, 1800, 1800},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("clamp_max(time(), 1400)", func(t *testing.T) {
 		t.Parallel()
 		q := `clamp_max(time(), 1400)`
@@ -1716,6 +1727,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`sign(time()-1400)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sign(time()-1400)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{-1, -1, 0, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`round(time()/1e3)`, func(t *testing.T) {
 		t.Parallel()
 		q := `round(time()/1e3)`
@@ -2763,7 +2785,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) {
+	t.Run(`histogram_quantile(zero-value-inf-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, (
 			label_set(100, "le", "+Inf"),
@@ -6307,6 +6329,7 @@ func TestExecError(t *testing.T) {
 	f(`abs()`)
 	f(`abs(1,2)`)
 	f(`absent(1, 2)`)
+	f(`clamp()`)
 	f(`clamp_max()`)
 	f(`clamp_min(1,2,3)`)
 	f(`hour(1,2)`)
@@ -6323,6 +6346,7 @@ func TestExecError(t *testing.T) {
 	f(`label_mismatch()`)
 	f(`round()`)
 	f(`round(1,2,3)`)
+	f(`sign()`)
 	f(`scalar()`)
 	f(`sort(1,2)`)
 	f(`sort_desc()`)
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index 20961e409..a893ce2e4 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -19,6 +19,7 @@ import (
 
 var transformFuncsKeepMetricGroup = map[string]bool{
 	"ceil":               true,
+	"clamp":              true,
 	"clamp_max":          true,
 	"clamp_min":          true,
 	"floor":              true,
@@ -44,6 +45,7 @@ var transformFuncs = map[string]transformFunc{
 	"abs":                newTransformFuncOneArg(transformAbs),
 	"absent":             transformAbsent,
 	"ceil":               newTransformFuncOneArg(transformCeil),
+	"clamp":              transformClamp,
 	"clamp_max":          transformClampMax,
 	"clamp_min":          transformClampMin,
 	"day_of_month":       newTransformFuncDateTime(transformDayOfMonth),
@@ -61,6 +63,7 @@ var transformFuncs = map[string]transformFunc{
 	"minute":             newTransformFuncDateTime(transformMinute),
 	"month":              newTransformFuncDateTime(transformMonth),
 	"round":              transformRound,
+	"sign":               transformSign,
 	"scalar":             transformScalar,
 	"sort":               newTransformFuncSort(false),
 	"sort_desc":          newTransformFuncSort(true),
@@ -215,6 +218,31 @@ func transformCeil(v float64) float64 {
 	return math.Ceil(v)
 }
 
+func transformClamp(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 3); err != nil {
+		return nil, err
+	}
+	mins, err := getScalar(args[1], 1)
+	if err != nil {
+		return nil, err
+	}
+	maxs, err := getScalar(args[2], 2)
+	if err != nil {
+		return nil, err
+	}
+	tf := func(values []float64) {
+		for i, v := range values {
+			if v > maxs[i] {
+				values[i] = maxs[i]
+			} else if v < mins[i] {
+				values[i] = mins[i]
+			}
+		}
+	}
+	return doTransformValues(args[0], tf, tfa.fe)
+}
+
 func transformClampMax(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
@@ -1569,6 +1597,25 @@ func transformRound(tfa *transformFuncArg) ([]*timeseries, error) {
 	return doTransformValues(args[0], tf, tfa.fe)
 }
 
+func transformSign(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 1); err != nil {
+		return nil, err
+	}
+	tf := func(values []float64) {
+		for i, v := range values {
+			sign := float64(0)
+			if v < 0 {
+				sign = -1
+			} else if v > 0 {
+				sign = 1
+			}
+			values[i] = sign
+		}
+	}
+	return doTransformValues(args[0], tf, tfa.fe)
+}
+
 func transformScalar(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if err := expectTransformArgsNum(args, 1); err != nil {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 192a75ff5..c0c85e44a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 # tip
 
+* FEATURE: add `sign(q)` and `clamp(q, min, max)` functions, which are planned to be added in [the upcoming Prometheus release](https://twitter.com/roidelapluie/status/1363428376162295811) . The `last_over_time(m[d])` function is already supported in [MetricsQL](https://victoriametrics.github.io/MetricsQL.html).
 * FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details.
 * FEATURE: expose io-related metrics at `/metrics` page for every VictoriaMetrics component:
   * `process_io_read_bytes_total` - the number of bytes read via io syscalls such as read and pread
diff --git a/go.mod b/go.mod
index fe007c0dd..fa4503144 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.12
 	github.com/VictoriaMetrics/metrics v1.15.0
-	github.com/VictoriaMetrics/metricsql v0.11.0
+	github.com/VictoriaMetrics/metricsql v0.12.0
 	github.com/aws/aws-sdk-go v1.37.12
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/cheggaaa/pb/v3 v3.0.6
diff --git a/go.sum b/go.sum
index 343c504a0..8dfcad85f 100644
--- a/go.sum
+++ b/go.sum
@@ -87,8 +87,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.15.0 h1:HGmGaILioC4vNk6UhkcwLIaDlg5y4MVganq1verl5js=
 github.com/VictoriaMetrics/metrics v1.15.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.11.0 h1:85zbY8NxWNALppctOAkWfLDC7dDFcvTEn5IHMKwOGag=
-github.com/VictoriaMetrics/metricsql v0.11.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.12.0 h1:NMIu0MPBmGP34g4RUjI1U0xW5XYp7IVNXe9KtZI3PFQ=
+github.com/VictoriaMetrics/metricsql v0.12.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go
index c1196badf..f8dd7cc84 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go
@@ -10,6 +10,7 @@ var transformFuncs = map[string]bool{
 	"abs":                true,
 	"absent":             true,
 	"ceil":               true,
+	"clamp":              true,
 	"clamp_max":          true,
 	"clamp_min":          true,
 	"day_of_month":       true,
@@ -28,6 +29,7 @@ var transformFuncs = map[string]bool{
 	"month":              true,
 	"round":              true,
 	"scalar":             true,
+	"sign":               true,
 	"sort":               true,
 	"sort_desc":          true,
 	"sqrt":               true,
diff --git a/vendor/modules.txt b/vendor/modules.txt
index fa5b55f16..dcecec4b1 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.15.0
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.11.0
+# github.com/VictoriaMetrics/metricsql v0.12.0
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/VividCortex/ewma v1.1.1

From f7049e2af7e7a83f997805e1b433b3e11b420211 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 24 Feb 2021 17:57:50 +0200
Subject: [PATCH 22/32] lib/promrelabel: optimize `labeldrop` and `labelkeep`
 relabeling for `prefix.*` and `prefix.+` regexps

---
 lib/promrelabel/relabel.go             | 16 +++++-
 lib/promrelabel/relabel_test.go        | 46 +++++++++++++++-
 lib/promrelabel/relabel_timing_test.go | 72 +++++++++++++++++++++++++-
 3 files changed, 129 insertions(+), 5 deletions(-)

diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index a161e55cc..477cf2bea 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -334,7 +334,21 @@ func (prc *parsedRelabelConfig) matchString(s string) bool {
 	if complete {
 		return prefix == s
 	}
-	return strings.HasPrefix(s, prefix) && prc.Regex.MatchString(s)
+	if !strings.HasPrefix(s, prefix) {
+		return false
+	}
+	reStr := prc.regexOriginal.String()
+	if strings.HasPrefix(reStr, prefix) {
+		// Fast path for `foo.*` and `bar.+` regexps
+		reSuffix := reStr[len(prefix):]
+		switch reSuffix {
+		case ".+", "(.+)":
+			return len(s) > len(prefix)
+		case ".*", "(.*)":
+			return true
+		}
+	}
+	return prc.Regex.MatchString(s)
 }
 
 func (prc *parsedRelabelConfig) expandCaptureGroups(template, source string, match []int) string {
diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go
index cc06887e7..c8033877a 100644
--- a/lib/promrelabel/relabel_test.go
+++ b/lib/promrelabel/relabel_test.go
@@ -713,7 +713,7 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		})
 	})
-	t.Run("labeldrop-regexp", func(t *testing.T) {
+	t.Run("labeldrop-prefix", func(t *testing.T) {
 		f(`
 - action: labeldrop
   regex: "dropme.*"
@@ -730,7 +730,49 @@ func TestApplyRelabelConfigs(t *testing.T) {
 		})
 		f(`
 - action: labeldrop
-  regex: "dropme.*"
+  regex: "dropme(.+)"
+`, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+			{
+				Name:  "dropme-please",
+				Value: "aaa",
+			},
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+		}, false, []prompbmarshal.Label{
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("labeldrop-regexp", func(t *testing.T) {
+		f(`
+- action: labeldrop
+  regex: ".*dropme.*"
+`, []prompbmarshal.Label{
+			{
+				Name:  "aaa",
+				Value: "bbb",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "aaa",
+				Value: "bbb",
+			},
+		})
+		f(`
+- action: labeldrop
+  regex: ".*dropme.*"
 `, []prompbmarshal.Label{
 			{
 				Name:  "xxx",
diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go
index 4904bb0ef..bdaf05986 100644
--- a/lib/promrelabel/relabel_timing_test.go
+++ b/lib/promrelabel/relabel_timing_test.go
@@ -495,10 +495,44 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
-	b.Run("labeldrop-match-regexp", func(b *testing.B) {
+	b.Run("labeldrop-match-prefix", func(b *testing.B) {
 		pcs := mustParseRelabelConfigs(`
 - action: labeldrop
   regex: "id.*"
+`)
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = pcs.Apply(labels, 0, true)
+				if len(labels) != 1 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
+				}
+				if labels[0].Name != "__name__" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
+				}
+				if labels[0].Value != "metric" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "metric"))
+				}
+			}
+		})
+	})
+	b.Run("labeldrop-match-regexp", func(b *testing.B) {
+		pcs := mustParseRelabelConfigs(`
+- action: labeldrop
+  regex: ".*id.*"
 `)
 		labelsOrig := []prompbmarshal.Label{
 			{
@@ -591,10 +625,44 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 			}
 		})
 	})
-	b.Run("labelkeep-match-regexp", func(b *testing.B) {
+	b.Run("labelkeep-match-prefix", func(b *testing.B) {
 		pcs := mustParseRelabelConfigs(`
 - action: labelkeep
   regex: "id.*"
+`)
+		labelsOrig := []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "metric",
+			},
+			{
+				Name:  "id",
+				Value: "foobar-random-string-here",
+			},
+		}
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = pcs.Apply(labels, 0, true)
+				if len(labels) != 1 {
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
+				}
+				if labels[0].Name != "id" {
+					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "id"))
+				}
+				if labels[0].Value != "foobar-random-string-here" {
+					panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
+				}
+			}
+		})
+	})
+	b.Run("labelkeep-match-regexp", func(b *testing.B) {
+		pcs := mustParseRelabelConfigs(`
+- action: labelkeep
+  regex: ".*id.*"
 `)
 		labelsOrig := []prompbmarshal.Label{
 			{

From d5f21f3f4b3371626eb1f40b0fca675dac32b19a Mon Sep 17 00:00:00 2001
From: dereksfoster99 <62961548+dereksfoster99@users.noreply.github.com>
Date: Thu, 25 Feb 2021 08:57:09 -0500
Subject: [PATCH 23/32] Native speaker edits. (#1088)

I made an effort to not change anything substantive.
---
 docs/Quick-Start.md | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/Quick-Start.md b/docs/Quick-Start.md
index df49809d3..e2f0f085c 100644
--- a/docs/Quick-Start.md
+++ b/docs/Quick-Start.md
@@ -1,31 +1,31 @@
 # Quick Start
 
-1. If you run Ubuntu, then just run `snap install victoriametrics` command in order to install and start VictoriaMetrics, then read [these docs](https://snapcraft.io/victoriametrics).
-   Otherwise download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
-   from [Docker hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/)
-   or [build it from sources](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-build-from-sources).
+1. If you run Ubuntu please run the `snap install victoriametrics` command to install and start VictoriaMetrics. Then read [these docs](https://snapcraft.io/victoriametrics).
+   Otherwise you can download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
+   or [Docker hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/)
+   or [build it from sources](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-build-from-sources). 
 
 2. This step isn't needed if you run VictoriaMetrics via `snap install victoriametrics` as described above.
-   Otherwise run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
-   and their default values. Default flag values should fit the majoirty of cases. The minimum required flags to configure are:
+   Otherwise, please run the binary or Docker image with your desired command-line flags. You can look at `-help` to see descriptions of all available flags
+   and their default values. The default flag values should fit the majority of cases. The minimum required flags that must be configured are:
 
-   * `-storageDataPath` - path to directory where VictoriaMetrics stores all the data.
+   * `-storageDataPath` - the path to directory where VictoriaMetrics stores your data.
    * `-retentionPeriod` - data retention.
 
-   For instance:
+   For example:
 
    `./victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics-data -retentionPeriod=3`
 
-   See [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) in order to configure VictoriaMetrics as OS service.
-   It is recommended setting up [VictoriaMetrics monitoring](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#monitoring).
+   Check [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) to configure VictoriaMetrics as an OS service.
+   We recommended setting up [VictoriaMetrics monitoring](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#monitoring).
 
-3. Configure [vmagent](https://victoriametrics.github.io/vmagent.html) or Prometheus to write data to VictoriaMetrics.
-   It is recommended to use `vmagent` instead of Prometheus, since it is more resource efficient. If you still prefer Prometheus, then
+3. Configure either [vmagent](https://victoriametrics.github.io/vmagent.html) or Prometheus to write data to VictoriaMetrics.
+   We recommended using `vmagent` instead of Prometheus because it is more resource efficient. If you still prefer Prometheus
    see [these instructions](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#prometheus-setup)
-   for details on how to configure Prometheus.
+   for details on how it may be properly configured.
 
-4. Configure Grafana to query VictoriaMetrics instead of Prometheus.
-   See [these instructions](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#grafana-setup).
+4. To configure Grafana to query VictoriaMetrics instead of Prometheus
+   please see [these instructions](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#grafana-setup).
 
 
 There is also [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) and [SaaS playground](https://play.victoriametrics.com/signIn).

From 438428b5b0a3f0b28e22667c4f334d54ff1af647 Mon Sep 17 00:00:00 2001
From: dereksfoster99 <62961548+dereksfoster99@users.noreply.github.com>
Date: Fri, 26 Feb 2021 09:39:15 -0500
Subject: [PATCH 24/32] Alphabetized names and added "native speaker" spin.
 (#1093)

Do we have the dates that each of these were written so that can be added next to each brand?
---
 docs/CaseStudies.md | 487 ++++++++++++++++++++++----------------------
 1 file changed, 239 insertions(+), 248 deletions(-)

diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md
index 99e22456b..c9c9a8e01 100644
--- a/docs/CaseStudies.md
+++ b/docs/CaseStudies.md
@@ -1,9 +1,9 @@
 # Case studies and talks
 
-Below are approved public case studies and talks from VictoriaMetrics users. Join our [community Slack channel](http://slack.victoriametrics.com/)
-and feel free asking for references, reviews and additional case studies from real VictoriaMetrics users there.
+Below please find public case studies and talks from VictoriaMetrics users. You can also join our [community Slack channel](http://slack.victoriametrics.com/)
+where you can chat with VictoriaMetrics users to get additional references, reviews and case studies.
 
-See also [articles about VictoriaMetrics from our users](https://victoriametrics.github.io/Articles.html#third-party-articles-and-slides).
+You can also read [articles about VictoriaMetrics from our users](https://victoriametrics.github.io/Articles.html#third-party-articles-and-slides).
 
 Alphabetically sorted links to case studies:
 
@@ -23,201 +23,141 @@ Alphabetically sorted links to case studies:
 * [zhihu](#zhihu)
 
 
-## zhihu
-
-[zhihu](https://www.zhihu.com) is the largest chinese question-and-answer website. We use VictoriaMetrics to store and use Graphite metrics, and we shared the [promate](https://github.com/zhihu/promate) solution in our [单机 20 亿指标，知乎 Graphite 极致优化！](https://qcon.infoq.cn/2020/shenzhen/presentation/2881)([slides](https://static001.geekbang.org/con/76/pdf/828698018/file/%E5%8D%95%E6%9C%BA%2020%20%E4%BA%BF%E6%8C%87%E6%A0%87%EF%BC%8C%E7%9F%A5%E4%B9%8E%20Graphite%20%E6%9E%81%E8%87%B4%E4%BC%98%E5%8C%96%EF%BC%81-%E7%86%8A%E8%B1%B9.pdf)) talk at [QCon 2020](https://qcon.infoq.cn/2020/shenzhen/).
-
-Numbers:
-
-- Active time series: ~2500 Million
-- Datapoints: ~20 Trillion
-- Ingestion rate: ~1800k/s
-- Disk usage: ~20 TiB
-- Index size: ~600 GiB
-- The average query rate is ~3k per second (mostly alert queries).
-- Query duration: median is ~40ms, 99th percentile is ~100ms.
-
-
 ## adidas
 
-See [slides](https://promcon.io/2019-munich/slides/remote-write-storage-wars.pdf) and [video](https://youtu.be/OsH6gPdxR4s)
+See our [slides](https://promcon.io/2019-munich/slides/remote-write-storage-wars.pdf) and [video](https://youtu.be/OsH6gPdxR4s)
 from [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk at [PromCon 2019](https://promcon.io/2019-munich/).
 VictoriaMetrics is compared to Thanos, Corex and M3DB in the talk.
 
+## Adsterra
 
-## CERN
+[Adsterra Network](https://adsterra.com) is a leading digital advertising agency that offers
+performance-based solutions for advertisers and media partners worldwide.
 
-The European Organization for Nuclear Research known as [CERN](https://home.cern/) uses VictoriaMetrics for real-time monitoring
-of the [CMS](https://home.cern/science/experiments/cms) detector system.
-According to [published talk](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
-VictoriaMetrics is used for the following purposes as a part of "CMS Monitoring cluster":
+We used to collect and store our metrics with Prometheus. Over time, the data volume on our servers
+and metrics increased to the point that we were forced to gradually reduce what we were retaining. When our retention got as low as 7 days
+we looked for alternative solutions. We chose between Thanos, VictoriaMetrics and Prometheus federation.
 
-* As long-term storage for messages consumed from the [NATS messaging system](https://nats.io/). Consumed messages are pushed directly to VictoriaMetrics via HTTP protocol
-* As long-term storage for Prometheus monitoring system (30 days retention policy, there are plans to increase it up to ½ year)
-* As a data source for visualizing metrics in Grafana.
+We ended up with the following configuration:
 
-R&D topic: Evaluate VictoraMetrics vs InfluxDB for large cardinality data.
+- Local instances of Prometheus with VictoriaMetrics as the remote storage on our backend servers.
+- A single Prometheus on our monitoring server scrapes metrics from other servers and writes to VictoriaMetrics.
+- A separate Prometheus that federates from other instances of Prometheus and processes alerts.
 
-See also [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf) publication from CERN with details about VictoriaMetrics usage.
+We learned that remote write protocol generated too much traffic and connections so after 8 months we started looking for alternatives.
+
+Around the same time, VictoriaMetrics released [vmagent](https://victoriametrics.github.io/vmagent.html).
+We tried to scrape all the metrics via a single instance of vmagent but it that didn't work because vmgent wasn't able to catch up with writes
+into VictoriaMetrics. We tested different options and end up with the following scheme:
+
+- We removed Prometheus from our setup.
+- VictoriaMetrics [can scrape targets](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-scrape-prometheus-exporters-such-as-node-exporter) as well
+so we removed vmagent. Now, VictoriaMetrics scrapes all the metrics from 110 jobs and 5531 targets.
+- We use [Promxy](https://github.com/jacksontj/promxy) for alerting.
+
+Such a scheme has generated the following benefits compared with Prometheus:
+
+- We can store more metrics.
+- We need less RAM and CPU for the same workload.
+
+Cons are the following:
+
+- VictoriaMetrics didn't support replication (it [supports replication now](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#replication-and-data-safety)) - we run an extra instance of VictoriaMetrics and Promxy in front of a VictoriaMetrics pair for high availability.
+- VictoriaMetrics stores 1 extra month for defined retention (if retention is set to N months, then VM stores N+1 months of data), but this is still better than other solutions.
+
+Here are some numbers from our single-node VictoriaMetrics setup:
+
+- active time series: 10M
+- ingestion rate: 800K samples/sec
+- total number of datapoints: more than 2 trillion
+- total number of entries in inverted index: more than 1 billion
+- daily time series churn rate: 2.6M
+- data size on disk: 1.5 TB
+- index size on disk: 27 GB
+- average datapoint size on disk: 0.75 bytes
+- range query rate: 16 rps
+- instant query rate: 25 rps
+- range query duration: max: 0.5s; median: 0.05s; 97th percentile: 0.29s
+- instant query duration: max: 2.1s; median: 0.04s; 97th percentile: 0.15s
+
+VictoriaMetrics consumes about 50GB of RAM.
+
+Setup:
+
+We have 2 single-node instances of VictoriaMetrics. The first instance collects and stores high-resolution metrics (10s scrape interval) for a month.
+The second instance collects and stores low-resolution metrics (300s scrape interval) for a month.
+We use Promxy + Alertmanager for global view and alerts evaluation.
 
 
-## COLOPL
+## ARNES
 
-[COLOPL](http://www.colopl.co.jp/en/) is Japaneese Game Development company. It started using VictoriaMetrics
-after evaulating the following remote storage solutions for Prometheus:
+[The Academic and Research Network of Slovenia](https://www.arnes.si/en/) (ARNES) is a public institute that provides network services to research,
+educational and cultural organizations enabling connections and cooperation with each other and with related organizations worldwide.
 
-* Cortex
-* Thanos
-* M3DB
-* VictoriaMetrics
+After using Cacti, Graphite and StatsD for years, we wanted to upgrade our monitoring stack to something that:
 
-See [slides](https://speakerdeck.com/inletorder/monitoring-platform-with-victoria-metrics) and [video](https://www.youtube.com/watch?v=hUpHIluxw80)
-from `Large-scale, super-load system monitoring platform built with VictoriaMetrics` talk at [Prometheus Meetup Tokyo #3](https://prometheus.connpass.com/event/157721/).
+- has native alerting support
+- can be run on-prem
+- has multi-dimensional metrics
+- has lower hardware requirements
+- is scalable
+- has a simple client that allows for provisioning and discovery with Puppet
 
+We hed been running Prometheus for about a year in a test environment and it was working well but there was a need/wish for a few more years of retention than the old system provided. We tested Thanos which was a bit resource hungry but worked great for about half a year.
+Then we discovered VictoriaMetrics. Our scale isn't that big so we don't have on-prem S3 and no Kubernetes. VM's single node instance provided
+the same result with far less maintenance overhead and lower hardware requirements.
 
-## Zerodha
-
-[Zerodha](https://zerodha.com/) is India's largest stock broker. Monitoring team at Zerodha faced with the following requirements:
-
-* Multiple K8s clusters to monitor
-* Consistent monitoring infra for each cluster across the fleet
-* Ability to handle billions of timeseries events at any point of time
-* Easier to operate and cost effective
-
-Thanos, Cortex and VictoriaMetrics were evaluated as a long-term storage for Prometheus. VictoriaMetrics has been selected due to the following reasons:
-
-* Blazing fast benchmarks for a single node setup.
-* Single binary mode. Easy to scale vertically, very less operational headache.
-* Considerable [improvements on creating Histograms](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350).
-* [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) gives us the ability to extend PromQL with more aggregation operators.
-* API is compatible with Prometheus, almost all standard PromQL queries just work out of the box.
-* Handles storage well, with periodic compaction. Makes it easy to take snapshots.
-
-See [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit) slides,
-[video](https://youtu.be/ZJQYW-cFOms) and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
-
-
-## Wix.com
-
-[Wix.com](https://en.wikipedia.org/wiki/Wix.com) is the leading web development platform.
-
-> We needed to redesign metric infrastructure from the ground up after the move to Kubernethes. A few approaches/designs have been tried before the one that works great has been chosen: Prometheus instance in every datacenter with 2 hours retention for local storage and remote write into [HA pair of single-node VictoriaMetrics instances](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#high-availability).
+After testing it a few months and with great support from the maintainers on [Slack](http://slack.victoriametrics.com/),
+we decided to go with it. VM's support for the ingestion of InfluxDB metrics was an additional bonus as our hardware team uses
+SNMPCollector to collect metrics from network devices and switching from InfluxDB to VictoriaMetrics required just a simple change in the config file. 
 
 Numbers:
 
-* The number of active time series per VictoriaMetrics instance is 50 millions.
-* The total number of time series per VictoriaMetrics instance is 5000 millions.
-* Ingestion rate per VictoriaMetrics instance is 1.1 millions data points per second.
-* The total number of datapoints per VictoriaMetrics instance is 8.5 trillions.
-* The average churn rate is 150 millions new time series per day.
-* The average query rate is ~150 per second (mostly alert queries).
-* Query duration: median is ~1ms, 99th percentile is ~1sec.
-* Retention: 3 months.
+- 2 single node instances per DC (one for Prometheus and one for InfluxDB metrics)
+- Active time series per VictoriaMetrics instance: ~500k (Prometheus) + ~320k (InfluxDB)
+- Ingestion rate per VictoriaMetrics instance: 45k/s (Prometheus) / 30k/s (InfluxDB)
+- Query duration: median ~5ms, 99th percentile ~45ms
+- Total number of datapoints per instance: 390B (Prometheus), 110B (InfluxDB)
+- Average datapoint size on drive: 0.4 bytes
+- Disk usage per VictoriaMetrics instance: 125GB (Prometheus), 185GB (InfluxDB)
+- Index size per VictoriaMetrics instance: 1.6GB (Prometheus), 1.2GB (InfluxDB)
 
-> Alternatives that we’ve played with before choosing VictoriaMetrics are: federated Prometheus, Cortex, IronDB and Thanos.
-> Points that were critical to us when we were choosing a central tsdb, in order of importance:
-
-* At least 3 month worth of history.
-* Raw data, no aggregation, no sampling.
-* High query speed.
-* Clean fail state for HA (multi-node clusters may return partial data resulting in false alerts).
-* Enough head room/scaling capacity for future growth, up to 100M active time series.
-* Ability to split DB replicas per workload. Alert queries go to one replica, user queries go to another (speed for users, effective cache).
-
-> Optimizing for those points and our specific workload VictoriaMetrics proved to be the best option. As an icing on a cake we’ve got [PromQL extensions](https://victoriametrics.github.io/MetricsQL.html) - `default 0` and `histogram` are my favorite ones, for example. What we specially like is having a lot of tsdb params easily available via config options, that makes tsdb easy to tune for specific use case. Also worth noting is a great community in [Slack channel](http://slack.victoriametrics.com/) and of course maintainer support.
-
-Alex Ulstein, Head of Monitoring, Wix.com
-
-
-## Wedos.com
-
-> [Wedos](https://www.wedos.com/) is the Biggest Czech Hosting. We have our own private data center, that holds only our servers and technologies. The second data center, where the servers will be cooled in an oil bath, is being built. We started using [cluster VictoriaMetrics](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html) to store Prometheus metrics from all our infrastructure after receiving positive references from our friends who successfully use VictoriaMetrics.
-
-Numbers:
-
-* The number of acitve time series: 5M.
-* Ingestion rate: 170K data points per second.
-* Query duration: median is ~2ms, 99th percentile is ~50ms.
-
-> We like configuration simplicity and zero maintenance for VictoriaMetrics - once installed and forgot about it. It works out of the box without any issues.
-
-
-## Synthesio
-
-[Synthesio](https://www.synthesio.com/) is the leading social intelligence tool for social media monitoring & social analytics.
-
-> We fully migrated from [Metrictank](https://grafana.com/oss/metrictank/)  to Victoria Metrics
-
-Numbers:
-- Single node
-- Active time series - 5 Million
-- Datapoints: 1.25 Trillion
-- Ingestion rate - 550k datapoints per second
-- Disk usage - 150gb
-- Index size - 3gb
-- Query duration 99th percentile - 147ms
-- Churn rate - 100 new time series per hour
-
-
-## MHI Vestas Offshore Wind
-
-The mission of [MHI Vestas Offshore Wind](http://www.mhivestasoffshore.com) is to co-develop offshore wind as an economically viable and sustainable energy resource to benefit future generations.
-
-MHI Vestas Offshore Wind is using VictoriaMetrics to ingest and visualize sensor data from offshore wind turbines. The very efficient storage and ability to backfill was key in chosing VictoriaMetrics. MHI Vestas Offshore Wind is running the cluster version of VictoriaMetrics on Kubernetes using the Helm charts for deployment to be able to scale up capacity as the solution will be rolled out.
-
-Numbers with current limited roll out:
-
-- Active time series: 270K
-- Ingestion rate: 70K/sec
-- Total number of datapoints: 850 billions
-- Data size on disk: 800 GiB
-- Retention time: 3 years
-
-
-## Dreamteam
-
-[Dreamteam](https://dreamteam.gg/) successfully uses single-node VictoriaMetrics in multiple environments.
-
-Numbers:
-
-* Active time series: from 350K to 725K.
-* Total number of time series: from 100M to 320M.
-* Total number of datapoints: from 120 billions to 155 billions.
-* Retention: 3 months.
-
-VictoriaMetrics in production environment runs on 2 M5 EC2 instances in "HA" mode, managed by Terraform and Ansible TF module.
-2 Prometheus instances are writing to both VMs, with 2 [Promxy](https://github.com/jacksontj/promxy) replicas
-as load balancer for reads.
+We are running 1 Prometheus, 1 VictoriaMetrics and 1 Grafana server in each datacenter on baremetal servers, scraping 350+ targets
+(and 3k+ devices collected via SNMPCollector sending metrics directly to VM). Each Prometheus is scraping all targets
+so we have all metrics in both VictoriaMetrics instances. We are using [Promxy](https://github.com/jacksontj/promxy) to deduplicate metrics from both instances.
+Grafana has an LB infront so if one DC has problems we can still view all metrics from both DCs on the other Grafana instance.
 
+We are still in the process of migration, but we are really happy with the whole stack. It has proven to be an essential tool
+for gathering insights into our services during COVID-19 and has enabled us to provide better service and identify problems faster.
 
 ## Brandwatch
 
 [Brandwatch](https://www.brandwatch.com/) is the world's pioneering digital consumer intelligence suite,
 helping over 2,000 of the world's most admired brands and agencies to make insightful, data-driven business decisions.
 
-The engineering department at Brandwatch has been using InfluxDB for storing application metrics for many years
-and when End-of-Life of InfluxDB version 1.x was announced we decided to re-evaluate our whole metrics collection and storage stack.
+The engineering department at Brandwatch has been using InfluxDB to store application metrics for many years
+but when End-of-Life of InfluxDB version 1.x was announced we decided to re-evaluate our entire metrics collection and storage stack.
 
-Main goals for the new metrics stack were:
+The main goals for the new metrics stack were:
 - improved performance
 - lower maintenance
 - support for native clustering in open source version
 - the less metrics shipment had to change, the better
-- achieving longer data retention would be great but not critical
+- longer data retention time period would be great but not critical
 
-We initially looked at CrateDB and TimescaleDB which both turned out to have limitations or requirements in the open source versions
-that made them unfit for our use case. Prometheus was also considered but push vs. pull metrics was a big change we did not want
+We initially tested CrateDB and TimescaleDB wand found that both had limitations or requirements in their open source versions
+that made them unfit for our use case. Prometheus was also considered but it's push vs. pull metrics was a big change we did not want
 to include in the already significant change.
 
 Once we found VictoriaMetrics it solved the following problems:
-- it is very light weight and we can now run virtual machines instead of dedicated hardware machines for metrics storage
-- very short startup time and any possible gaps in data can easily be filled in by using Promxy
-- we could continue using Telegraf as our metrics agent and ship identical metrics to both InfluxDB and VictoriaMetrics during a migration period (migration just about to start)
-- compression is really good so we can store more metrics and we can spin up new VictoriaMetrics instances
+- it is very lightweight and we can now run virtual machines instead of dedicated hardware machines for metrics storage
+- very short startup time and any possible gaps in data can easily be filled in using Promxy
+- we could continue using Telegraf as our metrics agent and ship identical metrics to both InfluxDB and VictoriaMetrics during the migration period (migration just about to start)
+- compression im VM is really good. We can store more metrics and we can easily spin up new VictoriaMetrics instances
 for new data and keep read-only nodes with older data if we need to extend our retention period further
 than single virtual machine disks allow and we can aggregate all the data from VictoriaMetrics with Promxy
 
-High availability is done the same way we did with InfluxDB, by running parallel single nodes of VictoriaMetrics.
+High availability is done the same way we did with InfluxDB by running parallel single nodes of VictoriaMetrics.
 
 Numbers:
 
@@ -234,116 +174,56 @@ Query rates are insignificant as we have concentrated on data ingestion so far.
 
 Anders Bomberg, Monitoring and Infrastructure Team Lead, brandwatch.com
 
+## CERN
 
-## Adsterra
+The European Organization for Nuclear Research better known as [CERN](https://home.cern/) uses VictoriaMetrics for real-time monitoring
+of the [CMS](https://home.cern/science/experiments/cms) detector system.
+According to [published talk](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
+VictoriaMetrics is used for the following purposes as a part of the "CMS Monitoring cluster":
 
-[Adsterra Network](https://adsterra.com) is a leading digital advertising company that offers
-performance-based solutions for advertisers and media partners worldwide.
+* As a long-term storage for messages ingested from the [NATS messaging system](https://nats.io/). Ingested messages are pushed directly to VictoriaMetrics via HTTP protocol
+* As a long-term storage for Prometheus monitoring system (30 days retention policy. There are plans to increase it up to ½ year)
+* As a data source for visualizing metrics in Grafana.
 
-We used to collect and store our metrics via Prometheus. Over time the amount of our servers
-and metrics increased so we were gradually reducing the retention. When retention became 7 days
-we started to look for alternative solutions. We were choosing among Thanos, VictoriaMetrics and Prometheus federation.
+R&D topic: Evaluate VictoraMetrics vs InfluxDB for large cardinality data.
 
-We end up with the following configuration:
-
-- Local Prometheus'es with VictoriaMetrics as remote storage on our backend servers.
-- A single Prometheus on our monitoring server scrapes metrics from other servers and writes to VictoriaMetrics.
-- A separate Prometheus that federates from other Prometheus'es and processes alerts.
-
-Turns out that remote write protocol generates too much traffic and connections. So after 8 months we started to look for alternatives.
-
-Around the same time VictoriaMetrics released [vmagent](https://victoriametrics.github.io/vmagent.html).
-We tried to scrape all the metrics via a single insance of vmagent. But that didn't work - vmgent wasn't able to catch up with writes
-into VictoriaMetrics. We tested different options and end up with the following scheme:
-
-- We removed Prometheus from our setup.
-- VictoriaMetrics [can scrape targets](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-scrape-prometheus-exporters-such-as-node-exporter) as well,
-so we removed vmagent. Now VictoriaMetrics scrapes all the metrics from 110 jobs and 5531 targets.
-- We use [Promxy](https://github.com/jacksontj/promxy) for alerting.
-
-Such a scheme has the following benefits comparing to Prometheus:
-
-- We can store more metrics.
-- We need less RAM and CPU for the same workload.
-
-Cons are the following:
-
-- VictoriaMetrics didn't support replication (it [supports replication now](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#replication-and-data-safety)) - we run extra instance of VictoriaMetrics and Promxy in front of VictoriaMetrics pair for high availability.
-- VictoriaMetrics stores 1 extra month for defined retention (if retention is set to N months, then VM stores N+1 months of data), but this is still better than other solutions.
-
-Some numbers from our single-node VictoriaMetrics setup:
-
-- active time series: 10M
-- ingestion rate: 800K samples/sec
-- total number of datapoints: more than 2 trillion
-- total number of entries in inverted index: more than 1 billion
-- daily time series churn rate: 2.6M
-- data size on disk: 1.5 TB
-- index size on disk: 27 GB
-- average datapoint size on disk: 0.75 bytes
-- range query rate: 16 rps
-- instant query rate: 25 rps
-- range query duration: max: 0.5s; median: 0.05s; 97th percentile: 0.29s
-- instant query duration: max: 2.1s; median: 0.04s; 97th percentile: 0.15s
-
-VictoriaMetrics consumes about 50GiB of RAM.
-
-Setup:
-
-We have 2 single-node instances of VictoriaMetircs. The first instance collects and stores high-resolution metrics (10s scrape interval) for a month.
-The second instance collects and stores low-resolution metrics (300s scrape interval) for a month.
-We use Promxy + Alertmanager for global view and alerts evaluation.
+Please also see [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf) publication from CERN with details about their VictoriaMetrics usage.
 
 
-## ARNES
+## COLOPL
 
-[The Academic and Research Network of Slovenia](https://www.arnes.si/en/) (ARNES) is a public institute that provides network services to research,
-educational and cultural organizations, and enables them to establish connections and cooperation with each other and with related organizations abroad.
+[COLOPL](http://www.colopl.co.jp/en/) is Japanese game development company. It started using VictoriaMetrics
+after evaulating the following remote storage solutions for Prometheus:
 
-After using Cacti, Graphite and StatsD for years, we wanted to upgrade our monitoring stack to something that:
+* Cortex
+* Thanos
+* M3DB
+* VictoriaMetrics
 
-- has native alerting support
-- can run on-prem
-- has multi-dimension metrics
-- lower hardware requirements
-- is scalable
-- simple client provisioning and discovery with Puppet
+See [slides](https://speakerdeck.com/inletorder/monitoring-platform-with-victoria-metrics) and [video](https://www.youtube.com/watch?v=hUpHIluxw80)
+from `Large-scale, super-load system monitoring platform built with VictoriaMetrics` talk at [Prometheus Meetup Tokyo #3](https://prometheus.connpass.com/event/157721/).
 
-We were running Prometheus for about a year in a test environment and it worked great. But there was a need/wish for a few years of retention time,
-like the old systems provided. We tested Thanos, which was a bit resource hungry back then, but it worked great for about half a year
-until we discovered VictoriaMetrics. As our scale is not that big, we don't have on-prem S3 and no Kubernetes, VM's single node instance provided
-the same result with less maintenance overhead and lower hardware requirements.
+## Dreamteam
 
-After testing it a few months and having great support from the maintainers on [Slack](http://slack.victoriametrics.com/),
-we decided to go with it. VM's support for ingesting InfluxDB metrics was an additional bonus, since our hardware team uses
-SNMPCollector to collect metrics from network devices and switching from InfluxDB to VictoriaMetrics was a simple change in the config file for them. 
+[Dreamteam](https://dreamteam.gg/) successfully uses single-node VictoriaMetrics in multiple environments.
 
 Numbers:
 
-- 2 single node instances per DC (one for prometheus and one for influxdb metrics)
-- Active time series per VictoriaMetrics instance: ~500k (prometheus) + ~320k (influxdb)
-- Ingestion rate per VictoriaMetrics instance: 45k/s (prometheus) / 30k/s (influxdb)
-- Query duration: median is ~5ms, 99th percentile is ~45ms
-- Total number of datapoints per instance: 390B (prometheus), 110B (influxdb)
-- Average datapoint size on drive: 0.4 bytes
-- Disk usage per VictoriaMetrics instance: 125GB (prometheus), 185GB (influxdb)
-- Index size per VictoriaMetrics instance: 1.6GB (prometheus), 1.2GB (influcdb)
-
-We are running 1 Prometheus, 1 VictoriaMetrics and 1 Grafana server in each datacenter on baremetal servers, scraping 350+ targets
-(and 3k+ devices collected via SNMPCollector sending metrics directly to VM). Each Prometheus is scraping all targets,
-so we have all metrics in both VictoriaMetrics instances. We are using [Promxy](https://github.com/jacksontj/promxy) to deduplicate metrics from both instances.
-Grafana has a LB infront, so if one DC has problems, we can still view all metrics from both DCs on the other Grafana instance.
-
-We are still in the process of migration, but we are really happy with the whole stack. It has proven as an essential piece
-for insight into our services during COVID-19 and has enabled us to provide better service and spot problems faster.
+* Active time series: from 350K to 725K.
+* Total number of time series: from 100M to 320M.
+* Total number of datapoints: from 120 billion to 155 billion.
+* Retention period: 3 months.
 
+VictoriaMetrics in production environment runs on 2 M5 EC2 instances in "HA" mode, managed by Terraform and Ansible TF module.
+2 Prometheus instances are writing to both VMs, with 2 [Promxy](https://github.com/jacksontj/promxy) replicas
+as the load balancer for reads.
 
 ## Idealo.de
 
 [idealo.de](https://www.idealo.de/) is the leading price comparison website in Germany. We use Prometheus for metrics on our container platform.
-When we introduced Prometheus at idealo we started with m3db as a longterm storage. In our setup m3db was quite unstable and consumed a lot of resources.
+When we introduced Prometheus at idealo we started with m3db as our longterm storage. In our setup, m3db was quite unstable and consumed a lot of resources.
 
-VictoriaMetrics runs very stable for us and uses only a fraction of the resources. Although we also increased our retention time from 1 month to 13 months.
+VictoriaMetrics in poroduction is very stable for us and uses only a fraction of the resources even though we also increased our retention period from 1 month to 13 months.
 
 Numbers:
 
@@ -354,3 +234,114 @@ Numbers:
 - The average query rate is ~20 per second. Response time for 99th quantile is 120ms.
 - Retention: 13 months.
 - Size of all datapoints: 3.5 TB
+
+
+## MHI Vestas Offshore Wind
+
+The mission of [MHI Vestas Offshore Wind](http://www.mhivestasoffshore.com) is to co-develop offshore wind as an economically viable and sustainable energy resource to benefit future generations.
+
+MHI Vestas Offshore Wind is using VictoriaMetrics to ingest and visualize sensor data from offshore wind turbines. The very efficient storage and ability to backfill was key in choosing VictoriaMetrics. MHI Vestas Offshore Wind is running the cluster version of VictoriaMetrics on Kubernetes using the Helm charts for deployment to be able to scale up capacity as the solution is rolled out.
+
+Numbers with current, limited roll out:
+
+- Active time series: 270K
+- Ingestion rate: 70K/sec
+- Total number of datapoints: 850 billion
+- Data size on disk: 800 GiB
+- Retention period: 3 years
+
+
+## Synthesio
+
+[Synthesio](https://www.synthesio.com/) is the leading social intelligence tool for social media monitoring and analytics.
+
+> We fully migrated from [Metrictank](https://grafana.com/oss/metrictank/)  to VictoriaMetrics
+
+Numbers:
+- Single node
+- Active time series - 5 Million
+- Datapoints: 1.25 Trillion
+- Ingestion rate - 550k datapoints per second
+- Disk usage - 150gb
+- Index size - 3gb
+- Query duration 99th percentile - 147ms
+- Churn rate - 100 new time series per hour
+
+## Wedos.com
+
+> [Wedos](https://www.wedos.com/) is the biggest hosting provider in the Czech Republic. We have our own private data center that holds our servers and technologies. We are in the process of building a second, stae of the art data center where the servers will be cooled in an oil bath. We started using [cluster VictoriaMetrics](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html) to store Prometheus metrics from all our infrastructure after receiving positive references from people who had successfully used VictoriaMetrics.
+
+Numbers:
+
+* The number of acitve time series: 5M.
+* Ingestion rate: 170K data points per second.
+* Query duration: median is ~2ms, 99th percentile is ~50ms.
+
+> We like that VictoriaMetrics is simple to configuree and requires zero maintenance. It works right out of the box and once it's set up you can just forget about it. 
+
+## Wix.com
+
+[Wix.com](https://en.wikipedia.org/wiki/Wix.com) is the leading web development platform.
+
+> We needed to redesign our metrics infrastructure from the ground up after the move to Kubernetes. We had tried out a few different options before landing on this solution which is working great. We have a Prometheus instance in every datacenter with 2 hours retention for local storage and remote write into [HA pair of single-node VictoriaMetrics instances](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#high-availability).
+
+Numbers:
+
+* The number of active time series per VictoriaMetrics instance is 50 millios.
+* The total number of time series per VictoriaMetrics instance is 5000 million.
+* Ingestion rate per VictoriaMetrics instance is 1.1 millions data points per second.
+* The total number of datapoints per VictoriaMetrics instance is 8.5 trillion.
+* The average churn rate is 150 millions new time series per day.
+* The average query rate is ~150 per second (mostly alert queries).
+* Query duration: median is ~1ms, 99th percentile is ~1sec.
+* Retention period: 3 months.
+
+> The alternatives that we tested prior to choosing VictoriaMetrics were: Prometheus federated, Cortex, IronDB and Thanos.
+> The items that were critical to us central tsdb, in order of importance were as follows:
+
+* At least 3 month worth of retention.
+* Raw data, no aggregation, no sampling.
+* High query speed.
+* Clean fail state for HA (multi-node clusters may return partial data resulting in false alerts).
+* Enough headroom/scaling capacity for future growth which is planned to be up to 100M active time series.
+* Ability to split DB replicas per workload. Alert queries go to one replica and user queries go to another (speed for users, effective cache).
+
+> Optimizing for those points and our specific workload, VictoriaMetrics proved to be the best option. As icing on the cake we’ve got [PromQL extensions](https://victoriametrics.github.io/MetricsQL.html) - `default 0` and `histogram` are my favorite ones. We really like having a lot of tsdb params easily available via config options which makes tsdb easy to tune for each specific use case. We've also found a great community in [Slack channel](http://slack.victoriametrics.com/) and responsive and helpful maintainer support.
+
+Alex Ulstein, Head of Monitoring, Wix.com
+
+## Zerodha
+
+[Zerodha](https://zerodha.com/) is India's largest stock broker. The monitoring team at Zerodha had the following requirements:
+
+* Multiple K8s clusters to monitor
+* Consistent monitoring infra for each cluster across the fleet
+* The ability to handle billions of timeseries events at any point of time
+* Easy to operate and cost effective
+
+Thanos, Cortex and VictoriaMetrics were evaluated as a long-term storage for Prometheus. VictoriaMetrics has been selected for the following reasons:
+
+* Blazingly fast benchmarks for a single node setup.
+* Single binary mode. Easy to scale vertically with far fewer operational headaches.
+* Considerable [improvements on creating Histograms](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350).
+* [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) gives us the ability to extend PromQL with more aggregation operators.
+* The API is compatible with Prometheus and nearly all standard PromQL queries work well out of the box.
+* Handles storage well, with periodic compaction which makes it easy to take snapshots.
+
+Please see [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit) slides,
+[video](https://youtu.be/ZJQYW-cFOms) and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
+
+
+## zhihu
+
+[zhihu](https://www.zhihu.com) is the largest Chinese question-and-answer website. We use VictoriaMetrics to store and use Graphite metrics. We shared the [promate](https://github.com/zhihu/promate) solution in our [单机 20 亿指标，知乎 Graphite 极致优化！](https://qcon.infoq.cn/2020/shenzhen/presentation/2881)([slides](https://static001.geekbang.org/con/76/pdf/828698018/file/%E5%8D%95%E6%9C%BA%2020%20%E4%BA%BF%E6%8C%87%E6%A0%87%EF%BC%8C%E7%9F%A5%E4%B9%8E%20Graphite%20%E6%9E%81%E8%87%B4%E4%BC%98%E5%8C%96%EF%BC%81-%E7%86%8A%E8%B1%B9.pdf)) talk at [QCon 2020](https://qcon.infoq.cn/2020/shenzhen/).
+
+Numbers:
+
+- Active time series: ~2500 Million
+- Datapoints: ~20 Trillion
+- Ingestion rate: ~1800k/s
+- Disk usage: ~20 TB
+- Index size: ~600 GB
+- The average query rate is ~3k per second (mostly alert queries).
+- Query duration: median is ~40ms, 99th percentile is ~100ms.

From f7b242540b6d95d81b4b8b41d44470e8d362ffea Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 12:46:28 +0200
Subject: [PATCH 25/32] lib/promscrape: reduce processing time for big number
 of discovered targets by processing them in parallel

---
 lib/promscrape/config.go | 61 ++++++++++++++++++++++++++++++----------
 1 file changed, 46 insertions(+), 15 deletions(-)

diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 4f97506b6..b0c8eb578 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -608,14 +609,42 @@ func appendGCEScrapeWork(dst []*ScrapeWork, sdc *gce.SDConfig, swc *scrapeWorkCo
 
 func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, targetLabels []map[string]string, sectionName string) []*ScrapeWork {
 	startTime := time.Now()
+	// Process targetLabels in parallel in order to reduce processing time for big number of targetLabels.
+	type result struct {
+		sw  *ScrapeWork
+		err error
+	}
+	resultCh := make(chan result)
+	workCh := make(chan map[string]string)
+	goroutines := cgroup.AvailableCPUs()
+	for i := 0; i < goroutines; i++ {
+		go func() {
+			for metaLabels := range workCh {
+				target := metaLabels["__address__"]
+				sw, err := getScrapeWork(swc, target, nil, metaLabels)
+				if err != nil {
+					err = fmt.Errorf("skipping target %q for job_name %q in %s because of error: %w", target, swc.jobName, sectionName, err)
+				}
+				resultCh <- result{
+					sw:  sw,
+					err: err,
+				}
+			}
+		}()
+	}
 	for _, metaLabels := range targetLabels {
-		target := metaLabels["__address__"]
-		var err error
-		dst, err = appendScrapeWork(dst, swc, target, nil, metaLabels)
-		if err != nil {
-			logger.Errorf("error when parsing `%s` target %q for `job_name` %q: %s; skipping it", sectionName, target, swc.jobName, err)
+		workCh <- metaLabels
+	}
+	close(workCh)
+	for range targetLabels {
+		r := <-resultCh
+		if r.err != nil {
+			logger.Errorf("%s", r.err)
 			continue
 		}
+		if r.sw != nil {
+			dst = append(dst, r.sw)
+		}
 	}
 	metrics.GetOrCreateHistogram(fmt.Sprintf("vm_promscrape_target_relabel_duration_seconds{type=%q}", sectionName)).UpdateDuration(startTime)
 	return dst
@@ -673,18 +702,20 @@ func (stc *StaticConfig) appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConf
 			logger.Errorf("`static_configs` target for `job_name` %q cannot be empty; skipping it", swc.jobName)
 			continue
 		}
-		var err error
-		dst, err = appendScrapeWork(dst, swc, target, stc.Labels, metaLabels)
+		sw, err := getScrapeWork(swc, target, stc.Labels, metaLabels)
 		if err != nil {
 			// Do not return this error, since other targets may be valid
 			logger.Errorf("error when parsing `static_configs` target %q for `job_name` %q: %s; skipping it", target, swc.jobName, err)
 			continue
 		}
+		if sw != nil {
+			dst = append(dst, sw)
+		}
 	}
 	return dst
 }
 
-func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) ([]*ScrapeWork, error) {
+func getScrapeWork(swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) (*ScrapeWork, error) {
 	labels := mergeLabels(swc.jobName, swc.scheme, target, swc.metricsPath, extraLabels, swc.externalLabels, metaLabels, swc.params)
 	var originalLabels []prompbmarshal.Label
 	if !*dropOriginalLabels {
@@ -703,7 +734,7 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 	if len(labels) == 0 {
 		// Drop target without labels.
 		droppedTargetsMap.Register(originalLabels)
-		return dst, nil
+		return nil, nil
 	}
 	// See https://www.robustperception.io/life-of-a-label
 	schemeRelabeled := promrelabel.GetLabelValueByName(labels, "__scheme__")
@@ -714,12 +745,12 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 	if len(addressRelabeled) == 0 {
 		// Drop target without scrape address.
 		droppedTargetsMap.Register(originalLabels)
-		return dst, nil
+		return nil, nil
 	}
 	if strings.Contains(addressRelabeled, "/") {
 		// Drop target with '/'
 		droppedTargetsMap.Register(originalLabels)
-		return dst, nil
+		return nil, nil
 	}
 	addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled)
 	metricsPathRelabeled := promrelabel.GetLabelValueByName(labels, "__metrics_path__")
@@ -737,7 +768,7 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 	paramsStr := url.Values(paramsRelabeled).Encode()
 	scrapeURL := fmt.Sprintf("%s://%s%s%s%s", schemeRelabeled, addressRelabeled, metricsPathRelabeled, optionalQuestion, paramsStr)
 	if _, err := url.Parse(scrapeURL); err != nil {
-		return dst, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q (%q), metrics_path=%q (%q) for `job_name` %q: %w",
+		return nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q (%q), metrics_path=%q (%q) for `job_name` %q: %w",
 			scrapeURL, swc.scheme, schemeRelabeled, target, addressRelabeled, swc.metricsPath, metricsPathRelabeled, swc.jobName, err)
 	}
 	// Set missing "instance" label according to https://www.robustperception.io/life-of-a-label
@@ -750,7 +781,7 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 	}
 	// Reduce memory usage by interning all the strings in labels.
 	internLabelStrings(labels)
-	dst = append(dst, &ScrapeWork{
+	sw := &ScrapeWork{
 		ScrapeURL:            scrapeURL,
 		ScrapeInterval:       swc.scrapeInterval,
 		ScrapeTimeout:        swc.scrapeTimeout,
@@ -768,8 +799,8 @@ func appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConfig, target string, e
 		ScrapeAlignInterval:  swc.scrapeAlignInterval,
 
 		jobNameOriginal: swc.jobName,
-	})
-	return dst, nil
+	}
+	return sw, nil
 }
 
 func internLabelStrings(labels []prompbmarshal.Label) {

From a93e6440016f61565d48ace2ce579759d0b6b51b Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 15:53:42 +0200
Subject: [PATCH 26/32] lib/promscrape: remove duplicate code a bit

---
 lib/promscrape/config.go                      | 93 ++++---------------
 lib/promscrape/discovery/consul/consul.go     |  2 +-
 lib/promscrape/discovery/dns/dns.go           |  2 +-
 .../discovery/dockerswarm/dockerswarm.go      |  2 +-
 lib/promscrape/discovery/ec2/ec2.go           |  2 +-
 lib/promscrape/discovery/eureka/eureka.go     |  2 +-
 lib/promscrape/discovery/gce/gce.go           |  2 +-
 .../discovery/kubernetes/kubernetes.go        |  2 +-
 .../discovery/openstack/openstack.go          |  4 +-
 9 files changed, 26 insertions(+), 85 deletions(-)

diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index b0c8eb578..b41bcc463 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -197,7 +197,7 @@ func (cfg *Config) getKubernetesSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.KubernetesSDConfigs {
 			sdc := &sc.KubernetesSDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendKubernetesScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "kubernetes_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -225,7 +225,7 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.OpenStackSDConfigs {
 			sdc := &sc.OpenStackSDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendOpenstackScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "openstack_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -253,7 +253,7 @@ func (cfg *Config) getDockerSwarmSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork
 		for j := range sc.DockerSwarmConfigs {
 			sdc := &sc.DockerSwarmConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendDockerSwarmScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "dockerswarm_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -281,7 +281,7 @@ func (cfg *Config) getConsulSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.ConsulSDConfigs {
 			sdc := &sc.ConsulSDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendConsulScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "consul_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -309,7 +309,7 @@ func (cfg *Config) getEurekaSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.EurekaSDConfigs {
 			sdc := &sc.EurekaSDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendEurekaScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "eureka_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -337,7 +337,7 @@ func (cfg *Config) getDNSSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.DNSSDConfigs {
 			sdc := &sc.DNSSDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendDNSScrapeWork(dst, sdc, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "dns_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -365,7 +365,7 @@ func (cfg *Config) getEC2SDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.EC2SDConfigs {
 			sdc := &sc.EC2SDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendEC2ScrapeWork(dst, sdc, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "ec2_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -393,7 +393,7 @@ func (cfg *Config) getGCESDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		for j := range sc.GCESDConfigs {
 			sdc := &sc.GCESDConfigs[j]
 			var okLocal bool
-			dst, okLocal = appendGCEScrapeWork(dst, sdc, sc.swc)
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "gce_sd_config")
 			if ok {
 				ok = okLocal
 			}
@@ -535,79 +535,20 @@ type scrapeWorkConfig struct {
 	scrapeAlignInterval  time.Duration
 }
 
-func appendKubernetesScrapeWork(dst []*ScrapeWork, sdc *kubernetes.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := kubernetes.GetLabels(sdc, baseDir)
-	if err != nil {
-		logger.Errorf("error when discovering kubernetes targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "kubernetes_sd_config"), true
+type targetLabelsGetter interface {
+	GetLabels(baseDir string) ([]map[string]string, error)
 }
 
-func appendOpenstackScrapeWork(dst []*ScrapeWork, sdc *openstack.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := openstack.GetLabels(sdc, baseDir)
+func appendSDScrapeWork(dst []*ScrapeWork, sdc targetLabelsGetter, baseDir string, swc *scrapeWorkConfig, discoveryType string) ([]*ScrapeWork, bool) {
+	targetLabels, err := sdc.GetLabels(baseDir)
 	if err != nil {
-		logger.Errorf("error when discovering openstack targets for `job_name` %q: %s; skipping it", swc.jobName, err)
+		logger.Errorf("skipping %s targets for job_name %q because of error: %s", discoveryType, swc.jobName, err)
 		return dst, false
 	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "openstack_sd_config"), true
+	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, discoveryType), true
 }
 
-func appendDockerSwarmScrapeWork(dst []*ScrapeWork, sdc *dockerswarm.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := dockerswarm.GetLabels(sdc, baseDir)
-	if err != nil {
-		logger.Errorf("error when discovering dockerswarm targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "dockerswarm_sd_config"), true
-}
-
-func appendConsulScrapeWork(dst []*ScrapeWork, sdc *consul.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := consul.GetLabels(sdc, baseDir)
-	if err != nil {
-		logger.Errorf("error when discovering consul targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "consul_sd_config"), true
-}
-
-func appendEurekaScrapeWork(dst []*ScrapeWork, sdc *eureka.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := eureka.GetLabels(sdc, baseDir)
-	if err != nil {
-		logger.Errorf("error when discovering eureka targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "eureka_sd_config"), true
-}
-
-func appendDNSScrapeWork(dst []*ScrapeWork, sdc *dns.SDConfig, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := dns.GetLabels(sdc)
-	if err != nil {
-		logger.Errorf("error when discovering dns targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "dns_sd_config"), true
-}
-
-func appendEC2ScrapeWork(dst []*ScrapeWork, sdc *ec2.SDConfig, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := ec2.GetLabels(sdc)
-	if err != nil {
-		logger.Errorf("error when discovering ec2 targets for `job_name` %q: %s; skipping it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "ec2_sd_config"), true
-}
-
-func appendGCEScrapeWork(dst []*ScrapeWork, sdc *gce.SDConfig, swc *scrapeWorkConfig) ([]*ScrapeWork, bool) {
-	targetLabels, err := gce.GetLabels(sdc)
-	if err != nil {
-		logger.Errorf("error when discovering gce targets for `job_name` %q: %s; skippint it", swc.jobName, err)
-		return dst, false
-	}
-	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "gce_sd_config"), true
-}
-
-func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, targetLabels []map[string]string, sectionName string) []*ScrapeWork {
+func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, targetLabels []map[string]string, discoveryType string) []*ScrapeWork {
 	startTime := time.Now()
 	// Process targetLabels in parallel in order to reduce processing time for big number of targetLabels.
 	type result struct {
@@ -623,7 +564,7 @@ func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, t
 				target := metaLabels["__address__"]
 				sw, err := getScrapeWork(swc, target, nil, metaLabels)
 				if err != nil {
-					err = fmt.Errorf("skipping target %q for job_name %q in %s because of error: %w", target, swc.jobName, sectionName, err)
+					err = fmt.Errorf("skipping %s target %q for job_name %q because of error: %w", discoveryType, target, swc.jobName, err)
 				}
 				resultCh <- result{
 					sw:  sw,
@@ -646,7 +587,7 @@ func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, t
 			dst = append(dst, r.sw)
 		}
 	}
-	metrics.GetOrCreateHistogram(fmt.Sprintf("vm_promscrape_target_relabel_duration_seconds{type=%q}", sectionName)).UpdateDuration(startTime)
+	metrics.GetOrCreateHistogram(fmt.Sprintf("vm_promscrape_target_relabel_duration_seconds{type=%q}", discoveryType)).UpdateDuration(startTime)
 	return dst
 }
 
diff --git a/lib/promscrape/discovery/consul/consul.go b/lib/promscrape/discovery/consul/consul.go
index 9c980f779..b0e81d062 100644
--- a/lib/promscrape/discovery/consul/consul.go
+++ b/lib/promscrape/discovery/consul/consul.go
@@ -29,7 +29,7 @@ type SDConfig struct {
 }
 
 // GetLabels returns Consul labels according to sdc.
-func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)
diff --git a/lib/promscrape/discovery/dns/dns.go b/lib/promscrape/discovery/dns/dns.go
index 40d8c580d..7d330a2d5 100644
--- a/lib/promscrape/discovery/dns/dns.go
+++ b/lib/promscrape/discovery/dns/dns.go
@@ -24,7 +24,7 @@ type SDConfig struct {
 }
 
 // GetLabels returns DNS labels according to sdc.
-func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	if len(sdc.Names) == 0 {
 		return nil, fmt.Errorf("`names` cannot be empty in `dns_sd_config`")
 	}
diff --git a/lib/promscrape/discovery/dockerswarm/dockerswarm.go b/lib/promscrape/discovery/dockerswarm/dockerswarm.go
index bd9564306..b58f68fcd 100644
--- a/lib/promscrape/discovery/dockerswarm/dockerswarm.go
+++ b/lib/promscrape/discovery/dockerswarm/dockerswarm.go
@@ -31,7 +31,7 @@ type Filter struct {
 }
 
 // GetLabels returns dockerswarm labels according to sdc.
-func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)
diff --git a/lib/promscrape/discovery/ec2/ec2.go b/lib/promscrape/discovery/ec2/ec2.go
index 8ffa1697e..6a830e163 100644
--- a/lib/promscrape/discovery/ec2/ec2.go
+++ b/lib/promscrape/discovery/ec2/ec2.go
@@ -31,7 +31,7 @@ type Filter struct {
 }
 
 // GetLabels returns ec2 labels according to sdc.
-func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)
diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go
index c8ebc55ef..c9759e6e0 100644
--- a/lib/promscrape/discovery/eureka/eureka.go
+++ b/lib/promscrape/discovery/eureka/eureka.go
@@ -82,7 +82,7 @@ type DataCenterInfo struct {
 }
 
 // GetLabels returns Eureka labels according to sdc.
-func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)
diff --git a/lib/promscrape/discovery/gce/gce.go b/lib/promscrape/discovery/gce/gce.go
index f0629d2ef..4234c7655 100644
--- a/lib/promscrape/discovery/gce/gce.go
+++ b/lib/promscrape/discovery/gce/gce.go
@@ -48,7 +48,7 @@ func (z *ZoneYAML) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // GetLabels returns gce labels according to sdc.
-func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)
diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go
index 2c7a48995..d1f29ed75 100644
--- a/lib/promscrape/discovery/kubernetes/kubernetes.go
+++ b/lib/promscrape/discovery/kubernetes/kubernetes.go
@@ -38,7 +38,7 @@ type Selector struct {
 }
 
 // GetLabels returns labels for the given sdc and baseDir.
-func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create API config: %w", err)
diff --git a/lib/promscrape/discovery/openstack/openstack.go b/lib/promscrape/discovery/openstack/openstack.go
index 0a1e9bd1d..f4b17bac3 100644
--- a/lib/promscrape/discovery/openstack/openstack.go
+++ b/lib/promscrape/discovery/openstack/openstack.go
@@ -31,8 +31,8 @@ type SDConfig struct {
 	Availability string              `yaml:"availability,omitempty"`
 }
 
-// GetLabels returns gce labels according to sdc.
-func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+// GetLabels returns OpenStack labels according to sdc.
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config: %w", err)

From 9b2246c29b860cd8a838d3e191f453ee8fcb5997 Mon Sep 17 00:00:00 2001
From: Nikolay <nik@victoriametrics.com>
Date: Fri, 26 Feb 2021 17:46:13 +0300
Subject: [PATCH 27/32] vmagent kubernetes watch stream discovery. (#1082)

* started work on sd for k8s

* continue work on watch sd

* fixes

* continue work

* continue work on sd k8s

* disable gzip

* fixes typos

* log errror

* minor fix

Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>
---
 lib/promscrape/config.go                      |  46 +-
 lib/promscrape/discovery/kubernetes/api.go    |  61 +--
 .../discovery/kubernetes/common_types.go      |   6 +
 .../discovery/kubernetes/endpoints.go         | 108 ++--
 .../discovery/kubernetes/endpoints_test.go    |   4 +-
 .../discovery/kubernetes/endpointslices.go    | 100 ++--
 .../kubernetes/endpointslices_test.go         |  16 +-
 lib/promscrape/discovery/kubernetes/framer.go |  56 ++
 .../discovery/kubernetes/ingress.go           |  76 +--
 .../discovery/kubernetes/kubernetes.go        |  28 +-
 lib/promscrape/discovery/kubernetes/node.go   |  47 +-
 lib/promscrape/discovery/kubernetes/pod.go    |  77 +--
 .../discovery/kubernetes/service.go           |  77 +--
 .../discovery/kubernetes/shared_cache.go      |  76 +++
 lib/promscrape/discovery/kubernetes/watch.go  | 510 ++++++++++++++++++
 lib/promscrape/scraper.go                     |   5 +-
 lib/promscrape/watch_handler.go               |  83 +++
 17 files changed, 989 insertions(+), 387 deletions(-)
 create mode 100644 lib/promscrape/discovery/kubernetes/framer.go
 create mode 100644 lib/promscrape/discovery/kubernetes/shared_cache.go
 create mode 100644 lib/promscrape/discovery/kubernetes/watch.go
 create mode 100644 lib/promscrape/watch_handler.go

diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index b41bcc463..ee2f4b2d9 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -48,6 +48,8 @@ type Config struct {
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
+	// used for data sync with kubernetes.
+	kwh *kubernetesWatchHandler
 }
 
 // GlobalConfig represents essential parts for `global` section of Prometheus config.
@@ -139,6 +141,7 @@ func loadConfig(path string) (cfg *Config, data []byte, err error) {
 	if err := cfgObj.parse(data, path); err != nil {
 		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
 	}
+	cfgObj.kwh = newKubernetesWatchHandler()
 	return &cfgObj, data, nil
 }
 
@@ -187,30 +190,41 @@ func getSWSByJob(sws []*ScrapeWork) map[string][]*ScrapeWork {
 }
 
 // getKubernetesSDScrapeWork returns `kubernetes_sd_configs` ScrapeWork from cfg.
-func (cfg *Config) getKubernetesSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
-	swsPrevByJob := getSWSByJob(prev)
+func getKubernetesSDScrapeWorkStream(cfg *Config, prev []*ScrapeWork) []*ScrapeWork {
+	cfg.kwh.startOnce.Do(func() {
+		go processKubernetesSyncEvents(cfg)
+	})
 	dst := make([]*ScrapeWork, 0, len(prev))
+	// updated access time.
+	cfg.kwh.mu.Lock()
+	cfg.kwh.lastAccessTime = time.Now()
+	cfg.kwh.mu.Unlock()
 	for i := range cfg.ScrapeConfigs {
 		sc := &cfg.ScrapeConfigs[i]
-		dstLen := len(dst)
-		ok := true
 		for j := range sc.KubernetesSDConfigs {
+			// generate set name
+			setKey := fmt.Sprintf("%d/%d/%s", i, j, sc.JobName)
+			cfg.kwh.mu.Lock()
+			cfg.kwh.sdcSet[setKey] = sc.swc
+			cfg.kwh.mu.Unlock()
 			sdc := &sc.KubernetesSDConfigs[j]
-			var okLocal bool
-			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "kubernetes_sd_config")
-			if ok {
-				ok = okLocal
+			ms, err := kubernetes.StartWatchOnce(cfg.kwh.watchCfg, setKey, sdc, cfg.baseDir)
+			if err != nil {
+				logger.Errorf("got unexpected error: %v", err)
 			}
-		}
-		if ok {
-			continue
-		}
-		swsPrev := swsPrevByJob[sc.swc.jobName]
-		if len(swsPrev) > 0 {
-			logger.Errorf("there were errors when discovering kubernetes targets for job %q, so preserving the previous targets", sc.swc.jobName)
-			dst = append(dst[:dstLen], swsPrev...)
+			dst = appendScrapeWorkForTargetLabels(dst, sc.swc, ms, "kubernetes_sd_config")
 		}
 	}
+	// dst will
+	if len(dst) > 0 {
+		return dst
+	}
+	// result from cache
+	cfg.kwh.mu.Lock()
+	for _, v := range cfg.kwh.swCache {
+		dst = append(dst, v...)
+	}
+	cfg.kwh.mu.Unlock()
 	return dst
 }
 
diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index 2c0463214..a2b3834d9 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -1,77 +1,42 @@
 package kubernetes
 
 import (
-	"fmt"
-	"net"
-	"os"
+	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
 // apiConfig contains config for API server
 type apiConfig struct {
-	client     *discoveryutils.Client
+	setName    string
 	namespaces []string
 	selectors  []Selector
+	wc         *watchClient
+	targetChan chan SyncEvent
+	watchOnce  sync.Once
 }
 
 var configMap = discoveryutils.NewConfigMap()
 
-func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
+func getAPIConfig(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(watchCfg, setName, sdc, baseDir) })
 	if err != nil {
 		return nil, err
 	}
 	return v.(*apiConfig), nil
 }
 
-func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
+func newAPIConfig(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	wc, err := newWatchClient(watchCfg.WG, sdc, baseDir)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse auth config: %w", err)
-	}
-	apiServer := sdc.APIServer
-	if len(apiServer) == 0 {
-		// Assume we run at k8s pod.
-		// Discover apiServer and auth config according to k8s docs.
-		// See https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller
-		host := os.Getenv("KUBERNETES_SERVICE_HOST")
-		port := os.Getenv("KUBERNETES_SERVICE_PORT")
-		if len(host) == 0 {
-			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_HOST env var; it must be defined when running in k8s; " +
-				"probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?")
-		}
-		if len(port) == 0 {
-			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_PORT env var; it must be defined when running in k8s; "+
-				"KUBERNETES_SERVICE_HOST=%q", host)
-		}
-		apiServer = "https://" + net.JoinHostPort(host, port)
-		tlsConfig := promauth.TLSConfig{
-			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
-		}
-		acNew, err := promauth.NewConfig(".", nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
-		if err != nil {
-			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
-		}
-		ac = acNew
-	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
+		return nil, err
 	}
 	cfg := &apiConfig{
-		client:     client,
+		setName:    setName,
+		targetChan: watchCfg.WatchChan,
+		wc:         wc,
 		namespaces: sdc.Namespaces.Names,
 		selectors:  sdc.Selectors,
 	}
 	return cfg, nil
 }
-
-func getAPIResponse(cfg *apiConfig, role, path string) ([]byte, error) {
-	query := joinSelectors(role, cfg.namespaces, cfg.selectors)
-	if len(query) > 0 {
-		path += "?" + query
-	}
-	return cfg.client.GetAPIResponse(path)
-}
diff --git a/lib/promscrape/discovery/kubernetes/common_types.go b/lib/promscrape/discovery/kubernetes/common_types.go
index d1bc21203..f1b8dcce9 100644
--- a/lib/promscrape/discovery/kubernetes/common_types.go
+++ b/lib/promscrape/discovery/kubernetes/common_types.go
@@ -19,6 +19,12 @@ type ObjectMeta struct {
 	OwnerReferences []OwnerReference
 }
 
+// listMetadata kubernetes list metadata
+// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#listmeta-v1-meta
+type listMetadata struct {
+	ResourceVersion string `json:"resourceVersion"`
+}
+
 func (om *ObjectMeta) registerLabelsAndAnnotations(prefix string, m map[string]string) {
 	for _, lb := range om.Labels {
 		ln := discoveryutils.SanitizeLabelName(lb.Name)
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 6f95a2a9d..2839202d0 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -3,70 +3,18 @@ package kubernetes
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getEndpointsLabels returns labels for k8s endpoints obtained from the given cfg.
-func getEndpointsLabels(cfg *apiConfig) ([]map[string]string, error) {
-	eps, err := getEndpoints(cfg)
-	if err != nil {
-		return nil, err
-	}
-	pods, err := getPods(cfg)
-	if err != nil {
-		return nil, err
-	}
-	svcs, err := getServices(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []map[string]string
-	for _, ep := range eps {
-		ms = ep.appendTargetLabels(ms, pods, svcs)
-	}
-	return ms, nil
-}
-
-func getEndpoints(cfg *apiConfig) ([]Endpoints, error) {
-	if len(cfg.namespaces) == 0 {
-		return getEndpointsByPath(cfg, "/api/v1/endpoints")
-	}
-	// Query /api/v1/namespaces/* for each namespace.
-	// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
-	cfgCopy := *cfg
-	namespaces := cfgCopy.namespaces
-	cfgCopy.namespaces = nil
-	cfg = &cfgCopy
-	var result []Endpoints
-	for _, ns := range namespaces {
-		path := fmt.Sprintf("/api/v1/namespaces/%s/endpoints", ns)
-		eps, err := getEndpointsByPath(cfg, path)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, eps...)
-	}
-	return result, nil
-}
-
-func getEndpointsByPath(cfg *apiConfig, path string) ([]Endpoints, error) {
-	data, err := getAPIResponse(cfg, "endpoints", path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain endpoints data from API server: %w", err)
-	}
-	epl, err := parseEndpointsList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse endpoints response from API server: %w", err)
-	}
-	return epl.Items, nil
-}
-
 // EndpointsList implements k8s endpoints list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslist-v1-core
 type EndpointsList struct {
-	Items []Endpoints
+	Items    []Endpoints
+	Metadata listMetadata `json:"metadata"`
 }
 
 // Endpoints implements k8s endpoints.
@@ -77,6 +25,10 @@ type Endpoints struct {
 	Subsets  []EndpointSubset
 }
 
+func (eps *Endpoints) key() string {
+	return eps.Metadata.Namespace + "/" + eps.Metadata.Name
+}
+
 // EndpointSubset implements k8s endpoint subset.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointsubset-v1-core
@@ -105,6 +57,10 @@ type ObjectReference struct {
 	Namespace string
 }
 
+func (or ObjectReference) key() string {
+	return or.Namespace + "/" + or.Name
+}
+
 // EndpointPort implements k8s endpoint port.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointport-v1beta1-discovery-k8s-io
@@ -127,13 +83,16 @@ func parseEndpointsList(data []byte) (*EndpointsList, error) {
 // appendTargetLabels appends labels for each endpoint in eps to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#endpoints
-func (eps *Endpoints) appendTargetLabels(ms []map[string]string, pods []Pod, svcs []Service) []map[string]string {
-	svc := getService(svcs, eps.Metadata.Namespace, eps.Metadata.Name)
+func (eps *Endpoints) appendTargetLabels(ms []map[string]string, podsCache, servicesCache *sync.Map) []map[string]string {
+	var svc *Service
+	if svco, ok := servicesCache.Load(eps.key()); ok {
+		svc = svco.(*Service)
+	}
 	podPortsSeen := make(map[*Pod][]int)
 	for _, ess := range eps.Subsets {
 		for _, epp := range ess.Ports {
-			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.Addresses, epp, pods, svc, "true")
-			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.NotReadyAddresses, epp, pods, svc, "false")
+			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.Addresses, epp, podsCache, svc, "true")
+			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.NotReadyAddresses, epp, podsCache, svc, "false")
 		}
 	}
 
@@ -169,9 +128,13 @@ func (eps *Endpoints) appendTargetLabels(ms []map[string]string, pods []Pod, svc
 }
 
 func appendEndpointLabelsForAddresses(ms []map[string]string, podPortsSeen map[*Pod][]int, eps *Endpoints, eas []EndpointAddress, epp EndpointPort,
-	pods []Pod, svc *Service, ready string) []map[string]string {
+	podsCache *sync.Map, svc *Service, ready string) []map[string]string {
 	for _, ea := range eas {
-		p := getPod(pods, ea.TargetRef.Namespace, ea.TargetRef.Name)
+		var p *Pod
+		if po, ok := podsCache.Load(ea.TargetRef.key()); ok {
+			p = po.(*Pod)
+		}
+		//p := getPod(pods, ea.TargetRef.Namespace, ea.TargetRef.Name)
 		m := getEndpointLabelsForAddressAndPort(podPortsSeen, eps, ea, epp, p, svc, ready)
 		ms = append(ms, m)
 	}
@@ -223,3 +186,24 @@ func getEndpointLabels(om ObjectMeta, ea EndpointAddress, epp EndpointPort, read
 	}
 	return m
 }
+
+func processEndpoints(cfg *apiConfig, sc *SharedKubernetesCache, p *Endpoints, action string) {
+	key := buildSyncKey("endpoints", cfg.setName, p.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		lbs := p.appendTargetLabels(nil, sc.Pods, sc.Services)
+		cfg.targetChan <- SyncEvent{
+			Labels:           lbs,
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index e5f0f4251..8f154b5c6 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -89,7 +90,8 @@ func TestParseEndpointsListSuccess(t *testing.T) {
 	endpoint := els.Items[0]
 
 	// Check endpoint.appendTargetLabels()
-	labelss := endpoint.appendTargetLabels(nil, nil, nil)
+	var pc, sc sync.Map
+	labelss := endpoint.appendTargetLabels(nil, &pc, &sc)
 	var sortedLabelss [][]prompbmarshal.Label
 	for _, labels := range labelss {
 		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices.go b/lib/promscrape/discovery/kubernetes/endpointslices.go
index f1f9da645..9a0a2c9d9 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices.go
@@ -4,69 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getEndpointSlicesLabels returns labels for k8s endpointSlices obtained from the given cfg.
-func getEndpointSlicesLabels(cfg *apiConfig) ([]map[string]string, error) {
-	eps, err := getEndpointSlices(cfg)
-	if err != nil {
-		return nil, err
-	}
-	pods, err := getPods(cfg)
-	if err != nil {
-		return nil, err
-	}
-	svcs, err := getServices(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []map[string]string
-	for _, ep := range eps {
-		ms = ep.appendTargetLabels(ms, pods, svcs)
-	}
-
-	return ms, nil
-}
-
-// getEndpointSlices retrieves endpointSlice with given apiConfig
-func getEndpointSlices(cfg *apiConfig) ([]EndpointSlice, error) {
-	if len(cfg.namespaces) == 0 {
-		return getEndpointSlicesByPath(cfg, "/apis/discovery.k8s.io/v1beta1/endpointslices")
-	}
-	// Query /api/v1/namespaces/* for each namespace.
-	// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
-	cfgCopy := *cfg
-	namespaces := cfgCopy.namespaces
-	cfgCopy.namespaces = nil
-	cfg = &cfgCopy
-	var result []EndpointSlice
-	for _, ns := range namespaces {
-		path := fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/namespaces/%s/endpointslices", ns)
-		eps, err := getEndpointSlicesByPath(cfg, path)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, eps...)
-	}
-	return result, nil
-}
-
-// getEndpointSlicesByPath retrieves endpointSlices from k8s api by given path
-func getEndpointSlicesByPath(cfg *apiConfig, path string) ([]EndpointSlice, error) {
-	data, err := getAPIResponse(cfg, "endpointslices", path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain endpointslices data from API server: %w", err)
-	}
-	epl, err := parseEndpointSlicesList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse endpointslices response from API server: %w", err)
-	}
-	return epl.Items, nil
-
-}
-
 // parseEndpointsList parses EndpointSliceList from data.
 func parseEndpointSlicesList(data []byte) (*EndpointSliceList, error) {
 	var esl EndpointSliceList
@@ -79,11 +22,17 @@ func parseEndpointSlicesList(data []byte) (*EndpointSliceList, error) {
 
 // appendTargetLabels injects labels for endPointSlice to slice map
 // follows TargetRef for enrich labels with pod and service metadata
-func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, pods []Pod, svcs []Service) []map[string]string {
-	svc := getService(svcs, eps.Metadata.Namespace, eps.Metadata.Name)
+func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, podsCache, servicesCache *sync.Map) []map[string]string {
+	var svc *Service
+	if s, ok := servicesCache.Load(eps.key()); ok {
+		svc = s.(*Service)
+	}
 	podPortsSeen := make(map[*Pod][]int)
 	for _, ess := range eps.Endpoints {
-		pod := getPod(pods, ess.TargetRef.Namespace, ess.TargetRef.Name)
+		var pod *Pod
+		if p, ok := podsCache.Load(ess.TargetRef.key()); ok {
+			pod = p.(*Pod)
+		}
 		for _, epp := range eps.Ports {
 			for _, addr := range ess.Addresses {
 				ms = append(ms, getEndpointSliceLabelsForAddressAndPort(podPortsSeen, addr, eps, ess, epp, pod, svc))
@@ -186,7 +135,8 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En
 // that groups service endpoints slices.
 // https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
 type EndpointSliceList struct {
-	Items []EndpointSlice
+	Items    []EndpointSlice
+	Metadata listMetadata `json:"metadata"`
 }
 
 // EndpointSlice - implements kubernetes endpoint slice.
@@ -198,6 +148,10 @@ type EndpointSlice struct {
 	Ports       []EndpointPort
 }
 
+func (eps EndpointSlice) key() string {
+	return eps.Metadata.Namespace + "/" + eps.Metadata.Name
+}
+
 // Endpoint implements kubernetes object endpoint for endpoint slice.
 // https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io
 type Endpoint struct {
@@ -213,3 +167,23 @@ type Endpoint struct {
 type EndpointConditions struct {
 	Ready bool
 }
+
+func processEndpointSlices(cfg *apiConfig, sc *SharedKubernetesCache, p *EndpointSlice, action string) {
+	key := buildSyncKey("endpointslices", cfg.setName, p.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		cfg.targetChan <- SyncEvent{
+			Labels:           p.appendTargetLabels(nil, sc.Pods, sc.Services),
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices_test.go b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
index 900751b74..b1c3731ef 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -186,7 +187,8 @@ func Test_parseEndpointSlicesListSuccess(t *testing.T) {
 	}
 
 	firstEsl := esl.Items[0]
-	got := firstEsl.appendTargetLabels(nil, nil, nil)
+	var pc, sc sync.Map
+	got := firstEsl.appendTargetLabels(nil, &pc, &sc)
 	sortedLables := [][]prompbmarshal.Label{}
 	for _, labels := range got {
 		sortedLables = append(sortedLables, discoveryutils.GetSortedLabels(labels))
@@ -439,7 +441,17 @@ func TestEndpointSlice_appendTargetLabels(t *testing.T) {
 				AddressType: tt.fields.AddressType,
 				Ports:       tt.fields.Ports,
 			}
-			got := eps.appendTargetLabels(tt.args.ms, tt.args.pods, tt.args.svcs)
+			pc := sync.Map{}
+			sc := sync.Map{}
+			for _, p := range tt.args.pods {
+				p := &p
+				pc.Store(p.key(), p)
+			}
+			for _, s := range tt.args.svcs {
+				s := &s
+				sc.Store(s.key(), s)
+			}
+			got := eps.appendTargetLabels(tt.args.ms, &pc, &sc)
 			var sortedLabelss [][]prompbmarshal.Label
 			for _, labels := range got {
 				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
diff --git a/lib/promscrape/discovery/kubernetes/framer.go b/lib/promscrape/discovery/kubernetes/framer.go
new file mode 100644
index 000000000..3aff2c927
--- /dev/null
+++ b/lib/promscrape/discovery/kubernetes/framer.go
@@ -0,0 +1,56 @@
+package kubernetes
+
+import (
+	"encoding/json"
+	"io"
+)
+
+type jsonFrameReader struct {
+	r         io.ReadCloser
+	decoder   *json.Decoder
+	remaining []byte
+}
+
+func newJSONFramedReader(r io.ReadCloser) io.ReadCloser {
+	return &jsonFrameReader{
+		r:       r,
+		decoder: json.NewDecoder(r),
+	}
+}
+
+// ReadFrame decodes the next JSON object in the stream, or returns an error. The returned
+// byte slice will be modified the next time ReadFrame is invoked and should not be altered.
+func (r *jsonFrameReader) Read(data []byte) (int, error) {
+	// Return whatever remaining data exists from an in progress frame
+	if n := len(r.remaining); n > 0 {
+		if n <= len(data) {
+			data = append(data[0:0], r.remaining...)
+			r.remaining = nil
+			return n, nil
+		}
+		n = len(data)
+		data = append(data[0:0], r.remaining[:n]...)
+		r.remaining = r.remaining[n:]
+		return n, io.ErrShortBuffer
+	}
+
+	n := len(data)
+	m := json.RawMessage(data[:0])
+	if err := r.decoder.Decode(&m); err != nil {
+		return 0, err
+	}
+
+	// If capacity of data is less than length of the message, decoder will allocate a new slice
+	// and set m to it, which means we need to copy the partial result back into data and preserve
+	// the remaining result for subsequent reads.
+	if len(m) > n {
+		data = append(data[0:0], m[:n]...)
+		r.remaining = m[n:]
+		return n, io.ErrShortBuffer
+	}
+	return len(m), nil
+}
+
+func (r *jsonFrameReader) Close() error {
+	return r.r.Close()
+}
diff --git a/lib/promscrape/discovery/kubernetes/ingress.go b/lib/promscrape/discovery/kubernetes/ingress.go
index 5aad6d903..7a17854df 100644
--- a/lib/promscrape/discovery/kubernetes/ingress.go
+++ b/lib/promscrape/discovery/kubernetes/ingress.go
@@ -3,60 +3,16 @@ package kubernetes
 import (
 	"encoding/json"
 	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// getIngressesLabels returns labels for k8s ingresses obtained from the given cfg.
-func getIngressesLabels(cfg *apiConfig) ([]map[string]string, error) {
-	igs, err := getIngresses(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []map[string]string
-	for _, ig := range igs {
-		ms = ig.appendTargetLabels(ms)
-	}
-	return ms, nil
-}
-
-func getIngresses(cfg *apiConfig) ([]Ingress, error) {
-	if len(cfg.namespaces) == 0 {
-		return getIngressesByPath(cfg, "/apis/extensions/v1beta1/ingresses")
-	}
-	// Query /api/v1/namespaces/* for each namespace.
-	// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
-	cfgCopy := *cfg
-	namespaces := cfgCopy.namespaces
-	cfgCopy.namespaces = nil
-	cfg = &cfgCopy
-	var result []Ingress
-	for _, ns := range namespaces {
-		path := fmt.Sprintf("/apis/extensions/v1beta1/namespaces/%s/ingresses", ns)
-		igs, err := getIngressesByPath(cfg, path)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, igs...)
-	}
-	return result, nil
-}
-
-func getIngressesByPath(cfg *apiConfig, path string) ([]Ingress, error) {
-	data, err := getAPIResponse(cfg, "ingress", path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain ingresses data from API server: %w", err)
-	}
-	igl, err := parseIngressList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse ingresses response from API server: %w", err)
-	}
-	return igl.Items, nil
-}
-
 // IngressList represents ingress list in k8s.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresslist-v1beta1-extensions
 type IngressList struct {
-	Items []Ingress
+	Items    []Ingress
+	Metadata listMetadata `json:"metadata"`
 }
 
 // Ingress represents ingress in k8s.
@@ -67,6 +23,10 @@ type Ingress struct {
 	Spec     IngressSpec
 }
 
+func (ig Ingress) key() string {
+	return ig.Metadata.Namespace + "/" + ig.Metadata.Name
+}
+
 // IngressSpec represents ingress spec in k8s.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressspec-v1beta1-extensions
@@ -164,3 +124,23 @@ func getIngressRulePaths(paths []HTTPIngressPath) []string {
 	}
 	return result
 }
+
+func processIngress(cfg *apiConfig, p *Ingress, action string) {
+	key := buildSyncKey("ingress", cfg.setName, p.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		cfg.targetChan <- SyncEvent{
+			Labels:           p.appendTargetLabels(nil),
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "ERROR":
+	default:
+		logger.Infof("unexpected action: %s", action)
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go
index d1f29ed75..aea328438 100644
--- a/lib/promscrape/discovery/kubernetes/kubernetes.go
+++ b/lib/promscrape/discovery/kubernetes/kubernetes.go
@@ -37,26 +37,16 @@ type Selector struct {
 	Field string `yaml:"field"`
 }
 
-// GetLabels returns labels for the given sdc and baseDir.
-func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
-	cfg, err := getAPIConfig(sdc, baseDir)
+// StartWatchOnce returns init labels for the given sdc and baseDir.
+// and starts watching for changes.
+func StartWatchOnce(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) ([]map[string]string, error) {
+	cfg, err := getAPIConfig(watchCfg, setName, sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create API config: %w", err)
 	}
-	switch sdc.Role {
-	case "node":
-		return getNodesLabels(cfg)
-	case "service":
-		return getServicesLabels(cfg)
-	case "pod":
-		return getPodsLabels(cfg)
-	case "endpoints":
-		return getEndpointsLabels(cfg)
-	case "endpointslices":
-		return getEndpointSlicesLabels(cfg)
-	case "ingress":
-		return getIngressesLabels(cfg)
-	default:
-		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `service`, `pod`, `endpoints` or `ingress`; skipping it", sdc.Role)
-	}
+	var ms []map[string]string
+	cfg.watchOnce.Do(func() {
+		ms = startWatcherByRole(watchCfg.Ctx, sdc.Role, cfg, watchCfg.SC)
+	})
+	return ms, nil
 }
diff --git a/lib/promscrape/discovery/kubernetes/node.go b/lib/promscrape/discovery/kubernetes/node.go
index 9a584c67e..134bdc77a 100644
--- a/lib/promscrape/discovery/kubernetes/node.go
+++ b/lib/promscrape/discovery/kubernetes/node.go
@@ -4,32 +4,16 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getNodesLabels returns labels for k8s nodes obtained from the given cfg.
-func getNodesLabels(cfg *apiConfig) ([]map[string]string, error) {
-	data, err := getAPIResponse(cfg, "node", "/api/v1/nodes")
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain nodes data from API server: %w", err)
-	}
-	nl, err := parseNodeList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse nodes response from API server: %w", err)
-	}
-	var ms []map[string]string
-	for _, n := range nl.Items {
-		// Do not apply namespaces, since they are missing in nodes.
-		ms = n.appendTargetLabels(ms)
-	}
-	return ms, nil
-}
-
 // NodeList represents NodeList from k8s API.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodelist-v1-core
 type NodeList struct {
-	Items []Node
+	Items    []Node
+	Metadata listMetadata `json:"metadata"`
 }
 
 // Node represents Node from k8s API.
@@ -40,6 +24,10 @@ type Node struct {
 	Status   NodeStatus
 }
 
+func (n Node) key() string {
+	return n.Metadata.Name
+}
+
 // NodeStatus represents NodeStatus from k8s API.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodestatus-v1-core
@@ -131,3 +119,24 @@ func getAddrByType(nas []NodeAddress, typ string) string {
 	}
 	return ""
 }
+
+func processNode(cfg *apiConfig, n *Node, action string) {
+	key := buildSyncKey("nodes", cfg.setName, n.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		lbs := n.appendTargetLabels(nil)
+		cfg.targetChan <- SyncEvent{
+			Labels:           lbs,
+			ConfigSectionSet: cfg.setName,
+			Key:              key,
+		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			ConfigSectionSet: cfg.setName,
+			Key:              key,
+		}
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go
index 3bc3495b0..1d5c384e4 100644
--- a/lib/promscrape/discovery/kubernetes/pod.go
+++ b/lib/promscrape/discovery/kubernetes/pod.go
@@ -6,61 +6,16 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getPodsLabels returns labels for k8s pods obtained from the given cfg
-func getPodsLabels(cfg *apiConfig) ([]map[string]string, error) {
-	pods, err := getPods(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []map[string]string
-	for _, p := range pods {
-		ms = p.appendTargetLabels(ms)
-	}
-	return ms, nil
-}
-
-func getPods(cfg *apiConfig) ([]Pod, error) {
-	if len(cfg.namespaces) == 0 {
-		return getPodsByPath(cfg, "/api/v1/pods")
-	}
-	// Query /api/v1/namespaces/* for each namespace.
-	// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
-	cfgCopy := *cfg
-	namespaces := cfgCopy.namespaces
-	cfgCopy.namespaces = nil
-	cfg = &cfgCopy
-	var result []Pod
-	for _, ns := range namespaces {
-		path := fmt.Sprintf("/api/v1/namespaces/%s/pods", ns)
-		pods, err := getPodsByPath(cfg, path)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, pods...)
-	}
-	return result, nil
-}
-
-func getPodsByPath(cfg *apiConfig, path string) ([]Pod, error) {
-	data, err := getAPIResponse(cfg, "pod", path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain pods data from API server: %w", err)
-	}
-	pl, err := parsePodList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse pods response from API server: %w", err)
-	}
-	return pl.Items, nil
-}
-
 // PodList implements k8s pod list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podlist-v1-core
 type PodList struct {
-	Items []Pod
+	Items    []Pod
+	Metadata listMetadata `json:"metadata"`
 }
 
 // Pod implements k8s pod.
@@ -72,6 +27,10 @@ type Pod struct {
 	Status   PodStatus
 }
 
+func (p Pod) key() string {
+	return p.Metadata.Namespace + "/" + p.Metadata.Name
+}
+
 // PodSpec implements k8s pod spec.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podspec-v1-core
@@ -211,12 +170,22 @@ func getPodReadyStatus(conds []PodCondition) string {
 	return "unknown"
 }
 
-func getPod(pods []Pod, namespace, name string) *Pod {
-	for i := range pods {
-		pod := &pods[i]
-		if pod.Metadata.Name == name && pod.Metadata.Namespace == namespace {
-			return pod
+func processPods(cfg *apiConfig, p *Pod, action string) {
+	key := buildSyncKey("pods", cfg.setName, p.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		cfg.targetChan <- SyncEvent{
+			Labels:           p.appendTargetLabels(nil),
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
 		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
 	}
-	return nil
 }
diff --git a/lib/promscrape/discovery/kubernetes/service.go b/lib/promscrape/discovery/kubernetes/service.go
index 5ce94a4c9..3723493bb 100644
--- a/lib/promscrape/discovery/kubernetes/service.go
+++ b/lib/promscrape/discovery/kubernetes/service.go
@@ -4,61 +4,16 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getServicesLabels returns labels for k8s services obtained from the given cfg.
-func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
-	svcs, err := getServices(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []map[string]string
-	for _, svc := range svcs {
-		ms = svc.appendTargetLabels(ms)
-	}
-	return ms, nil
-}
-
-func getServices(cfg *apiConfig) ([]Service, error) {
-	if len(cfg.namespaces) == 0 {
-		return getServicesByPath(cfg, "/api/v1/services")
-	}
-	// Query /api/v1/namespaces/* for each namespace.
-	// This fixes authorization issue at https://github.com/VictoriaMetrics/VictoriaMetrics/issues/432
-	cfgCopy := *cfg
-	namespaces := cfgCopy.namespaces
-	cfgCopy.namespaces = nil
-	cfg = &cfgCopy
-	var result []Service
-	for _, ns := range namespaces {
-		path := fmt.Sprintf("/api/v1/namespaces/%s/services", ns)
-		svcs, err := getServicesByPath(cfg, path)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, svcs...)
-	}
-	return result, nil
-}
-
-func getServicesByPath(cfg *apiConfig, path string) ([]Service, error) {
-	data, err := getAPIResponse(cfg, "service", path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot obtain services data from API server: %w", err)
-	}
-	sl, err := parseServiceList(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse services response from API server: %w", err)
-	}
-	return sl.Items, nil
-}
-
 // ServiceList is k8s service list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicelist-v1-core
 type ServiceList struct {
-	Items []Service
+	Items    []Service
+	Metadata listMetadata `json:"metadata"`
 }
 
 // Service is k8s service.
@@ -69,6 +24,10 @@ type Service struct {
 	Spec     ServiceSpec
 }
 
+func (s Service) key() string {
+	return s.Metadata.Namespace + "/" + s.Metadata.Name
+}
+
 // ServiceSpec is k8s service spec.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core
@@ -127,12 +86,22 @@ func (s *Service) appendCommonLabels(m map[string]string) {
 	s.Metadata.registerLabelsAndAnnotations("__meta_kubernetes_service", m)
 }
 
-func getService(svcs []Service, namespace, name string) *Service {
-	for i := range svcs {
-		svc := &svcs[i]
-		if svc.Metadata.Name == name && svc.Metadata.Namespace == namespace {
-			return svc
+func processService(cfg *apiConfig, svc *Service, action string) {
+	key := buildSyncKey("service", cfg.setName, svc.key())
+	switch action {
+	case "ADDED", "MODIFIED":
+		cfg.targetChan <- SyncEvent{
+			Labels:           svc.appendTargetLabels(nil),
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
 		}
+	case "DELETED":
+		cfg.targetChan <- SyncEvent{
+			Key:              key,
+			ConfigSectionSet: cfg.setName,
+		}
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
 	}
-	return nil
 }
diff --git a/lib/promscrape/discovery/kubernetes/shared_cache.go b/lib/promscrape/discovery/kubernetes/shared_cache.go
new file mode 100644
index 000000000..72c763c39
--- /dev/null
+++ b/lib/promscrape/discovery/kubernetes/shared_cache.go
@@ -0,0 +1,76 @@
+package kubernetes
+
+import (
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// SharedKubernetesCache holds cache of kubernetes objects for current config.
+type SharedKubernetesCache struct {
+	Endpoints       *sync.Map
+	EndpointsSlices *sync.Map
+	Pods            *sync.Map
+	Services        *sync.Map
+}
+
+// NewSharedKubernetesCache returns new cache.
+func NewSharedKubernetesCache() *SharedKubernetesCache {
+	return &SharedKubernetesCache{
+		Endpoints:       new(sync.Map),
+		EndpointsSlices: new(sync.Map),
+		Pods:            new(sync.Map),
+		Services:        new(sync.Map),
+	}
+}
+
+func updatePodCache(cache *sync.Map, p *Pod, action string) {
+	switch action {
+	case "ADDED":
+		cache.Store(p.key(), p)
+	case "DELETED":
+		cache.Delete(p.key())
+	case "MODIFIED":
+		cache.Store(p.key(), p)
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+}
+
+func updateServiceCache(cache *sync.Map, p *Service, action string) {
+	switch action {
+	case "ADDED", "MODIFIED":
+		cache.Store(p.key(), p)
+	case "DELETED":
+		cache.Delete(p.key())
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+
+}
+
+func updateEndpointsCache(cache *sync.Map, p *Endpoints, action string) {
+	switch action {
+	case "ADDED", "MODIFIED":
+		cache.Store(p.key(), p)
+	case "DELETED":
+		cache.Delete(p.key())
+	case "ERROR":
+	default:
+		logger.Warnf("unexpected action: %s", action)
+	}
+}
+
+func updateEndpointsSliceCache(cache *sync.Map, p *EndpointSlice, action string) {
+	switch action {
+	case "ADDED", "MODIFIED":
+		cache.Store(p.key(), p)
+	case "DELETED":
+		cache.Delete(p.key())
+	case "ERROR":
+	default:
+		logger.Infof("unexpected action: %s", action)
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/watch.go b/lib/promscrape/discovery/kubernetes/watch.go
new file mode 100644
index 000000000..1cec161a7
--- /dev/null
+++ b/lib/promscrape/discovery/kubernetes/watch.go
@@ -0,0 +1,510 @@
+package kubernetes
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+// SyncEvent represent kubernetes resource watch event.
+type SyncEvent struct {
+	// object type + set name + ns + name
+	// must be unique.
+	Key string
+	// Labels targets labels for given resource
+	Labels []map[string]string
+	// job name + position id
+	ConfigSectionSet string
+}
+
+type watchResponse struct {
+	Action string          `json:"type"`
+	Object json.RawMessage `json:"object"`
+}
+
+// WatchConfig holds objects for watch handler start.
+type WatchConfig struct {
+	Ctx       context.Context
+	SC        *SharedKubernetesCache
+	WG        *sync.WaitGroup
+	WatchChan chan SyncEvent
+}
+
+// NewWatchConfig returns new config with given context.
+func NewWatchConfig(ctx context.Context) *WatchConfig {
+	return &WatchConfig{
+		Ctx:       ctx,
+		SC:        NewSharedKubernetesCache(),
+		WG:        new(sync.WaitGroup),
+		WatchChan: make(chan SyncEvent, 100),
+	}
+}
+
+func buildSyncKey(objType string, setName string, objKey string) string {
+	return objType + "/" + setName + "/" + objKey
+}
+
+func startWatcherByRole(ctx context.Context, role string, cfg *apiConfig, sc *SharedKubernetesCache) []map[string]string {
+	var ms []map[string]string
+	switch role {
+	case "pod":
+		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
+			var p Pod
+			if err := json.Unmarshal(wr.Object, &p); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processPods(cfg, &p, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			pods, err := parsePodList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, pod := range pods.Items {
+				ms = pod.appendTargetLabels(ms)
+				processPods(cfg, &pod, "ADDED")
+			}
+			return pods.Metadata.ResourceVersion, nil
+		})
+	case "node":
+		startWatchForObject(ctx, cfg, "nodes", func(wr *watchResponse) {
+			var n Node
+			if err := json.Unmarshal(wr.Object, &n); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processNode(cfg, &n, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			nodes, err := parseNodeList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, node := range nodes.Items {
+				processNode(cfg, &node, "ADDED")
+				ms = node.appendTargetLabels(ms)
+			}
+			return nodes.Metadata.ResourceVersion, nil
+		})
+	case "endpoints":
+		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
+			var p Pod
+			if err := json.Unmarshal(wr.Object, &p); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			updatePodCache(sc.Pods, &p, wr.Action)
+			if wr.Action == "MODIFIED" {
+				eps, ok := sc.Endpoints.Load(p.key())
+				if ok {
+					ep := eps.(*Endpoints)
+					processEndpoints(cfg, sc, ep, wr.Action)
+				}
+			}
+		}, func(bytes []byte) (string, error) {
+			pods, err := parsePodList(bytes)
+			if err != nil {
+				return "", err
+			}
+			for _, pod := range pods.Items {
+				updatePodCache(sc.Pods, &pod, "ADDED")
+			}
+			return pods.Metadata.ResourceVersion, nil
+		})
+		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
+			var svc Service
+			if err := json.Unmarshal(wr.Object, &svc); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			updateServiceCache(sc.Services, &svc, wr.Action)
+			if wr.Action == "MODIFIED" {
+				linkedEps, ok := sc.Endpoints.Load(svc.key())
+				if ok {
+					ep := linkedEps.(*Endpoints)
+					processEndpoints(cfg, sc, ep, wr.Action)
+				}
+			}
+		}, func(bytes []byte) (string, error) {
+			svcs, err := parseServiceList(bytes)
+			if err != nil {
+				return "", err
+			}
+			for _, svc := range svcs.Items {
+				updateServiceCache(sc.Services, &svc, "ADDED")
+			}
+			return svcs.Metadata.ResourceVersion, nil
+		})
+		startWatchForObject(ctx, cfg, "endpoints", func(wr *watchResponse) {
+			var eps Endpoints
+			if err := json.Unmarshal(wr.Object, &eps); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processEndpoints(cfg, sc, &eps, wr.Action)
+			updateEndpointsCache(sc.Endpoints, &eps, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			eps, err := parseEndpointsList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, ep := range eps.Items {
+				ms = ep.appendTargetLabels(ms, sc.Pods, sc.Services)
+				processEndpoints(cfg, sc, &ep, "ADDED")
+				updateEndpointsCache(sc.Endpoints, &ep, "ADDED")
+			}
+			return eps.Metadata.ResourceVersion, nil
+		})
+	case "service":
+		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
+			var svc Service
+			if err := json.Unmarshal(wr.Object, &svc); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processService(cfg, &svc, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			svcs, err := parseServiceList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, svc := range svcs.Items {
+				processService(cfg, &svc, "ADDED")
+				ms = svc.appendTargetLabels(ms)
+			}
+			return svcs.Metadata.ResourceVersion, nil
+		})
+	case "ingress":
+		startWatchForObject(ctx, cfg, "ingresses", func(wr *watchResponse) {
+			var ig Ingress
+			if err := json.Unmarshal(wr.Object, &ig); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processIngress(cfg, &ig, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			igs, err := parseIngressList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, ig := range igs.Items {
+				processIngress(cfg, &ig, "ADDED")
+				ms = ig.appendTargetLabels(ms)
+			}
+			return igs.Metadata.ResourceVersion, nil
+		})
+	case "endpointslices":
+		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
+			var p Pod
+			if err := json.Unmarshal(wr.Object, &p); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			updatePodCache(sc.Pods, &p, wr.Action)
+			if wr.Action == "MODIFIED" {
+				eps, ok := sc.EndpointsSlices.Load(p.key())
+				if ok {
+					ep := eps.(*EndpointSlice)
+					processEndpointSlices(cfg, sc, ep, wr.Action)
+				}
+			}
+		}, func(bytes []byte) (string, error) {
+			pods, err := parsePodList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, pod := range pods.Items {
+				updatePodCache(sc.Pods, &pod, "ADDED")
+			}
+			return pods.Metadata.ResourceVersion, nil
+		})
+		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
+			var svc Service
+			if err := json.Unmarshal(wr.Object, &svc); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			updateServiceCache(sc.Services, &svc, wr.Action)
+			if wr.Action == "MODIFIED" {
+				linkedEps, ok := sc.EndpointsSlices.Load(svc.key())
+				if ok {
+					ep := linkedEps.(*EndpointSlice)
+					processEndpointSlices(cfg, sc, ep, wr.Action)
+				}
+			}
+		}, func(bytes []byte) (string, error) {
+			svcs, err := parseServiceList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, svc := range svcs.Items {
+				updateServiceCache(sc.Services, &svc, "ADDED")
+			}
+			return svcs.Metadata.ResourceVersion, nil
+		})
+		startWatchForObject(ctx, cfg, "endpointslices", func(wr *watchResponse) {
+			var eps EndpointSlice
+			if err := json.Unmarshal(wr.Object, &eps); err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return
+			}
+			processEndpointSlices(cfg, sc, &eps, wr.Action)
+			updateEndpointsSliceCache(sc.EndpointsSlices, &eps, wr.Action)
+		}, func(bytes []byte) (string, error) {
+			epss, err := parseEndpointSlicesList(bytes)
+			if err != nil {
+				logger.Errorf("failed to parse object, err: %v", err)
+				return "", err
+			}
+			for _, eps := range epss.Items {
+				ms = eps.appendTargetLabels(ms, sc.Pods, sc.Services)
+				processEndpointSlices(cfg, sc, &eps, "ADDED")
+			}
+			return epss.Metadata.ResourceVersion, nil
+		})
+	default:
+		logger.Errorf("unexpected role: %s", role)
+	}
+	return ms
+}
+
+func startWatchForObject(ctx context.Context, cfg *apiConfig, objectName string, wh func(wr *watchResponse), getSync func([]byte) (string, error)) {
+	if len(cfg.namespaces) > 0 {
+		for _, ns := range cfg.namespaces {
+			path := fmt.Sprintf("/api/v1/namespaces/%s/%s", ns, objectName)
+			// special case.
+			if objectName == "endpointslices" {
+				path = fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/namespaces/%s/%s", ns, objectName)
+			}
+			query := joinSelectors(objectName, nil, cfg.selectors)
+			if len(query) > 0 {
+				path += "?" + query
+			}
+			data, err := cfg.wc.getBlockingAPIResponse(path)
+			if err != nil {
+				logger.Errorf("cannot get latest resource version: %v", err)
+			}
+			version, err := getSync(data)
+			if err != nil {
+				logger.Errorf("cannot get latest resource version: %v", err)
+			}
+			cfg.wc.wg.Add(1)
+			go func(path, version string) {
+				cfg.wc.startWatchForResource(ctx, path, wh, version)
+			}(path, version)
+		}
+	} else {
+		path := "/api/v1/" + objectName
+		if objectName == "endpointslices" {
+			// special case.
+			path = fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/%s", objectName)
+		}
+		query := joinSelectors(objectName, nil, cfg.selectors)
+		if len(query) > 0 {
+			path += "?" + query
+		}
+		data, err := cfg.wc.getBlockingAPIResponse(path)
+		if err != nil {
+			logger.Errorf("cannot get latest resource version: %v", err)
+		}
+		version, err := getSync(data)
+		if err != nil {
+			logger.Errorf("cannot get latest resource version: %v", err)
+		}
+		cfg.wc.wg.Add(1)
+		go func() {
+			cfg.wc.startWatchForResource(ctx, path, wh, version)
+		}()
+	}
+}
+
+type watchClient struct {
+	c         *http.Client
+	ac        *promauth.Config
+	apiServer string
+	wg        *sync.WaitGroup
+}
+
+func (wc *watchClient) startWatchForResource(ctx context.Context, path string, wh func(wr *watchResponse), initResourceVersion string) {
+	defer wc.wg.Done()
+	path += "?watch=1"
+	maxBackOff := time.Second * 30
+	backoff := time.Second
+	for {
+		err := wc.getStreamAPIResponse(ctx, path, initResourceVersion, wh)
+		if errors.Is(err, context.Canceled) {
+			return
+		}
+		if !errors.Is(err, io.EOF) {
+			logger.Errorf("got unexpected error : %v", err)
+		}
+		// reset version.
+		initResourceVersion = ""
+		if backoff < maxBackOff {
+			backoff += time.Second * 5
+		}
+		time.Sleep(backoff)
+	}
+}
+
+func (wc *watchClient) getBlockingAPIResponse(path string) ([]byte, error) {
+	req, err := http.NewRequest("GET", wc.apiServer+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept-Encoding", "gzip")
+	if wc.ac != nil && wc.ac.Authorization != "" {
+		req.Header.Set("Authorization", wc.ac.Authorization)
+	}
+	resp, err := wc.c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("get unexpected code: %d, at blocking api request path: %q", resp.StatusCode, path)
+	}
+	if ce := resp.Header.Get("Content-Encoding"); ce == "gzip" {
+		gr, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("cannot create gzip reader: %w", err)
+		}
+		return ioutil.ReadAll(gr)
+	}
+	return ioutil.ReadAll(resp.Body)
+}
+
+func (wc *watchClient) getStreamAPIResponse(ctx context.Context, path, resouceVersion string, wh func(wr *watchResponse)) error {
+	if resouceVersion != "" {
+		path += "&resourceVersion=" + resouceVersion
+	}
+	req, err := http.NewRequestWithContext(ctx, "GET", wc.apiServer+path, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Accept-Encoding", "gzip")
+	if wc.ac != nil && wc.ac.Authorization != "" {
+		req.Header.Set("Authorization", wc.ac.Authorization)
+	}
+	resp, err := wc.c.Do(req)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+	br := resp.Body
+	if ce := resp.Header.Get("Content-Encoding"); ce == "gzip" {
+		br, err = gzip.NewReader(resp.Body)
+		if err != nil {
+			return fmt.Errorf("cannot create gzip reader: %w", err)
+		}
+	}
+	r := newJSONFramedReader(br)
+	for {
+		b := make([]byte, 1024)
+		b, err := readJSONObject(r, b)
+		if err != nil {
+			return err
+		}
+		var rObject watchResponse
+		err = json.Unmarshal(b, &rObject)
+		if err != nil {
+			logger.Errorf("failed to parse watch api response as json, err %v, response: %v", err, string(b))
+			continue
+		}
+		wh(&rObject)
+	}
+}
+
+func readJSONObject(r io.Reader, b []byte) ([]byte, error) {
+	offset := 0
+	for {
+		n, err := r.Read(b[offset:])
+		if err == io.ErrShortBuffer {
+			if n == 0 {
+				return nil, fmt.Errorf("got short buffer with n=0, cap=%d", cap(b))
+			}
+			// double buffer..
+			b = bytesutil.Resize(b, len(b)*2)
+			offset += n
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		offset += n
+		break
+	}
+	return b[:offset], nil
+}
+
+func newWatchClient(wg *sync.WaitGroup, sdc *SDConfig, baseDir string) (*watchClient, error) {
+	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
+	}
+	apiServer := sdc.APIServer
+	if len(apiServer) == 0 {
+		// Assume we run at k8s pod.
+		// Discover apiServer and auth config according to k8s docs.
+		// See https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller
+		host := os.Getenv("KUBERNETES_SERVICE_HOST")
+		port := os.Getenv("KUBERNETES_SERVICE_PORT")
+		if len(host) == 0 {
+			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_HOST env var; it must be defined when running in k8s; " +
+				"probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?")
+		}
+		if len(port) == 0 {
+			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_PORT env var; it must be defined when running in k8s; "+
+				"KUBERNETES_SERVICE_HOST=%q", host)
+		}
+		apiServer = "https://" + net.JoinHostPort(host, port)
+		tlsConfig := promauth.TLSConfig{
+			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+		}
+		acNew, err := promauth.NewConfig(".", nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
+		}
+		ac = acNew
+	}
+	var proxy func(*http.Request) (*url.URL, error)
+	if proxyURL := sdc.ProxyURL.URL(); proxyURL != nil {
+		proxy = http.ProxyURL(proxyURL)
+	}
+	c := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig:     ac.NewTLSConfig(),
+			Proxy:               proxy,
+			TLSHandshakeTimeout: 10 * time.Second,
+			IdleConnTimeout:     2 * time.Minute,
+		},
+	}
+	wc := watchClient{
+		c:         c,
+		apiServer: apiServer,
+		ac:        ac,
+		wg:        wg,
+	}
+	return &wc, nil
+}
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 5e3aef789..ddef4861d 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -23,6 +23,7 @@ var (
 	kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+
 		"This works only if `kubernetes_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details")
+
 	openstackSDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*time.Second, "Interval for checking for changes in openstack API server. "+
 		"This works only if `openstack_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details")
@@ -97,7 +98,9 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	scs := newScrapeConfigs(pushData)
 	scs.add("static_configs", 0, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })
 	scs.add("file_sd_configs", *fileSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getFileSDScrapeWork(swsPrev) })
-	scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
+	scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork {
+		return getKubernetesSDScrapeWorkStream(cfg, swsPrev)
+	})
 	scs.add("openstack_sd_configs", *openstackSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
 	scs.add("consul_sd_configs", *consul.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getConsulSDScrapeWork(swsPrev) })
 	scs.add("eureka_sd_configs", *eurekaSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) })
diff --git a/lib/promscrape/watch_handler.go b/lib/promscrape/watch_handler.go
new file mode 100644
index 000000000..d940e08fe
--- /dev/null
+++ b/lib/promscrape/watch_handler.go
@@ -0,0 +1,83 @@
+package promscrape
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
+)
+
+type kubernetesWatchHandler struct {
+	ctx       context.Context
+	cancel    context.CancelFunc
+	startOnce sync.Once
+	watchCfg  *kubernetes.WatchConfig
+	// guards cache and set
+	mu             sync.Mutex
+	lastAccessTime time.Time
+	swCache        map[string][]*ScrapeWork
+	sdcSet         map[string]*scrapeWorkConfig
+}
+
+func newKubernetesWatchHandler() *kubernetesWatchHandler {
+	ctx, cancel := context.WithCancel(context.Background())
+	kwh := &kubernetesWatchHandler{
+		ctx:      ctx,
+		cancel:   cancel,
+		swCache:  map[string][]*ScrapeWork{},
+		sdcSet:   map[string]*scrapeWorkConfig{},
+		watchCfg: kubernetes.NewWatchConfig(ctx),
+	}
+	go kwh.waitForStop()
+	return kwh
+}
+
+func (ksc *kubernetesWatchHandler) waitForStop() {
+	t := time.NewTicker(time.Second * 5)
+	for range t.C {
+		ksc.mu.Lock()
+		lastTime := time.Since(ksc.lastAccessTime)
+		ksc.mu.Unlock()
+		if lastTime > *kubernetesSDCheckInterval*30 {
+			t1 := time.Now()
+			ksc.cancel()
+			ksc.watchCfg.WG.Wait()
+			close(ksc.watchCfg.WatchChan)
+			logger.Infof("stopped kubernetes api watcher handler, after: %.3f seconds", time.Since(t1).Seconds())
+			ksc.watchCfg.SC = nil
+			t.Stop()
+			return
+		}
+	}
+}
+
+func processKubernetesSyncEvents(cfg *Config) {
+	for {
+		select {
+		case <-cfg.kwh.ctx.Done():
+			return
+		case se, ok := <-cfg.kwh.watchCfg.WatchChan:
+			if !ok {
+				return
+			}
+			if se.Labels == nil {
+				cfg.kwh.mu.Lock()
+				delete(cfg.kwh.swCache, se.Key)
+				cfg.kwh.mu.Unlock()
+				continue
+			}
+			cfg.kwh.mu.Lock()
+			swc, ok := cfg.kwh.sdcSet[se.ConfigSectionSet]
+			cfg.kwh.mu.Unlock()
+			if !ok {
+				logger.Fatalf("bug config section not found: %v", se.ConfigSectionSet)
+			}
+			ms := appendScrapeWorkForTargetLabels(nil, swc, se.Labels, "kubernetes_sd_config")
+			cfg.kwh.mu.Lock()
+			cfg.kwh.swCache[se.Key] = ms
+			cfg.kwh.mu.Unlock()
+		}
+	}
+}

From c8f2f9b2e8c665b36011e26bc99353c67ad85df6 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 16:54:03 +0200
Subject: [PATCH 28/32] lib/promscrape: cleanup after
 9b2246c29b860cd8a838d3e191f453ee8fcb5997

Main points:

* Revert changes outside lib/promscrape/discovery/kuberntes . These changes can be applied later in a separate commit
* Minimize changes in lib/promscrape/discovery/kubernetes compared to a93e6440016f61565d48ace2ce579759d0b6b51b
* Corner case fixes.
---
 docs/CHANGELOG.md                             |   1 +
 lib/promscrape/config.go                      |  46 +-
 lib/promscrape/discovery/kubernetes/api.go    | 487 ++++++++++++++++-
 .../discovery/kubernetes/api_test.go          | 162 ++++++
 .../discovery/kubernetes/common_types.go      |  39 +-
 .../discovery/kubernetes/endpoints.go         | 106 ++--
 .../discovery/kubernetes/endpoints_test.go    |  25 +-
 .../discovery/kubernetes/endpointslices.go    |  93 ++--
 .../kubernetes/endpointslices_test.go         | 376 ++++---------
 lib/promscrape/discovery/kubernetes/framer.go |  56 --
 .../discovery/kubernetes/ingress.go           |  82 +--
 .../discovery/kubernetes/ingress_test.go      |  24 +-
 .../discovery/kubernetes/kubernetes.go        |  28 +-
 lib/promscrape/discovery/kubernetes/node.go   |  82 +--
 .../discovery/kubernetes/node_test.go         | 151 +++---
 lib/promscrape/discovery/kubernetes/pod.go    |  81 +--
 .../discovery/kubernetes/pod_test.go          |  30 +-
 .../discovery/kubernetes/service.go           |  81 +--
 .../discovery/kubernetes/service_test.go      |  69 +--
 .../discovery/kubernetes/shared_cache.go      |  76 ---
 lib/promscrape/discovery/kubernetes/watch.go  | 510 ------------------
 lib/promscrape/scraper.go                     |   5 +-
 lib/promscrape/watch_handler.go               |  83 ---
 23 files changed, 1166 insertions(+), 1527 deletions(-)
 create mode 100644 lib/promscrape/discovery/kubernetes/api_test.go
 delete mode 100644 lib/promscrape/discovery/kubernetes/framer.go
 delete mode 100644 lib/promscrape/discovery/kubernetes/shared_cache.go
 delete mode 100644 lib/promscrape/discovery/kubernetes/watch.go
 delete mode 100644 lib/promscrape/watch_handler.go

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c0c85e44a..b9ce762de 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -11,6 +11,7 @@
   * `process_io_write_syscalls_total` - the number of write syscalls such as write and pwrite
   * `process_io_storage_read_bytes_total` - the number of bytes read from storage layer
   * `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
+* FEATURE: vmagent: use watch API for Kuberntes service discovery. This should reduce load on Kuberntes API server when it tracks big number of objects (for example, 10K pods). This should also reduce the time needed for k8s targets discovery.
 * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
 * FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
 * FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index ee2f4b2d9..b41bcc463 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -48,8 +48,6 @@ type Config struct {
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
-	// used for data sync with kubernetes.
-	kwh *kubernetesWatchHandler
 }
 
 // GlobalConfig represents essential parts for `global` section of Prometheus config.
@@ -141,7 +139,6 @@ func loadConfig(path string) (cfg *Config, data []byte, err error) {
 	if err := cfgObj.parse(data, path); err != nil {
 		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
 	}
-	cfgObj.kwh = newKubernetesWatchHandler()
 	return &cfgObj, data, nil
 }
 
@@ -190,41 +187,30 @@ func getSWSByJob(sws []*ScrapeWork) map[string][]*ScrapeWork {
 }
 
 // getKubernetesSDScrapeWork returns `kubernetes_sd_configs` ScrapeWork from cfg.
-func getKubernetesSDScrapeWorkStream(cfg *Config, prev []*ScrapeWork) []*ScrapeWork {
-	cfg.kwh.startOnce.Do(func() {
-		go processKubernetesSyncEvents(cfg)
-	})
+func (cfg *Config) getKubernetesSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
+	swsPrevByJob := getSWSByJob(prev)
 	dst := make([]*ScrapeWork, 0, len(prev))
-	// updated access time.
-	cfg.kwh.mu.Lock()
-	cfg.kwh.lastAccessTime = time.Now()
-	cfg.kwh.mu.Unlock()
 	for i := range cfg.ScrapeConfigs {
 		sc := &cfg.ScrapeConfigs[i]
+		dstLen := len(dst)
+		ok := true
 		for j := range sc.KubernetesSDConfigs {
-			// generate set name
-			setKey := fmt.Sprintf("%d/%d/%s", i, j, sc.JobName)
-			cfg.kwh.mu.Lock()
-			cfg.kwh.sdcSet[setKey] = sc.swc
-			cfg.kwh.mu.Unlock()
 			sdc := &sc.KubernetesSDConfigs[j]
-			ms, err := kubernetes.StartWatchOnce(cfg.kwh.watchCfg, setKey, sdc, cfg.baseDir)
-			if err != nil {
-				logger.Errorf("got unexpected error: %v", err)
+			var okLocal bool
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "kubernetes_sd_config")
+			if ok {
+				ok = okLocal
 			}
-			dst = appendScrapeWorkForTargetLabels(dst, sc.swc, ms, "kubernetes_sd_config")
+		}
+		if ok {
+			continue
+		}
+		swsPrev := swsPrevByJob[sc.swc.jobName]
+		if len(swsPrev) > 0 {
+			logger.Errorf("there were errors when discovering kubernetes targets for job %q, so preserving the previous targets", sc.swc.jobName)
+			dst = append(dst[:dstLen], swsPrev...)
 		}
 	}
-	// dst will
-	if len(dst) > 0 {
-		return dst
-	}
-	// result from cache
-	cfg.kwh.mu.Lock()
-	for _, v := range cfg.kwh.swCache {
-		dst = append(dst, v...)
-	}
-	cfg.kwh.mu.Unlock()
 	return dst
 }
 
diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index a2b3834d9..816d5e194 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -1,42 +1,497 @@
 package kubernetes
 
 import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
 	"sync"
+	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+var apiServerTimeout = flag.Duration("promscrape.kubernetes.apiServerTimeout", 2*time.Minute, "Timeout for requests to Kuberntes API server")
+
 // apiConfig contains config for API server
 type apiConfig struct {
-	setName    string
-	namespaces []string
-	selectors  []Selector
-	wc         *watchClient
-	targetChan chan SyncEvent
-	watchOnce  sync.Once
+	aw *apiWatcher
 }
 
 var configMap = discoveryutils.NewConfigMap()
 
-func getAPIConfig(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(watchCfg, setName, sdc, baseDir) })
+func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
 	if err != nil {
 		return nil, err
 	}
 	return v.(*apiConfig), nil
 }
 
-func newAPIConfig(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	wc, err := newWatchClient(watchCfg.WG, sdc, baseDir)
+func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
+	apiServer := sdc.APIServer
+	if len(apiServer) == 0 {
+		// Assume we run at k8s pod.
+		// Discover apiServer and auth config according to k8s docs.
+		// See https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller
+		host := os.Getenv("KUBERNETES_SERVICE_HOST")
+		port := os.Getenv("KUBERNETES_SERVICE_PORT")
+		if len(host) == 0 {
+			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_HOST env var; it must be defined when running in k8s; " +
+				"probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?")
+		}
+		if len(port) == 0 {
+			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_PORT env var; it must be defined when running in k8s; "+
+				"KUBERNETES_SERVICE_HOST=%q", host)
+		}
+		apiServer = "https://" + net.JoinHostPort(host, port)
+		tlsConfig := promauth.TLSConfig{
+			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+		}
+		acNew, err := promauth.NewConfig(".", nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
+		}
+		ac = acNew
+	}
+	if !strings.Contains(apiServer, "://") {
+		proto := "http"
+		if sdc.TLSConfig != nil {
+			proto = "https"
+		}
+		apiServer = proto + "://" + apiServer
+	}
+	for strings.HasSuffix(apiServer, "/") {
+		apiServer = apiServer[:len(apiServer)-1]
+	}
+	var proxy func(*http.Request) (*url.URL, error)
+	if proxyURL := sdc.ProxyURL.URL(); proxyURL != nil {
+		proxy = http.ProxyURL(proxyURL)
+	}
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig:     ac.NewTLSConfig(),
+			Proxy:               proxy,
+			TLSHandshakeTimeout: 10 * time.Second,
+			IdleConnTimeout:     *apiServerTimeout,
+		},
+		Timeout: *apiServerTimeout,
+	}
+	aw := newAPIWatcher(client, apiServer, ac.Authorization, sdc.Namespaces.Names, sdc.Selectors)
 	cfg := &apiConfig{
-		setName:    setName,
-		targetChan: watchCfg.WatchChan,
-		wc:         wc,
-		namespaces: sdc.Namespaces.Names,
-		selectors:  sdc.Selectors,
+		aw: aw,
 	}
 	return cfg, nil
 }
+
+// WatchEvent is a watch event returned from API server endpoints if `watch=1` query arg is set.
+//
+// See https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+type WatchEvent struct {
+	Type   string
+	Object json.RawMessage
+}
+
+// object is any Kubernetes object.
+type object interface {
+	key() string
+}
+
+// parseObjectFunc must parse object from the given data.
+type parseObjectFunc func(data []byte) (object, error)
+
+// parseObjectListFunc must parse objectList from the given data.
+type parseObjectListFunc func(data []byte) (map[string]object, ListMeta, error)
+
+// apiWatcher is used for watching for Kuberntes object changes and caching their latest states.
+type apiWatcher struct {
+	// The client used for watching for object changes
+	client *http.Client
+
+	// Kubenetes API server address in the form http://api-server
+	apiServer string
+
+	// The contents for `Authorization` HTTP request header
+	authorization string
+
+	// Namespaces to watch
+	namespaces []string
+
+	// Selectors to apply during watch
+	selectors []Selector
+
+	// mu protects watchersByURL and lastAccessTime
+	mu sync.Mutex
+
+	// a map of watchers keyed by request paths
+	watchersByURL map[string]*urlWatcher
+
+	// The last time the apiWatcher was queried for cached objects.
+	// It is used for stopping unused watchers.
+	lastAccessTime time.Time
+}
+
+func newAPIWatcher(client *http.Client, apiServer, authorization string, namespaces []string, selectors []Selector) *apiWatcher {
+	return &apiWatcher{
+		apiServer:     apiServer,
+		authorization: authorization,
+		client:        client,
+		namespaces:    namespaces,
+		selectors:     selectors,
+
+		watchersByURL: make(map[string]*urlWatcher),
+
+		lastAccessTime: time.Now(),
+	}
+}
+
+// getObjectByRole returns an object with the given (namespace, name) key and the given role.
+func (aw *apiWatcher) getObjectByRole(role, namespace, name string) object {
+	if aw == nil {
+		return nil
+	}
+	key := namespace + "/" + name
+	aw.startWatchersForRole(role)
+	var o object
+	aw.mu.Lock()
+	for _, uw := range aw.watchersByURL {
+		if uw.role != role {
+			continue
+		}
+		uw.mu.Lock()
+		o = uw.objectsByKey[key]
+		uw.mu.Unlock()
+		if o != nil {
+			break
+		}
+	}
+	aw.lastAccessTime = time.Now()
+	aw.mu.Unlock()
+	return o
+}
+
+// getObjectsByRole returns all the objects for the given role.
+func (aw *apiWatcher) getObjectsByRole(role string) []object {
+	aw.startWatchersForRole(role)
+	var os []object
+	aw.mu.Lock()
+	for _, uw := range aw.watchersByURL {
+		if uw.role != role {
+			continue
+		}
+		uw.mu.Lock()
+		for _, o := range uw.objectsByKey {
+			os = append(os, o)
+		}
+		uw.mu.Unlock()
+	}
+	aw.lastAccessTime = time.Now()
+	aw.mu.Unlock()
+	return os
+}
+
+func (aw *apiWatcher) startWatchersForRole(role string) {
+	parseObject, parseObjectList := getObjectParsersForRole(role)
+	paths := getAPIPaths(role, aw.namespaces, aw.selectors)
+	for _, path := range paths {
+		apiURL := aw.apiServer + path
+		aw.startWatcherForURL(role, apiURL, parseObject, parseObjectList)
+	}
+}
+
+func (aw *apiWatcher) startWatcherForURL(role, apiURL string, parseObject parseObjectFunc, parseObjectList parseObjectListFunc) {
+	aw.mu.Lock()
+	defer aw.mu.Unlock()
+	if aw.watchersByURL[apiURL] != nil {
+		// Watcher for the given path already exists.
+		return
+	}
+	uw := aw.newURLWatcher(role, apiURL, parseObject, parseObjectList)
+	resourceVersion := uw.reloadObjects()
+	aw.watchersByURL[apiURL] = uw
+	go func() {
+		uw.watchForUpdates(resourceVersion)
+		aw.mu.Lock()
+		delete(aw.watchersByURL, apiURL)
+		aw.mu.Unlock()
+	}()
+}
+
+// needStop returns true if aw wasn't used for long time.
+func (aw *apiWatcher) needStop() bool {
+	aw.mu.Lock()
+	defer aw.mu.Unlock()
+	return time.Since(aw.lastAccessTime) > 5*time.Minute
+}
+
+// doRequest performs http request to the given requestURL.
+func (aw *apiWatcher) doRequest(requestURL string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", requestURL, nil)
+	if err != nil {
+		logger.Fatalf("cannot create a request for %q: %s", requestURL, err)
+	}
+	if aw.authorization != "" {
+		req.Header.Set("Authorization", aw.authorization)
+	}
+	return aw.client.Do(req)
+}
+
+// urlWatcher watches for an apiURL and updates object states in objectsByKey.
+type urlWatcher struct {
+	role   string
+	apiURL string
+
+	parseObject     parseObjectFunc
+	parseObjectList parseObjectListFunc
+
+	// mu protects objectsByKey
+	mu sync.Mutex
+
+	// objectsByKey contains the latest state for objects obtained from apiURL
+	objectsByKey map[string]object
+
+	// the parent apiWatcher
+	aw *apiWatcher
+}
+
+func (aw *apiWatcher) newURLWatcher(role, apiURL string, parseObject parseObjectFunc, parseObjectList parseObjectListFunc) *urlWatcher {
+	return &urlWatcher{
+		role:   role,
+		apiURL: apiURL,
+
+		parseObject:     parseObject,
+		parseObjectList: parseObjectList,
+
+		objectsByKey: make(map[string]object),
+
+		aw: aw,
+	}
+}
+
+// reloadObjects reloads objects to the latest state and returns resourceVersion for the latest state.
+func (uw *urlWatcher) reloadObjects() string {
+	requestURL := uw.apiURL
+	resp, err := uw.aw.doRequest(requestURL)
+	if err != nil {
+		logger.Errorf("error when performing a request to %q: %s", requestURL, err)
+		return ""
+	}
+	body, _ := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		logger.Errorf("unexpected status code for request to %q: %d; want %d; response: %q", requestURL, resp.StatusCode, http.StatusOK, body)
+		return ""
+	}
+	objectsByKey, metadata, err := uw.parseObjectList(body)
+	if err != nil {
+		logger.Errorf("cannot parse response from %q: %s", requestURL, err)
+		return ""
+	}
+	uw.mu.Lock()
+	uw.objectsByKey = objectsByKey
+	uw.mu.Unlock()
+	return metadata.ResourceVersion
+}
+
+// watchForUpdates watches for object updates starting from resourceVersion and updates the corresponding objects to the latest state.
+//
+// See https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+func (uw *urlWatcher) watchForUpdates(resourceVersion string) {
+	aw := uw.aw
+	backoffDelay := time.Second
+	maxBackoffDelay := 30 * time.Second
+	backoffSleep := func() {
+		time.Sleep(backoffDelay)
+		backoffDelay *= 2
+		if backoffDelay > maxBackoffDelay {
+			backoffDelay = maxBackoffDelay
+		}
+	}
+	apiURL := uw.apiURL
+	delimiter := "?"
+	if strings.Contains(apiURL, "?") {
+		delimiter = "&"
+	}
+	timeoutSeconds := time.Duration(0.9 * float64(aw.client.Timeout)).Seconds()
+	apiURL += delimiter + "watch=1&timeoutSeconds=" + strconv.Itoa(int(timeoutSeconds))
+	logger.Infof("started watcher for %q", apiURL)
+	for {
+		if aw.needStop() {
+			logger.Infof("stopped unused watcher for %q", apiURL)
+			return
+		}
+		requestURL := apiURL
+		if resourceVersion != "" {
+			requestURL += "&resourceVersion=" + url.QueryEscape(resourceVersion) + "&resourceVersionMatch=NotOlderThan"
+		}
+		resp, err := aw.doRequest(requestURL)
+		if err != nil {
+			logger.Errorf("error when performing a request to %q: %s", requestURL, err)
+			backoffSleep()
+			// There is no sense in reloading resources on non-http errors.
+			continue
+		}
+		if resp.StatusCode != http.StatusOK {
+			body, _ := ioutil.ReadAll(resp.Body)
+			_ = resp.Body.Close()
+			logger.Errorf("unexpected status code for request to %q: %d; want %d; response: %q", requestURL, resp.StatusCode, http.StatusOK, body)
+			if resp.StatusCode == 410 {
+				// Update stale resourceVersion. See https://kubernetes.io/docs/reference/using-api/api-concepts/#410-gone-responses
+				resourceVersion = uw.reloadObjects()
+				backoffDelay = time.Second
+			} else {
+				backoffSleep()
+				// There is no sense in reloading resources on non-410 status codes.
+			}
+			continue
+		}
+		backoffDelay = time.Second
+		err = uw.readObjectUpdateStream(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				// The stream has been closed (probably due to timeout)
+				backoffSleep()
+				continue
+			}
+			logger.Errorf("error when reading WatchEvent stream from %q: %s", requestURL, err)
+			backoffSleep()
+			// There is no sense in reloading resources on non-http errors.
+			continue
+		}
+	}
+}
+
+// readObjectUpdateStream reads Kuberntes watch events from r and updates locally cached objects according to the received events.
+func (uw *urlWatcher) readObjectUpdateStream(r io.Reader) error {
+	d := json.NewDecoder(r)
+	var we WatchEvent
+	for {
+		if err := d.Decode(&we); err != nil {
+			return err
+		}
+		o, err := uw.parseObject(we.Object)
+		if err != nil {
+			return err
+		}
+		key := o.key()
+		switch we.Type {
+		case "ADDED", "MODIFIED":
+			uw.mu.Lock()
+			uw.objectsByKey[key] = o
+			uw.mu.Unlock()
+		case "DELETED":
+			uw.mu.Lock()
+			delete(uw.objectsByKey, key)
+			uw.mu.Unlock()
+		default:
+			return fmt.Errorf("unexpected WatchEvent type %q for role %q", we.Type, uw.role)
+		}
+	}
+}
+
+func getAPIPaths(role string, namespaces []string, selectors []Selector) []string {
+	objectName := getObjectNameByRole(role)
+	if objectName == "nodes" || len(namespaces) == 0 {
+		query := joinSelectors(role, selectors)
+		path := getAPIPath(objectName, "", query)
+		return []string{path}
+	}
+	query := joinSelectors(role, selectors)
+	paths := make([]string, len(namespaces))
+	for i, namespace := range namespaces {
+		paths[i] = getAPIPath(objectName, namespace, query)
+	}
+	return paths
+}
+
+func getAPIPath(objectName, namespace, query string) string {
+	suffix := objectName
+	if namespace != "" {
+		suffix = "namespaces/" + namespace + "/" + objectName
+	}
+	if len(query) > 0 {
+		suffix += "?" + query
+	}
+	if objectName == "endpointslices" {
+		return "/apis/discovery.k8s.io/v1beta1/" + suffix
+	}
+	return "/api/v1/" + suffix
+}
+
+func joinSelectors(role string, selectors []Selector) string {
+	var labelSelectors, fieldSelectors []string
+	for _, s := range selectors {
+		if s.Role != role {
+			continue
+		}
+		if s.Label != "" {
+			labelSelectors = append(labelSelectors, s.Label)
+		}
+		if s.Field != "" {
+			fieldSelectors = append(fieldSelectors, s.Field)
+		}
+	}
+	var args []string
+	if len(labelSelectors) > 0 {
+		args = append(args, "labelSelector="+url.QueryEscape(strings.Join(labelSelectors, ",")))
+	}
+	if len(fieldSelectors) > 0 {
+		args = append(args, "fieldSelector="+url.QueryEscape(strings.Join(fieldSelectors, ",")))
+	}
+	return strings.Join(args, "&")
+}
+
+func getObjectNameByRole(role string) string {
+	switch role {
+	case "node":
+		return "nodes"
+	case "pod":
+		return "pods"
+	case "service":
+		return "services"
+	case "endpoints":
+		return "endpoints"
+	case "endpointslices":
+		return "endpointslices"
+	case "ingress":
+		return "ingresses"
+	default:
+		logger.Panicf("BUG: unknonw role=%q", role)
+		return ""
+	}
+}
+
+func getObjectParsersForRole(role string) (parseObjectFunc, parseObjectListFunc) {
+	switch role {
+	case "node":
+		return parseNode, parseNodeList
+	case "pod":
+		return parsePod, parsePodList
+	case "service":
+		return parseService, parseServiceList
+	case "endpoints":
+		return parseEndpoints, parseEndpointsList
+	case "endpointslices":
+		return parseEndpointSlice, parseEndpointSliceList
+	case "ingress":
+		return parseIngress, parseIngressList
+	default:
+		logger.Panicf("BUG: unsupported role=%q", role)
+		return nil, nil
+	}
+}
diff --git a/lib/promscrape/discovery/kubernetes/api_test.go b/lib/promscrape/discovery/kubernetes/api_test.go
new file mode 100644
index 000000000..1a21eb70d
--- /dev/null
+++ b/lib/promscrape/discovery/kubernetes/api_test.go
@@ -0,0 +1,162 @@
+package kubernetes
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGetAPIPaths(t *testing.T) {
+	f := func(role string, namespaces []string, selectors []Selector, expectedPaths []string) {
+		t.Helper()
+		paths := getAPIPaths(role, namespaces, selectors)
+		if !reflect.DeepEqual(paths, expectedPaths) {
+			t.Fatalf("unexpected paths; got\n%q\nwant\n%q", paths, expectedPaths)
+		}
+	}
+
+	// role=node
+	f("node", nil, nil, []string{"/api/v1/nodes"})
+	f("node", []string{"foo", "bar"}, nil, []string{"/api/v1/nodes"})
+	f("node", nil, []Selector{
+		{
+			Role:  "pod",
+			Label: "foo",
+			Field: "bar",
+		},
+	}, []string{"/api/v1/nodes"})
+	f("node", nil, []Selector{
+		{
+			Role:  "node",
+			Label: "foo",
+			Field: "bar",
+		},
+	}, []string{"/api/v1/nodes?labelSelector=foo&fieldSelector=bar"})
+	f("node", []string{"x", "y"}, []Selector{
+		{
+			Role:  "node",
+			Label: "foo",
+			Field: "bar",
+		},
+	}, []string{"/api/v1/nodes?labelSelector=foo&fieldSelector=bar"})
+
+	// role=pod
+	f("pod", nil, nil, []string{"/api/v1/pods"})
+	f("pod", []string{"foo", "bar"}, nil, []string{
+		"/api/v1/namespaces/foo/pods",
+		"/api/v1/namespaces/bar/pods",
+	})
+	f("pod", nil, []Selector{
+		{
+			Role:  "node",
+			Label: "foo",
+		},
+	}, []string{"/api/v1/pods"})
+	f("pod", nil, []Selector{
+		{
+			Role:  "pod",
+			Label: "foo",
+		},
+		{
+			Role:  "pod",
+			Label: "x",
+			Field: "y",
+		},
+	}, []string{"/api/v1/pods?labelSelector=foo%2Cx&fieldSelector=y"})
+	f("pod", []string{"x", "y"}, []Selector{
+		{
+			Role:  "pod",
+			Label: "foo",
+		},
+		{
+			Role:  "pod",
+			Label: "x",
+			Field: "y",
+		},
+	}, []string{
+		"/api/v1/namespaces/x/pods?labelSelector=foo%2Cx&fieldSelector=y",
+		"/api/v1/namespaces/y/pods?labelSelector=foo%2Cx&fieldSelector=y",
+	})
+
+	// role=service
+	f("service", nil, nil, []string{"/api/v1/services"})
+	f("service", []string{"x", "y"}, nil, []string{
+		"/api/v1/namespaces/x/services",
+		"/api/v1/namespaces/y/services",
+	})
+	f("service", nil, []Selector{
+		{
+			Role:  "node",
+			Label: "foo",
+		},
+		{
+			Role:  "service",
+			Field: "bar",
+		},
+	}, []string{"/api/v1/services?fieldSelector=bar"})
+	f("service", []string{"x", "y"}, []Selector{
+		{
+			Role:  "service",
+			Label: "abc=de",
+		},
+	}, []string{
+		"/api/v1/namespaces/x/services?labelSelector=abc%3Dde",
+		"/api/v1/namespaces/y/services?labelSelector=abc%3Dde",
+	})
+
+	// role=endpoints
+	f("endpoints", nil, nil, []string{"/api/v1/endpoints"})
+	f("endpoints", []string{"x", "y"}, nil, []string{
+		"/api/v1/namespaces/x/endpoints",
+		"/api/v1/namespaces/y/endpoints",
+	})
+	f("endpoints", []string{"x", "y"}, []Selector{
+		{
+			Role:  "endpoints",
+			Label: "bbb",
+		},
+		{
+			Role:  "node",
+			Label: "aa",
+		},
+	}, []string{
+		"/api/v1/namespaces/x/endpoints?labelSelector=bbb",
+		"/api/v1/namespaces/y/endpoints?labelSelector=bbb",
+	})
+
+	// role=endpointslices
+	f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1beta1/endpointslices"})
+	f("endpointslices", []string{"x", "y"}, []Selector{
+		{
+			Role:  "endpointslices",
+			Field: "field",
+			Label: "label",
+		},
+	}, []string{
+		"/apis/discovery.k8s.io/v1beta1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field",
+		"/apis/discovery.k8s.io/v1beta1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field",
+	})
+
+	// role=ingress
+	f("ingress", nil, nil, []string{"/api/v1/ingresses"})
+	f("ingress", []string{"x", "y"}, []Selector{
+		{
+			Role:  "node",
+			Field: "xyay",
+		},
+		{
+			Role:  "ingress",
+			Field: "abc",
+		},
+		{
+			Role:  "ingress",
+			Label: "cde",
+		},
+		{
+			Role:  "ingress",
+			Label: "baaa",
+		},
+	}, []string{
+		"/api/v1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+		"/api/v1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+	})
+}
diff --git a/lib/promscrape/discovery/kubernetes/common_types.go b/lib/promscrape/discovery/kubernetes/common_types.go
index f1b8dcce9..be93bbb4a 100644
--- a/lib/promscrape/discovery/kubernetes/common_types.go
+++ b/lib/promscrape/discovery/kubernetes/common_types.go
@@ -1,9 +1,6 @@
 package kubernetes
 
 import (
-	"net/url"
-	"strings"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -19,10 +16,14 @@ type ObjectMeta struct {
 	OwnerReferences []OwnerReference
 }
 
-// listMetadata kubernetes list metadata
+func (om *ObjectMeta) key() string {
+	return om.Namespace + "/" + om.Name
+}
+
+// ListMeta is a Kubernetes list metadata
 // https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#listmeta-v1-meta
-type listMetadata struct {
-	ResourceVersion string `json:"resourceVersion"`
+type ListMeta struct {
+	ResourceVersion string
 }
 
 func (om *ObjectMeta) registerLabelsAndAnnotations(prefix string, m map[string]string) {
@@ -53,29 +54,3 @@ type OwnerReference struct {
 type DaemonEndpoint struct {
 	Port int
 }
-
-func joinSelectors(role string, namespaces []string, selectors []Selector) string {
-	var labelSelectors, fieldSelectors []string
-	for _, ns := range namespaces {
-		fieldSelectors = append(fieldSelectors, "metadata.namespace="+ns)
-	}
-	for _, s := range selectors {
-		if s.Role != role {
-			continue
-		}
-		if s.Label != "" {
-			labelSelectors = append(labelSelectors, s.Label)
-		}
-		if s.Field != "" {
-			fieldSelectors = append(fieldSelectors, s.Field)
-		}
-	}
-	var args []string
-	if len(labelSelectors) > 0 {
-		args = append(args, "labelSelector="+url.QueryEscape(strings.Join(labelSelectors, ",")))
-	}
-	if len(fieldSelectors) > 0 {
-		args = append(args, "fieldSelector="+url.QueryEscape(strings.Join(fieldSelectors, ",")))
-	}
-	return strings.Join(args, "&")
-}
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 2839202d0..2b6881140 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -3,18 +3,59 @@ package kubernetes
 import (
 	"encoding/json"
 	"fmt"
-	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+// getEndpointsLabels returns labels for k8s endpoints obtained from the given cfg.
+func getEndpointsLabels(cfg *apiConfig) []map[string]string {
+	epss := getEndpoints(cfg)
+	var ms []map[string]string
+	for _, eps := range epss {
+		ms = eps.appendTargetLabels(ms, cfg.aw)
+	}
+	return ms
+}
+
+func getEndpoints(cfg *apiConfig) []*Endpoints {
+	os := cfg.aw.getObjectsByRole("endpoint")
+	epss := make([]*Endpoints, len(os))
+	for i, o := range os {
+		epss[i] = o.(*Endpoints)
+	}
+	return epss
+}
+
+func (eps *Endpoints) key() string {
+	return eps.Metadata.key()
+}
+
+func parseEndpointsList(data []byte) (map[string]object, ListMeta, error) {
+	var epsl EndpointsList
+	if err := json.Unmarshal(data, &epsl); err != nil {
+		return nil, epsl.Metadata, fmt.Errorf("cannot unmarshal EndpointsList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, eps := range epsl.Items {
+		objectsByKey[eps.key()] = eps
+	}
+	return objectsByKey, epsl.Metadata, nil
+}
+
+func parseEndpoints(data []byte) (object, error) {
+	var eps Endpoints
+	if err := json.Unmarshal(data, &eps); err != nil {
+		return nil, err
+	}
+	return &eps, nil
+}
+
 // EndpointsList implements k8s endpoints list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslist-v1-core
 type EndpointsList struct {
-	Items    []Endpoints
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*Endpoints
 }
 
 // Endpoints implements k8s endpoints.
@@ -25,10 +66,6 @@ type Endpoints struct {
 	Subsets  []EndpointSubset
 }
 
-func (eps *Endpoints) key() string {
-	return eps.Metadata.Namespace + "/" + eps.Metadata.Name
-}
-
 // EndpointSubset implements k8s endpoint subset.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointsubset-v1-core
@@ -57,10 +94,6 @@ type ObjectReference struct {
 	Namespace string
 }
 
-func (or ObjectReference) key() string {
-	return or.Namespace + "/" + or.Name
-}
-
 // EndpointPort implements k8s endpoint port.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointport-v1beta1-discovery-k8s-io
@@ -71,28 +104,19 @@ type EndpointPort struct {
 	Protocol    string
 }
 
-// parseEndpointsList parses EndpointsList from data.
-func parseEndpointsList(data []byte) (*EndpointsList, error) {
-	var esl EndpointsList
-	if err := json.Unmarshal(data, &esl); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal EndpointsList from %q: %w", data, err)
-	}
-	return &esl, nil
-}
-
 // appendTargetLabels appends labels for each endpoint in eps to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#endpoints
-func (eps *Endpoints) appendTargetLabels(ms []map[string]string, podsCache, servicesCache *sync.Map) []map[string]string {
+func (eps *Endpoints) appendTargetLabels(ms []map[string]string, aw *apiWatcher) []map[string]string {
 	var svc *Service
-	if svco, ok := servicesCache.Load(eps.key()); ok {
-		svc = svco.(*Service)
+	if o := aw.getObjectByRole("service", eps.Metadata.Namespace, eps.Metadata.Name); o != nil {
+		svc = o.(*Service)
 	}
 	podPortsSeen := make(map[*Pod][]int)
 	for _, ess := range eps.Subsets {
 		for _, epp := range ess.Ports {
-			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.Addresses, epp, podsCache, svc, "true")
-			ms = appendEndpointLabelsForAddresses(ms, podPortsSeen, eps, ess.NotReadyAddresses, epp, podsCache, svc, "false")
+			ms = appendEndpointLabelsForAddresses(ms, aw, podPortsSeen, eps, ess.Addresses, epp, svc, "true")
+			ms = appendEndpointLabelsForAddresses(ms, aw, podPortsSeen, eps, ess.NotReadyAddresses, epp, svc, "false")
 		}
 	}
 
@@ -127,14 +151,13 @@ func (eps *Endpoints) appendTargetLabels(ms []map[string]string, podsCache, serv
 	return ms
 }
 
-func appendEndpointLabelsForAddresses(ms []map[string]string, podPortsSeen map[*Pod][]int, eps *Endpoints, eas []EndpointAddress, epp EndpointPort,
-	podsCache *sync.Map, svc *Service, ready string) []map[string]string {
+func appendEndpointLabelsForAddresses(ms []map[string]string, aw *apiWatcher, podPortsSeen map[*Pod][]int, eps *Endpoints,
+	eas []EndpointAddress, epp EndpointPort, svc *Service, ready string) []map[string]string {
 	for _, ea := range eas {
 		var p *Pod
-		if po, ok := podsCache.Load(ea.TargetRef.key()); ok {
-			p = po.(*Pod)
+		if o := aw.getObjectByRole("pod", ea.TargetRef.Namespace, ea.TargetRef.Name); o != nil {
+			p = o.(*Pod)
 		}
-		//p := getPod(pods, ea.TargetRef.Namespace, ea.TargetRef.Name)
 		m := getEndpointLabelsForAddressAndPort(podPortsSeen, eps, ea, epp, p, svc, ready)
 		ms = append(ms, m)
 	}
@@ -186,24 +209,3 @@ func getEndpointLabels(om ObjectMeta, ea EndpointAddress, epp EndpointPort, read
 	}
 	return m
 }
-
-func processEndpoints(cfg *apiConfig, sc *SharedKubernetesCache, p *Endpoints, action string) {
-	key := buildSyncKey("endpoints", cfg.setName, p.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		lbs := p.appendTargetLabels(nil, sc.Pods, sc.Services)
-		cfg.targetChan <- SyncEvent{
-			Labels:           lbs,
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index 8f154b5c6..bd00ebdcf 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"reflect"
-	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -12,12 +11,12 @@ import (
 func TestParseEndpointsListFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		els, err := parseEndpointsList([]byte(s))
+		objectsByKey, _, err := parseEndpointsList([]byte(s))
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if els != nil {
-			t.Fatalf("unexpected non-nil EnpointsList: %v", els)
+		if len(objectsByKey) != 0 {
+			t.Fatalf("unexpected non-empty objectsByKey: %v", objectsByKey)
 		}
 	}
 	f(``)
@@ -80,22 +79,18 @@ func TestParseEndpointsListSuccess(t *testing.T) {
   ]
 }
 `
-	els, err := parseEndpointsList([]byte(data))
+	objectsByKey, meta, err := parseEndpointsList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if len(els.Items) != 1 {
-		t.Fatalf("unexpected length of EndpointsList.Items; got %d; want %d", len(els.Items), 1)
+	expectedResourceVersion := "128055"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	endpoint := els.Items[0]
 
-	// Check endpoint.appendTargetLabels()
-	var pc, sc sync.Map
-	labelss := endpoint.appendTargetLabels(nil, &pc, &sc)
-	var sortedLabelss [][]prompbmarshal.Label
-	for _, labels := range labelss {
-		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
-	}
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*Endpoints).appendTargetLabels(nil, nil)
+	})
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.17.0.2:8443",
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices.go b/lib/promscrape/discovery/kubernetes/endpointslices.go
index 9a0a2c9d9..2246be105 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices.go
@@ -4,38 +4,70 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
-	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// parseEndpointsList parses EndpointSliceList from data.
-func parseEndpointSlicesList(data []byte) (*EndpointSliceList, error) {
-	var esl EndpointSliceList
-	if err := json.Unmarshal(data, &esl); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal EndpointSliceList from %q: %w", data, err)
+// getEndpointSlicesLabels returns labels for k8s endpointSlices obtained from the given cfg.
+func getEndpointSlicesLabels(cfg *apiConfig) []map[string]string {
+	epss := getEndpointSlices(cfg)
+	var ms []map[string]string
+	for _, eps := range epss {
+		ms = eps.appendTargetLabels(ms, cfg.aw)
 	}
+	return ms
+}
 
-	return &esl, nil
+// getEndpointSlices retrieves endpointSlice with given apiConfig
+func getEndpointSlices(cfg *apiConfig) []*EndpointSlice {
+	os := cfg.aw.getObjectsByRole("endpointslices")
+	epss := make([]*EndpointSlice, len(os))
+	for i, o := range os {
+		epss[i] = o.(*EndpointSlice)
+	}
+	return epss
+}
+
+func (eps *EndpointSlice) key() string {
+	return eps.Metadata.key()
+}
+
+func parseEndpointSliceList(data []byte) (map[string]object, ListMeta, error) {
+	var epsl EndpointSliceList
+	if err := json.Unmarshal(data, &epsl); err != nil {
+		return nil, epsl.Metadata, fmt.Errorf("cannot unmarshal EndpointSliceList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, eps := range epsl.Items {
+		objectsByKey[eps.key()] = eps
+	}
+	return objectsByKey, epsl.Metadata, nil
+}
+
+func parseEndpointSlice(data []byte) (object, error) {
+	var eps EndpointSlice
+	if err := json.Unmarshal(data, &eps); err != nil {
+		return nil, err
+	}
+	return &eps, nil
 }
 
 // appendTargetLabels injects labels for endPointSlice to slice map
 // follows TargetRef for enrich labels with pod and service metadata
-func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, podsCache, servicesCache *sync.Map) []map[string]string {
+func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, aw *apiWatcher) []map[string]string {
 	var svc *Service
-	if s, ok := servicesCache.Load(eps.key()); ok {
-		svc = s.(*Service)
+	if o := aw.getObjectByRole("service", eps.Metadata.Namespace, eps.Metadata.Name); o != nil {
+		svc = o.(*Service)
 	}
 	podPortsSeen := make(map[*Pod][]int)
 	for _, ess := range eps.Endpoints {
-		var pod *Pod
-		if p, ok := podsCache.Load(ess.TargetRef.key()); ok {
-			pod = p.(*Pod)
+		var p *Pod
+		if o := aw.getObjectByRole("pod", ess.TargetRef.Namespace, ess.TargetRef.Name); o != nil {
+			p = o.(*Pod)
 		}
 		for _, epp := range eps.Ports {
 			for _, addr := range ess.Addresses {
-				ms = append(ms, getEndpointSliceLabelsForAddressAndPort(podPortsSeen, addr, eps, ess, epp, pod, svc))
+				ms = append(ms, getEndpointSliceLabelsForAddressAndPort(podPortsSeen, addr, eps, ess, epp, p, svc))
 			}
 
 		}
@@ -70,13 +102,12 @@ func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, podsCache,
 		}
 	}
 	return ms
-
 }
 
 // getEndpointSliceLabelsForAddressAndPort gets labels for endpointSlice
 // from  address, Endpoint and EndpointPort
 // enriches labels with TargetRef
-// pod appended to seen Ports
+// p appended to seen Ports
 // if TargetRef matches
 func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr string, eps *EndpointSlice, ea Endpoint, epp EndpointPort, p *Pod, svc *Service) map[string]string {
 	m := getEndpointSliceLabels(eps, addr, ea, epp)
@@ -135,8 +166,8 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En
 // that groups service endpoints slices.
 // https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
 type EndpointSliceList struct {
-	Items    []EndpointSlice
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*EndpointSlice
 }
 
 // EndpointSlice - implements kubernetes endpoint slice.
@@ -148,10 +179,6 @@ type EndpointSlice struct {
 	Ports       []EndpointPort
 }
 
-func (eps EndpointSlice) key() string {
-	return eps.Metadata.Namespace + "/" + eps.Metadata.Name
-}
-
 // Endpoint implements kubernetes object endpoint for endpoint slice.
 // https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io
 type Endpoint struct {
@@ -167,23 +194,3 @@ type Endpoint struct {
 type EndpointConditions struct {
 	Ready bool
 }
-
-func processEndpointSlices(cfg *apiConfig, sc *SharedKubernetesCache, p *EndpointSlice, action string) {
-	key := buildSyncKey("endpointslices", cfg.setName, p.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		cfg.targetChan <- SyncEvent{
-			Labels:           p.appendTargetLabels(nil, sc.Pods, sc.Services),
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices_test.go b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
index b1c3731ef..a89eccf1b 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
@@ -2,21 +2,20 @@ package kubernetes
 
 import (
 	"reflect"
-	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-func Test_parseEndpointSlicesListFail(t *testing.T) {
+func TestParseEndpointSliceListFail(t *testing.T) {
 	f := func(data string) {
-		eslList, err := parseEndpointSlicesList([]byte(data))
+		objectsByKey, _, err := parseEndpointSliceList([]byte(data))
 		if err == nil {
 			t.Errorf("unexpected result, test must fail! data: %s", data)
 		}
-		if eslList != nil {
-			t.Errorf("endpointSliceList must be nil, got: %v", eslList)
+		if len(objectsByKey) != 0 {
+			t.Errorf("EndpointSliceList must be emptry, got: %v", objectsByKey)
 		}
 	}
 
@@ -29,7 +28,7 @@ func Test_parseEndpointSlicesListFail(t *testing.T) {
 
 }
 
-func Test_parseEndpointSlicesListSuccess(t *testing.T) {
+func TestParseEndpointSliceListSuccess(t *testing.T) {
 	data := `{
   "kind": "EndpointSliceList",
   "apiVersion": "discovery.k8s.io/v1beta1",
@@ -177,23 +176,19 @@ func Test_parseEndpointSlicesListSuccess(t *testing.T) {
     }
   ]
 }`
-	esl, err := parseEndpointSlicesList([]byte(data))
+	objectsByKey, meta, err := parseEndpointSliceList([]byte(data))
 	if err != nil {
 		t.Errorf("cannot parse data for EndpointSliceList: %v", err)
 		return
 	}
-	if len(esl.Items) != 2 {
-		t.Fatalf("expected 2 items at endpointSliceList, got: %d", len(esl.Items))
+	expectedResourceVersion := "1177"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-
-	firstEsl := esl.Items[0]
-	var pc, sc sync.Map
-	got := firstEsl.appendTargetLabels(nil, &pc, &sc)
-	sortedLables := [][]prompbmarshal.Label{}
-	for _, labels := range got {
-		sortedLables = append(sortedLables, discoveryutils.GetSortedLabels(labels))
-	}
-	expectedLabels := [][]prompbmarshal.Label{
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*EndpointSlice).appendTargetLabels(nil, nil)
+	})
+	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.18.0.2:6443",
 			"__meta_kubernetes_endpointslice_address_type":              "IPv4",
@@ -203,263 +198,94 @@ func Test_parseEndpointSlicesListSuccess(t *testing.T) {
 			"__meta_kubernetes_endpointslice_port_name":                 "https",
 			"__meta_kubernetes_endpointslice_port_protocol":             "TCP",
 			"__meta_kubernetes_namespace":                               "default",
-		})}
-	if !reflect.DeepEqual(sortedLables, expectedLabels) {
-		t.Fatalf("unexpected labels,\ngot:\n%v,\nwant:\n%v", sortedLables, expectedLabels)
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.3:53",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-z8czk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "53",
+			"__meta_kubernetes_endpointslice_port_name":                                        "dns-tcp",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "TCP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.3:9153",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-z8czk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "9153",
+			"__meta_kubernetes_endpointslice_port_name":                                        "metrics",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "TCP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.3:53",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-z8czk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "53",
+			"__meta_kubernetes_endpointslice_port_name":                                        "dns",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "UDP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.4:53",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-kpbhk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "53",
+			"__meta_kubernetes_endpointslice_port_name":                                        "dns-tcp",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "TCP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.4:9153",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-kpbhk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "9153",
+			"__meta_kubernetes_endpointslice_port_name":                                        "metrics",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "TCP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.244.0.4:53",
+			"__meta_kubernetes_endpointslice_address_target_kind":                              "Pod",
+			"__meta_kubernetes_endpointslice_address_target_name":                              "coredns-66bff467f8-kpbhk",
+			"__meta_kubernetes_endpointslice_address_type":                                     "IPv4",
+			"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                        "true",
+			"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_io_hostname":         "kind-control-plane",
+			"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname": "true",
+			"__meta_kubernetes_endpointslice_name":                                             "kube-dns-22mvb",
+			"__meta_kubernetes_endpointslice_port":                                             "53",
+			"__meta_kubernetes_endpointslice_port_name":                                        "dns",
+			"__meta_kubernetes_endpointslice_port_protocol":                                    "UDP",
+			"__meta_kubernetes_namespace":                                                      "kube-system",
+		}),
+	}
+	if !reflect.DeepEqual(sortedLabelss, expectedLabelss) {
+		t.Fatalf("unexpected labels,\ngot:\n%v,\nwant:\n%v", sortedLabelss, expectedLabelss)
 	}
 
 }
-
-func TestEndpointSlice_appendTargetLabels(t *testing.T) {
-	type fields struct {
-		Metadata    ObjectMeta
-		Endpoints   []Endpoint
-		AddressType string
-		Ports       []EndpointPort
-	}
-	type args struct {
-		ms   []map[string]string
-		pods []Pod
-		svcs []Service
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   [][]prompbmarshal.Label
-	}{
-		{
-			name: "simple eps",
-			args: args{},
-			fields: fields{
-				Metadata: ObjectMeta{
-					Name:      "fake-esl",
-					Namespace: "default",
-				},
-				AddressType: "ipv4",
-				Endpoints: []Endpoint{
-					{Addresses: []string{"127.0.0.1"},
-						Hostname:   "node-1",
-						Topology:   map[string]string{"kubernetes.topoligy.io/zone": "gce-1"},
-						Conditions: EndpointConditions{Ready: true},
-						TargetRef: ObjectReference{
-							Kind:      "Pod",
-							Namespace: "default",
-							Name:      "main-pod",
-						},
-					},
-				},
-				Ports: []EndpointPort{
-					{
-						Name:        "http",
-						Port:        8085,
-						AppProtocol: "http",
-						Protocol:    "tcp",
-					},
-				},
-			},
-			want: [][]prompbmarshal.Label{
-				discoveryutils.GetSortedLabels(map[string]string{
-					"__address__": "127.0.0.1:8085",
-					"__meta_kubernetes_endpointslice_address_target_kind":                                   "Pod",
-					"__meta_kubernetes_endpointslice_address_target_name":                                   "main-pod",
-					"__meta_kubernetes_endpointslice_address_type":                                          "ipv4",
-					"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                             "true",
-					"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_topoligy_io_zone":         "gce-1",
-					"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_topoligy_io_zone": "true",
-					"__meta_kubernetes_endpointslice_endpoint_hostname":                                     "node-1",
-					"__meta_kubernetes_endpointslice_name":                                                  "fake-esl",
-					"__meta_kubernetes_endpointslice_port":                                                  "8085",
-					"__meta_kubernetes_endpointslice_port_app_protocol":                                     "http",
-					"__meta_kubernetes_endpointslice_port_name":                                             "http",
-					"__meta_kubernetes_endpointslice_port_protocol":                                         "tcp",
-					"__meta_kubernetes_namespace":                                                           "default",
-				}),
-			},
-		},
-		{
-			name: "eps with pods and services",
-			args: args{
-				pods: []Pod{
-					{
-						Metadata: ObjectMeta{
-							UID:       "some-pod-uuid",
-							Namespace: "monitoring",
-							Name:      "main-pod",
-							Labels: discoveryutils.GetSortedLabels(map[string]string{
-								"pod-label-1": "pod-value-1",
-								"pod-label-2": "pod-value-2",
-							}),
-							Annotations: discoveryutils.GetSortedLabels(map[string]string{
-								"pod-annotations-1": "annotation-value-1",
-							}),
-						},
-						Status: PodStatus{PodIP: "192.168.11.5", HostIP: "172.15.1.1"},
-						Spec: PodSpec{NodeName: "node-2", Containers: []Container{
-							{
-								Name: "container-1",
-								Ports: []ContainerPort{
-									{
-										ContainerPort: 8085,
-										Protocol:      "tcp",
-										Name:          "http",
-									},
-									{
-										ContainerPort: 8011,
-										Protocol:      "udp",
-										Name:          "dns",
-									},
-								},
-							},
-						}},
-					},
-				},
-				svcs: []Service{
-					{
-						Spec: ServiceSpec{Type: "ClusterIP", Ports: []ServicePort{
-							{
-								Name:     "http",
-								Protocol: "tcp",
-								Port:     8085,
-							},
-						}},
-						Metadata: ObjectMeta{
-							Name:      "custom-esl",
-							Namespace: "monitoring",
-							Labels: discoveryutils.GetSortedLabels(map[string]string{
-								"service-label-1": "value-1",
-								"service-label-2": "value-2",
-							}),
-						},
-					},
-				},
-			},
-			fields: fields{
-				Metadata: ObjectMeta{
-					Name:      "custom-esl",
-					Namespace: "monitoring",
-				},
-				AddressType: "ipv4",
-				Endpoints: []Endpoint{
-					{Addresses: []string{"127.0.0.1"},
-						Hostname:   "node-1",
-						Topology:   map[string]string{"kubernetes.topoligy.io/zone": "gce-1"},
-						Conditions: EndpointConditions{Ready: true},
-						TargetRef: ObjectReference{
-							Kind:      "Pod",
-							Namespace: "monitoring",
-							Name:      "main-pod",
-						},
-					},
-				},
-				Ports: []EndpointPort{
-					{
-						Name:        "http",
-						Port:        8085,
-						AppProtocol: "http",
-						Protocol:    "tcp",
-					},
-				},
-			},
-			want: [][]prompbmarshal.Label{
-				discoveryutils.GetSortedLabels(map[string]string{
-					"__address__": "127.0.0.1:8085",
-					"__meta_kubernetes_endpointslice_address_target_kind":                                   "Pod",
-					"__meta_kubernetes_endpointslice_address_target_name":                                   "main-pod",
-					"__meta_kubernetes_endpointslice_address_type":                                          "ipv4",
-					"__meta_kubernetes_endpointslice_endpoint_conditions_ready":                             "true",
-					"__meta_kubernetes_endpointslice_endpoint_topology_kubernetes_topoligy_io_zone":         "gce-1",
-					"__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_topoligy_io_zone": "true",
-					"__meta_kubernetes_endpointslice_endpoint_hostname":                                     "node-1",
-					"__meta_kubernetes_endpointslice_name":                                                  "custom-esl",
-					"__meta_kubernetes_endpointslice_port":                                                  "8085",
-					"__meta_kubernetes_endpointslice_port_app_protocol":                                     "http",
-					"__meta_kubernetes_endpointslice_port_name":                                             "http",
-					"__meta_kubernetes_endpointslice_port_protocol":                                         "tcp",
-					"__meta_kubernetes_namespace":                                                           "monitoring",
-					"__meta_kubernetes_pod_annotation_pod_annotations_1":                                    "annotation-value-1",
-					"__meta_kubernetes_pod_annotationpresent_pod_annotations_1":                             "true",
-					"__meta_kubernetes_pod_container_name":                                                  "container-1",
-					"__meta_kubernetes_pod_container_port_name":                                             "http",
-					"__meta_kubernetes_pod_container_port_number":                                           "8085",
-					"__meta_kubernetes_pod_container_port_protocol":                                         "tcp",
-					"__meta_kubernetes_pod_host_ip":                                                         "172.15.1.1",
-					"__meta_kubernetes_pod_ip":                                                              "192.168.11.5",
-					"__meta_kubernetes_pod_label_pod_label_1":                                               "pod-value-1",
-					"__meta_kubernetes_pod_label_pod_label_2":                                               "pod-value-2",
-					"__meta_kubernetes_pod_labelpresent_pod_label_1":                                        "true",
-					"__meta_kubernetes_pod_labelpresent_pod_label_2":                                        "true",
-					"__meta_kubernetes_pod_name":                                                            "main-pod",
-					"__meta_kubernetes_pod_node_name":                                                       "node-2",
-					"__meta_kubernetes_pod_phase":                                                           "",
-					"__meta_kubernetes_pod_ready":                                                           "unknown",
-					"__meta_kubernetes_pod_uid":                                                             "some-pod-uuid",
-					"__meta_kubernetes_service_cluster_ip":                                                  "",
-					"__meta_kubernetes_service_label_service_label_1":                                       "value-1",
-					"__meta_kubernetes_service_label_service_label_2":                                       "value-2",
-					"__meta_kubernetes_service_labelpresent_service_label_1":                                "true",
-					"__meta_kubernetes_service_labelpresent_service_label_2":                                "true",
-					"__meta_kubernetes_service_name":                                                        "custom-esl",
-					"__meta_kubernetes_service_type":                                                        "ClusterIP",
-				}),
-				discoveryutils.GetSortedLabels(map[string]string{
-					"__address__":                 "192.168.11.5:8011",
-					"__meta_kubernetes_namespace": "monitoring",
-					"__meta_kubernetes_pod_annotation_pod_annotations_1":        "annotation-value-1",
-					"__meta_kubernetes_pod_annotationpresent_pod_annotations_1": "true",
-					"__meta_kubernetes_pod_container_name":                      "container-1",
-					"__meta_kubernetes_pod_container_port_name":                 "dns",
-					"__meta_kubernetes_pod_container_port_number":               "8011",
-					"__meta_kubernetes_pod_container_port_protocol":             "udp",
-					"__meta_kubernetes_pod_host_ip":                             "172.15.1.1",
-					"__meta_kubernetes_pod_ip":                                  "192.168.11.5",
-					"__meta_kubernetes_pod_label_pod_label_1":                   "pod-value-1",
-					"__meta_kubernetes_pod_label_pod_label_2":                   "pod-value-2",
-					"__meta_kubernetes_pod_labelpresent_pod_label_1":            "true",
-					"__meta_kubernetes_pod_labelpresent_pod_label_2":            "true",
-					"__meta_kubernetes_pod_name":                                "main-pod",
-					"__meta_kubernetes_pod_node_name":                           "node-2",
-					"__meta_kubernetes_pod_phase":                               "",
-					"__meta_kubernetes_pod_ready":                               "unknown",
-					"__meta_kubernetes_pod_uid":                                 "some-pod-uuid",
-					"__meta_kubernetes_service_cluster_ip":                      "",
-					"__meta_kubernetes_service_label_service_label_1":           "value-1",
-					"__meta_kubernetes_service_label_service_label_2":           "value-2",
-					"__meta_kubernetes_service_labelpresent_service_label_1":    "true",
-					"__meta_kubernetes_service_labelpresent_service_label_2":    "true",
-					"__meta_kubernetes_service_name":                            "custom-esl",
-					"__meta_kubernetes_service_type":                            "ClusterIP",
-				}),
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			eps := &EndpointSlice{
-				Metadata:    tt.fields.Metadata,
-				Endpoints:   tt.fields.Endpoints,
-				AddressType: tt.fields.AddressType,
-				Ports:       tt.fields.Ports,
-			}
-			pc := sync.Map{}
-			sc := sync.Map{}
-			for _, p := range tt.args.pods {
-				p := &p
-				pc.Store(p.key(), p)
-			}
-			for _, s := range tt.args.svcs {
-				s := &s
-				sc.Store(s.key(), s)
-			}
-			got := eps.appendTargetLabels(tt.args.ms, &pc, &sc)
-			var sortedLabelss [][]prompbmarshal.Label
-			for _, labels := range got {
-				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
-			}
-
-			if !reflect.DeepEqual(sortedLabelss, tt.want) {
-				t.Errorf("got unxpected labels: \ngot:\n %v, \nexpect:\n %v", sortedLabelss, tt.want)
-			}
-		})
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/framer.go b/lib/promscrape/discovery/kubernetes/framer.go
deleted file mode 100644
index 3aff2c927..000000000
--- a/lib/promscrape/discovery/kubernetes/framer.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package kubernetes
-
-import (
-	"encoding/json"
-	"io"
-)
-
-type jsonFrameReader struct {
-	r         io.ReadCloser
-	decoder   *json.Decoder
-	remaining []byte
-}
-
-func newJSONFramedReader(r io.ReadCloser) io.ReadCloser {
-	return &jsonFrameReader{
-		r:       r,
-		decoder: json.NewDecoder(r),
-	}
-}
-
-// ReadFrame decodes the next JSON object in the stream, or returns an error. The returned
-// byte slice will be modified the next time ReadFrame is invoked and should not be altered.
-func (r *jsonFrameReader) Read(data []byte) (int, error) {
-	// Return whatever remaining data exists from an in progress frame
-	if n := len(r.remaining); n > 0 {
-		if n <= len(data) {
-			data = append(data[0:0], r.remaining...)
-			r.remaining = nil
-			return n, nil
-		}
-		n = len(data)
-		data = append(data[0:0], r.remaining[:n]...)
-		r.remaining = r.remaining[n:]
-		return n, io.ErrShortBuffer
-	}
-
-	n := len(data)
-	m := json.RawMessage(data[:0])
-	if err := r.decoder.Decode(&m); err != nil {
-		return 0, err
-	}
-
-	// If capacity of data is less than length of the message, decoder will allocate a new slice
-	// and set m to it, which means we need to copy the partial result back into data and preserve
-	// the remaining result for subsequent reads.
-	if len(m) > n {
-		data = append(data[0:0], m[:n]...)
-		r.remaining = m[n:]
-		return n, io.ErrShortBuffer
-	}
-	return len(m), nil
-}
-
-func (r *jsonFrameReader) Close() error {
-	return r.r.Close()
-}
diff --git a/lib/promscrape/discovery/kubernetes/ingress.go b/lib/promscrape/discovery/kubernetes/ingress.go
index 7a17854df..f69c6673d 100644
--- a/lib/promscrape/discovery/kubernetes/ingress.go
+++ b/lib/promscrape/discovery/kubernetes/ingress.go
@@ -3,16 +3,57 @@ package kubernetes
 import (
 	"encoding/json"
 	"fmt"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
+// getIngressesLabels returns labels for k8s ingresses obtained from the given cfg.
+func getIngressesLabels(cfg *apiConfig) []map[string]string {
+	igs := getIngresses(cfg)
+	var ms []map[string]string
+	for _, ig := range igs {
+		ms = ig.appendTargetLabels(ms)
+	}
+	return ms
+}
+
+func getIngresses(cfg *apiConfig) []*Ingress {
+	os := cfg.aw.getObjectsByRole("ingress")
+	igs := make([]*Ingress, len(os))
+	for i, o := range os {
+		igs[i] = o.(*Ingress)
+	}
+	return igs
+}
+
+func (ig *Ingress) key() string {
+	return ig.Metadata.key()
+}
+
+func parseIngressList(data []byte) (map[string]object, ListMeta, error) {
+	var igl IngressList
+	if err := json.Unmarshal(data, &igl); err != nil {
+		return nil, igl.Metadata, fmt.Errorf("cannot unmarshal IngressList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, ig := range igl.Items {
+		objectsByKey[ig.key()] = ig
+	}
+	return objectsByKey, igl.Metadata, nil
+}
+
+func parseIngress(data []byte) (object, error) {
+	var ig Ingress
+	if err := json.Unmarshal(data, &ig); err != nil {
+		return nil, err
+	}
+	return &ig, nil
+}
+
 // IngressList represents ingress list in k8s.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresslist-v1beta1-extensions
 type IngressList struct {
-	Items    []Ingress
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*Ingress
 }
 
 // Ingress represents ingress in k8s.
@@ -23,10 +64,6 @@ type Ingress struct {
 	Spec     IngressSpec
 }
 
-func (ig Ingress) key() string {
-	return ig.Metadata.Namespace + "/" + ig.Metadata.Name
-}
-
 // IngressSpec represents ingress spec in k8s.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressspec-v1beta1-extensions
@@ -64,15 +101,6 @@ type HTTPIngressPath struct {
 	Path string
 }
 
-// parseIngressList parses IngressList from data.
-func parseIngressList(data []byte) (*IngressList, error) {
-	var il IngressList
-	if err := json.Unmarshal(data, &il); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal IngressList from %q: %w", data, err)
-	}
-	return &il, nil
-}
-
 // appendTargetLabels appends labels for Ingress ig to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ingress
@@ -124,23 +152,3 @@ func getIngressRulePaths(paths []HTTPIngressPath) []string {
 	}
 	return result
 }
-
-func processIngress(cfg *apiConfig, p *Ingress, action string) {
-	key := buildSyncKey("ingress", cfg.setName, p.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		cfg.targetChan <- SyncEvent{
-			Labels:           p.appendTargetLabels(nil),
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "ERROR":
-	default:
-		logger.Infof("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/ingress_test.go b/lib/promscrape/discovery/kubernetes/ingress_test.go
index 575abb5f8..8007c6376 100644
--- a/lib/promscrape/discovery/kubernetes/ingress_test.go
+++ b/lib/promscrape/discovery/kubernetes/ingress_test.go
@@ -11,12 +11,12 @@ import (
 func TestParseIngressListFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		nls, err := parseIngressList([]byte(s))
+		objectsByKey, _, err := parseIngressList([]byte(s))
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if nls != nil {
-			t.Fatalf("unexpected non-nil IngressList: %v", nls)
+		if len(objectsByKey) != 0 {
+			t.Fatalf("unexpected non-empty IngressList: %v", objectsByKey)
 		}
 	}
 	f(``)
@@ -71,21 +71,17 @@ func TestParseIngressListSuccess(t *testing.T) {
     }
   ]
 }`
-	igs, err := parseIngressList([]byte(data))
+	objectsByKey, meta, err := parseIngressList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if len(igs.Items) != 1 {
-		t.Fatalf("unexpected length of IngressList.Items; got %d; want %d", len(igs.Items), 1)
-	}
-	ig := igs.Items[0]
-
-	// Check ig.appendTargetLabels()
-	labelss := ig.appendTargetLabels(nil)
-	var sortedLabelss [][]prompbmarshal.Label
-	for _, labels := range labelss {
-		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
+	expectedResourceVersion := "351452"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*Ingress).appendTargetLabels(nil)
+	})
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "foobar",
diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go
index aea328438..b6c7a99c4 100644
--- a/lib/promscrape/discovery/kubernetes/kubernetes.go
+++ b/lib/promscrape/discovery/kubernetes/kubernetes.go
@@ -37,16 +37,26 @@ type Selector struct {
 	Field string `yaml:"field"`
 }
 
-// StartWatchOnce returns init labels for the given sdc and baseDir.
-// and starts watching for changes.
-func StartWatchOnce(watchCfg *WatchConfig, setName string, sdc *SDConfig, baseDir string) ([]map[string]string, error) {
-	cfg, err := getAPIConfig(watchCfg, setName, sdc, baseDir)
+// GetLabels returns labels for the given sdc and baseDir.
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
+	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create API config: %w", err)
 	}
-	var ms []map[string]string
-	cfg.watchOnce.Do(func() {
-		ms = startWatcherByRole(watchCfg.Ctx, sdc.Role, cfg, watchCfg.SC)
-	})
-	return ms, nil
+	switch sdc.Role {
+	case "node":
+		return getNodesLabels(cfg), nil
+	case "pod":
+		return getPodsLabels(cfg), nil
+	case "service":
+		return getServicesLabels(cfg), nil
+	case "endpoints":
+		return getEndpointsLabels(cfg), nil
+	case "endpointslices":
+		return getEndpointSlicesLabels(cfg), nil
+	case "ingress":
+		return getIngressesLabels(cfg), nil
+	default:
+		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslices` or `ingress`; skipping it", sdc.Role)
+	}
 }
diff --git a/lib/promscrape/discovery/kubernetes/node.go b/lib/promscrape/discovery/kubernetes/node.go
index 134bdc77a..720a58f04 100644
--- a/lib/promscrape/discovery/kubernetes/node.go
+++ b/lib/promscrape/discovery/kubernetes/node.go
@@ -4,16 +4,58 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+// getNodesLabels returns labels for k8s nodes obtained from the given cfg
+func getNodesLabels(cfg *apiConfig) []map[string]string {
+	nodes := getNodes(cfg)
+	var ms []map[string]string
+	for _, n := range nodes {
+		ms = n.appendTargetLabels(ms)
+	}
+	return ms
+}
+
+func getNodes(cfg *apiConfig) []*Node {
+	os := cfg.aw.getObjectsByRole("node")
+	ns := make([]*Node, len(os))
+	for i, o := range os {
+		ns[i] = o.(*Node)
+	}
+	return ns
+}
+
+func (n *Node) key() string {
+	return n.Metadata.key()
+}
+
+func parseNodeList(data []byte) (map[string]object, ListMeta, error) {
+	var nl NodeList
+	if err := json.Unmarshal(data, &nl); err != nil {
+		return nil, nl.Metadata, fmt.Errorf("cannot unmarshal NodeList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, n := range nl.Items {
+		objectsByKey[n.key()] = n
+	}
+	return objectsByKey, nl.Metadata, nil
+}
+
+func parseNode(data []byte) (object, error) {
+	var n Node
+	if err := json.Unmarshal(data, &n); err != nil {
+		return nil, err
+	}
+	return &n, nil
+}
+
 // NodeList represents NodeList from k8s API.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodelist-v1-core
 type NodeList struct {
-	Items    []Node
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*Node
 }
 
 // Node represents Node from k8s API.
@@ -24,10 +66,6 @@ type Node struct {
 	Status   NodeStatus
 }
 
-func (n Node) key() string {
-	return n.Metadata.Name
-}
-
 // NodeStatus represents NodeStatus from k8s API.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodestatus-v1-core
@@ -51,15 +89,6 @@ type NodeDaemonEndpoints struct {
 	KubeletEndpoint DaemonEndpoint
 }
 
-// parseNodeList parses NodeList from data.
-func parseNodeList(data []byte) (*NodeList, error) {
-	var nl NodeList
-	if err := json.Unmarshal(data, &nl); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal NodeList from %q: %w", data, err)
-	}
-	return &nl, nil
-}
-
 // appendTargetLabels appends labels for the given Node n to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#node
@@ -119,24 +148,3 @@ func getAddrByType(nas []NodeAddress, typ string) string {
 	}
 	return ""
 }
-
-func processNode(cfg *apiConfig, n *Node, action string) {
-	key := buildSyncKey("nodes", cfg.setName, n.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		lbs := n.appendTargetLabels(nil)
-		cfg.targetChan <- SyncEvent{
-			Labels:           lbs,
-			ConfigSectionSet: cfg.setName,
-			Key:              key,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			ConfigSectionSet: cfg.setName,
-			Key:              key,
-		}
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/node_test.go b/lib/promscrape/discovery/kubernetes/node_test.go
index c632b8123..deed59fc4 100644
--- a/lib/promscrape/discovery/kubernetes/node_test.go
+++ b/lib/promscrape/discovery/kubernetes/node_test.go
@@ -4,18 +4,19 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
 func TestParseNodeListFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		nls, err := parseNodeList([]byte(s))
+		objectsByKey, _, err := parseNodeList([]byte(s))
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if nls != nil {
-			t.Fatalf("unexpected non-nil NodeList: %v", nls)
+		if len(objectsByKey) != 0 {
+			t.Fatalf("unexpected non-empty objectsByKey: %v", objectsByKey)
 		}
 	}
 	f(``)
@@ -226,97 +227,69 @@ func TestParseNodeListSuccess(t *testing.T) {
   ]
 }
 `
-	nls, err := parseNodeList([]byte(data))
+	objectsByKey, meta, err := parseNodeList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if len(nls.Items) != 1 {
-		t.Fatalf("unexpected length of NodeList.Items; got %d; want %d", len(nls.Items), 1)
+	expectedResourceVersion := "22627"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	node := nls.Items[0]
-	meta := node.Metadata
-	if meta.Name != "m01" {
-		t.Fatalf("unexpected ObjectMeta.Name; got %q; want %q", meta.Name, "m01")
-	}
-	expectedLabels := discoveryutils.GetSortedLabels(map[string]string{
-		"beta.kubernetes.io/arch":        "amd64",
-		"beta.kubernetes.io/os":          "linux",
-		"kubernetes.io/arch":             "amd64",
-		"kubernetes.io/hostname":         "m01",
-		"kubernetes.io/os":               "linux",
-		"minikube.k8s.io/commit":         "eb13446e786c9ef70cb0a9f85a633194e62396a1",
-		"minikube.k8s.io/name":           "minikube",
-		"minikube.k8s.io/updated_at":     "2020_03_16T22_44_27_0700",
-		"minikube.k8s.io/version":        "v1.8.2",
-		"node-role.kubernetes.io/master": "",
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*Node).appendTargetLabels(nil)
 	})
-	if !reflect.DeepEqual(meta.Labels, expectedLabels) {
-		t.Fatalf("unexpected ObjectMeta.Labels\ngot\n%v\nwant\n%v", meta.Labels, expectedLabels)
+	expectedLabelss := [][]prompbmarshal.Label{
+		discoveryutils.GetSortedLabels(map[string]string{
+			"instance":                    "m01",
+			"__address__":                 "172.17.0.2:10250",
+			"__meta_kubernetes_node_name": "m01",
+
+			"__meta_kubernetes_node_label_beta_kubernetes_io_arch":        "amd64",
+			"__meta_kubernetes_node_label_beta_kubernetes_io_os":          "linux",
+			"__meta_kubernetes_node_label_kubernetes_io_arch":             "amd64",
+			"__meta_kubernetes_node_label_kubernetes_io_hostname":         "m01",
+			"__meta_kubernetes_node_label_kubernetes_io_os":               "linux",
+			"__meta_kubernetes_node_label_minikube_k8s_io_commit":         "eb13446e786c9ef70cb0a9f85a633194e62396a1",
+			"__meta_kubernetes_node_label_minikube_k8s_io_name":           "minikube",
+			"__meta_kubernetes_node_label_minikube_k8s_io_updated_at":     "2020_03_16T22_44_27_0700",
+			"__meta_kubernetes_node_label_minikube_k8s_io_version":        "v1.8.2",
+			"__meta_kubernetes_node_label_node_role_kubernetes_io_master": "",
+
+			"__meta_kubernetes_node_labelpresent_beta_kubernetes_io_arch":        "true",
+			"__meta_kubernetes_node_labelpresent_beta_kubernetes_io_os":          "true",
+			"__meta_kubernetes_node_labelpresent_kubernetes_io_arch":             "true",
+			"__meta_kubernetes_node_labelpresent_kubernetes_io_hostname":         "true",
+			"__meta_kubernetes_node_labelpresent_kubernetes_io_os":               "true",
+			"__meta_kubernetes_node_labelpresent_minikube_k8s_io_commit":         "true",
+			"__meta_kubernetes_node_labelpresent_minikube_k8s_io_name":           "true",
+			"__meta_kubernetes_node_labelpresent_minikube_k8s_io_updated_at":     "true",
+			"__meta_kubernetes_node_labelpresent_minikube_k8s_io_version":        "true",
+			"__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_master": "true",
+
+			"__meta_kubernetes_node_annotation_kubeadm_alpha_kubernetes_io_cri_socket":                 "/var/run/dockershim.sock",
+			"__meta_kubernetes_node_annotation_node_alpha_kubernetes_io_ttl":                           "0",
+			"__meta_kubernetes_node_annotation_volumes_kubernetes_io_controller_managed_attach_detach": "true",
+
+			"__meta_kubernetes_node_annotationpresent_kubeadm_alpha_kubernetes_io_cri_socket":                 "true",
+			"__meta_kubernetes_node_annotationpresent_node_alpha_kubernetes_io_ttl":                           "true",
+			"__meta_kubernetes_node_annotationpresent_volumes_kubernetes_io_controller_managed_attach_detach": "true",
+
+			"__meta_kubernetes_node_address_InternalIP": "172.17.0.2",
+			"__meta_kubernetes_node_address_Hostname":   "m01",
+		}),
 	}
-	expectedAnnotations := discoveryutils.GetSortedLabels(map[string]string{
-		"kubeadm.alpha.kubernetes.io/cri-socket":                 "/var/run/dockershim.sock",
-		"node.alpha.kubernetes.io/ttl":                           "0",
-		"volumes.kubernetes.io/controller-managed-attach-detach": "true",
-	})
-	if !reflect.DeepEqual(meta.Annotations, expectedAnnotations) {
-		t.Fatalf("unexpected ObjectMeta.Annotations\ngot\n%v\nwant\n%v", meta.Annotations, expectedAnnotations)
-	}
-	status := node.Status
-	expectedAddresses := []NodeAddress{
-		{
-			Type:    "InternalIP",
-			Address: "172.17.0.2",
-		},
-		{
-			Type:    "Hostname",
-			Address: "m01",
-		},
-	}
-	if !reflect.DeepEqual(status.Addresses, expectedAddresses) {
-		t.Fatalf("unexpected addresses\ngot\n%v\nwant\n%v", status.Addresses, expectedAddresses)
-	}
-
-	// Check node.appendTargetLabels()
-	labels := discoveryutils.GetSortedLabels(node.appendTargetLabels(nil)[0])
-	expectedLabels = discoveryutils.GetSortedLabels(map[string]string{
-		"instance":                    "m01",
-		"__address__":                 "172.17.0.2:10250",
-		"__meta_kubernetes_node_name": "m01",
-
-		"__meta_kubernetes_node_label_beta_kubernetes_io_arch":        "amd64",
-		"__meta_kubernetes_node_label_beta_kubernetes_io_os":          "linux",
-		"__meta_kubernetes_node_label_kubernetes_io_arch":             "amd64",
-		"__meta_kubernetes_node_label_kubernetes_io_hostname":         "m01",
-		"__meta_kubernetes_node_label_kubernetes_io_os":               "linux",
-		"__meta_kubernetes_node_label_minikube_k8s_io_commit":         "eb13446e786c9ef70cb0a9f85a633194e62396a1",
-		"__meta_kubernetes_node_label_minikube_k8s_io_name":           "minikube",
-		"__meta_kubernetes_node_label_minikube_k8s_io_updated_at":     "2020_03_16T22_44_27_0700",
-		"__meta_kubernetes_node_label_minikube_k8s_io_version":        "v1.8.2",
-		"__meta_kubernetes_node_label_node_role_kubernetes_io_master": "",
-
-		"__meta_kubernetes_node_labelpresent_beta_kubernetes_io_arch":        "true",
-		"__meta_kubernetes_node_labelpresent_beta_kubernetes_io_os":          "true",
-		"__meta_kubernetes_node_labelpresent_kubernetes_io_arch":             "true",
-		"__meta_kubernetes_node_labelpresent_kubernetes_io_hostname":         "true",
-		"__meta_kubernetes_node_labelpresent_kubernetes_io_os":               "true",
-		"__meta_kubernetes_node_labelpresent_minikube_k8s_io_commit":         "true",
-		"__meta_kubernetes_node_labelpresent_minikube_k8s_io_name":           "true",
-		"__meta_kubernetes_node_labelpresent_minikube_k8s_io_updated_at":     "true",
-		"__meta_kubernetes_node_labelpresent_minikube_k8s_io_version":        "true",
-		"__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_master": "true",
-
-		"__meta_kubernetes_node_annotation_kubeadm_alpha_kubernetes_io_cri_socket":                 "/var/run/dockershim.sock",
-		"__meta_kubernetes_node_annotation_node_alpha_kubernetes_io_ttl":                           "0",
-		"__meta_kubernetes_node_annotation_volumes_kubernetes_io_controller_managed_attach_detach": "true",
-
-		"__meta_kubernetes_node_annotationpresent_kubeadm_alpha_kubernetes_io_cri_socket":                 "true",
-		"__meta_kubernetes_node_annotationpresent_node_alpha_kubernetes_io_ttl":                           "true",
-		"__meta_kubernetes_node_annotationpresent_volumes_kubernetes_io_controller_managed_attach_detach": "true",
-
-		"__meta_kubernetes_node_address_InternalIP": "172.17.0.2",
-		"__meta_kubernetes_node_address_Hostname":   "m01",
-	})
-	if !reflect.DeepEqual(labels, expectedLabels) {
-		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", labels, expectedLabels)
+	if !reflect.DeepEqual(sortedLabelss, expectedLabelss) {
+		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss)
 	}
 }
+
+func getSortedLabelss(objectsByKey map[string]object, getLabelss func(o object) []map[string]string) [][]prompbmarshal.Label {
+	var result [][]prompbmarshal.Label
+	for _, o := range objectsByKey {
+		labelss := getLabelss(o)
+		for _, labels := range labelss {
+			result = append(result, discoveryutils.GetSortedLabels(labels))
+		}
+	}
+	return result
+}
diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go
index 1d5c384e4..cb2b00877 100644
--- a/lib/promscrape/discovery/kubernetes/pod.go
+++ b/lib/promscrape/discovery/kubernetes/pod.go
@@ -6,16 +6,58 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+// getPodsLabels returns labels for k8s pods obtained from the given cfg
+func getPodsLabels(cfg *apiConfig) []map[string]string {
+	pods := getPods(cfg)
+	var ms []map[string]string
+	for _, p := range pods {
+		ms = p.appendTargetLabels(ms)
+	}
+	return ms
+}
+
+func getPods(cfg *apiConfig) []*Pod {
+	os := cfg.aw.getObjectsByRole("pod")
+	ps := make([]*Pod, len(os))
+	for i, o := range os {
+		ps[i] = o.(*Pod)
+	}
+	return ps
+}
+
+func (p *Pod) key() string {
+	return p.Metadata.key()
+}
+
+func parsePodList(data []byte) (map[string]object, ListMeta, error) {
+	var pl PodList
+	if err := json.Unmarshal(data, &pl); err != nil {
+		return nil, pl.Metadata, fmt.Errorf("cannot unmarshal PodList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, p := range pl.Items {
+		objectsByKey[p.key()] = p
+	}
+	return objectsByKey, pl.Metadata, nil
+}
+
+func parsePod(data []byte) (object, error) {
+	var p Pod
+	if err := json.Unmarshal(data, &p); err != nil {
+		return nil, err
+	}
+	return &p, nil
+}
+
 // PodList implements k8s pod list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podlist-v1-core
 type PodList struct {
-	Items    []Pod
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*Pod
 }
 
 // Pod implements k8s pod.
@@ -27,10 +69,6 @@ type Pod struct {
 	Status   PodStatus
 }
 
-func (p Pod) key() string {
-	return p.Metadata.Namespace + "/" + p.Metadata.Name
-}
-
 // PodSpec implements k8s pod spec.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podspec-v1-core
@@ -73,15 +111,6 @@ type PodCondition struct {
 	Status string
 }
 
-// parsePodList parses PodList from data.
-func parsePodList(data []byte) (*PodList, error) {
-	var pl PodList
-	if err := json.Unmarshal(data, &pl); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal PodList from %q: %w", data, err)
-	}
-	return &pl, nil
-}
-
 // appendTargetLabels appends labels for each port of the given Pod p to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
@@ -169,23 +198,3 @@ func getPodReadyStatus(conds []PodCondition) string {
 	}
 	return "unknown"
 }
-
-func processPods(cfg *apiConfig, p *Pod, action string) {
-	key := buildSyncKey("pods", cfg.setName, p.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		cfg.targetChan <- SyncEvent{
-			Labels:           p.appendTargetLabels(nil),
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go
index 7462f0014..74c5456ca 100644
--- a/lib/promscrape/discovery/kubernetes/pod_test.go
+++ b/lib/promscrape/discovery/kubernetes/pod_test.go
@@ -11,12 +11,12 @@ import (
 func TestParsePodListFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		nls, err := parsePodList([]byte(s))
+		objectsByKey, _, err := parsePodList([]byte(s))
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if nls != nil {
-			t.Fatalf("unexpected non-nil PodList: %v", nls)
+		if len(objectsByKey) != 0 {
+			t.Fatalf("unexpected non-empty objectsByKey: %v", objectsByKey)
 		}
 	}
 	f(``)
@@ -228,22 +228,18 @@ func TestParsePodListSuccess(t *testing.T) {
   ]
 }
 `
-	pls, err := parsePodList([]byte(data))
+	objectsByKey, meta, err := parsePodList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if len(pls.Items) != 1 {
-		t.Fatalf("unexpected length of PodList.Items; got %d; want %d", len(pls.Items), 1)
+	expectedResourceVersion := "72425"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	pod := pls.Items[0]
-
-	// Check pod.appendTargetLabels()
-	labelss := pod.appendTargetLabels(nil)
-	var sortedLabelss [][]prompbmarshal.Label
-	for _, labels := range labelss {
-		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
-	}
-	expectedLabels := [][]prompbmarshal.Label{
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*Pod).appendTargetLabels(nil)
+	})
+	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.17.0.2:1234",
 
@@ -280,7 +276,7 @@ func TestParsePodListSuccess(t *testing.T) {
 			"__meta_kubernetes_pod_annotationpresent_kubernetes_io_config_source": "true",
 		}),
 	}
-	if !reflect.DeepEqual(sortedLabelss, expectedLabels) {
-		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabels)
+	if !reflect.DeepEqual(sortedLabelss, expectedLabelss) {
+		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss)
 	}
 }
diff --git a/lib/promscrape/discovery/kubernetes/service.go b/lib/promscrape/discovery/kubernetes/service.go
index 3723493bb..c23569bab 100644
--- a/lib/promscrape/discovery/kubernetes/service.go
+++ b/lib/promscrape/discovery/kubernetes/service.go
@@ -4,16 +4,58 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+// getServicesLabels returns labels for k8s services obtained from the given cfg
+func getServicesLabels(cfg *apiConfig) []map[string]string {
+	svcs := getServices(cfg)
+	var ms []map[string]string
+	for _, svc := range svcs {
+		ms = svc.appendTargetLabels(ms)
+	}
+	return ms
+}
+
+func getServices(cfg *apiConfig) []*Service {
+	os := cfg.aw.getObjectsByRole("service")
+	svcs := make([]*Service, len(os))
+	for i, o := range os {
+		svcs[i] = o.(*Service)
+	}
+	return svcs
+}
+
+func (svc *Service) key() string {
+	return svc.Metadata.key()
+}
+
+func parseServiceList(data []byte) (map[string]object, ListMeta, error) {
+	var sl ServiceList
+	if err := json.Unmarshal(data, &sl); err != nil {
+		return nil, sl.Metadata, fmt.Errorf("cannot unmarshal ServiceList from %q: %w", data, err)
+	}
+	objectsByKey := make(map[string]object)
+	for _, svc := range sl.Items {
+		objectsByKey[svc.key()] = svc
+	}
+	return objectsByKey, sl.Metadata, nil
+}
+
+func parseService(data []byte) (object, error) {
+	var svc Service
+	if err := json.Unmarshal(data, &svc); err != nil {
+		return nil, err
+	}
+	return &svc, nil
+}
+
 // ServiceList is k8s service list.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicelist-v1-core
 type ServiceList struct {
-	Items    []Service
-	Metadata listMetadata `json:"metadata"`
+	Metadata ListMeta
+	Items    []*Service
 }
 
 // Service is k8s service.
@@ -24,10 +66,6 @@ type Service struct {
 	Spec     ServiceSpec
 }
 
-func (s Service) key() string {
-	return s.Metadata.Namespace + "/" + s.Metadata.Name
-}
-
 // ServiceSpec is k8s service spec.
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core
@@ -47,15 +85,6 @@ type ServicePort struct {
 	Port     int
 }
 
-// parseServiceList parses ServiceList from data.
-func parseServiceList(data []byte) (*ServiceList, error) {
-	var sl ServiceList
-	if err := json.Unmarshal(data, &sl); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal ServiceList from %q: %w", data, err)
-	}
-	return &sl, nil
-}
-
 // appendTargetLabels appends labels for each port of the given Service s to ms and returns the result.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#service
@@ -85,23 +114,3 @@ func (s *Service) appendCommonLabels(m map[string]string) {
 	}
 	s.Metadata.registerLabelsAndAnnotations("__meta_kubernetes_service", m)
 }
-
-func processService(cfg *apiConfig, svc *Service, action string) {
-	key := buildSyncKey("service", cfg.setName, svc.key())
-	switch action {
-	case "ADDED", "MODIFIED":
-		cfg.targetChan <- SyncEvent{
-			Labels:           svc.appendTargetLabels(nil),
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "DELETED":
-		cfg.targetChan <- SyncEvent{
-			Key:              key,
-			ConfigSectionSet: cfg.setName,
-		}
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/service_test.go b/lib/promscrape/discovery/kubernetes/service_test.go
index 2f3c1f7e7..4718fe6ed 100644
--- a/lib/promscrape/discovery/kubernetes/service_test.go
+++ b/lib/promscrape/discovery/kubernetes/service_test.go
@@ -11,12 +11,12 @@ import (
 func TestParseServiceListFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		nls, err := parseServiceList([]byte(s))
+		objectsByKey, _, err := parseServiceList([]byte(s))
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if nls != nil {
-			t.Fatalf("unexpected non-nil ServiceList: %v", nls)
+		if len(objectsByKey) != 0 {
+			t.Fatalf("unexpected non-empty objectsByKey: %v", objectsByKey)
 		}
 	}
 	f(``)
@@ -89,68 +89,17 @@ func TestParseServiceListSuccess(t *testing.T) {
   ]
 }
 `
-	sls, err := parseServiceList([]byte(data))
+	objectsByKey, meta, err := parseServiceList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if len(sls.Items) != 1 {
-		t.Fatalf("unexpected length of ServiceList.Items; got %d; want %d", len(sls.Items), 1)
+	expectedResourceVersion := "60485"
+	if meta.ResourceVersion != expectedResourceVersion {
+		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	service := sls.Items[0]
-	meta := service.Metadata
-	if meta.Name != "kube-dns" {
-		t.Fatalf("unexpected ObjectMeta.Name; got %q; want %q", meta.Name, "kube-dns")
-	}
-	expectedLabels := discoveryutils.GetSortedLabels(map[string]string{
-		"k8s-app":                       "kube-dns",
-		"kubernetes.io/cluster-service": "true",
-		"kubernetes.io/name":            "KubeDNS",
+	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
+		return o.(*Service).appendTargetLabels(nil)
 	})
-	if !reflect.DeepEqual(meta.Labels, expectedLabels) {
-		t.Fatalf("unexpected ObjectMeta.Labels\ngot\n%v\nwant\n%v", meta.Labels, expectedLabels)
-	}
-	expectedAnnotations := discoveryutils.GetSortedLabels(map[string]string{
-		"prometheus.io/port":   "9153",
-		"prometheus.io/scrape": "true",
-	})
-	if !reflect.DeepEqual(meta.Annotations, expectedAnnotations) {
-		t.Fatalf("unexpected ObjectMeta.Annotations\ngot\n%v\nwant\n%v", meta.Annotations, expectedAnnotations)
-	}
-	spec := service.Spec
-	expectedClusterIP := "10.96.0.10"
-	if spec.ClusterIP != expectedClusterIP {
-		t.Fatalf("unexpected clusterIP; got %q; want %q", spec.ClusterIP, expectedClusterIP)
-	}
-	if spec.Type != "ClusterIP" {
-		t.Fatalf("unexpected type; got %q; want %q", spec.Type, "ClusterIP")
-	}
-	expectedPorts := []ServicePort{
-		{
-			Name:     "dns",
-			Protocol: "UDP",
-			Port:     53,
-		},
-		{
-			Name:     "dns-tcp",
-			Protocol: "TCP",
-			Port:     53,
-		},
-		{
-			Name:     "metrics",
-			Protocol: "TCP",
-			Port:     9153,
-		},
-	}
-	if !reflect.DeepEqual(spec.Ports, expectedPorts) {
-		t.Fatalf("unexpected ports\ngot\n%v\nwant\n%v", spec.Ports, expectedPorts)
-	}
-
-	// Check service.appendTargetLabels()
-	labelss := service.appendTargetLabels(nil)
-	var sortedLabelss [][]prompbmarshal.Label
-	for _, labels := range labelss {
-		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
-	}
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__":                             "kube-dns.kube-system.svc:53",
diff --git a/lib/promscrape/discovery/kubernetes/shared_cache.go b/lib/promscrape/discovery/kubernetes/shared_cache.go
deleted file mode 100644
index 72c763c39..000000000
--- a/lib/promscrape/discovery/kubernetes/shared_cache.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package kubernetes
-
-import (
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-)
-
-// SharedKubernetesCache holds cache of kubernetes objects for current config.
-type SharedKubernetesCache struct {
-	Endpoints       *sync.Map
-	EndpointsSlices *sync.Map
-	Pods            *sync.Map
-	Services        *sync.Map
-}
-
-// NewSharedKubernetesCache returns new cache.
-func NewSharedKubernetesCache() *SharedKubernetesCache {
-	return &SharedKubernetesCache{
-		Endpoints:       new(sync.Map),
-		EndpointsSlices: new(sync.Map),
-		Pods:            new(sync.Map),
-		Services:        new(sync.Map),
-	}
-}
-
-func updatePodCache(cache *sync.Map, p *Pod, action string) {
-	switch action {
-	case "ADDED":
-		cache.Store(p.key(), p)
-	case "DELETED":
-		cache.Delete(p.key())
-	case "MODIFIED":
-		cache.Store(p.key(), p)
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
-
-func updateServiceCache(cache *sync.Map, p *Service, action string) {
-	switch action {
-	case "ADDED", "MODIFIED":
-		cache.Store(p.key(), p)
-	case "DELETED":
-		cache.Delete(p.key())
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-
-}
-
-func updateEndpointsCache(cache *sync.Map, p *Endpoints, action string) {
-	switch action {
-	case "ADDED", "MODIFIED":
-		cache.Store(p.key(), p)
-	case "DELETED":
-		cache.Delete(p.key())
-	case "ERROR":
-	default:
-		logger.Warnf("unexpected action: %s", action)
-	}
-}
-
-func updateEndpointsSliceCache(cache *sync.Map, p *EndpointSlice, action string) {
-	switch action {
-	case "ADDED", "MODIFIED":
-		cache.Store(p.key(), p)
-	case "DELETED":
-		cache.Delete(p.key())
-	case "ERROR":
-	default:
-		logger.Infof("unexpected action: %s", action)
-	}
-}
diff --git a/lib/promscrape/discovery/kubernetes/watch.go b/lib/promscrape/discovery/kubernetes/watch.go
deleted file mode 100644
index 1cec161a7..000000000
--- a/lib/promscrape/discovery/kubernetes/watch.go
+++ /dev/null
@@ -1,510 +0,0 @@
-package kubernetes
-
-import (
-	"compress/gzip"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-)
-
-// SyncEvent represent kubernetes resource watch event.
-type SyncEvent struct {
-	// object type + set name + ns + name
-	// must be unique.
-	Key string
-	// Labels targets labels for given resource
-	Labels []map[string]string
-	// job name + position id
-	ConfigSectionSet string
-}
-
-type watchResponse struct {
-	Action string          `json:"type"`
-	Object json.RawMessage `json:"object"`
-}
-
-// WatchConfig holds objects for watch handler start.
-type WatchConfig struct {
-	Ctx       context.Context
-	SC        *SharedKubernetesCache
-	WG        *sync.WaitGroup
-	WatchChan chan SyncEvent
-}
-
-// NewWatchConfig returns new config with given context.
-func NewWatchConfig(ctx context.Context) *WatchConfig {
-	return &WatchConfig{
-		Ctx:       ctx,
-		SC:        NewSharedKubernetesCache(),
-		WG:        new(sync.WaitGroup),
-		WatchChan: make(chan SyncEvent, 100),
-	}
-}
-
-func buildSyncKey(objType string, setName string, objKey string) string {
-	return objType + "/" + setName + "/" + objKey
-}
-
-func startWatcherByRole(ctx context.Context, role string, cfg *apiConfig, sc *SharedKubernetesCache) []map[string]string {
-	var ms []map[string]string
-	switch role {
-	case "pod":
-		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
-			var p Pod
-			if err := json.Unmarshal(wr.Object, &p); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processPods(cfg, &p, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			pods, err := parsePodList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, pod := range pods.Items {
-				ms = pod.appendTargetLabels(ms)
-				processPods(cfg, &pod, "ADDED")
-			}
-			return pods.Metadata.ResourceVersion, nil
-		})
-	case "node":
-		startWatchForObject(ctx, cfg, "nodes", func(wr *watchResponse) {
-			var n Node
-			if err := json.Unmarshal(wr.Object, &n); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processNode(cfg, &n, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			nodes, err := parseNodeList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, node := range nodes.Items {
-				processNode(cfg, &node, "ADDED")
-				ms = node.appendTargetLabels(ms)
-			}
-			return nodes.Metadata.ResourceVersion, nil
-		})
-	case "endpoints":
-		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
-			var p Pod
-			if err := json.Unmarshal(wr.Object, &p); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			updatePodCache(sc.Pods, &p, wr.Action)
-			if wr.Action == "MODIFIED" {
-				eps, ok := sc.Endpoints.Load(p.key())
-				if ok {
-					ep := eps.(*Endpoints)
-					processEndpoints(cfg, sc, ep, wr.Action)
-				}
-			}
-		}, func(bytes []byte) (string, error) {
-			pods, err := parsePodList(bytes)
-			if err != nil {
-				return "", err
-			}
-			for _, pod := range pods.Items {
-				updatePodCache(sc.Pods, &pod, "ADDED")
-			}
-			return pods.Metadata.ResourceVersion, nil
-		})
-		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
-			var svc Service
-			if err := json.Unmarshal(wr.Object, &svc); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			updateServiceCache(sc.Services, &svc, wr.Action)
-			if wr.Action == "MODIFIED" {
-				linkedEps, ok := sc.Endpoints.Load(svc.key())
-				if ok {
-					ep := linkedEps.(*Endpoints)
-					processEndpoints(cfg, sc, ep, wr.Action)
-				}
-			}
-		}, func(bytes []byte) (string, error) {
-			svcs, err := parseServiceList(bytes)
-			if err != nil {
-				return "", err
-			}
-			for _, svc := range svcs.Items {
-				updateServiceCache(sc.Services, &svc, "ADDED")
-			}
-			return svcs.Metadata.ResourceVersion, nil
-		})
-		startWatchForObject(ctx, cfg, "endpoints", func(wr *watchResponse) {
-			var eps Endpoints
-			if err := json.Unmarshal(wr.Object, &eps); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processEndpoints(cfg, sc, &eps, wr.Action)
-			updateEndpointsCache(sc.Endpoints, &eps, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			eps, err := parseEndpointsList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, ep := range eps.Items {
-				ms = ep.appendTargetLabels(ms, sc.Pods, sc.Services)
-				processEndpoints(cfg, sc, &ep, "ADDED")
-				updateEndpointsCache(sc.Endpoints, &ep, "ADDED")
-			}
-			return eps.Metadata.ResourceVersion, nil
-		})
-	case "service":
-		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
-			var svc Service
-			if err := json.Unmarshal(wr.Object, &svc); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processService(cfg, &svc, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			svcs, err := parseServiceList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, svc := range svcs.Items {
-				processService(cfg, &svc, "ADDED")
-				ms = svc.appendTargetLabels(ms)
-			}
-			return svcs.Metadata.ResourceVersion, nil
-		})
-	case "ingress":
-		startWatchForObject(ctx, cfg, "ingresses", func(wr *watchResponse) {
-			var ig Ingress
-			if err := json.Unmarshal(wr.Object, &ig); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processIngress(cfg, &ig, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			igs, err := parseIngressList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, ig := range igs.Items {
-				processIngress(cfg, &ig, "ADDED")
-				ms = ig.appendTargetLabels(ms)
-			}
-			return igs.Metadata.ResourceVersion, nil
-		})
-	case "endpointslices":
-		startWatchForObject(ctx, cfg, "pods", func(wr *watchResponse) {
-			var p Pod
-			if err := json.Unmarshal(wr.Object, &p); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			updatePodCache(sc.Pods, &p, wr.Action)
-			if wr.Action == "MODIFIED" {
-				eps, ok := sc.EndpointsSlices.Load(p.key())
-				if ok {
-					ep := eps.(*EndpointSlice)
-					processEndpointSlices(cfg, sc, ep, wr.Action)
-				}
-			}
-		}, func(bytes []byte) (string, error) {
-			pods, err := parsePodList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, pod := range pods.Items {
-				updatePodCache(sc.Pods, &pod, "ADDED")
-			}
-			return pods.Metadata.ResourceVersion, nil
-		})
-		startWatchForObject(ctx, cfg, "services", func(wr *watchResponse) {
-			var svc Service
-			if err := json.Unmarshal(wr.Object, &svc); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			updateServiceCache(sc.Services, &svc, wr.Action)
-			if wr.Action == "MODIFIED" {
-				linkedEps, ok := sc.EndpointsSlices.Load(svc.key())
-				if ok {
-					ep := linkedEps.(*EndpointSlice)
-					processEndpointSlices(cfg, sc, ep, wr.Action)
-				}
-			}
-		}, func(bytes []byte) (string, error) {
-			svcs, err := parseServiceList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, svc := range svcs.Items {
-				updateServiceCache(sc.Services, &svc, "ADDED")
-			}
-			return svcs.Metadata.ResourceVersion, nil
-		})
-		startWatchForObject(ctx, cfg, "endpointslices", func(wr *watchResponse) {
-			var eps EndpointSlice
-			if err := json.Unmarshal(wr.Object, &eps); err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return
-			}
-			processEndpointSlices(cfg, sc, &eps, wr.Action)
-			updateEndpointsSliceCache(sc.EndpointsSlices, &eps, wr.Action)
-		}, func(bytes []byte) (string, error) {
-			epss, err := parseEndpointSlicesList(bytes)
-			if err != nil {
-				logger.Errorf("failed to parse object, err: %v", err)
-				return "", err
-			}
-			for _, eps := range epss.Items {
-				ms = eps.appendTargetLabels(ms, sc.Pods, sc.Services)
-				processEndpointSlices(cfg, sc, &eps, "ADDED")
-			}
-			return epss.Metadata.ResourceVersion, nil
-		})
-	default:
-		logger.Errorf("unexpected role: %s", role)
-	}
-	return ms
-}
-
-func startWatchForObject(ctx context.Context, cfg *apiConfig, objectName string, wh func(wr *watchResponse), getSync func([]byte) (string, error)) {
-	if len(cfg.namespaces) > 0 {
-		for _, ns := range cfg.namespaces {
-			path := fmt.Sprintf("/api/v1/namespaces/%s/%s", ns, objectName)
-			// special case.
-			if objectName == "endpointslices" {
-				path = fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/namespaces/%s/%s", ns, objectName)
-			}
-			query := joinSelectors(objectName, nil, cfg.selectors)
-			if len(query) > 0 {
-				path += "?" + query
-			}
-			data, err := cfg.wc.getBlockingAPIResponse(path)
-			if err != nil {
-				logger.Errorf("cannot get latest resource version: %v", err)
-			}
-			version, err := getSync(data)
-			if err != nil {
-				logger.Errorf("cannot get latest resource version: %v", err)
-			}
-			cfg.wc.wg.Add(1)
-			go func(path, version string) {
-				cfg.wc.startWatchForResource(ctx, path, wh, version)
-			}(path, version)
-		}
-	} else {
-		path := "/api/v1/" + objectName
-		if objectName == "endpointslices" {
-			// special case.
-			path = fmt.Sprintf("/apis/discovery.k8s.io/v1beta1/%s", objectName)
-		}
-		query := joinSelectors(objectName, nil, cfg.selectors)
-		if len(query) > 0 {
-			path += "?" + query
-		}
-		data, err := cfg.wc.getBlockingAPIResponse(path)
-		if err != nil {
-			logger.Errorf("cannot get latest resource version: %v", err)
-		}
-		version, err := getSync(data)
-		if err != nil {
-			logger.Errorf("cannot get latest resource version: %v", err)
-		}
-		cfg.wc.wg.Add(1)
-		go func() {
-			cfg.wc.startWatchForResource(ctx, path, wh, version)
-		}()
-	}
-}
-
-type watchClient struct {
-	c         *http.Client
-	ac        *promauth.Config
-	apiServer string
-	wg        *sync.WaitGroup
-}
-
-func (wc *watchClient) startWatchForResource(ctx context.Context, path string, wh func(wr *watchResponse), initResourceVersion string) {
-	defer wc.wg.Done()
-	path += "?watch=1"
-	maxBackOff := time.Second * 30
-	backoff := time.Second
-	for {
-		err := wc.getStreamAPIResponse(ctx, path, initResourceVersion, wh)
-		if errors.Is(err, context.Canceled) {
-			return
-		}
-		if !errors.Is(err, io.EOF) {
-			logger.Errorf("got unexpected error : %v", err)
-		}
-		// reset version.
-		initResourceVersion = ""
-		if backoff < maxBackOff {
-			backoff += time.Second * 5
-		}
-		time.Sleep(backoff)
-	}
-}
-
-func (wc *watchClient) getBlockingAPIResponse(path string) ([]byte, error) {
-	req, err := http.NewRequest("GET", wc.apiServer+path, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Accept-Encoding", "gzip")
-	if wc.ac != nil && wc.ac.Authorization != "" {
-		req.Header.Set("Authorization", wc.ac.Authorization)
-	}
-	resp, err := wc.c.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("get unexpected code: %d, at blocking api request path: %q", resp.StatusCode, path)
-	}
-	if ce := resp.Header.Get("Content-Encoding"); ce == "gzip" {
-		gr, err := gzip.NewReader(resp.Body)
-		if err != nil {
-			return nil, fmt.Errorf("cannot create gzip reader: %w", err)
-		}
-		return ioutil.ReadAll(gr)
-	}
-	return ioutil.ReadAll(resp.Body)
-}
-
-func (wc *watchClient) getStreamAPIResponse(ctx context.Context, path, resouceVersion string, wh func(wr *watchResponse)) error {
-	if resouceVersion != "" {
-		path += "&resourceVersion=" + resouceVersion
-	}
-	req, err := http.NewRequestWithContext(ctx, "GET", wc.apiServer+path, nil)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Accept-Encoding", "gzip")
-	if wc.ac != nil && wc.ac.Authorization != "" {
-		req.Header.Set("Authorization", wc.ac.Authorization)
-	}
-	resp, err := wc.c.Do(req)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
-	}
-	br := resp.Body
-	if ce := resp.Header.Get("Content-Encoding"); ce == "gzip" {
-		br, err = gzip.NewReader(resp.Body)
-		if err != nil {
-			return fmt.Errorf("cannot create gzip reader: %w", err)
-		}
-	}
-	r := newJSONFramedReader(br)
-	for {
-		b := make([]byte, 1024)
-		b, err := readJSONObject(r, b)
-		if err != nil {
-			return err
-		}
-		var rObject watchResponse
-		err = json.Unmarshal(b, &rObject)
-		if err != nil {
-			logger.Errorf("failed to parse watch api response as json, err %v, response: %v", err, string(b))
-			continue
-		}
-		wh(&rObject)
-	}
-}
-
-func readJSONObject(r io.Reader, b []byte) ([]byte, error) {
-	offset := 0
-	for {
-		n, err := r.Read(b[offset:])
-		if err == io.ErrShortBuffer {
-			if n == 0 {
-				return nil, fmt.Errorf("got short buffer with n=0, cap=%d", cap(b))
-			}
-			// double buffer..
-			b = bytesutil.Resize(b, len(b)*2)
-			offset += n
-			continue
-		}
-		if err != nil {
-			return nil, err
-		}
-		offset += n
-		break
-	}
-	return b[:offset], nil
-}
-
-func newWatchClient(wg *sync.WaitGroup, sdc *SDConfig, baseDir string) (*watchClient, error) {
-	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse auth config: %w", err)
-	}
-	apiServer := sdc.APIServer
-	if len(apiServer) == 0 {
-		// Assume we run at k8s pod.
-		// Discover apiServer and auth config according to k8s docs.
-		// See https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller
-		host := os.Getenv("KUBERNETES_SERVICE_HOST")
-		port := os.Getenv("KUBERNETES_SERVICE_PORT")
-		if len(host) == 0 {
-			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_HOST env var; it must be defined when running in k8s; " +
-				"probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?")
-		}
-		if len(port) == 0 {
-			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_PORT env var; it must be defined when running in k8s; "+
-				"KUBERNETES_SERVICE_HOST=%q", host)
-		}
-		apiServer = "https://" + net.JoinHostPort(host, port)
-		tlsConfig := promauth.TLSConfig{
-			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
-		}
-		acNew, err := promauth.NewConfig(".", nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
-		if err != nil {
-			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
-		}
-		ac = acNew
-	}
-	var proxy func(*http.Request) (*url.URL, error)
-	if proxyURL := sdc.ProxyURL.URL(); proxyURL != nil {
-		proxy = http.ProxyURL(proxyURL)
-	}
-	c := &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig:     ac.NewTLSConfig(),
-			Proxy:               proxy,
-			TLSHandshakeTimeout: 10 * time.Second,
-			IdleConnTimeout:     2 * time.Minute,
-		},
-	}
-	wc := watchClient{
-		c:         c,
-		apiServer: apiServer,
-		ac:        ac,
-		wg:        wg,
-	}
-	return &wc, nil
-}
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index ddef4861d..5e3aef789 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -23,7 +23,6 @@ var (
 	kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+
 		"This works only if `kubernetes_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details")
-
 	openstackSDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*time.Second, "Interval for checking for changes in openstack API server. "+
 		"This works only if `openstack_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details")
@@ -98,9 +97,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	scs := newScrapeConfigs(pushData)
 	scs.add("static_configs", 0, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })
 	scs.add("file_sd_configs", *fileSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getFileSDScrapeWork(swsPrev) })
-	scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork {
-		return getKubernetesSDScrapeWorkStream(cfg, swsPrev)
-	})
+	scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
 	scs.add("openstack_sd_configs", *openstackSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
 	scs.add("consul_sd_configs", *consul.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getConsulSDScrapeWork(swsPrev) })
 	scs.add("eureka_sd_configs", *eurekaSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) })
diff --git a/lib/promscrape/watch_handler.go b/lib/promscrape/watch_handler.go
deleted file mode 100644
index d940e08fe..000000000
--- a/lib/promscrape/watch_handler.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package promscrape
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
-)
-
-type kubernetesWatchHandler struct {
-	ctx       context.Context
-	cancel    context.CancelFunc
-	startOnce sync.Once
-	watchCfg  *kubernetes.WatchConfig
-	// guards cache and set
-	mu             sync.Mutex
-	lastAccessTime time.Time
-	swCache        map[string][]*ScrapeWork
-	sdcSet         map[string]*scrapeWorkConfig
-}
-
-func newKubernetesWatchHandler() *kubernetesWatchHandler {
-	ctx, cancel := context.WithCancel(context.Background())
-	kwh := &kubernetesWatchHandler{
-		ctx:      ctx,
-		cancel:   cancel,
-		swCache:  map[string][]*ScrapeWork{},
-		sdcSet:   map[string]*scrapeWorkConfig{},
-		watchCfg: kubernetes.NewWatchConfig(ctx),
-	}
-	go kwh.waitForStop()
-	return kwh
-}
-
-func (ksc *kubernetesWatchHandler) waitForStop() {
-	t := time.NewTicker(time.Second * 5)
-	for range t.C {
-		ksc.mu.Lock()
-		lastTime := time.Since(ksc.lastAccessTime)
-		ksc.mu.Unlock()
-		if lastTime > *kubernetesSDCheckInterval*30 {
-			t1 := time.Now()
-			ksc.cancel()
-			ksc.watchCfg.WG.Wait()
-			close(ksc.watchCfg.WatchChan)
-			logger.Infof("stopped kubernetes api watcher handler, after: %.3f seconds", time.Since(t1).Seconds())
-			ksc.watchCfg.SC = nil
-			t.Stop()
-			return
-		}
-	}
-}
-
-func processKubernetesSyncEvents(cfg *Config) {
-	for {
-		select {
-		case <-cfg.kwh.ctx.Done():
-			return
-		case se, ok := <-cfg.kwh.watchCfg.WatchChan:
-			if !ok {
-				return
-			}
-			if se.Labels == nil {
-				cfg.kwh.mu.Lock()
-				delete(cfg.kwh.swCache, se.Key)
-				cfg.kwh.mu.Unlock()
-				continue
-			}
-			cfg.kwh.mu.Lock()
-			swc, ok := cfg.kwh.sdcSet[se.ConfigSectionSet]
-			cfg.kwh.mu.Unlock()
-			if !ok {
-				logger.Fatalf("bug config section not found: %v", se.ConfigSectionSet)
-			}
-			ms := appendScrapeWorkForTargetLabels(nil, swc, se.Labels, "kubernetes_sd_config")
-			cfg.kwh.mu.Lock()
-			cfg.kwh.swCache[se.Key] = ms
-			cfg.kwh.mu.Unlock()
-		}
-	}
-}

From 19712fc2bd504841aaf8f80f90aa3797f963be5c Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 17:00:08 +0200
Subject: [PATCH 29/32] lib/promscrape/discovery/kubernetes: errcheck fix

---
 lib/promscrape/discovery/kubernetes/service.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/promscrape/discovery/kubernetes/service.go b/lib/promscrape/discovery/kubernetes/service.go
index c23569bab..9d1c5fbed 100644
--- a/lib/promscrape/discovery/kubernetes/service.go
+++ b/lib/promscrape/discovery/kubernetes/service.go
@@ -26,8 +26,8 @@ func getServices(cfg *apiConfig) []*Service {
 	return svcs
 }
 
-func (svc *Service) key() string {
-	return svc.Metadata.key()
+func (s *Service) key() string {
+	return s.Metadata.key()
 }
 
 func parseServiceList(data []byte) (map[string]object, ListMeta, error) {
@@ -36,18 +36,18 @@ func parseServiceList(data []byte) (map[string]object, ListMeta, error) {
 		return nil, sl.Metadata, fmt.Errorf("cannot unmarshal ServiceList from %q: %w", data, err)
 	}
 	objectsByKey := make(map[string]object)
-	for _, svc := range sl.Items {
-		objectsByKey[svc.key()] = svc
+	for _, s := range sl.Items {
+		objectsByKey[s.key()] = s
 	}
 	return objectsByKey, sl.Metadata, nil
 }
 
 func parseService(data []byte) (object, error) {
-	var svc Service
-	if err := json.Unmarshal(data, &svc); err != nil {
+	var s Service
+	if err := json.Unmarshal(data, &s); err != nil {
 		return nil, err
 	}
-	return &svc, nil
+	return &s, nil
 }
 
 // ServiceList is k8s service list.

From 815666e6a607bea310a684f267b626de4d593619 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 20:21:27 +0200
Subject: [PATCH 30/32] lib/promscrape/discovery/kubernetes: cache target
 labels

This should reduce CPU usage on repeated SDConfig.GetLabels() calls.
---
 lib/promscrape/discovery/kubernetes/api.go    | 47 ++++++++++---------
 .../discovery/kubernetes/endpoints.go         | 24 ++--------
 .../discovery/kubernetes/endpoints_test.go    |  4 +-
 .../discovery/kubernetes/endpointslices.go    | 28 ++---------
 .../kubernetes/endpointslices_test.go         |  4 +-
 .../discovery/kubernetes/ingress.go           | 24 ++--------
 .../discovery/kubernetes/ingress_test.go      |  4 +-
 .../discovery/kubernetes/kubernetes.go        | 14 +-----
 lib/promscrape/discovery/kubernetes/node.go   | 27 ++---------
 .../discovery/kubernetes/node_test.go         |  8 ++--
 lib/promscrape/discovery/kubernetes/pod.go    | 26 ++--------
 .../discovery/kubernetes/pod_test.go          |  4 +-
 .../discovery/kubernetes/service.go           | 24 ++--------
 .../discovery/kubernetes/service_test.go      |  4 +-
 14 files changed, 58 insertions(+), 184 deletions(-)

diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index 816d5e194..3212cf3d3 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -109,6 +109,7 @@ type WatchEvent struct {
 // object is any Kubernetes object.
 type object interface {
 	key() string
+	getTargetLabels(aw *apiWatcher) []map[string]string
 }
 
 // parseObjectFunc must parse object from the given data.
@@ -159,6 +160,23 @@ func newAPIWatcher(client *http.Client, apiServer, authorization string, namespa
 	}
 }
 
+// getLabelsForRole returns all the sets of labels for the given role.
+func (aw *apiWatcher) getLabelsForRole(role string) []map[string]string {
+	var ms []map[string]string
+	aw.mu.Lock()
+	for _, uw := range aw.watchersByURL {
+		if uw.role != role {
+			continue
+		}
+		uw.mu.Lock()
+		for _, labels := range uw.labelsByKey {
+			ms = append(ms, labels...)
+		}
+		uw.mu.Unlock()
+	}
+	return ms
+}
+
 // getObjectByRole returns an object with the given (namespace, name) key and the given role.
 func (aw *apiWatcher) getObjectByRole(role, namespace, name string) object {
 	if aw == nil {
@@ -184,26 +202,6 @@ func (aw *apiWatcher) getObjectByRole(role, namespace, name string) object {
 	return o
 }
 
-// getObjectsByRole returns all the objects for the given role.
-func (aw *apiWatcher) getObjectsByRole(role string) []object {
-	aw.startWatchersForRole(role)
-	var os []object
-	aw.mu.Lock()
-	for _, uw := range aw.watchersByURL {
-		if uw.role != role {
-			continue
-		}
-		uw.mu.Lock()
-		for _, o := range uw.objectsByKey {
-			os = append(os, o)
-		}
-		uw.mu.Unlock()
-	}
-	aw.lastAccessTime = time.Now()
-	aw.mu.Unlock()
-	return os
-}
-
 func (aw *apiWatcher) startWatchersForRole(role string) {
 	parseObject, parseObjectList := getObjectParsersForRole(role)
 	paths := getAPIPaths(role, aw.namespaces, aw.selectors)
@@ -258,11 +256,12 @@ type urlWatcher struct {
 	parseObject     parseObjectFunc
 	parseObjectList parseObjectListFunc
 
-	// mu protects objectsByKey
+	// mu protects objectsByKey and labelsByKey
 	mu sync.Mutex
 
 	// objectsByKey contains the latest state for objects obtained from apiURL
 	objectsByKey map[string]object
+	labelsByKey  map[string][]map[string]string
 
 	// the parent apiWatcher
 	aw *apiWatcher
@@ -277,6 +276,7 @@ func (aw *apiWatcher) newURLWatcher(role, apiURL string, parseObject parseObject
 		parseObjectList: parseObjectList,
 
 		objectsByKey: make(map[string]object),
+		labelsByKey:  make(map[string][]map[string]string),
 
 		aw: aw,
 	}
@@ -394,9 +394,14 @@ func (uw *urlWatcher) readObjectUpdateStream(r io.Reader) error {
 			uw.mu.Lock()
 			uw.objectsByKey[key] = o
 			uw.mu.Unlock()
+			labels := o.getTargetLabels(uw.aw)
+			uw.mu.Lock()
+			uw.labelsByKey[key] = labels
+			uw.mu.Unlock()
 		case "DELETED":
 			uw.mu.Lock()
 			delete(uw.objectsByKey, key)
+			delete(uw.labelsByKey, key)
 			uw.mu.Unlock()
 		default:
 			return fmt.Errorf("unexpected WatchEvent type %q for role %q", we.Type, uw.role)
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 2b6881140..003cceca5 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -7,25 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getEndpointsLabels returns labels for k8s endpoints obtained from the given cfg.
-func getEndpointsLabels(cfg *apiConfig) []map[string]string {
-	epss := getEndpoints(cfg)
-	var ms []map[string]string
-	for _, eps := range epss {
-		ms = eps.appendTargetLabels(ms, cfg.aw)
-	}
-	return ms
-}
-
-func getEndpoints(cfg *apiConfig) []*Endpoints {
-	os := cfg.aw.getObjectsByRole("endpoint")
-	epss := make([]*Endpoints, len(os))
-	for i, o := range os {
-		epss[i] = o.(*Endpoints)
-	}
-	return epss
-}
-
 func (eps *Endpoints) key() string {
 	return eps.Metadata.key()
 }
@@ -104,15 +85,16 @@ type EndpointPort struct {
 	Protocol    string
 }
 
-// appendTargetLabels appends labels for each endpoint in eps to ms and returns the result.
+// getTargetLabels returns labels for each endpoint in eps.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#endpoints
-func (eps *Endpoints) appendTargetLabels(ms []map[string]string, aw *apiWatcher) []map[string]string {
+func (eps *Endpoints) getTargetLabels(aw *apiWatcher) []map[string]string {
 	var svc *Service
 	if o := aw.getObjectByRole("service", eps.Metadata.Namespace, eps.Metadata.Name); o != nil {
 		svc = o.(*Service)
 	}
 	podPortsSeen := make(map[*Pod][]int)
+	var ms []map[string]string
 	for _, ess := range eps.Subsets {
 		for _, epp := range ess.Ports {
 			ms = appendEndpointLabelsForAddresses(ms, aw, podPortsSeen, eps, ess.Addresses, epp, svc, "true")
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index bd00ebdcf..05a74183b 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -88,9 +88,7 @@ func TestParseEndpointsListSuccess(t *testing.T) {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
 
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*Endpoints).appendTargetLabels(nil, nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.17.0.2:8443",
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices.go b/lib/promscrape/discovery/kubernetes/endpointslices.go
index 2246be105..cec87ce13 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices.go
@@ -8,26 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getEndpointSlicesLabels returns labels for k8s endpointSlices obtained from the given cfg.
-func getEndpointSlicesLabels(cfg *apiConfig) []map[string]string {
-	epss := getEndpointSlices(cfg)
-	var ms []map[string]string
-	for _, eps := range epss {
-		ms = eps.appendTargetLabels(ms, cfg.aw)
-	}
-	return ms
-}
-
-// getEndpointSlices retrieves endpointSlice with given apiConfig
-func getEndpointSlices(cfg *apiConfig) []*EndpointSlice {
-	os := cfg.aw.getObjectsByRole("endpointslices")
-	epss := make([]*EndpointSlice, len(os))
-	for i, o := range os {
-		epss[i] = o.(*EndpointSlice)
-	}
-	return epss
-}
-
 func (eps *EndpointSlice) key() string {
 	return eps.Metadata.key()
 }
@@ -52,14 +32,16 @@ func parseEndpointSlice(data []byte) (object, error) {
 	return &eps, nil
 }
 
-// appendTargetLabels injects labels for endPointSlice to slice map
-// follows TargetRef for enrich labels with pod and service metadata
-func (eps *EndpointSlice) appendTargetLabels(ms []map[string]string, aw *apiWatcher) []map[string]string {
+// getTargetLabels returns labels for eps.
+//
+// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#endpointslices
+func (eps *EndpointSlice) getTargetLabels(aw *apiWatcher) []map[string]string {
 	var svc *Service
 	if o := aw.getObjectByRole("service", eps.Metadata.Namespace, eps.Metadata.Name); o != nil {
 		svc = o.(*Service)
 	}
 	podPortsSeen := make(map[*Pod][]int)
+	var ms []map[string]string
 	for _, ess := range eps.Endpoints {
 		var p *Pod
 		if o := aw.getObjectByRole("pod", ess.TargetRef.Namespace, ess.TargetRef.Name); o != nil {
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices_test.go b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
index a89eccf1b..d587fd93f 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
@@ -185,9 +185,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
 	if meta.ResourceVersion != expectedResourceVersion {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*EndpointSlice).appendTargetLabels(nil, nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.18.0.2:6443",
diff --git a/lib/promscrape/discovery/kubernetes/ingress.go b/lib/promscrape/discovery/kubernetes/ingress.go
index f69c6673d..38aef79d3 100644
--- a/lib/promscrape/discovery/kubernetes/ingress.go
+++ b/lib/promscrape/discovery/kubernetes/ingress.go
@@ -5,25 +5,6 @@ import (
 	"fmt"
 )
 
-// getIngressesLabels returns labels for k8s ingresses obtained from the given cfg.
-func getIngressesLabels(cfg *apiConfig) []map[string]string {
-	igs := getIngresses(cfg)
-	var ms []map[string]string
-	for _, ig := range igs {
-		ms = ig.appendTargetLabels(ms)
-	}
-	return ms
-}
-
-func getIngresses(cfg *apiConfig) []*Ingress {
-	os := cfg.aw.getObjectsByRole("ingress")
-	igs := make([]*Ingress, len(os))
-	for i, o := range os {
-		igs[i] = o.(*Ingress)
-	}
-	return igs
-}
-
 func (ig *Ingress) key() string {
 	return ig.Metadata.key()
 }
@@ -101,16 +82,17 @@ type HTTPIngressPath struct {
 	Path string
 }
 
-// appendTargetLabels appends labels for Ingress ig to ms and returns the result.
+// getTargetLabels returns labels for ig.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ingress
-func (ig *Ingress) appendTargetLabels(ms []map[string]string) []map[string]string {
+func (ig *Ingress) getTargetLabels(aw *apiWatcher) []map[string]string {
 	tlsHosts := make(map[string]bool)
 	for _, tls := range ig.Spec.TLS {
 		for _, host := range tls.Hosts {
 			tlsHosts[host] = true
 		}
 	}
+	var ms []map[string]string
 	for _, r := range ig.Spec.Rules {
 		paths := getIngressRulePaths(r.HTTP.Paths)
 		scheme := "http"
diff --git a/lib/promscrape/discovery/kubernetes/ingress_test.go b/lib/promscrape/discovery/kubernetes/ingress_test.go
index 8007c6376..8046d8a24 100644
--- a/lib/promscrape/discovery/kubernetes/ingress_test.go
+++ b/lib/promscrape/discovery/kubernetes/ingress_test.go
@@ -79,9 +79,7 @@ func TestParseIngressListSuccess(t *testing.T) {
 	if meta.ResourceVersion != expectedResourceVersion {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*Ingress).appendTargetLabels(nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "foobar",
diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go
index b6c7a99c4..504fef616 100644
--- a/lib/promscrape/discovery/kubernetes/kubernetes.go
+++ b/lib/promscrape/discovery/kubernetes/kubernetes.go
@@ -44,18 +44,8 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 		return nil, fmt.Errorf("cannot create API config: %w", err)
 	}
 	switch sdc.Role {
-	case "node":
-		return getNodesLabels(cfg), nil
-	case "pod":
-		return getPodsLabels(cfg), nil
-	case "service":
-		return getServicesLabels(cfg), nil
-	case "endpoints":
-		return getEndpointsLabels(cfg), nil
-	case "endpointslices":
-		return getEndpointSlicesLabels(cfg), nil
-	case "ingress":
-		return getIngressesLabels(cfg), nil
+	case "node", "pod", "service", "endpoints", "endpointslices", "ingress":
+		return cfg.aw.getLabelsForRole(sdc.Role), nil
 	default:
 		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslices` or `ingress`; skipping it", sdc.Role)
 	}
diff --git a/lib/promscrape/discovery/kubernetes/node.go b/lib/promscrape/discovery/kubernetes/node.go
index 720a58f04..653a99e73 100644
--- a/lib/promscrape/discovery/kubernetes/node.go
+++ b/lib/promscrape/discovery/kubernetes/node.go
@@ -8,24 +8,6 @@ import (
 )
 
 // getNodesLabels returns labels for k8s nodes obtained from the given cfg
-func getNodesLabels(cfg *apiConfig) []map[string]string {
-	nodes := getNodes(cfg)
-	var ms []map[string]string
-	for _, n := range nodes {
-		ms = n.appendTargetLabels(ms)
-	}
-	return ms
-}
-
-func getNodes(cfg *apiConfig) []*Node {
-	os := cfg.aw.getObjectsByRole("node")
-	ns := make([]*Node, len(os))
-	for i, o := range os {
-		ns[i] = o.(*Node)
-	}
-	return ns
-}
-
 func (n *Node) key() string {
 	return n.Metadata.key()
 }
@@ -89,14 +71,14 @@ type NodeDaemonEndpoints struct {
 	KubeletEndpoint DaemonEndpoint
 }
 
-// appendTargetLabels appends labels for the given Node n to ms and returns the result.
+// getTargetLabels returs labels for the given n.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#node
-func (n *Node) appendTargetLabels(ms []map[string]string) []map[string]string {
+func (n *Node) getTargetLabels(aw *apiWatcher) []map[string]string {
 	addr := getNodeAddr(n.Status.Addresses)
 	if len(addr) == 0 {
 		// Skip node without address
-		return ms
+		return nil
 	}
 	addr = discoveryutils.JoinHostPort(addr, n.Status.DaemonEndpoints.KubeletEndpoint.Port)
 	m := map[string]string{
@@ -114,8 +96,7 @@ func (n *Node) appendTargetLabels(ms []map[string]string) []map[string]string {
 		ln := discoveryutils.SanitizeLabelName(a.Type)
 		m["__meta_kubernetes_node_address_"+ln] = a.Address
 	}
-	ms = append(ms, m)
-	return ms
+	return []map[string]string{m}
 }
 
 func getNodeAddr(nas []NodeAddress) string {
diff --git a/lib/promscrape/discovery/kubernetes/node_test.go b/lib/promscrape/discovery/kubernetes/node_test.go
index deed59fc4..e63cbe4b4 100644
--- a/lib/promscrape/discovery/kubernetes/node_test.go
+++ b/lib/promscrape/discovery/kubernetes/node_test.go
@@ -235,9 +235,7 @@ func TestParseNodeListSuccess(t *testing.T) {
 	if meta.ResourceVersion != expectedResourceVersion {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*Node).appendTargetLabels(nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"instance":                    "m01",
@@ -283,10 +281,10 @@ func TestParseNodeListSuccess(t *testing.T) {
 	}
 }
 
-func getSortedLabelss(objectsByKey map[string]object, getLabelss func(o object) []map[string]string) [][]prompbmarshal.Label {
+func getSortedLabelss(objectsByKey map[string]object) [][]prompbmarshal.Label {
 	var result [][]prompbmarshal.Label
 	for _, o := range objectsByKey {
-		labelss := getLabelss(o)
+		labelss := o.getTargetLabels(nil)
 		for _, labels := range labelss {
 			result = append(result, discoveryutils.GetSortedLabels(labels))
 		}
diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go
index cb2b00877..80864d25e 100644
--- a/lib/promscrape/discovery/kubernetes/pod.go
+++ b/lib/promscrape/discovery/kubernetes/pod.go
@@ -9,25 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getPodsLabels returns labels for k8s pods obtained from the given cfg
-func getPodsLabels(cfg *apiConfig) []map[string]string {
-	pods := getPods(cfg)
-	var ms []map[string]string
-	for _, p := range pods {
-		ms = p.appendTargetLabels(ms)
-	}
-	return ms
-}
-
-func getPods(cfg *apiConfig) []*Pod {
-	os := cfg.aw.getObjectsByRole("pod")
-	ps := make([]*Pod, len(os))
-	for i, o := range os {
-		ps[i] = o.(*Pod)
-	}
-	return ps
-}
-
 func (p *Pod) key() string {
 	return p.Metadata.key()
 }
@@ -111,14 +92,15 @@ type PodCondition struct {
 	Status string
 }
 
-// appendTargetLabels appends labels for each port of the given Pod p to ms and returns the result.
+// getTargetLabels returns labels for each port of the given p.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
-func (p *Pod) appendTargetLabels(ms []map[string]string) []map[string]string {
+func (p *Pod) getTargetLabels(aw *apiWatcher) []map[string]string {
 	if len(p.Status.PodIP) == 0 {
 		// Skip pod without IP
-		return ms
+		return nil
 	}
+	var ms []map[string]string
 	ms = appendPodLabels(ms, p, p.Spec.Containers, "false")
 	ms = appendPodLabels(ms, p, p.Spec.InitContainers, "true")
 	return ms
diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go
index 74c5456ca..204a8f102 100644
--- a/lib/promscrape/discovery/kubernetes/pod_test.go
+++ b/lib/promscrape/discovery/kubernetes/pod_test.go
@@ -236,9 +236,7 @@ func TestParsePodListSuccess(t *testing.T) {
 	if meta.ResourceVersion != expectedResourceVersion {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*Pod).appendTargetLabels(nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "172.17.0.2:1234",
diff --git a/lib/promscrape/discovery/kubernetes/service.go b/lib/promscrape/discovery/kubernetes/service.go
index 9d1c5fbed..c129816e6 100644
--- a/lib/promscrape/discovery/kubernetes/service.go
+++ b/lib/promscrape/discovery/kubernetes/service.go
@@ -7,25 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// getServicesLabels returns labels for k8s services obtained from the given cfg
-func getServicesLabels(cfg *apiConfig) []map[string]string {
-	svcs := getServices(cfg)
-	var ms []map[string]string
-	for _, svc := range svcs {
-		ms = svc.appendTargetLabels(ms)
-	}
-	return ms
-}
-
-func getServices(cfg *apiConfig) []*Service {
-	os := cfg.aw.getObjectsByRole("service")
-	svcs := make([]*Service, len(os))
-	for i, o := range os {
-		svcs[i] = o.(*Service)
-	}
-	return svcs
-}
-
 func (s *Service) key() string {
 	return s.Metadata.key()
 }
@@ -85,11 +66,12 @@ type ServicePort struct {
 	Port     int
 }
 
-// appendTargetLabels appends labels for each port of the given Service s to ms and returns the result.
+// getTargetLabels returns labels for each port of the given s.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#service
-func (s *Service) appendTargetLabels(ms []map[string]string) []map[string]string {
+func (s *Service) getTargetLabels(aw *apiWatcher) []map[string]string {
 	host := fmt.Sprintf("%s.%s.svc", s.Metadata.Name, s.Metadata.Namespace)
+	var ms []map[string]string
 	for _, sp := range s.Spec.Ports {
 		addr := discoveryutils.JoinHostPort(host, sp.Port)
 		m := map[string]string{
diff --git a/lib/promscrape/discovery/kubernetes/service_test.go b/lib/promscrape/discovery/kubernetes/service_test.go
index 4718fe6ed..56508e5f3 100644
--- a/lib/promscrape/discovery/kubernetes/service_test.go
+++ b/lib/promscrape/discovery/kubernetes/service_test.go
@@ -97,9 +97,7 @@ func TestParseServiceListSuccess(t *testing.T) {
 	if meta.ResourceVersion != expectedResourceVersion {
 		t.Fatalf("unexpected resource version; got %s; want %s", meta.ResourceVersion, expectedResourceVersion)
 	}
-	sortedLabelss := getSortedLabelss(objectsByKey, func(o object) []map[string]string {
-		return o.(*Service).appendTargetLabels(nil)
-	})
+	sortedLabelss := getSortedLabelss(objectsByKey)
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__":                             "kube-dns.kube-system.svc:53",

From ed8441ec5240f3cd2d70c6372f1b13b8761c83f5 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 21:41:54 +0200
Subject: [PATCH 31/32] lib/promscrape: cache ScrapeWork

This should reduce the time needed for updating big number of scrape targets.
---
 lib/promscrape/config.go                   | 92 +++++++++++++++++++++-
 lib/promscrape/discovery/kubernetes/api.go | 11 ++-
 2 files changed, 96 insertions(+), 7 deletions(-)

diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index b41bcc463..351637cf9 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -6,12 +6,15 @@ import (
 	"io/ioutil"
 	"net/url"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -510,6 +513,8 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		disableKeepAlive:     sc.DisableKeepAlive,
 		streamParse:          sc.StreamParse,
 		scrapeAlignInterval:  sc.ScrapeAlignInterval,
+
+		cache: newScrapeWorkCache(),
 	}
 	return swc, nil
 }
@@ -533,6 +538,52 @@ type scrapeWorkConfig struct {
 	disableKeepAlive     bool
 	streamParse          bool
 	scrapeAlignInterval  time.Duration
+
+	cache *scrapeWorkCache
+}
+
+type scrapeWorkCache struct {
+	mu              sync.Mutex
+	m               map[string]*scrapeWorkEntry
+	lastCleanupTime uint64
+}
+
+type scrapeWorkEntry struct {
+	sw             *ScrapeWork
+	lastAccessTime uint64
+}
+
+func newScrapeWorkCache() *scrapeWorkCache {
+	return &scrapeWorkCache{
+		m: make(map[string]*scrapeWorkEntry),
+	}
+}
+
+func (swc *scrapeWorkCache) Get(key string) *ScrapeWork {
+	currentTime := fasttime.UnixTimestamp()
+	swc.mu.Lock()
+	swe := swc.m[key]
+	swe.lastAccessTime = currentTime
+	swc.mu.Unlock()
+	return swe.sw
+}
+
+func (swc *scrapeWorkCache) Set(key string, sw *ScrapeWork) {
+	currentTime := fasttime.UnixTimestamp()
+	swc.mu.Lock()
+	swc.m[key] = &scrapeWorkEntry{
+		sw:             sw,
+		lastAccessTime: currentTime,
+	}
+	if currentTime > swc.lastCleanupTime+10*60 {
+		for k, swe := range swc.m {
+			if currentTime > swe.lastAccessTime+2*60 {
+				delete(swc.m, k)
+			}
+		}
+		swc.lastCleanupTime = currentTime
+	}
+	swc.mu.Unlock()
 }
 
 type targetLabelsGetter interface {
@@ -562,7 +613,7 @@ func appendScrapeWorkForTargetLabels(dst []*ScrapeWork, swc *scrapeWorkConfig, t
 		go func() {
 			for metaLabels := range workCh {
 				target := metaLabels["__address__"]
-				sw, err := getScrapeWork(swc, target, nil, metaLabels)
+				sw, err := swc.getScrapeWork(target, nil, metaLabels)
 				if err != nil {
 					err = fmt.Errorf("skipping %s target %q for job_name %q because of error: %w", discoveryType, target, swc.jobName, err)
 				}
@@ -643,7 +694,7 @@ func (stc *StaticConfig) appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConf
 			logger.Errorf("`static_configs` target for `job_name` %q cannot be empty; skipping it", swc.jobName)
 			continue
 		}
-		sw, err := getScrapeWork(swc, target, stc.Labels, metaLabels)
+		sw, err := swc.getScrapeWork(target, stc.Labels, metaLabels)
 		if err != nil {
 			// Do not return this error, since other targets may be valid
 			logger.Errorf("error when parsing `static_configs` target %q for `job_name` %q: %s; skipping it", target, swc.jobName, err)
@@ -656,7 +707,42 @@ func (stc *StaticConfig) appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConf
 	return dst
 }
 
-func getScrapeWork(swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) (*ScrapeWork, error) {
+func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabels map[string]string) (*ScrapeWork, error) {
+	key := getScrapeWorkKey(extraLabels, metaLabels)
+	if sw := swc.cache.Get(key); sw != nil {
+		return sw, nil
+	}
+	sw, err := swc.getScrapeWorkReal(target, extraLabels, metaLabels)
+	if err != nil {
+		swc.cache.Set(key, sw)
+	}
+	return sw, err
+}
+
+func getScrapeWorkKey(extraLabels, metaLabels map[string]string) string {
+	var b []byte
+	b = appendSortedKeyValuePairs(b, extraLabels)
+	b = appendSortedKeyValuePairs(b, metaLabels)
+	return string(b)
+}
+
+func appendSortedKeyValuePairs(dst []byte, m map[string]string) []byte {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		dst = strconv.AppendQuote(dst, k)
+		dst = append(dst, ':')
+		dst = strconv.AppendQuote(dst, m[k])
+		dst = append(dst, ',')
+	}
+	dst = append(dst, '\n')
+	return dst
+}
+
+func (swc *scrapeWorkConfig) getScrapeWorkReal(target string, extraLabels, metaLabels map[string]string) (*ScrapeWork, error) {
 	labels := mergeLabels(swc.jobName, swc.scheme, target, swc.metricsPath, extraLabels, swc.externalLabels, metaLabels, swc.params)
 	var originalLabels []prompbmarshal.Label
 	if !*dropOriginalLabels {
diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index 3212cf3d3..a11e528c7 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
@@ -143,7 +144,7 @@ type apiWatcher struct {
 
 	// The last time the apiWatcher was queried for cached objects.
 	// It is used for stopping unused watchers.
-	lastAccessTime time.Time
+	lastAccessTime uint64
 }
 
 func newAPIWatcher(client *http.Client, apiServer, authorization string, namespaces []string, selectors []Selector) *apiWatcher {
@@ -156,7 +157,7 @@ func newAPIWatcher(client *http.Client, apiServer, authorization string, namespa
 
 		watchersByURL: make(map[string]*urlWatcher),
 
-		lastAccessTime: time.Now(),
+		lastAccessTime: fasttime.UnixTimestamp(),
 	}
 }
 
@@ -174,6 +175,8 @@ func (aw *apiWatcher) getLabelsForRole(role string) []map[string]string {
 		}
 		uw.mu.Unlock()
 	}
+	aw.lastAccessTime = fasttime.UnixTimestamp()
+	aw.mu.Unlock()
 	return ms
 }
 
@@ -197,7 +200,7 @@ func (aw *apiWatcher) getObjectByRole(role, namespace, name string) object {
 			break
 		}
 	}
-	aw.lastAccessTime = time.Now()
+	aw.lastAccessTime = fasttime.UnixTimestamp()
 	aw.mu.Unlock()
 	return o
 }
@@ -233,7 +236,7 @@ func (aw *apiWatcher) startWatcherForURL(role, apiURL string, parseObject parseO
 func (aw *apiWatcher) needStop() bool {
 	aw.mu.Lock()
 	defer aw.mu.Unlock()
-	return time.Since(aw.lastAccessTime) > 5*time.Minute
+	return fasttime.UnixTimestamp() > aw.lastAccessTime+5*60
 }
 
 // doRequest performs http request to the given requestURL.

From d86e9b49c4ecea74d7fe6b655307d9107043d74d Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 26 Feb 2021 22:53:41 +0200
Subject: [PATCH 32/32] app/vmselect/promql: increase accuracy for
 `buckets_limit()` function for small limits by skipping the first and the
 last buckets during merge

The first and the last buckets are usually `[0 ... leMin]` and `(leMax ... +Inf)`. If they are merged with adjancent buckets,
then the resulting accuracy can suffer.
---
 app/vmselect/promql/exec_test.go | 36 +++++++++++++++++++++++---------
 app/vmselect/promql/transform.go | 13 +++++++++---
 docs/CHANGELOG.md                |  1 +
 3 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 0bc3efc56..9a3664064 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -3337,14 +3337,14 @@ func TestExecSuccess(t *testing.T) {
 		)))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{52, 52, 52, 52, 52, 52},
+			Values:     []float64{9, 9, 9, 9, 9, 9},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.MetricGroup = []byte("metric")
 		r1.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("le"),
-				Value: []byte("200"),
+				Value: []byte("10"),
 			},
 			{
 				Key:   []byte("x"),
@@ -3353,11 +3353,27 @@ func TestExecSuccess(t *testing.T) {
 		}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{100, 100, 100, 100, 100, 100},
+			Values:     []float64{98, 98, 98, 98, 98, 98},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.MetricGroup = []byte("metric")
 		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("300"),
+			},
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{100, 100, 100, 100, 100, 100},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("le"),
 				Value: []byte("inf"),
@@ -3367,7 +3383,7 @@ func TestExecSuccess(t *testing.T) {
 				Value: []byte("y"),
 			},
 		}
-		resultExpected := []netstorage.Result{r1, r2}
+		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
 	t.Run(`prometheus_buckets(missing-vmrange)`, func(t *testing.T) {
@@ -4175,11 +4191,11 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort_desc(
+		q := `sort_by_label(
 			buckets_limit(
 				3,
 				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
-			)
+			), "le"
 		)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -4194,24 +4210,24 @@ func TestExecSuccess(t *testing.T) {
 		}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{24, 22, 26, 25, 24, 24},
+			Values:     []float64{0, 0, 0, 0, 0, 0},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("le"),
-				Value: []byte("1.896e+00"),
+				Value: []byte("1.000e+00"),
 			},
 		}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{11, 12, 11, 7, 11, 13},
+			Values:     []float64{40, 40, 40, 40, 40, 40},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("le"),
-				Value: []byte("1.468e+00"),
+				Value: []byte("2.448e+00"),
 			},
 		}
 		resultExpected := []netstorage.Result{r1, r2, r3}
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index a893ce2e4..714f3cf80 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -343,6 +343,11 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 	if limit <= 0 {
 		return nil, nil
 	}
+	if limit < 3 {
+		// Preserve the first and the last bucket for better accuracy,
+		// since these buckets are usually `[0...leMin]` and `(leMax ... +Inf]`
+		limit = 3
+	}
 	tss := vmrangeBucketsToLE(args[1])
 	if len(tss) == 0 {
 		return nil, nil
@@ -404,15 +409,18 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 			}
 		}
 		for len(leGroup) > limit {
+			// Preserve the first and the last bucket for better accuracy,
+			// since these buckets are usually `[0...leMin]` and `(leMax ... +Inf]`
 			xxMinIdx := 0
-			for i, xx := range leGroup {
+			for i, xx := range leGroup[1 : len(leGroup)-1] {
 				if xx.hits < leGroup[xxMinIdx].hits {
 					xxMinIdx = i
 				}
 			}
+			xxMinIdx++
 			// Merge the leGroup[xxMinIdx] bucket with the smallest adjacent bucket in order to preserve
 			// the maximum accuracy.
-			if xxMinIdx+1 == len(leGroup) || (xxMinIdx > 0 && leGroup[xxMinIdx-1].hits < leGroup[xxMinIdx+1].hits) {
+			if xxMinIdx > 1 && leGroup[xxMinIdx-1].hits < leGroup[xxMinIdx+1].hits {
 				xxMinIdx--
 			}
 			leGroup[xxMinIdx+1].hits += leGroup[xxMinIdx].hits
@@ -578,7 +586,6 @@ func transformHistogramShare(tfa *transformFuncArg) ([]*timeseries, error) {
 	m := groupLeTimeseries(tss)
 
 	// Calculate share for les
-
 	share := func(i int, les []float64, xss []leTimeseries) (q, lower, upper float64) {
 		leReq := les[i]
 		if math.IsNaN(leReq) || len(xss) == 0 {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b9ce762de..17d70b8cd 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,7 @@
 * FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
 * FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
 * FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details.
+* FEATURE: increase accuracy for `buckets_limit(limit, buckets)` results for small `limit` values. See [MetricsQL docs](https://victoriametrics.github.io/MetricsQL.html) for details.
 
 
 * BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065