From 705b31c351599d6536d34d4bc795265de399ae47 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 18 Sep 2023 17:45:10 +0200
Subject: [PATCH] lib/promscrape/discovery/kubernetes: wait for 10 seconds
 before checking whether the urlWatcher must be stopped

This should prevent from excess urlWatcher churn on config reload, since it leads to removal of all the apiWatchers
before creating new apiWatchers. So, every config reload would lead to stopping of all the previous urlWatchers
and starting new urlWatchers.

The new logic gives 10 seconds for config reload before stopping unused urlWatchers.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4861
---
 .../discovery/kubernetes/api_watcher.go         | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index d49795942c..d358113279 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -446,11 +446,10 @@ func (gw *groupWatcher) registerPendingAPIWatchers() {
 func (gw *groupWatcher) unsubscribeAPIWatcher(aw *apiWatcher) {
 	gw.mu.Lock()
 	defer gw.mu.Unlock()
-	for key, uw := range gw.m {
+	for _, uw := range gw.m {
 		uw.unsubscribeAPIWatcherLocked(aw)
-		if (len(uw.aws) + len(uw.awsPending)) == 0 {
-			uw.cancel()
-			delete(gw.m, key)
+		if len(uw.aws)+len(uw.awsPending) == 0 {
+			time.AfterFunc(10*time.Second, uw.stopIfNoUsers)
 		}
 	}
 }
@@ -524,6 +523,16 @@ func newURLWatcher(role, apiURL string, gw *groupWatcher) *urlWatcher {
 	return uw
 }
 
+func (uw *urlWatcher) stopIfNoUsers() {
+	gw := uw.gw
+	gw.mu.Lock()
+	if len(uw.aws)+len(uw.awsPending) == 0 {
+		uw.cancel()
+		delete(gw.m, uw.apiURL)
+	}
+	gw.mu.Unlock()
+}
+
 func (uw *urlWatcher) subscribeAPIWatcherLocked(aw *apiWatcher) {
 	if _, ok := uw.aws[aw]; !ok {
 		if _, ok := uw.awsPending[aw]; !ok {