From 705b31c351599d6536d34d4bc795265de399ae47 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin <valyala@victoriametrics.com> Date: Mon, 18 Sep 2023 17:45:10 +0200 Subject: [PATCH] lib/promscrape/discovery/kubernetes: wait for 10 seconds before checking whether the urlWatcher must be stopped This should prevent from excess urlWatcher churn on config reload, since it leads to removal of all the apiWatchers before creating new apiWatchers. So, every config reload would lead to stopping of all the previous urlWatchers and starting new urlWatchers. The new logic gives 10 seconds for config reload before stopping unused urlWatchers. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4850 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4861 --- .../discovery/kubernetes/api_watcher.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go index d49795942c..d358113279 100644 --- a/lib/promscrape/discovery/kubernetes/api_watcher.go +++ b/lib/promscrape/discovery/kubernetes/api_watcher.go @@ -446,11 +446,10 @@ func (gw *groupWatcher) registerPendingAPIWatchers() { func (gw *groupWatcher) unsubscribeAPIWatcher(aw *apiWatcher) { gw.mu.Lock() defer gw.mu.Unlock() - for key, uw := range gw.m { + for _, uw := range gw.m { uw.unsubscribeAPIWatcherLocked(aw) - if (len(uw.aws) + len(uw.awsPending)) == 0 { - uw.cancel() - delete(gw.m, key) + if len(uw.aws)+len(uw.awsPending) == 0 { + time.AfterFunc(10*time.Second, uw.stopIfNoUsers) } } } @@ -524,6 +523,16 @@ func newURLWatcher(role, apiURL string, gw *groupWatcher) *urlWatcher { return uw } +func (uw *urlWatcher) stopIfNoUsers() { + gw := uw.gw + gw.mu.Lock() + if len(uw.aws)+len(uw.awsPending) == 0 { + uw.cancel() + delete(gw.m, uw.apiURL) + } + gw.mu.Unlock() +} + func (uw *urlWatcher) subscribeAPIWatcherLocked(aw *apiWatcher) { if _, ok := uw.aws[aw]; !ok { if _, ok := uw.awsPending[aw]; !ok {