github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-11 15:34:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape: retry service discovery in a few seconds if it starts returning 0 targets

Browse source 
This should reduce recovery time from temporary issues during service discovery
This commit is contained in:
Aliaksandr Valialkin 2021-03-14 21:53:20 +02:00
parent bf15d6a6a2
commit 83edbb7cab
1 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
lib/promscrape/scraper.go
View file

@ -231,11 +231,17 @@ func (scfg *scrapeConfig) run() {
cfg := <-scfg.cfgCh cfg := <-scfg.cfgCh
var swsPrev []*ScrapeWork var swsPrev []*ScrapeWork
updateScrapeWork := func(cfg *Config) { updateScrapeWork := func(cfg *Config) {
startTime := time.Now() for {
sws := scfg.getScrapeWork(cfg, swsPrev) startTime := time.Now()
sg.update(sws) sws := scfg.getScrapeWork(cfg, swsPrev)
swsPrev = sws retry := sg.update(sws)
scfg.discoveryDuration.UpdateDuration(startTime) swsPrev = sws
scfg.discoveryDuration.UpdateDuration(startTime)
if !retry {
return
}
time.Sleep(2 * time.Second)
}
} }
updateScrapeWork(cfg) updateScrapeWork(cfg)
atomic.AddInt32(&PendingScrapeConfigs, -1) atomic.AddInt32(&PendingScrapeConfigs, -1)
@ -295,7 +301,7 @@ func (sg *scraperGroup) stop() {
sg.wg.Wait() sg.wg.Wait()
} }
func (sg *scraperGroup) update(sws []*ScrapeWork) { func (sg *scraperGroup) update(sws []*ScrapeWork) (retry bool) {
sg.mLock.Lock() sg.mLock.Lock()
defer sg.mLock.Unlock() defer sg.mLock.Unlock()
@ -352,6 +358,7 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) {
sg.changesCount.Add(additionsCount + deletionsCount) sg.changesCount.Add(additionsCount + deletionsCount)
logger.Infof("%s: added targets: %d, removed targets: %d; total targets: %d", sg.name, additionsCount, deletionsCount, len(sg.m)) logger.Infof("%s: added targets: %d, removed targets: %d; total targets: %d", sg.name, additionsCount, deletionsCount, len(sg.m))
} }
return deletionsCount > 0 && len(sg.m) == 0
} }
type scraper struct { type scraper struct {