lib/promscrape: move common code for discovery api config map handling into discoveryutils

lib/promscrape/discovery/ec2/api.go

@@ -8,8 +8,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
@@ -23,55 +21,16 @@ type apiConfig struct {
 	port      int
 }
 
+var configMap = discoveryutils.NewConfigMap()
+
 func getAPIConfig(sdc *SDConfig) (*apiConfig, error) {
-	apiConfigMapLock.Lock()
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc) })
-	defer apiConfigMapLock.Unlock()
-
-	if !hasAPIConfigMapCleaner {
-		hasAPIConfigMapCleaner = true
-		go apiConfigMapCleaner()
-	}
-
-	e := apiConfigMap[sdc]
-	if e != nil {
-		e.lastAccessTime = time.Now()
-		return e.cfg, nil
-	}
-	cfg, err := newAPIConfig(sdc)
 	if err != nil {
 		return nil, err
 	}
-	apiConfigMap[sdc] = &apiConfigMapEntry{
+	return v.(*apiConfig), nil
-		cfg:            cfg,
-		lastAccessTime: time.Now(),
-	}
-	return cfg, nil
 }
 
-func apiConfigMapCleaner() {
-	tc := time.NewTicker(15 * time.Minute)
-	for currentTime := range tc.C {
-		apiConfigMapLock.Lock()
-		for k, e := range apiConfigMap {
-			if currentTime.Sub(e.lastAccessTime) > 10*time.Minute {
-				delete(apiConfigMap, k)
-			}
-		}
-		apiConfigMapLock.Unlock()
-	}
-}
-
-type apiConfigMapEntry struct {
-	cfg            *apiConfig
-	lastAccessTime time.Time
-}
-
-var (
-	apiConfigMap           = make(map[*SDConfig]*apiConfigMapEntry)
-	apiConfigMapLock       sync.Mutex
-	hasAPIConfigMapCleaner bool
-)
-
 func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	region := sdc.Region
 	if len(region) == 0 {

lib/promscrape/discovery/gce/api.go

@@ -7,8 +7,6 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
@@ -24,55 +22,16 @@ type apiConfig struct {
 	port         int
 }
 
+var configMap = discoveryutils.NewConfigMap()
+
 func getAPIConfig(sdc *SDConfig) (*apiConfig, error) {
-	apiConfigMapLock.Lock()
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc) })
-	defer apiConfigMapLock.Unlock()
-
-	if !hasAPIConfigMapCleaner {
-		hasAPIConfigMapCleaner = true
-		go apiConfigMapCleaner()
-	}
-
-	e := apiConfigMap[sdc]
-	if e != nil {
-		e.lastAccessTime = time.Now()
-		return e.cfg, nil
-	}
-	cfg, err := newAPIConfig(sdc)
 	if err != nil {
 		return nil, err
 	}
-	apiConfigMap[sdc] = &apiConfigMapEntry{
+	return v.(*apiConfig), nil
-		cfg:            cfg,
-		lastAccessTime: time.Now(),
-	}
-	return cfg, nil
 }
 
-func apiConfigMapCleaner() {
-	tc := time.NewTicker(15 * time.Minute)
-	for currentTime := range tc.C {
-		apiConfigMapLock.Lock()
-		for k, e := range apiConfigMap {
-			if currentTime.Sub(e.lastAccessTime) > 10*time.Minute {
-				delete(apiConfigMap, k)
-			}
-		}
-		apiConfigMapLock.Unlock()
-	}
-}
-
-type apiConfigMapEntry struct {
-	cfg            *apiConfig
-	lastAccessTime time.Time
-}
-
-var (
-	apiConfigMap           = make(map[*SDConfig]*apiConfigMapEntry)
-	apiConfigMapLock       sync.Mutex
-	hasAPIConfigMapCleaner bool
-)
-
 func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	ctx := context.Background()
 	client, err := google.DefaultClient(ctx, "https://www.googleapis.com/auth/compute.readonly")

lib/promscrape/discovery/kubernetes/api.go

@@ -6,11 +6,11 @@ import (
 	"net"
 	"os"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 	"github.com/VictoriaMetrics/fasthttp"
 )
 
@@ -24,55 +24,16 @@ type apiConfig struct {
 	selectors  []Selector
 }
 
+var configMap = discoveryutils.NewConfigMap()
+
 func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	apiConfigMapLock.Lock()
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
-	defer apiConfigMapLock.Unlock()
-
-	if !hasAPIConfigMapCleaner {
-		hasAPIConfigMapCleaner = true
-		go apiConfigMapCleaner()
-	}
-
-	e := apiConfigMap[sdc]
-	if e != nil {
-		e.lastAccessTime = time.Now()
-		return e.cfg, nil
-	}
-	cfg, err := newAPIConfig(sdc, baseDir)
 	if err != nil {
 		return nil, err
 	}
-	apiConfigMap[sdc] = &apiConfigMapEntry{
+	return v.(*apiConfig), nil
-		cfg:            cfg,
-		lastAccessTime: time.Now(),
-	}
-	return cfg, nil
 }
 
-func apiConfigMapCleaner() {
-	tc := time.NewTicker(15 * time.Minute)
-	for currentTime := range tc.C {
-		apiConfigMapLock.Lock()
-		for k, e := range apiConfigMap {
-			if currentTime.Sub(e.lastAccessTime) > 10*time.Minute {
-				delete(apiConfigMap, k)
-			}
-		}
-		apiConfigMapLock.Unlock()
-	}
-}
-
-type apiConfigMapEntry struct {
-	cfg            *apiConfig
-	lastAccessTime time.Time
-}
-
-var (
-	apiConfigMap           = make(map[*SDConfig]*apiConfigMapEntry)
-	apiConfigMapLock       sync.Mutex
-	hasAPIConfigMapCleaner bool
-)
-
 func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
 	if err != nil {

lib/promscrape/discoveryutils/config_map.go

@@ -0,0 +1,70 @@
+package discoveryutils
+
+import (
+	"sync"
+	"time"
+)
+
+// ConfigMap is a map for storing discovery api configs.
+//
+// It automatically removes old configs which weren't accessed recently.
+type ConfigMap struct {
+	mu         sync.Mutex
+	m          map[interface{}]*configMapEntry
+	hasCleaner bool
+}
+
+// NewConfigMap creates ConfigMap
+func NewConfigMap() *ConfigMap {
+	return &ConfigMap{
+		m: make(map[interface{}]*configMapEntry),
+	}
+}
+
+// Get returns config for the given key.
+//
+// Key must be a pointer.
+//
+// It creates new config map with newConfig() call if cm doesn't contain config under the given key.
+func (cm *ConfigMap) Get(key interface{}, newConfig func() (interface{}, error)) (interface{}, error) {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	if !cm.hasCleaner {
+		cm.hasCleaner = true
+		go cm.cleaner()
+	}
+
+	e := cm.m[key]
+	if e != nil {
+		e.lastAccessTime = time.Now()
+		return e.cfg, nil
+	}
+	cfg, err := newConfig()
+	if err != nil {
+		return nil, err
+	}
+	cm.m[key] = &configMapEntry{
+		cfg:            cfg,
+		lastAccessTime: time.Now(),
+	}
+	return cfg, nil
+}
+
+func (cm *ConfigMap) cleaner() {
+	tc := time.NewTicker(15 * time.Minute)
+	for currentTime := range tc.C {
+		cm.mu.Lock()
+		for k, e := range cm.m {
+			if currentTime.Sub(e.lastAccessTime) > 10*time.Minute {
+				delete(cm.m, k)
+			}
+		}
+		cm.mu.Unlock()
+	}
+}
+
+type configMapEntry struct {
+	cfg            interface{}
+	lastAccessTime time.Time
+}