From ed91fe1d9b0b1dffa31acbeee7ca35aa1213e0cc Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 4 May 2020 16:21:24 +0300 Subject: [PATCH] lib/promscrape: move common code for discovery api config map handling into discoveryutils --- lib/promscrape/discovery/ec2/api.go | 49 ++------------- lib/promscrape/discovery/gce/api.go | 49 ++------------- lib/promscrape/discovery/kubernetes/api.go | 49 ++------------- lib/promscrape/discoveryutils/config_map.go | 70 +++++++++++++++++++++ 4 files changed, 83 insertions(+), 134 deletions(-) create mode 100644 lib/promscrape/discoveryutils/config_map.go diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go index e200e0591..e640fa2fa 100644 --- a/lib/promscrape/discovery/ec2/api.go +++ b/lib/promscrape/discovery/ec2/api.go @@ -8,8 +8,6 @@ import ( "net/url" "os" "strings" - "sync" - "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) @@ -23,55 +21,16 @@ type apiConfig struct { port int } +var configMap = discoveryutils.NewConfigMap() + func getAPIConfig(sdc *SDConfig) (*apiConfig, error) { - apiConfigMapLock.Lock() - defer apiConfigMapLock.Unlock() - - if !hasAPIConfigMapCleaner { - hasAPIConfigMapCleaner = true - go apiConfigMapCleaner() - } - - e := apiConfigMap[sdc] - if e != nil { - e.lastAccessTime = time.Now() - return e.cfg, nil - } - cfg, err := newAPIConfig(sdc) + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc) }) if err != nil { return nil, err } - apiConfigMap[sdc] = &apiConfigMapEntry{ - cfg: cfg, - lastAccessTime: time.Now(), - } - return cfg, nil + return v.(*apiConfig), nil } -func apiConfigMapCleaner() { - tc := time.NewTicker(15 * time.Minute) - for currentTime := range tc.C { - apiConfigMapLock.Lock() - for k, e := range apiConfigMap { - if currentTime.Sub(e.lastAccessTime) > 10*time.Minute { - delete(apiConfigMap, k) - } - } - apiConfigMapLock.Unlock() - } -} - -type apiConfigMapEntry struct { - cfg *apiConfig - lastAccessTime time.Time -} - -var ( - apiConfigMap = make(map[*SDConfig]*apiConfigMapEntry) - apiConfigMapLock sync.Mutex - hasAPIConfigMapCleaner bool -) - func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { region := sdc.Region if len(region) == 0 { diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go index f95782f80..b8aba9c1c 100644 --- a/lib/promscrape/discovery/gce/api.go +++ b/lib/promscrape/discovery/gce/api.go @@ -7,8 +7,6 @@ import ( "net/http" "net/url" "strings" - "sync" - "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" @@ -24,55 +22,16 @@ type apiConfig struct { port int } +var configMap = discoveryutils.NewConfigMap() + func getAPIConfig(sdc *SDConfig) (*apiConfig, error) { - apiConfigMapLock.Lock() - defer apiConfigMapLock.Unlock() - - if !hasAPIConfigMapCleaner { - hasAPIConfigMapCleaner = true - go apiConfigMapCleaner() - } - - e := apiConfigMap[sdc] - if e != nil { - e.lastAccessTime = time.Now() - return e.cfg, nil - } - cfg, err := newAPIConfig(sdc) + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc) }) if err != nil { return nil, err } - apiConfigMap[sdc] = &apiConfigMapEntry{ - cfg: cfg, - lastAccessTime: time.Now(), - } - return cfg, nil + return v.(*apiConfig), nil } -func apiConfigMapCleaner() { - tc := time.NewTicker(15 * time.Minute) - for currentTime := range tc.C { - apiConfigMapLock.Lock() - for k, e := range apiConfigMap { - if currentTime.Sub(e.lastAccessTime) > 10*time.Minute { - delete(apiConfigMap, k) - } - } - apiConfigMapLock.Unlock() - } -} - -type apiConfigMapEntry struct { - cfg *apiConfig - lastAccessTime time.Time -} - -var ( - apiConfigMap = make(map[*SDConfig]*apiConfigMapEntry) - apiConfigMapLock sync.Mutex - hasAPIConfigMapCleaner bool -) - func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { ctx := context.Background() client, err := google.DefaultClient(ctx, "https://www.googleapis.com/auth/compute.readonly") diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go index c12cdcb6f..1b254821a 100644 --- a/lib/promscrape/discovery/kubernetes/api.go +++ b/lib/promscrape/discovery/kubernetes/api.go @@ -6,11 +6,11 @@ import ( "net" "os" "strings" - "sync" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" "github.com/VictoriaMetrics/fasthttp" ) @@ -24,55 +24,16 @@ type apiConfig struct { selectors []Selector } +var configMap = discoveryutils.NewConfigMap() + func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { - apiConfigMapLock.Lock() - defer apiConfigMapLock.Unlock() - - if !hasAPIConfigMapCleaner { - hasAPIConfigMapCleaner = true - go apiConfigMapCleaner() - } - - e := apiConfigMap[sdc] - if e != nil { - e.lastAccessTime = time.Now() - return e.cfg, nil - } - cfg, err := newAPIConfig(sdc, baseDir) + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) }) if err != nil { return nil, err } - apiConfigMap[sdc] = &apiConfigMapEntry{ - cfg: cfg, - lastAccessTime: time.Now(), - } - return cfg, nil + return v.(*apiConfig), nil } -func apiConfigMapCleaner() { - tc := time.NewTicker(15 * time.Minute) - for currentTime := range tc.C { - apiConfigMapLock.Lock() - for k, e := range apiConfigMap { - if currentTime.Sub(e.lastAccessTime) > 10*time.Minute { - delete(apiConfigMap, k) - } - } - apiConfigMapLock.Unlock() - } -} - -type apiConfigMapEntry struct { - cfg *apiConfig - lastAccessTime time.Time -} - -var ( - apiConfigMap = make(map[*SDConfig]*apiConfigMapEntry) - apiConfigMapLock sync.Mutex - hasAPIConfigMapCleaner bool -) - func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig) if err != nil { diff --git a/lib/promscrape/discoveryutils/config_map.go b/lib/promscrape/discoveryutils/config_map.go new file mode 100644 index 000000000..d16c95261 --- /dev/null +++ b/lib/promscrape/discoveryutils/config_map.go @@ -0,0 +1,70 @@ +package discoveryutils + +import ( + "sync" + "time" +) + +// ConfigMap is a map for storing discovery api configs. +// +// It automatically removes old configs which weren't accessed recently. +type ConfigMap struct { + mu sync.Mutex + m map[interface{}]*configMapEntry + hasCleaner bool +} + +// NewConfigMap creates ConfigMap +func NewConfigMap() *ConfigMap { + return &ConfigMap{ + m: make(map[interface{}]*configMapEntry), + } +} + +// Get returns config for the given key. +// +// Key must be a pointer. +// +// It creates new config map with newConfig() call if cm doesn't contain config under the given key. +func (cm *ConfigMap) Get(key interface{}, newConfig func() (interface{}, error)) (interface{}, error) { + cm.mu.Lock() + defer cm.mu.Unlock() + + if !cm.hasCleaner { + cm.hasCleaner = true + go cm.cleaner() + } + + e := cm.m[key] + if e != nil { + e.lastAccessTime = time.Now() + return e.cfg, nil + } + cfg, err := newConfig() + if err != nil { + return nil, err + } + cm.m[key] = &configMapEntry{ + cfg: cfg, + lastAccessTime: time.Now(), + } + return cfg, nil +} + +func (cm *ConfigMap) cleaner() { + tc := time.NewTicker(15 * time.Minute) + for currentTime := range tc.C { + cm.mu.Lock() + for k, e := range cm.m { + if currentTime.Sub(e.lastAccessTime) > 10*time.Minute { + delete(cm.m, k) + } + } + cm.mu.Unlock() + } +} + +type configMapEntry struct { + cfg interface{} + lastAccessTime time.Time +}