github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-09 15:27:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape/discovery/gce: discover per-zone instances for gce_sd_config in parallel. This should reduce discovery latency

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-05-06 15:00:02 +03:00
parent 8665c2edb1
commit 9f39e618ed
2 changed files with 32 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/promscrape/discovery/gce/gce.go
View file

@ -53,9 +53,6 @@ func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot get API config: %s", err) return nil, fmt.Errorf("cannot get API config: %s", err)
} }
ms, err := getInstancesLabels(cfg) ms := getInstancesLabels(cfg)
if err != nil {
return nil, fmt.Errorf("error when fetching instances data from GCE: %s", err)
}
return ms, nil return ms, nil
} }

42
lib/promscrape/discovery/gce/instance.go
View file

@ -6,32 +6,48 @@ import (
"net/http" "net/http"
"strings" "strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
) )
// getInstancesLabels returns labels for gce instances obtained from the given cfg // getInstancesLabels returns labels for gce instances obtained from the given cfg
func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) { func getInstancesLabels(cfg *apiConfig) []map[string]string {
insts, err := getInstances(cfg) insts := getInstances(cfg)
if err != nil {
return nil, err
}
var ms []map[string]string var ms []map[string]string
for _, inst := range insts { for _, inst := range insts {
ms = inst.appendTargetLabels(ms, cfg.project, cfg.tagSeparator, cfg.port) ms = inst.appendTargetLabels(ms, cfg.project, cfg.tagSeparator, cfg.port)
} }
return ms, nil return ms
} }
func getInstances(cfg *apiConfig) ([]Instance, error) { func getInstances(cfg *apiConfig) []Instance {
var insts []Instance // Collect instances for each zone in parallel
type result struct {
zone string
insts []Instance
err error
}
ch := make(chan result, len(cfg.zones))
for _, zone := range cfg.zones { for _, zone := range cfg.zones {
zoneInsts, err := getInstancesForProjectAndZone(cfg.client, cfg.project, zone, cfg.filter) go func(zone string) {
if err != nil { insts, err := getInstancesForProjectAndZone(cfg.client, cfg.project, zone, cfg.filter)
return nil, err ch <- result{
zone: zone,
insts: insts,
err: err,
} }
insts = append(insts, zoneInsts...) }(zone)
} }
return insts, nil var insts []Instance
for range cfg.zones {
r := <-ch
if r.err != nil {
logger.Errorf("cannot collect instances from zone %q: %s", r.zone, r.err)
continue
}
insts = append(insts, r.insts...)
}
return insts
} }
func getInstancesForProjectAndZone(client *http.Client, project, zone, filter string) ([]Instance, error) { func getInstancesForProjectAndZone(client *http.Client, project, zone, filter string) ([]Instance, error) {