github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape: split docker and dockerswarm service discovery code bases, since they have very little in common

Browse source 
This is a follow up after c85a5b7fcb
This commit is contained in:
Aliaksandr Valialkin 2021-06-25 13:20:18 +03:00
parent a69045e440
commit cb5453953f
21 changed files with 491 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -6,6 +6,7 @@ sort: 15
## tip
* FEATURE: vmagent: add service discovery for Docker (aka [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config)). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1402).
* FEATURE: vmagent: add service discovery for DigitalOcean (aka [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1367).
* FEATURE: vmagent: change the default value for `-remoteWrite.queues` from 4 to `2 * numCPUs`. This should reduce scrape duration for highly loaded vmagent, which scrapes tens of thousands of targets. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1385).
* FEATURE: vmagent: show the number of samples the target returned during the last scrape on `/targets` and `/api/v1/targets` pages. This should simplify debugging targets, which may return too big or too low number of samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1377).

27
lib/promscrape/config.go
View file

@ -22,6 +22,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/digitalocean"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/docker"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
@ -119,19 +120,19 @@ type ScrapeConfig struct {
MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
SampleLimit int `yaml:"sample_limit,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
DockerSDConfigs []docker.DockerSDConfig `yaml:"docker_sd_configs,omitempty"`
DockerSwarmSDConfigs []docker.DockerSwarmSDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
DockerSDConfigs []docker.SDConfig `yaml:"docker_sd_configs,omitempty"`
DockerSwarmSDConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
// These options are supported only by lib/promscrape.
RelabelDebug bool `yaml:"relabel_debug,omitempty"`

15
lib/promscrape/discovery/docker/api.go
View file

@ -20,7 +20,7 @@ type apiConfig struct {
filtersQueryArg string
}
func getAPIConfigFromDockerSDConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
if err != nil {
return nil, err
@ -28,19 +28,14 @@ func getAPIConfigFromDockerSDConfig(sdc *DockerSDConfig, baseDir string) (*apiCo
return v.(*apiConfig), nil
}
func getAPIConfigFromDockerSwarmSDConfig(sdc *DockerSwarmSDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(&sdc.DockerSDConfig, baseDir) })
if err != nil {
return nil, err
}
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
filtersQueryArg: getFiltersQueryArg(sdc.Filters),
}
if cfg.port == 0 {
cfg.port = 80
}
ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
if err != nil {
return nil, fmt.Errorf("cannot parse auth config: %w", err)

87
lib/promscrape/discovery/docker/containers.go → lib/promscrape/discovery/docker/container.go
View file

@ -31,12 +31,10 @@ type container struct {
}
func getContainersLabels(cfg *apiConfig) ([]map[string]string, error) {
networkLabels, err := getNetworksLabels(cfg, "__meta_docker_")
networkLabels, err := getNetworksLabelsByNetworkID(cfg)
if err != nil {
return nil, err
}
containers, err := getContainers(cfg)
if err != nil {
return nil, err
@ -62,71 +60,52 @@ func parseContainers(data []byte) ([]container, error) {
func addContainersLabels(containers []container, networkLabels map[string]map[string]string, defaultPort int) []map[string]string {
var ms []map[string]string
for _, c := range containers {
if c.Names == nil || len(c.Names) == 0 {
for i := range containers {
c := &containers[i]
if len(c.Names) == 0 {
continue
}
commonLabels := map[string]string{
"__meta_docker_container_id": c.Id,
"__meta_docker_container_name": c.Names[0],
"__meta_docker_container_network_mode": c.HostConfig.NetworkMode,
}
for k, v := range c.Labels {
commonLabels["__meta_docker_container_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for _, network := range c.NetworkSettings.Networks {
for _, n := range c.NetworkSettings.Networks {
var added bool
for _, port := range c.Ports {
if port.Type != "tcp" {
for _, p := range c.Ports {
if p.Type != "tcp" {
continue
}
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
"__meta_docker_port_private": strconv.FormatInt(int64(port.PrivatePort), 10),
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(n.IPAddress, p.PrivatePort),
"__meta_docker_network_ip": n.IPAddress,
"__meta_docker_port_private": strconv.Itoa(p.PrivatePort),
}
if port.PublicPort > 0 {
labels["__meta_docker_port_public"] = strconv.FormatInt(int64(port.PublicPort), 10)
labels["__meta_docker_port_public_ip"] = port.IP
if p.PublicPort > 0 {
m["__meta_docker_port_public"] = strconv.Itoa(p.PublicPort)
m["__meta_docker_port_public_ip"] = p.IP
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, port.PrivatePort)
ms = append(ms, labels)
addCommonLabels(m, c, networkLabels[n.NetworkID])
ms = append(ms, m)
added = true
}
if !added {
// Use fallback port when no exposed ports are available or if all are non-TCP
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(n.IPAddress, defaultPort),
"__meta_docker_network_ip": n.IPAddress,
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, defaultPort)
ms = append(ms, labels)
addCommonLabels(m, c, networkLabels[n.NetworkID])
ms = append(ms, m)
}
}
}
return ms
}
func addCommonLabels(m map[string]string, c *container, networkLabels map[string]string) {
m["__meta_docker_container_id"] = c.Id
m["__meta_docker_container_name"] = c.Names[0]
m["__meta_docker_container_network_mode"] = c.HostConfig.NetworkMode
for k, v := range c.Labels {
m["__meta_docker_container_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for k, v := range networkLabels {
m[k] = v
}
}

2
lib/promscrape/discovery/docker/containers_test.go → lib/promscrape/discovery/docker/container_test.go
View file

@ -309,7 +309,7 @@ func Test_addContainerLabels(t *testing.T) {
if err != nil {
t.Fatalf("fail to parse networks: %v", err)
}
networkLabels := getNetworkLabelsGroupByNetworkID(networks, "__meta_docker_")
networkLabels := getNetworkLabelsByNetworkID(networks)
tests := []struct {
name string

15
lib/promscrape/discovery/docker/docker.go
View file

@ -14,15 +14,10 @@ var SDCheckInterval = flag.Duration("promscrape.dockerSDCheckInterval", 30*time.
"This works only if docker_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details")
// SwarmSDCheckInterval defines interval for dockerswarm targets refresh.
var SwarmSDCheckInterval = flag.Duration("promscrape.dockerswarmSDCheckInterval", 30*time.Second, "Interval for checking for changes in dockerswarm. "+
"This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details")
// DockerSDConfig defines the `docker_sd` section for Docker based discovery
// SDConfig defines the `docker_sd` section for Docker based discovery
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config
type DockerSDConfig struct {
type SDConfig struct {
Host string `yaml:"host"`
Port int `yaml:"port,omitempty"`
Filters []Filter `yaml:"filters,omitempty"`
@ -40,8 +35,8 @@ type Filter struct {
}
// GetLabels returns docker labels according to sdc.
func (sdc *DockerSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSDConfig(sdc, baseDir)
func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
@ -49,6 +44,6 @@ func (sdc *DockerSDConfig) GetLabels(baseDir string) ([]map[string]string, error
}
// MustStop stops further usage for sdc.
func (sdc *DockerSDConfig) MustStop() {
func (sdc *SDConfig) MustStop() {
configMap.Delete(sdc)
}

37
lib/promscrape/discovery/docker/dockerswarm.go
View file

@ -1,37 +0,0 @@
package docker
import (
"fmt"
)
// DockerSwarmSDConfig represents docker swarm service discovery configuration
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type DockerSwarmSDConfig struct {
Role string `yaml:"role"`
DockerSDConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *DockerSwarmSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSwarmSDConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
}
// MustStop stops further usage for sdc.
func (sdc *DockerSwarmSDConfig) MustStop() {
configMap.Delete(sdc)
}

20
lib/promscrape/discovery/docker/network.go
View file

@ -18,18 +18,18 @@ type network struct {
Labels map[string]string
}
func getNetworksLabels(cfg *apiConfig, labelPrefix string) (map[string]map[string]string, error) {
func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return getNetworkLabelsGroupByNetworkID(networks, labelPrefix), nil
return getNetworkLabelsByNetworkID(networks), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
resp, err := cfg.getAPIResponse("/networks")
if err != nil {
return nil, fmt.Errorf("cannot query docker/dockerswarm api for networks: %w", err)
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
}
return parseNetworks(resp)
}
@ -42,18 +42,18 @@ func parseNetworks(data []byte) ([]network, error) {
return networks, nil
}
func getNetworkLabelsGroupByNetworkID(networks []network, labelPrefix string) map[string]map[string]string {
func getNetworkLabelsByNetworkID(networks []network) map[string]map[string]string {
ms := make(map[string]map[string]string)
for _, network := range networks {
m := map[string]string{
labelPrefix + "network_id": network.ID,
labelPrefix + "network_name": network.Name,
labelPrefix + "network_internal": strconv.FormatBool(network.Internal),
labelPrefix + "network_ingress": strconv.FormatBool(network.Ingress),
labelPrefix + "network_scope": network.Scope,
"__meta_docker_network_id": network.ID,
"__meta_docker_network_name": network.Name,
"__meta_docker_network_internal": strconv.FormatBool(network.Internal),
"__meta_docker_network_ingress": strconv.FormatBool(network.Ingress),
"__meta_docker_network_scope": network.Scope,
}
for k, v := range network.Labels {
m[labelPrefix+"network_label_"+discoveryutils.SanitizeLabelName(k)] = v
m["__meta_docker_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms[network.ID] = m
}

14
lib/promscrape/discovery/docker/network_test.go
View file

@ -35,18 +35,18 @@ func Test_addNetworkLabels(t *testing.T) {
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
"__meta_docker_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_docker_network_ingress": "true",
"__meta_docker_network_internal": "false",
"__meta_docker_network_label_key1": "value1",
"__meta_docker_network_name": "ingress",
"__meta_docker_network_scope": "swarm",
})},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := getNetworkLabelsGroupByNetworkID(tt.args.networks, "__meta_dockerswarm_")
got := getNetworkLabelsByNetworkID(tt.args.networks)
var networkIDs []string
for networkID := range got {
networkIDs = append(networkIDs, networkID)

86
lib/promscrape/discovery/dockerswarm/api.go Normal file
View file

@ -0,0 +1,86 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"net/url"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
var configMap = discoveryutils.NewConfigMap()
type apiConfig struct {
client *discoveryutils.Client
port int
// filtersQueryArg contains escaped `filters` query arg to add to each request to Docker Swarm API.
filtersQueryArg string
}
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
if err != nil {
return nil, err
}
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
filtersQueryArg: getFiltersQueryArg(sdc.Filters),
}
if cfg.port == 0 {
cfg.port = 80
}
ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
if err != nil {
return nil, fmt.Errorf("cannot parse auth config: %w", err)
}
proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir)
if err != nil {
return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
}
client, err := discoveryutils.NewClient(sdc.Host, ac, sdc.ProxyURL, proxyAC)
if err != nil {
return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.Host, err)
}
cfg.client = client
return cfg, nil
}
func (cfg *apiConfig) getAPIResponse(path string) ([]byte, error) {
if len(cfg.filtersQueryArg) > 0 {
separator := "?"
if strings.Contains(path, "?") {
separator = "&"
}
path += separator + "filters=" + cfg.filtersQueryArg
}
return cfg.client.GetAPIResponse(path)
}
func getFiltersQueryArg(filters []Filter) string {
if len(filters) == 0 {
return ""
}
m := make(map[string]map[string]bool)
for _, f := range filters {
x := m[f.Name]
if x == nil {
x = make(map[string]bool)
m[f.Name] = x
}
for _, value := range f.Values {
x[value] = true
}
}
buf, err := json.Marshal(m)
if err != nil {
logger.Panicf("BUG: unexpected error in json.Marshal: %s", err)
}
return url.QueryEscape(string(buf))
}

26
lib/promscrape/discovery/dockerswarm/api_test.go Normal file
View file

@ -0,0 +1,26 @@
package dockerswarm
import (
"testing"
)
func TestGetFiltersQueryArg(t *testing.T) {
f := func(filters []Filter, queryArgExpected string) {
t.Helper()
queryArg := getFiltersQueryArg(filters)
if queryArg != queryArgExpected {
t.Fatalf("unexpected query arg; got %s; want %s", queryArg, queryArgExpected)
}
}
f(nil, "")
f([]Filter{
{
Name: "name",
Values: []string{"foo", "bar"},
},
{
Name: "xxx",
Values: []string{"aa"},
},
}, "%7B%22name%22%3A%7B%22bar%22%3Atrue%2C%22foo%22%3Atrue%7D%2C%22xxx%22%3A%7B%22aa%22%3Atrue%7D%7D")
}

59
lib/promscrape/discovery/dockerswarm/dockerswarm.go Normal file
View file

@ -0,0 +1,59 @@
package dockerswarm
import (
"flag"
"fmt"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
)
// SDCheckInterval defines interval for dockerswarm targets refresh.
var SDCheckInterval = flag.Duration("promscrape.dockerswarmSDCheckInterval", 30*time.Second, "Interval for checking for changes in dockerswarm. "+
"This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details")
// SDConfig represents docker swarm service discovery configuration
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type SDConfig struct {
Host string `yaml:"host"`
Role string `yaml:"role"`
Port int `yaml:"port,omitempty"`
Filters []Filter `yaml:"filters,omitempty"`
HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
ProxyURL proxy.URL `yaml:"proxy_url,omitempty"`
ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
}
// Filter is a filter, which can be passed to SDConfig.
type Filter struct {
Name string `yaml:"name"`
Values []string `yaml:"values"`
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
}
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
configMap.Delete(sdc)
}

61
lib/promscrape/discovery/dockerswarm/network.go Normal file
View file

@ -0,0 +1,61 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://docs.docker.com/engine/api/v1.40/#tag/Network
type network struct {
ID string
Name string
Scope string
Internal bool
Ingress bool
Labels map[string]string
}
func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return getNetworkLabelsByNetworkID(networks), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
resp, err := cfg.getAPIResponse("/networks")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
}
return parseNetworks(resp)
}
func parseNetworks(data []byte) ([]network, error) {
var networks []network
if err := json.Unmarshal(data, &networks); err != nil {
return nil, fmt.Errorf("cannot parse networks: %w", err)
}
return networks, nil
}
func getNetworkLabelsByNetworkID(networks []network) map[string]map[string]string {
ms := make(map[string]map[string]string)
for _, network := range networks {
m := map[string]string{
"__meta_dockerswarm_network_id": network.ID,
"__meta_dockerswarm_network_name": network.Name,
"__meta_dockerswarm_network_internal": strconv.FormatBool(network.Internal),
"__meta_dockerswarm_network_ingress": strconv.FormatBool(network.Ingress),
"__meta_dockerswarm_network_scope": network.Scope,
}
for k, v := range network.Labels {
m["__meta_dockerswarm_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms[network.ID] = m
}
return ms
}

173
lib/promscrape/discovery/dockerswarm/network_test.go Normal file
View file

@ -0,0 +1,173 @@
package dockerswarm
import (
"reflect"
"sort"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_addNetworkLabels(t *testing.T) {
type args struct {
networks []network
}
tests := []struct {
name string
args args
want [][]prompbmarshal.Label
}{
{
name: "ingress network",
args: args{
networks: []network{
{
ID: "qs0hog6ldlei9ct11pr3c77v1",
Ingress: true,
Scope: "swarm",
Name: "ingress",
Labels: map[string]string{
"key1": "value1",
},
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
})},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := getNetworkLabelsByNetworkID(tt.args.networks)
var networkIDs []string
for networkID := range got {
networkIDs = append(networkIDs, networkID)
}
sort.Strings(networkIDs)
var sortedLabelss [][]prompbmarshal.Label
for _, networkID := range networkIDs {
labels := got[networkID]
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("addNetworkLabels() \ngot %v, \nwant %v", sortedLabelss, tt.want)
}
})
}
}
func Test_parseNetworks(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []network
wantErr bool
}{
{
name: "parse two networks",
args: args{
data: []byte(`[
{
"Name": "ingress",
"Id": "qs0hog6ldlei9ct11pr3c77v1",
"Created": "2020-10-06T08:39:58.957083331Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.0.0/24",
"Gateway": "10.0.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": true,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": null,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4096"
},
"Labels": {
"key1": "value1"
}
},
{
"Name": "host",
"Id": "317f0384d7e5f5c26304a0b04599f9f54bc08def4d0535059ece89955e9c4b7b",
"Created": "2020-10-06T08:39:52.843373136Z",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {
"key": "value"
}
}
]`),
},
want: []network{
{
ID: "qs0hog6ldlei9ct11pr3c77v1",
Ingress: true,
Scope: "swarm",
Name: "ingress",
Labels: map[string]string{
"key1": "value1",
},
},
{
ID: "317f0384d7e5f5c26304a0b04599f9f54bc08def4d0535059ece89955e9c4b7b",
Scope: "local",
Name: "host",
Labels: map[string]string{
"key": "value",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseNetworks(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseNetworks() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseNetworks() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}

2
lib/promscrape/discovery/docker/nodes.go → lib/promscrape/discovery/dockerswarm/nodes.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"encoding/json"

2
lib/promscrape/discovery/docker/nodes_test.go → lib/promscrape/discovery/dockerswarm/nodes_test.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"reflect"

4
lib/promscrape/discovery/docker/services.go → lib/promscrape/discovery/dockerswarm/services.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"encoding/json"
@ -51,7 +51,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networksLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
networksLabels, err := getNetworksLabelsByNetworkID(cfg)
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/docker/services_test.go → lib/promscrape/discovery/dockerswarm/services_test.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"reflect"

4
lib/promscrape/discovery/docker/tasks.go → lib/promscrape/discovery/dockerswarm/tasks.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"encoding/json"
@ -44,7 +44,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networkLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
networkLabels, err := getNetworksLabelsByNetworkID(cfg)
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/docker/tasks_test.go → lib/promscrape/discovery/dockerswarm/tasks_test.go
View file

@ -1,4 +1,4 @@
package docker
package dockerswarm
import (
"reflect"

3
lib/promscrape/scraper.go
View file

@ -15,6 +15,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/digitalocean"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/docker"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
@ -94,7 +95,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
scs.add("digitalocean_sd_configs", *digitalocean.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDigitalOceanDScrapeWork(swsPrev) })
scs.add("dns_sd_configs", *dns.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
scs.add("docker_sd_configs", *docker.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSDScrapeWork(swsPrev) })
scs.add("dockerswarm_sd_configs", *docker.SwarmSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
scs.add("dockerswarm_sd_configs", *dockerswarm.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
scs.add("ec2_sd_configs", *ec2.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
scs.add("eureka_sd_configs", *eureka.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) })
scs.add("file_sd_configs", *fileSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getFileSDScrapeWork(swsPrev) })