github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

Support Docker ServiceDiscovery (#1402)

Browse source 
* add docker discovery

* add test

* add labels test and add scrape work

* remove TODO

* refactor to merge apiConfig and sdConfig

* apply suggestion
This commit is contained in:
Lu Jiajing 2021-06-25 16:42:47 +08:00 committed by GitHub
parent 434e33da9b
commit c85a5b7fcb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 665 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
lib/promscrape/config.go
View file

@ -21,7 +21,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/digitalocean"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/docker"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
@ -119,18 +119,19 @@ type ScrapeConfig struct {
MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
SampleLimit int `yaml:"sample_limit,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
DockerSwarmSDConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
DockerSDConfigs []docker.DockerSDConfig `yaml:"docker_sd_configs,omitempty"`
DockerSwarmSDConfigs []docker.DockerSwarmSDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
// These options are supported only by lib/promscrape.
RelabelDebug bool `yaml:"relabel_debug,omitempty"`
@ -174,6 +175,9 @@ func (sc *ScrapeConfig) mustStop() {
for i := range sc.EurekaSDConfigs {
sc.EurekaSDConfigs[i].MustStop()
}
for i := range sc.DockerSDConfigs {
sc.DockerSDConfigs[i].MustStop()
}
for i := range sc.DockerSwarmSDConfigs {
sc.DockerSwarmSDConfigs[i].MustStop()
}
@ -341,6 +345,34 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
return dst
}
// getDockerSDScrapeWork returns `docker_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getDockerSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
swsPrevByJob := getSWSByJob(prev)
dst := make([]*ScrapeWork, 0, len(prev))
for i := range cfg.ScrapeConfigs {
sc := &cfg.ScrapeConfigs[i]
dstLen := len(dst)
ok := true
for j := range sc.DockerSDConfigs {
sdc := &sc.DockerSDConfigs[j]
var okLocal bool
dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "docker_sd_config")
if ok {
ok = okLocal
}
}
if ok {
continue
}
swsPrev := swsPrevByJob[sc.swc.jobName]
if len(swsPrev) > 0 {
logger.Errorf("there were errors when discovering docker targets for job %q, so preserving the previous targets", sc.swc.jobName)
dst = append(dst[:dstLen], swsPrev...)
}
}
return dst
}
// getDockerSwarmSDScrapeWork returns `dockerswarm_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getDockerSwarmSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
swsPrevByJob := getSWSByJob(prev)

14
lib/promscrape/discovery/dockerswarm/api.go → lib/promscrape/discovery/docker/api.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -20,7 +20,7 @@ type apiConfig struct {
filtersQueryArg string
}
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
func getAPIConfigFromDockerSDConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
if err != nil {
return nil, err
@ -28,7 +28,15 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
func getAPIConfigFromDockerSwarmSDConfig(sdc *DockerSwarmSDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(&sdc.DockerSDConfig, baseDir) })
if err != nil {
return nil, err
}
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
filtersQueryArg: getFiltersQueryArg(sdc.Filters),

2
lib/promscrape/discovery/dockerswarm/api_test.go → lib/promscrape/discovery/docker/api_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"testing"

132
lib/promscrape/discovery/docker/containers.go Normal file
View file

@ -0,0 +1,132 @@
package docker
import (
"encoding/json"
"fmt"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://github.com/moby/moby/blob/314759dc2f4745925d8dec6d15acc7761c6e5c92/docs/api/v1.41.yaml#L4024
type container struct {
Id string
Names []string
Labels map[string]string
Ports []struct {
IP string
PrivatePort int
PublicPort int
Type string
}
HostConfig struct {
NetworkMode string
}
NetworkSettings struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}
}
func getContainersLabels(cfg *apiConfig) ([]map[string]string, error) {
networkLabels, err := getNetworksLabels(cfg, "__meta_docker_")
if err != nil {
return nil, err
}
containers, err := getContainers(cfg)
if err != nil {
return nil, err
}
return addContainersLabels(containers, networkLabels, cfg.port), nil
}
func getContainers(cfg *apiConfig) ([]container, error) {
resp, err := cfg.getAPIResponse("/containers/json")
if err != nil {
return nil, fmt.Errorf("cannot query dockerd api for containers: %w", err)
}
return parseContainers(resp)
}
func parseContainers(data []byte) ([]container, error) {
var containers []container
if err := json.Unmarshal(data, &containers); err != nil {
return nil, fmt.Errorf("cannot parse containers: %w", err)
}
return containers, nil
}
func addContainersLabels(containers []container, networkLabels map[string]map[string]string, defaultPort int) []map[string]string {
var ms []map[string]string
for _, c := range containers {
if c.Names == nil || len(c.Names) == 0 {
continue
}
commonLabels := map[string]string{
"__meta_docker_container_id": c.Id,
"__meta_docker_container_name": c.Names[0],
"__meta_docker_container_network_mode": c.HostConfig.NetworkMode,
}
for k, v := range c.Labels {
commonLabels["__meta_docker_container_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for _, network := range c.NetworkSettings.Networks {
var added bool
for _, port := range c.Ports {
if port.Type != "tcp" {
continue
}
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
"__meta_docker_port_private": strconv.FormatInt(int64(port.PrivatePort), 10),
}
if port.PublicPort > 0 {
labels["__meta_docker_port_public"] = strconv.FormatInt(int64(port.PublicPort), 10)
labels["__meta_docker_port_public_ip"] = port.IP
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, port.PrivatePort)
ms = append(ms, labels)
added = true
}
if !added {
// Use fallback port when no exposed ports are available or if all are non-TCP
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, defaultPort)
ms = append(ms, labels)
}
}
}
return ms
}

404
lib/promscrape/discovery/docker/containers_test.go Normal file
View file

@ -0,0 +1,404 @@
package docker
import (
"reflect"
"testing"
)
func Test_parseContainers(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []container
wantErr bool
}{
{
name: "parse two containers",
args: args{
data: []byte(`[
{
"Id": "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
"Names": [
"/crow-server"
],
"ImageID": "sha256:11045371758ccf9468d807d53d1e1faa100c8ebabe87296bc107d52bdf983378",
"Created": 1624440429,
"Ports": [
{
"IP": "0.0.0.0",
"PrivatePort": 8080,
"PublicPort": 18081,
"Type": "tcp"
}
],
"Labels": {
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2"
},
"State": "running",
"Status": "Up 2 hours",
"HostConfig": {
"NetworkMode": "bridge"
},
"NetworkSettings": {
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"EndpointID": "2bcd751c98578ad1ce3d6798077a3e110535f3dcb0b0735cc84bd1e03905d907",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null
}
}
}
},
{
"Id": "0e0f72a6eb7d9fb443f0426a66f7b8dd7d3283ab7e3a308b2bed584ac03a33dc",
"Names": [
"/crow-web"
],
"ImageID": "sha256:36e04dd2f67950179ab62a4ebd1b8b741232263fa1f210496a53335d5820b3af",
"Created": 1618302442,
"Ports": [
{
"IP": "0.0.0.0",
"PrivatePort": 8080,
"PublicPort": 18082,
"Type": "tcp"
}
],
"Labels": {
"com.docker.compose.config-hash": "d99ebd0fde8512366c2d78c367e95ddc74528bb60b7cf0c991c9f4835981e00e",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowweb",
"com.docker.compose.service": "crow-web",
"com.docker.compose.version": "1.11.2"
},
"State": "running",
"Status": "Up 2 months",
"HostConfig": {
"NetworkMode": "bridge"
},
"NetworkSettings": {
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"EndpointID": "78d751dfa31923d73e001e5dc5751ad6f9f5ffeb88056fc950f0a952d7f24302",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null
}
}
}
}
]`),
},
want: []container{
{
Id: "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
Names: []string{"/crow-server"},
Labels: map[string]string{
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18081,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.2",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
{
Id: "0e0f72a6eb7d9fb443f0426a66f7b8dd7d3283ab7e3a308b2bed584ac03a33dc",
Names: []string{"/crow-web"},
Labels: map[string]string{
"com.docker.compose.config-hash": "d99ebd0fde8512366c2d78c367e95ddc74528bb60b7cf0c991c9f4835981e00e",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowweb",
"com.docker.compose.service": "crow-web",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18082,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.3",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseContainers(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseContainers() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseNetworks() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}
func Test_addContainerLabels(t *testing.T) {
data := []byte(`[
{
"Name": "host",
"Id": "6a1989488dcb847c052eda939924d997457d5ecd994f76f35472996c4c75279a",
"Created": "2020-08-18T17:18:18.439033107+08:00",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
},
{
"Name": "none",
"Id": "c9668d06973d976527e913ba207a3819275649f347390379ec8356db375cfde3",
"Created": "2020-08-18T17:18:18.428827132+08:00",
"Scope": "local",
"Driver": "null",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
},
{
"Name": "bridge",
"Id": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"Created": "2021-03-18T14:36:04.290821903+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.17.0.0/16",
"Gateway": "172.17.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {
"com.docker.network.bridge.default_bridge": "true",
"com.docker.network.bridge.enable_icc": "true",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
"com.docker.network.bridge.name": "docker0",
"com.docker.network.driver.mtu": "1500"
},
"Labels": {}
}
]`)
networks, err := parseNetworks(data)
if err != nil {
t.Fatalf("fail to parse networks: %v", err)
}
networkLabels := getNetworkLabelsGroupByNetworkID(networks, "__meta_docker_")
tests := []struct {
name string
c container
want []map[string]string
wantErr bool
}{
{
name: "get labels from a container",
c: container{
Id: "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
Names: []string{"/crow-server"},
Labels: map[string]string{
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18081,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.2",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
want: []map[string]string{
{
"__address__": "172.17.0.2:8080",
"__meta_docker_container_id": "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
"__meta_docker_container_label_com_docker_compose_config_hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"__meta_docker_container_label_com_docker_compose_container_number": "1",
"__meta_docker_container_label_com_docker_compose_oneoff": "False",
"__meta_docker_container_label_com_docker_compose_project": "crowserver",
"__meta_docker_container_label_com_docker_compose_service": "crow-server",
"__meta_docker_container_label_com_docker_compose_version": "1.11.2",
"__meta_docker_container_name": "/crow-server",
"__meta_docker_container_network_mode": "bridge",
"__meta_docker_network_id": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.17.0.2",
"__meta_docker_network_name": "bridge",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "8080",
"__meta_docker_port_public": "18081",
"__meta_docker_port_public_ip": "0.0.0.0",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
labelsMap := addContainersLabels([]container{tt.c}, networkLabels, 80)
if (err != nil) != tt.wantErr {
t.Errorf("addContainersLabels() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(labelsMap, tt.want) {
t.Errorf("addContainersLabels() \ngot %v, \nwant %v", labelsMap, tt.want)
}
})
}
}

30
lib/promscrape/discovery/dockerswarm/dockerswarm.go → lib/promscrape/discovery/docker/docker.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"fmt"
@ -7,19 +7,18 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
)
// SDConfig represents docker swarm service discovery configuration
// DockerSDConfig defines the `docker_sd` section for Docker based discovery
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type SDConfig struct {
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config
type DockerSDConfig struct {
Host string `yaml:"host"`
Role string `yaml:"role"`
Port int `yaml:"port,omitempty"`
Filters []Filter `yaml:"filters,omitempty"`
HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
ProxyURL proxy.URL `yaml:"proxy_url,omitempty"`
ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
// refresh_interval is obtained from `-promscrape.dockerSDCheckInterval` command-line option
}
// Filter is a filter, which can be passed to SDConfig.
@ -28,25 +27,16 @@ type Filter struct {
Values []string `yaml:"values"`
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
// GetLabels returns docker labels according to sdc.
func (sdc *DockerSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSDConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
return getContainersLabels(cfg)
}
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
func (sdc *DockerSDConfig) MustStop() {
configMap.Delete(sdc)
}

37
lib/promscrape/discovery/docker/dockerswarm.go Normal file
View file

@ -0,0 +1,37 @@
package docker
import (
"fmt"
)
// DockerSwarmSDConfig represents docker swarm service discovery configuration
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type DockerSwarmSDConfig struct {
Role string `yaml:"role"`
DockerSDConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *DockerSwarmSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSwarmSDConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
}
// MustStop stops further usage for sdc.
func (sdc *DockerSwarmSDConfig) MustStop() {
configMap.Delete(sdc)
}

22
lib/promscrape/discovery/dockerswarm/network.go → lib/promscrape/discovery/docker/network.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -18,18 +18,18 @@ type network struct {
Labels map[string]string
}
func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, error) {
func getNetworksLabels(cfg *apiConfig, labelPrefix string) (map[string]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return getNetworkLabelsByNetworkID(networks), nil
return getNetworkLabelsGroupByNetworkID(networks, labelPrefix), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
resp, err := cfg.getAPIResponse("/networks")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
return nil, fmt.Errorf("cannot query docker/dockerswarm api for networks: %w", err)
}
return parseNetworks(resp)
}
@ -42,18 +42,18 @@ func parseNetworks(data []byte) ([]network, error) {
return networks, nil
}
func getNetworkLabelsByNetworkID(networks []network) map[string]map[string]string {
func getNetworkLabelsGroupByNetworkID(networks []network, labelPrefix string) map[string]map[string]string {
ms := make(map[string]map[string]string)
for _, network := range networks {
m := map[string]string{
"__meta_dockerswarm_network_id": network.ID,
"__meta_dockerswarm_network_name": network.Name,
"__meta_dockerswarm_network_internal": strconv.FormatBool(network.Internal),
"__meta_dockerswarm_network_ingress": strconv.FormatBool(network.Ingress),
"__meta_dockerswarm_network_scope": network.Scope,
labelPrefix + "network_id": network.ID,
labelPrefix + "network_name": network.Name,
labelPrefix + "network_internal": strconv.FormatBool(network.Internal),
labelPrefix + "network_ingress": strconv.FormatBool(network.Ingress),
labelPrefix + "network_scope": network.Scope,
}
for k, v := range network.Labels {
m["__meta_dockerswarm_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
m[labelPrefix+"network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms[network.ID] = m
}

4
lib/promscrape/discovery/dockerswarm/network_test.go → lib/promscrape/discovery/docker/network_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"
@ -46,7 +46,7 @@ func Test_addNetworkLabels(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := getNetworkLabelsByNetworkID(tt.args.networks)
got := getNetworkLabelsGroupByNetworkID(tt.args.networks, "__meta_dockerswarm_")
var networkIDs []string
for networkID := range got {
networkIDs = append(networkIDs, networkID)

2
lib/promscrape/discovery/dockerswarm/nodes.go → lib/promscrape/discovery/docker/nodes.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"

2
lib/promscrape/discovery/dockerswarm/nodes_test.go → lib/promscrape/discovery/docker/nodes_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

4
lib/promscrape/discovery/dockerswarm/services.go → lib/promscrape/discovery/docker/services.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -51,7 +51,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networksLabels, err := getNetworksLabelsByNetworkID(cfg)
networksLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/dockerswarm/services_test.go → lib/promscrape/discovery/docker/services_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

5
lib/promscrape/discovery/dockerswarm/tasks.go → lib/promscrape/discovery/docker/tasks.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -7,7 +7,6 @@ import (
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
@ -45,7 +44,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networkLabels, err := getNetworksLabelsByNetworkID(cfg)
networkLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/dockerswarm/tasks_test.go → lib/promscrape/discovery/docker/tasks_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

4
lib/promscrape/scraper.go
View file

@ -39,6 +39,9 @@ var (
gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+
"This works only if gce_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details")
dockerSDCheckInterval = flag.Duration("promscrape.dockerSDCheckInterval", 30*time.Second, "Interval for checking for changes in docker. "+
"This works only if docker_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details")
dockerswarmSDCheckInterval = flag.Duration("promscrape.dockerswarmSDCheckInterval", 30*time.Second, "Interval for checking for changes in dockerswarm. "+
"This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details")
@ -114,6 +117,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
scs.add("docker_sd_configs", *dockerSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSDScrapeWork(swsPrev) })
scs.add("dockerswarm_sd_configs", *dockerswarmSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
scs.add("digitalocean_sd_configs", *digitaloceanSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDigitalOceanDScrapeWork(swsPrev) })
scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) })