github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

Support Docker ServiceDiscovery (#1402)

Browse source 
* add docker discovery

* add test

* add labels test and add scrape work

* remove TODO

* refactor to merge apiConfig and sdConfig

* apply suggestion
This commit is contained in:
Lu Jiajing 2021-06-25 16:42:47 +08:00 committed by Aliaksandr Valialkin
parent 501429c3ff
commit 12b4cbb68f
16 changed files with 665 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
lib/promscrape/config.go
View file

@ -21,7 +21,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/digitalocean"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/docker"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
@ -119,18 +119,19 @@ type ScrapeConfig struct {
MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
SampleLimit int `yaml:"sample_limit,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
DockerSwarmSDConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
DockerSDConfigs []docker.DockerSDConfig `yaml:"docker_sd_configs,omitempty"`
DockerSwarmSDConfigs []docker.DockerSwarmSDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"`
// These options are supported only by lib/promscrape.
RelabelDebug bool `yaml:"relabel_debug,omitempty"`
@ -174,6 +175,9 @@ func (sc *ScrapeConfig) mustStop() {
for i := range sc.EurekaSDConfigs {
sc.EurekaSDConfigs[i].MustStop()
}
for i := range sc.DockerSDConfigs {
sc.DockerSDConfigs[i].MustStop()
}
for i := range sc.DockerSwarmSDConfigs {
sc.DockerSwarmSDConfigs[i].MustStop()
}
@ -341,6 +345,34 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
return dst
}
// getDockerSDScrapeWork returns `docker_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getDockerSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
swsPrevByJob := getSWSByJob(prev)
dst := make([]*ScrapeWork, 0, len(prev))
for i := range cfg.ScrapeConfigs {
sc := &cfg.ScrapeConfigs[i]
dstLen := len(dst)
ok := true
for j := range sc.DockerSDConfigs {
sdc := &sc.DockerSDConfigs[j]
var okLocal bool
dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "docker_sd_config")
if ok {
ok = okLocal
}
}
if ok {
continue
}
swsPrev := swsPrevByJob[sc.swc.jobName]
if len(swsPrev) > 0 {
logger.Errorf("there were errors when discovering docker targets for job %q, so preserving the previous targets", sc.swc.jobName)
dst = append(dst[:dstLen], swsPrev...)
}
}
return dst
}
// getDockerSwarmSDScrapeWork returns `dockerswarm_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getDockerSwarmSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
swsPrevByJob := getSWSByJob(prev)

14
lib/promscrape/discovery/dockerswarm/api.go → lib/promscrape/discovery/docker/api.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -20,7 +20,7 @@ type apiConfig struct {
filtersQueryArg string
}
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
func getAPIConfigFromDockerSDConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
if err != nil {
return nil, err
@ -28,7 +28,15 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
func getAPIConfigFromDockerSwarmSDConfig(sdc *DockerSwarmSDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(&sdc.DockerSDConfig, baseDir) })
if err != nil {
return nil, err
}
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *DockerSDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
filtersQueryArg: getFiltersQueryArg(sdc.Filters),

2
lib/promscrape/discovery/dockerswarm/api_test.go → lib/promscrape/discovery/docker/api_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"testing"

132
lib/promscrape/discovery/docker/containers.go Normal file
View file

@ -0,0 +1,132 @@
package docker
import (
"encoding/json"
"fmt"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://github.com/moby/moby/blob/314759dc2f4745925d8dec6d15acc7761c6e5c92/docs/api/v1.41.yaml#L4024
type container struct {
Id string
Names []string
Labels map[string]string
Ports []struct {
IP string
PrivatePort int
PublicPort int
Type string
}
HostConfig struct {
NetworkMode string
}
NetworkSettings struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}
}
func getContainersLabels(cfg *apiConfig) ([]map[string]string, error) {
networkLabels, err := getNetworksLabels(cfg, "__meta_docker_")
if err != nil {
return nil, err
}
containers, err := getContainers(cfg)
if err != nil {
return nil, err
}
return addContainersLabels(containers, networkLabels, cfg.port), nil
}
func getContainers(cfg *apiConfig) ([]container, error) {
resp, err := cfg.getAPIResponse("/containers/json")
if err != nil {
return nil, fmt.Errorf("cannot query dockerd api for containers: %w", err)
}
return parseContainers(resp)
}
func parseContainers(data []byte) ([]container, error) {
var containers []container
if err := json.Unmarshal(data, &containers); err != nil {
return nil, fmt.Errorf("cannot parse containers: %w", err)
}
return containers, nil
}
func addContainersLabels(containers []container, networkLabels map[string]map[string]string, defaultPort int) []map[string]string {
var ms []map[string]string
for _, c := range containers {
if c.Names == nil || len(c.Names) == 0 {
continue
}
commonLabels := map[string]string{
"__meta_docker_container_id": c.Id,
"__meta_docker_container_name": c.Names[0],
"__meta_docker_container_network_mode": c.HostConfig.NetworkMode,
}
for k, v := range c.Labels {
commonLabels["__meta_docker_container_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for _, network := range c.NetworkSettings.Networks {
var added bool
for _, port := range c.Ports {
if port.Type != "tcp" {
continue
}
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
"__meta_docker_port_private": strconv.FormatInt(int64(port.PrivatePort), 10),
}
if port.PublicPort > 0 {
labels["__meta_docker_port_public"] = strconv.FormatInt(int64(port.PublicPort), 10)
labels["__meta_docker_port_public_ip"] = port.IP
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, port.PrivatePort)
ms = append(ms, labels)
added = true
}
if !added {
// Use fallback port when no exposed ports are available or if all are non-TCP
labels := map[string]string{
"__meta_docker_network_ip": network.IPAddress,
}
for k, v := range commonLabels {
labels[k] = v
}
for k, v := range networkLabels[network.NetworkID] {
labels[k] = v
}
labels["__address__"] = discoveryutils.JoinHostPort(network.IPAddress, defaultPort)
ms = append(ms, labels)
}
}
}
return ms
}

404
lib/promscrape/discovery/docker/containers_test.go Normal file
View file

@ -0,0 +1,404 @@
package docker
import (
"reflect"
"testing"
)
func Test_parseContainers(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []container
wantErr bool
}{
{
name: "parse two containers",
args: args{
data: []byte(`[
{
"Id": "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
"Names": [
"/crow-server"
],
"ImageID": "sha256:11045371758ccf9468d807d53d1e1faa100c8ebabe87296bc107d52bdf983378",
"Created": 1624440429,
"Ports": [
{
"IP": "0.0.0.0",
"PrivatePort": 8080,
"PublicPort": 18081,
"Type": "tcp"
}
],
"Labels": {
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2"
},
"State": "running",
"Status": "Up 2 hours",
"HostConfig": {
"NetworkMode": "bridge"
},
"NetworkSettings": {
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"EndpointID": "2bcd751c98578ad1ce3d6798077a3e110535f3dcb0b0735cc84bd1e03905d907",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null
}
}
}
},
{
"Id": "0e0f72a6eb7d9fb443f0426a66f7b8dd7d3283ab7e3a308b2bed584ac03a33dc",
"Names": [
"/crow-web"
],
"ImageID": "sha256:36e04dd2f67950179ab62a4ebd1b8b741232263fa1f210496a53335d5820b3af",
"Created": 1618302442,
"Ports": [
{
"IP": "0.0.0.0",
"PrivatePort": 8080,
"PublicPort": 18082,
"Type": "tcp"
}
],
"Labels": {
"com.docker.compose.config-hash": "d99ebd0fde8512366c2d78c367e95ddc74528bb60b7cf0c991c9f4835981e00e",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowweb",
"com.docker.compose.service": "crow-web",
"com.docker.compose.version": "1.11.2"
},
"State": "running",
"Status": "Up 2 months",
"HostConfig": {
"NetworkMode": "bridge"
},
"NetworkSettings": {
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"EndpointID": "78d751dfa31923d73e001e5dc5751ad6f9f5ffeb88056fc950f0a952d7f24302",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null
}
}
}
}
]`),
},
want: []container{
{
Id: "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
Names: []string{"/crow-server"},
Labels: map[string]string{
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18081,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.2",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
{
Id: "0e0f72a6eb7d9fb443f0426a66f7b8dd7d3283ab7e3a308b2bed584ac03a33dc",
Names: []string{"/crow-web"},
Labels: map[string]string{
"com.docker.compose.config-hash": "d99ebd0fde8512366c2d78c367e95ddc74528bb60b7cf0c991c9f4835981e00e",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowweb",
"com.docker.compose.service": "crow-web",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18082,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.3",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseContainers(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseContainers() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseNetworks() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}
func Test_addContainerLabels(t *testing.T) {
data := []byte(`[
{
"Name": "host",
"Id": "6a1989488dcb847c052eda939924d997457d5ecd994f76f35472996c4c75279a",
"Created": "2020-08-18T17:18:18.439033107+08:00",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
},
{
"Name": "none",
"Id": "c9668d06973d976527e913ba207a3819275649f347390379ec8356db375cfde3",
"Created": "2020-08-18T17:18:18.428827132+08:00",
"Scope": "local",
"Driver": "null",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
},
{
"Name": "bridge",
"Id": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"Created": "2021-03-18T14:36:04.290821903+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.17.0.0/16",
"Gateway": "172.17.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {
"com.docker.network.bridge.default_bridge": "true",
"com.docker.network.bridge.enable_icc": "true",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
"com.docker.network.bridge.name": "docker0",
"com.docker.network.driver.mtu": "1500"
},
"Labels": {}
}
]`)
networks, err := parseNetworks(data)
if err != nil {
t.Fatalf("fail to parse networks: %v", err)
}
networkLabels := getNetworkLabelsGroupByNetworkID(networks, "__meta_docker_")
tests := []struct {
name string
c container
want []map[string]string
wantErr bool
}{
{
name: "get labels from a container",
c: container{
Id: "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
Names: []string{"/crow-server"},
Labels: map[string]string{
"com.docker.compose.config-hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"com.docker.compose.container-number": "1",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "crowserver",
"com.docker.compose.service": "crow-server",
"com.docker.compose.version": "1.11.2",
},
Ports: []struct {
IP string
PrivatePort int
PublicPort int
Type string
}{{
IP: "0.0.0.0",
PrivatePort: 8080,
PublicPort: 18081,
Type: "tcp",
}},
HostConfig: struct {
NetworkMode string
}{
NetworkMode: "bridge",
},
NetworkSettings: struct {
Networks map[string]struct {
IPAddress string
NetworkID string
}
}{
Networks: map[string]struct {
IPAddress string
NetworkID string
}{
"bridge": {
IPAddress: "172.17.0.2",
NetworkID: "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
},
},
},
},
want: []map[string]string{
{
"__address__": "172.17.0.2:8080",
"__meta_docker_container_id": "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
"__meta_docker_container_label_com_docker_compose_config_hash": "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
"__meta_docker_container_label_com_docker_compose_container_number": "1",
"__meta_docker_container_label_com_docker_compose_oneoff": "False",
"__meta_docker_container_label_com_docker_compose_project": "crowserver",
"__meta_docker_container_label_com_docker_compose_service": "crow-server",
"__meta_docker_container_label_com_docker_compose_version": "1.11.2",
"__meta_docker_container_name": "/crow-server",
"__meta_docker_container_network_mode": "bridge",
"__meta_docker_network_id": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.17.0.2",
"__meta_docker_network_name": "bridge",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "8080",
"__meta_docker_port_public": "18081",
"__meta_docker_port_public_ip": "0.0.0.0",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
labelsMap := addContainersLabels([]container{tt.c}, networkLabels, 80)
if (err != nil) != tt.wantErr {
t.Errorf("addContainersLabels() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(labelsMap, tt.want) {
t.Errorf("addContainersLabels() \ngot %v, \nwant %v", labelsMap, tt.want)
}
})
}
}

30
lib/promscrape/discovery/dockerswarm/dockerswarm.go → lib/promscrape/discovery/docker/docker.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"fmt"
@ -7,19 +7,18 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
)
// SDConfig represents docker swarm service discovery configuration
// DockerSDConfig defines the `docker_sd` section for Docker based discovery
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type SDConfig struct {
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config
type DockerSDConfig struct {
Host string `yaml:"host"`
Role string `yaml:"role"`
Port int `yaml:"port,omitempty"`
Filters []Filter `yaml:"filters,omitempty"`
HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
ProxyURL proxy.URL `yaml:"proxy_url,omitempty"`
ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
// refresh_interval is obtained from `-promscrape.dockerSDCheckInterval` command-line option
}
// Filter is a filter, which can be passed to SDConfig.
@ -28,25 +27,16 @@ type Filter struct {
Values []string `yaml:"values"`
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
// GetLabels returns docker labels according to sdc.
func (sdc *DockerSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSDConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
return getContainersLabels(cfg)
}
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
func (sdc *DockerSDConfig) MustStop() {
configMap.Delete(sdc)
}

37
lib/promscrape/discovery/docker/dockerswarm.go Normal file
View file

@ -0,0 +1,37 @@
package docker
import (
"fmt"
)
// DockerSwarmSDConfig represents docker swarm service discovery configuration
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type DockerSwarmSDConfig struct {
Role string `yaml:"role"`
DockerSDConfig `yaml:",inline"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
}
// GetLabels returns dockerswarm labels according to sdc.
func (sdc *DockerSwarmSDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfigFromDockerSwarmSDConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
}
// MustStop stops further usage for sdc.
func (sdc *DockerSwarmSDConfig) MustStop() {
configMap.Delete(sdc)
}

22
lib/promscrape/discovery/dockerswarm/network.go → lib/promscrape/discovery/docker/network.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -18,18 +18,18 @@ type network struct {
Labels map[string]string
}
func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, error) {
func getNetworksLabels(cfg *apiConfig, labelPrefix string) (map[string]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return getNetworkLabelsByNetworkID(networks), nil
return getNetworkLabelsGroupByNetworkID(networks, labelPrefix), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
resp, err := cfg.getAPIResponse("/networks")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
return nil, fmt.Errorf("cannot query docker/dockerswarm api for networks: %w", err)
}
return parseNetworks(resp)
}
@ -42,18 +42,18 @@ func parseNetworks(data []byte) ([]network, error) {
return networks, nil
}
func getNetworkLabelsByNetworkID(networks []network) map[string]map[string]string {
func getNetworkLabelsGroupByNetworkID(networks []network, labelPrefix string) map[string]map[string]string {
ms := make(map[string]map[string]string)
for _, network := range networks {
m := map[string]string{
"__meta_dockerswarm_network_id": network.ID,
"__meta_dockerswarm_network_name": network.Name,
"__meta_dockerswarm_network_internal": strconv.FormatBool(network.Internal),
"__meta_dockerswarm_network_ingress": strconv.FormatBool(network.Ingress),
"__meta_dockerswarm_network_scope": network.Scope,
labelPrefix + "network_id": network.ID,
labelPrefix + "network_name": network.Name,
labelPrefix + "network_internal": strconv.FormatBool(network.Internal),
labelPrefix + "network_ingress": strconv.FormatBool(network.Ingress),
labelPrefix + "network_scope": network.Scope,
}
for k, v := range network.Labels {
m["__meta_dockerswarm_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
m[labelPrefix+"network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms[network.ID] = m
}

4
lib/promscrape/discovery/dockerswarm/network_test.go → lib/promscrape/discovery/docker/network_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"
@ -46,7 +46,7 @@ func Test_addNetworkLabels(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := getNetworkLabelsByNetworkID(tt.args.networks)
got := getNetworkLabelsGroupByNetworkID(tt.args.networks, "__meta_dockerswarm_")
var networkIDs []string
for networkID := range got {
networkIDs = append(networkIDs, networkID)

2
lib/promscrape/discovery/dockerswarm/nodes.go → lib/promscrape/discovery/docker/nodes.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"

2
lib/promscrape/discovery/dockerswarm/nodes_test.go → lib/promscrape/discovery/docker/nodes_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

4
lib/promscrape/discovery/dockerswarm/services.go → lib/promscrape/discovery/docker/services.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -51,7 +51,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networksLabels, err := getNetworksLabelsByNetworkID(cfg)
networksLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/dockerswarm/services_test.go → lib/promscrape/discovery/docker/services_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

5
lib/promscrape/discovery/dockerswarm/tasks.go → lib/promscrape/discovery/docker/tasks.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"encoding/json"
@ -7,7 +7,6 @@ import (
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
@ -45,7 +44,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networkLabels, err := getNetworksLabelsByNetworkID(cfg)
networkLabels, err := getNetworksLabels(cfg, "__meta_dockerswarm_")
if err != nil {
return nil, err
}

2
lib/promscrape/discovery/dockerswarm/tasks_test.go → lib/promscrape/discovery/docker/tasks_test.go
View file

@ -1,4 +1,4 @@
package dockerswarm
package docker
import (
"reflect"

4
lib/promscrape/scraper.go
View file

@ -39,6 +39,9 @@ var (
gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+
"This works only if gce_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details")
dockerSDCheckInterval = flag.Duration("promscrape.dockerSDCheckInterval", 30*time.Second, "Interval for checking for changes in docker. "+
"This works only if docker_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details")
dockerswarmSDCheckInterval = flag.Duration("promscrape.dockerswarmSDCheckInterval", 30*time.Second, "Interval for checking for changes in dockerswarm. "+
"This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details")
@ -114,6 +117,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
scs.add("docker_sd_configs", *dockerSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSDScrapeWork(swsPrev) })
scs.add("dockerswarm_sd_configs", *dockerswarmSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
scs.add("digitalocean_sd_configs", *digitaloceanSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDigitalOceanDScrapeWork(swsPrev) })
scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) })