github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape: code prettifying after 9bd9f67718

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-10-12 16:12:36 +03:00
parent 7f96712b38
commit 938b3b7ed1
14 changed files with 150 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/vmagent/README.md
View file

@ -151,7 +151,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
* `openstack_sd_configs` - for scraping OpenStack targets.
See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
* `dockerswarm_sd_configs` - for scraping dockerswarm targets.
* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.

1
docs/Single-server-VictoriaMetrics.md
View file

@ -295,6 +295,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
In the future other `*_sd_config` types will be supported.

2
docs/vmagent.md
View file

@ -151,6 +151,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
* `openstack_sd_configs` - for scraping OpenStack targets.
See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.

4
lib/promscrape/discovery/dockerswarm/api.go
View file

@ -26,11 +26,11 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
}
config, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
if err != nil {
return nil, err
}
client, err := discoveryutils.NewClient(sdc.Host, config)
client, err := discoveryutils.NewClient(sdc.Host, ac)
if err != nil {
return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.Host, err)
}

19
lib/promscrape/discovery/dockerswarm/dockerswarm.go
View file

@ -11,28 +11,17 @@ import (
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type SDConfig struct {
Host string `yaml:"host"`
// TODO: add support for proxy_url
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
Role string `yaml:"role"`
Port int `yaml:"port"`
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth"`
BearerToken string `yaml:"bearer_token"`
BearerTokenFile string `yaml:"bearer_token_file"`
}
// joinLabels adds labels to destination from source with given key from destination matching given value.
func joinLabels(source []map[string]string, destination map[string]string, key, value string) map[string]string {
for _, sourceLabels := range source {
if sourceLabels[key] == value {
for k, v := range sourceLabels {
destination[k] = v
}
return destination
}
}
return destination
}
// GetLabels returns gce labels according to sdc.
// GetLabels returns dockerswarm labels according to sdc.
func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
if err != nil {

12
lib/promscrape/discovery/dockerswarm/network.go
View file

@ -18,12 +18,12 @@ type network struct {
Labels map[string]string
}
func getNetworksLabels(cfg *apiConfig) ([]map[string]string, error) {
func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return addNetworkLabels(networks), nil
return getNetworkLabelsByNetworkID(networks), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
@ -42,20 +42,20 @@ func parseNetworks(data []byte) ([]network, error) {
return networks, nil
}
func addNetworkLabels(networks []network) []map[string]string {
var ms []map[string]string
func getNetworkLabelsByNetworkID(networks []network) map[string]map[string]string {
ms := make(map[string]map[string]string)
for _, network := range networks {
m := map[string]string{
"__meta_dockerswarm_network_id": network.ID,
"__meta_dockerswarm_network_name": network.Name,
"__meta_dockerswarm_network_scope": network.Scope,
"__meta_dockerswarm_network_internal": strconv.FormatBool(network.Internal),
"__meta_dockerswarm_network_ingress": strconv.FormatBool(network.Ingress),
"__meta_dockerswarm_network_scope": network.Scope,
}
for k, v := range network.Labels {
m["__meta_dockerswarm_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms = append(ms, m)
ms[network.ID] = m
}
return ms
}

11
lib/promscrape/discovery/dockerswarm/network_test.go
View file

@ -2,6 +2,7 @@ package dockerswarm
import (
"reflect"
"sort"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@ -45,9 +46,15 @@ func Test_addNetworkLabels(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := addNetworkLabels(tt.args.networks)
got := getNetworkLabelsByNetworkID(tt.args.networks)
var networkIDs []string
for networkID := range got {
networkIDs = append(networkIDs, networkID)
}
sort.Strings(networkIDs)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
for _, networkID := range networkIDs {
labels := got[networkID]
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {

12
lib/promscrape/discovery/dockerswarm/nodes.go
View file

@ -30,7 +30,7 @@ type node struct {
Message string
Addr string
}
ManagerStatus *struct {
ManagerStatus struct {
Leader bool
Reachability string
Addr string
@ -66,21 +66,19 @@ func addNodeLabels(nodes []node, port int) []map[string]string {
for _, node := range nodes {
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(node.Status.Addr, port),
"__meta_dockerswarm_node_id": node.ID,
"__meta_dockerswarm_node_address": node.Status.Addr,
"__meta_dockerswarm_node_availability": node.Spec.Availability,
"__meta_dockerswarm_node_engine_version": node.Description.Engine.EngineVersion,
"__meta_dockerswarm_node_hostname": node.Description.Hostname,
"__meta_dockerswarm_node_id": node.ID,
"__meta_dockerswarm_node_manager_address": node.ManagerStatus.Addr,
"__meta_dockerswarm_node_manager_leader": fmt.Sprintf("%t", node.ManagerStatus.Leader),
"__meta_dockerswarm_node_manager_reachability": node.ManagerStatus.Reachability,
"__meta_dockerswarm_node_platform_architecture": node.Description.Platform.Architecture,
"__meta_dockerswarm_node_platform_os": node.Description.Platform.OS,
"__meta_dockerswarm_node_role": node.Spec.Role,
"__meta_dockerswarm_node_status": node.Status.State,
}
if node.ManagerStatus != nil {
m["__meta_dockerswarm_node_manager_address"] = node.ManagerStatus.Addr
m["__meta_dockerswarm_node_manager_manager_reachability"] = node.ManagerStatus.Reachability
m["__meta_dockerswarm_node_manager_leader"] = fmt.Sprintf("%t", node.ManagerStatus.Leader)
}
for k, v := range node.Spec.Labels {
m["__meta_dockerswarm_node_label_"+discoveryutils.SanitizeLabelName(k)] = v
}

3
lib/promscrape/discovery/dockerswarm/nodes_test.go
View file

@ -161,6 +161,9 @@ func Test_addNodeLabels(t *testing.T) {
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_manager_address": "",
"__meta_dockerswarm_node_manager_leader": "false",
"__meta_dockerswarm_node_manager_reachability": "",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",

48
lib/promscrape/discovery/dockerswarm/services.go
View file

@ -27,7 +27,7 @@ type service struct {
Replicated interface{}
}
}
UpdateStatus *struct {
UpdateStatus struct {
State string
}
Endpoint struct {
@ -51,7 +51,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networksLabels, err := getNetworksLabels(cfg)
networksLabels, err := getNetworksLabelsByNetworkID(cfg)
if err != nil {
return nil, err
}
@ -84,56 +84,58 @@ func getServiceMode(svc service) string {
return ""
}
func addServicesLabels(services []service, networksLabels []map[string]string, port int) []map[string]string {
func addServicesLabels(services []service, networksLabels map[string]map[string]string, port int) []map[string]string {
var ms []map[string]string
for _, service := range services {
m := map[string]string{
commonLabels := map[string]string{
"__meta_dockerswarm_service_id": service.ID,
"__meta_dockerswarm_service_name": service.Spec.Name,
"__meta_dockerswarm_service_mode": getServiceMode(service),
"__meta_dockerswarm_service_task_container_hostname": service.Spec.TaskTemplate.ContainerSpec.Hostname,
"__meta_dockerswarm_service_task_container_image": service.Spec.TaskTemplate.ContainerSpec.Image,
"__meta_dockerswarm_service_mode": getServiceMode(service),
}
if service.UpdateStatus != nil {
m["__meta_dockerswarm_service_updating_status"] = service.UpdateStatus.State
"__meta_dockerswarm_service_updating_status": service.UpdateStatus.State,
}
for k, v := range service.Spec.Labels {
m["__meta_dockerswarm_service_label_"+discoveryutils.SanitizeLabelName(k)] = v
commonLabels["__meta_dockerswarm_service_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for _, vip := range service.Endpoint.VirtualIPs {
var added bool
ip, _, err := net.ParseCIDR(vip.Addr)
if err != nil {
logger.Errorf("cannot parse: %q as cidr for service label add, err: %v", vip.Addr, err)
continue
}
added := false
for _, ep := range service.Endpoint.Ports {
if ep.Protocol != "tcp" {
continue
}
lbls := map[string]string{
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort),
"__meta_dockerswarm_service_endpoint_port_name": ep.Name,
"__meta_dockerswarm_service_endpoint_port_publish_mode": ep.PublishMode,
"__address__": discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort),
}
for k, v := range m {
lbls[k] = v
for k, v := range commonLabels {
m[k] = v
}
for k, v := range networksLabels[vip.NetworkID] {
m[k] = v
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", vip.NetworkID)
added = true
ms = append(ms, lbls)
ms = append(ms, m)
}
if !added {
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(ip.String(), port),
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", vip.NetworkID)
lbls["__address__"] = discoveryutils.JoinHostPort(ip.String(), port)
ms = append(ms, lbls)
for k, v := range commonLabels {
m[k] = v
}
for k, v := range networksLabels[vip.NetworkID] {
m[k] = v
}
ms = append(ms, m)
}
}
}
return ms
}

7
lib/promscrape/discovery/dockerswarm/services_test.go
View file

@ -172,7 +172,7 @@ func Test_parseServicesResponse(t *testing.T) {
func Test_addServicesLabels(t *testing.T) {
type args struct {
services []service
networksLabels []map[string]string
networksLabels map[string]map[string]string
port int
}
tests := []struct {
@ -184,8 +184,8 @@ func Test_addServicesLabels(t *testing.T) {
name: "add 2 services with network labels join",
args: args{
port: 9100,
networksLabels: []map[string]string{
{
networksLabels: map[string]map[string]string{
"qs0hog6ldlei9ct11pr3c77v1": {
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
@ -275,6 +275,7 @@ func Test_addServicesLabels(t *testing.T) {
"__meta_dockerswarm_service_name": "redis2",
"__meta_dockerswarm_service_task_container_hostname": "node1",
"__meta_dockerswarm_service_task_container_image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
"__meta_dockerswarm_service_updating_status": "",
})},
},
}

85
lib/promscrape/discovery/dockerswarm/tasks.go
View file

@ -26,7 +26,7 @@ type task struct {
}
Status struct {
State string
ContainerStatus *struct {
ContainerStatus struct {
ContainerID string
}
PortStatus struct {
@ -45,7 +45,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
if err != nil {
return nil, err
}
networkLabels, err := getNetworksLabels(cfg)
networkLabels, err := getNetworksLabelsByNetworkID(cfg)
if err != nil {
return nil, err
}
@ -73,20 +73,18 @@ func parseTasks(data []byte) ([]task, error) {
return tasks, nil
}
func addTasksLabels(tasks []task, nodesLabels, servicesLabels, networksLabels []map[string]string, services []service, port int) []map[string]string {
func addTasksLabels(tasks []task, nodesLabels, servicesLabels []map[string]string, networksLabels map[string]map[string]string, services []service, port int) []map[string]string {
var ms []map[string]string
for _, task := range tasks {
m := map[string]string{
commonLabels := map[string]string{
"__meta_dockerswarm_task_id": task.ID,
"__meta_dockerswarm_task_container_id": task.Status.ContainerStatus.ContainerID,
"__meta_dockerswarm_task_desired_state": task.DesiredState,
"__meta_dockerswarm_task_state": task.Status.State,
"__meta_dockerswarm_task_slot": strconv.Itoa(task.Slot),
}
if task.Status.ContainerStatus != nil {
m["__meta_dockerswarm_task_container_id"] = task.Status.ContainerStatus.ContainerID
"__meta_dockerswarm_task_state": task.Status.State,
}
for k, v := range task.Labels {
m["__meta_dockerswarm_task_label_"+discoveryutils.SanitizeLabelName(k)] = v
commonLabels["__meta_dockerswarm_task_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
var svcPorts []portConfig
for i, v := range services {
@ -95,20 +93,21 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels, networksLabels []
break
}
}
m = joinLabels(servicesLabels, m, "__meta_dockerswarm_service_id", task.ServiceID)
m = joinLabels(nodesLabels, m, "__meta_dockerswarm_node_id", task.NodeID)
addLabels(commonLabels, servicesLabels, "__meta_dockerswarm_service_id", task.ServiceID)
addLabels(commonLabels, nodesLabels, "__meta_dockerswarm_node_id", task.NodeID)
for _, port := range task.Status.PortStatus.Ports {
if port.Protocol != "tcp" {
continue
}
lbls := make(map[string]string, len(m))
lbls["__meta_dockerswarm_task_port_publish_mode"] = port.PublishMode
lbls["__address__"] = discoveryutils.JoinHostPort(m["__meta_dockerswarm_node_address"], port.PublishedPort)
for k, v := range m {
lbls[k] = v
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(commonLabels["__meta_dockerswarm_node_address"], port.PublishedPort),
"__meta_dockerswarm_task_port_publish_mode": port.PublishMode,
}
ms = append(ms, lbls)
for k, v := range commonLabels {
m[k] = v
}
ms = append(ms, m)
}
for _, na := range task.NetworksAttachments {
for _, address := range na.Addresses {
@ -117,33 +116,51 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels, networksLabels []
logger.Errorf("cannot parse task network attachments address: %s as net CIDR: %v", address, err)
continue
}
var added bool
for _, v := range svcPorts {
if v.Protocol != "tcp" {
added := false
for _, ep := range svcPorts {
if ep.Protocol != "tcp" {
continue
}
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
m := map[string]string{
"__address": discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort),
"__meta_dockerswarm_task_port_publish_mode": ep.PublishMode,
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", na.Network.ID)
lbls["__address"] = discoveryutils.JoinHostPort(ip.String(), v.PublishedPort)
lbls["__meta_dockerswarm_task_port_publish_mode"] = v.PublishMode
ms = append(ms, lbls)
for k, v := range commonLabels {
m[k] = v
}
for k, v := range networksLabels[na.Network.ID] {
m[k] = v
}
ms = append(ms, m)
added = true
}
if !added {
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(ip.String(), port),
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", na.Network.ID)
lbls["__address__"] = discoveryutils.JoinHostPort(ip.String(), port)
ms = append(ms, lbls)
for k, v := range commonLabels {
m[k] = v
}
for k, v := range networksLabels[na.Network.ID] {
m[k] = v
}
ms = append(ms, m)
}
}
}
}
return ms
}
// addLabels adds lables from src to dst if they contain the given `key: value` pair.
func addLabels(dst map[string]string, src []map[string]string, key, value string) {
for _, m := range src {
if m[key] != value {
continue
}
for k, v := range m {
dst[k] = v
}
return
}
}

29
lib/promscrape/discovery/dockerswarm/tasks_test.go
View file

@ -27,7 +27,9 @@ func Test_parseTasks(t *testing.T) {
"Version": {
"Index": 23
},
"Labels": {},
"Labels": {
"label1": "value1"
},
"Spec": {
"ContainerSpec": {
"Image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
@ -68,16 +70,18 @@ func Test_parseTasks(t *testing.T) {
ID: "t4rdm7j2y9yctbrksiwvsgpu5",
ServiceID: "t91nf284wzle1ya09lqvyjgnq",
NodeID: "qauwmifceyvqs0sipvzu8oslu",
Labels: map[string]string{},
Labels: map[string]string{
"label1": "value1",
},
DesiredState: "running",
Slot: 1,
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
ContainerStatus struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerStatus: struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{}},
@ -104,7 +108,7 @@ func Test_addTasksLabels(t *testing.T) {
tasks []task
nodesLabels []map[string]string
servicesLabels []map[string]string
networksLabels []map[string]string
networksLabels map[string]map[string]string
services []service
port int
}
@ -127,11 +131,11 @@ func Test_addTasksLabels(t *testing.T) {
Slot: 1,
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
ContainerStatus struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerStatus: struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{
@ -208,18 +212,18 @@ func Test_addTasksLabels(t *testing.T) {
},
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
ContainerStatus struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerStatus: struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{}},
},
},
networksLabels: []map[string]string{
{
networksLabels: map[string]map[string]string{
"qs0hog6ldlei9ct11pr3c77v1": {
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
@ -288,7 +292,8 @@ func Test_addTasksLabels(t *testing.T) {
NetworkID string
Addr string
}
}{Ports: []portConfig{
}{
Ports: []portConfig{
{
Protocol: "tcp",
Name: "redis",

4
lib/promscrape/discovery/openstack/api.go
View file

@ -75,12 +75,12 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
port: sdc.Port,
}
if sdc.TLSConfig != nil {
config, err := promauth.NewConfig(baseDir, nil, "", "", sdc.TLSConfig)
ac, err := promauth.NewConfig(baseDir, nil, "", "", sdc.TLSConfig)
if err != nil {
return nil, err
}
cfg.client.Transport = &http.Transport{
TLSClientConfig: config.NewTLSConfig(),
TLSClientConfig: ac.NewTLSConfig(),
}
}
// use public compute endpoint by default