From 8aee209c5387e47ed00aa748c3eb28b906ba114f Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 9 Jan 2023 09:07:40 +0100 Subject: [PATCH] lib/promscrape: remove `datacenter` field from nomad_sd_config (#3612) Looks like `datacenter` field isn't part of `/v1/services` API. See https://developer.hashicorp.com/nomad/api-docs/services#list-services and https://developer.hashicorp.com/nomad/api-docs/services#read-service Related issues: https://github.com/traefik/traefik/issues/9109 https://github.com/prometheus/prometheus/issues/11776 Signed-off-by: hagen1778 --- docs/sd_configs.md | 5 - lib/promscrape/discovery/nomad/agent.go | 28 -- lib/promscrape/discovery/nomad/agent_test.go | 477 ------------------- lib/promscrape/discovery/nomad/api.go | 22 +- lib/promscrape/discovery/nomad/nomad.go | 7 +- lib/promscrape/discovery/nomad/watch.go | 9 +- 6 files changed, 8 insertions(+), 540 deletions(-) delete mode 100644 lib/promscrape/discovery/nomad/agent.go delete mode 100644 lib/promscrape/discovery/nomad/agent_test.go diff --git a/docs/sd_configs.md b/docs/sd_configs.md index 2cfdc1bd1..c171bc8bc 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -888,11 +888,6 @@ scrape_configs: # If the token isn't specified, then it is read from the NOMAD_TOKEN environment var. # token: "..." - # datacenter is an optional Nomad API datacenter. - # If the datacenter isn't specified, then it is read from Nomad server. - # See https://developer.hashicorp.com/nomad/api-docs/agent#query-self - # datacenter: "..." - # namespace is an optional Nomad namespace. # If the namespace isn't specified, then it is read from NOMAD_NAMESPACE environment var. # namespace: "..." diff --git a/lib/promscrape/discovery/nomad/agent.go b/lib/promscrape/discovery/nomad/agent.go deleted file mode 100644 index 115a100c0..000000000 --- a/lib/promscrape/discovery/nomad/agent.go +++ /dev/null @@ -1,28 +0,0 @@ -package nomad - -import ( - "encoding/json" - "fmt" -) - -// Agent is Nomad agent. -// -// See https://developer.hashicorp.com/nomad/api-docs/agent -type Agent struct { - Config AgentConfig -} - -// AgentConfig is Nomad agent config. -// -// See https://developer.hashicorp.com/nomad/api-docs/agent -type AgentConfig struct { - Datacenter string -} - -func parseAgent(data []byte) (*Agent, error) { - var a Agent - if err := json.Unmarshal(data, &a); err != nil { - return nil, fmt.Errorf("cannot unmarshal agent info from %q: %w", data, err) - } - return &a, nil -} diff --git a/lib/promscrape/discovery/nomad/agent_test.go b/lib/promscrape/discovery/nomad/agent_test.go deleted file mode 100644 index 77e086736..000000000 --- a/lib/promscrape/discovery/nomad/agent_test.go +++ /dev/null @@ -1,477 +0,0 @@ -package nomad - -import ( - "reflect" - "testing" -) - -func TestParseAgentFailure(t *testing.T) { - f := func(s string) { - t.Helper() - a, err := parseAgent([]byte(s)) - if err == nil { - t.Fatalf("expecting non-nil error") - } - if a != nil { - t.Fatalf("unexpected non-nil Agent: %v", a) - } - } - f(``) - f(`[1,23]`) -} - -func TestParseAgentSuccess(t *testing.T) { - data := ` - { - "config": { - "ACL": { - "Enabled": false, - "PolicyTTL": 30000000000, - "ReplicationToken": "", - "RoleTTL": 30000000000, - "TokenMaxExpirationTTL": 0, - "TokenMinExpirationTTL": 0, - "TokenTTL": 30000000000 - }, - "Addresses": { - "HTTP": "0.0.0.0", - "RPC": "0.0.0.0", - "Serf": "0.0.0.0" - }, - "AdvertiseAddrs": { - "HTTP": "192.168.29.76:4646", - "RPC": "192.168.29.76:4647", - "Serf": "192.168.29.76:4648" - }, - "Audit": { - "Enabled": null, - "Filters": null, - "Sinks": null - }, - "Autopilot": { - "CleanupDeadServers": null, - "DisableUpgradeMigration": null, - "EnableCustomUpgrades": null, - "EnableRedundancyZones": null, - "LastContactThreshold": 200000000, - "MaxTrailingLogs": 250, - "MinQuorum": 0, - "ServerStabilizationTime": 10000000000 - }, - "BindAddr": "0.0.0.0", - "Client": { - "AllocDir": "", - "Artifact": { - "GCSTimeout": "30m", - "GitTimeout": "30m", - "HTTPMaxSize": "100GB", - "HTTPReadTimeout": "30m", - "HgTimeout": "30m", - "S3Timeout": "30m" - }, - "BindWildcardDefaultHostNetwork": true, - "BridgeNetworkName": "", - "BridgeNetworkSubnet": "", - "CNIConfigDir": "/opt/cni/config", - "CNIPath": "/opt/cni/bin", - "CgroupParent": "", - "ChrootEnv": { - "/run/resolvconf": "/run/resolvconf", - "/sbin": "/sbin", - "/usr": "/usr", - "/bin/": "/bin/", - "/etc/": "/etc/", - "/lib": "/lib", - "/lib32": "/lib32", - "/lib64": "/lib64" - }, - "ClientMaxPort": 14512, - "ClientMinPort": 14000, - "CpuCompute": 0, - "DisableRemoteExec": false, - "Enabled": true, - "GCDiskUsageThreshold": 80, - "GCInodeUsageThreshold": 70, - "GCInterval": 60000000000, - "GCMaxAllocs": 50, - "GCParallelDestroys": 2, - "HostNetworks": null, - "HostVolumes": null, - "MaxDynamicPort": 32000, - "MaxKillTimeout": "30s", - "MemoryMB": 0, - "Meta": { - "stack": "zerodha", - "env": "dev" - }, - "MinDynamicPort": 20000, - "NetworkInterface": "", - "NetworkSpeed": 0, - "NoHostUUID": true, - "NodeClass": "", - "NomadServiceDiscovery": true, - "Options": {}, - "ReserveableCores": "", - "Reserved": { - "CPU": 0, - "Cores": "2", - "DiskMB": 1024, - "MemoryMB": 1024, - "ReservedPorts": "22" - }, - "ServerJoin": { - "RetryInterval": 30000000000, - "RetryJoin": [], - "RetryMaxAttempts": 0, - "StartJoin": null - }, - "Servers": null, - "StateDir": "", - "TemplateConfig": { - "BlockQueryWaitTime": null, - "BlockQueryWaitTimeHCL": "", - "ConsulRetry": null, - "DisableSandbox": false, - "FunctionBlacklist": null, - "FunctionDenylist": [ - "plugin", - "writeToFile" - ], - "MaxStale": null, - "MaxStaleHCL": "", - "NomadRetry": null, - "VaultRetry": null - } - }, - "Consul": { - "Addr": "127.0.0.1:8500", - "AllowUnauthenticated": true, - "Auth": "", - "AutoAdvertise": true, - "CAFile": "", - "CertFile": "", - "ChecksUseAdvertise": false, - "ClientAutoJoin": true, - "ClientHTTPCheckName": "Nomad Client HTTP Check", - "ClientServiceName": "nomad-client", - "EnableSSL": false, - "GRPCAddr": "", - "KeyFile": "", - "Namespace": "", - "ServerAutoJoin": true, - "ServerHTTPCheckName": "Nomad Server HTTP Check", - "ServerRPCCheckName": "Nomad Server RPC Check", - "ServerSerfCheckName": "Nomad Server Serf Check", - "ServerServiceName": "nomad", - "ShareSSL": null, - "Tags": null, - "Timeout": 5000000000, - "Token": "", - "VerifySSL": true - }, - "DataDir": "/opt/nomad/data", - "Datacenter": "dc1", - "DevMode": false, - "DisableAnonymousSignature": false, - "DisableUpdateCheck": false, - "EnableDebug": false, - "EnableSyslog": false, - "Files": [ - "nomad.hcl" - ], - "HTTPAPIResponseHeaders": {}, - "LeaveOnInt": false, - "LeaveOnTerm": false, - "Limits": { - "HTTPMaxConnsPerClient": 100, - "HTTPSHandshakeTimeout": "5s", - "RPCHandshakeTimeout": "5s", - "RPCMaxConnsPerClient": 100 - }, - "LogFile": "", - "LogJson": false, - "LogLevel": "INFO", - "LogRotateBytes": 0, - "LogRotateDuration": "", - "LogRotateMaxFiles": 0, - "NodeName": "", - "PluginDir": "/opt/nomad/data/plugins", - "Plugins": [ - { - "Args": null, - "Config": { - "allow_privileged": true, - "volumes": [ - { - "enabled": true - } - ], - "extra_labels": [ - "job_name", - "job_id", - "task_group_name", - "task_name", - "namespace", - "node_name", - "node_id" - ] - }, - "Name": "docker" - }, - { - "Args": null, - "Config": { - "enabled": true, - "no_cgroups": true - }, - "Name": "raw_exec" - } - ], - "Ports": { - "HTTP": 4646, - "RPC": 4647, - "Serf": 4648 - }, - "Region": "global", - "Sentinel": { - "Imports": null - }, - "Server": { - "ACLTokenGCThreshold": "", - "AuthoritativeRegion": "", - "BootstrapExpect": 1, - "CSIPluginGCThreshold": "", - "CSIVolumeClaimGCThreshold": "", - "DataDir": "", - "DefaultSchedulerConfig": null, - "DeploymentGCThreshold": "", - "DeploymentQueryRateLimit": 0, - "EnableEventBroker": true, - "Enabled": true, - "EnabledSchedulers": null, - "EvalGCThreshold": "", - "EventBufferSize": 100, - "FailoverHeartbeatTTL": 0, - "HeartbeatGrace": 0, - "JobGCInterval": "", - "JobGCThreshold": "", - "LicenseEnv": "", - "LicensePath": "", - "MaxHeartbeatsPerSecond": 0, - "MinHeartbeatTTL": 0, - "NodeGCThreshold": "", - "NonVotingServer": false, - "NumSchedulers": null, - "PlanRejectionTracker": { - "Enabled": false, - "NodeThreshold": 100, - "NodeWindow": 300000000000 - }, - "RaftBoltConfig": null, - "RaftMultiplier": null, - "RaftProtocol": 3, - "RedundancyZone": "", - "RejoinAfterLeave": false, - "RetryInterval": 0, - "RetryJoin": [], - "RetryMaxAttempts": 0, - "RootKeyGCInterval": "", - "RootKeyGCThreshold": "", - "RootKeyRotationThreshold": "", - "Search": { - "FuzzyEnabled": true, - "LimitQuery": 20, - "LimitResults": 100, - "MinTermLength": 2 - }, - "ServerJoin": { - "RetryInterval": 30000000000, - "RetryJoin": [], - "RetryMaxAttempts": 0, - "StartJoin": null - }, - "StartJoin": [], - "UpgradeVersion": "" - }, - "SyslogFacility": "LOCAL0", - "TLSConfig": { - "CAFile": "", - "CertFile": "", - "Checksum": "", - "EnableHTTP": false, - "EnableRPC": false, - "KeyFile": "", - "KeyLoader": {}, - "RPCUpgradeMode": false, - "TLSCipherSuites": "", - "TLSMinVersion": "", - "TLSPreferServerCipherSuites": false, - "VerifyHTTPSClient": false, - "VerifyServerHostname": false - }, - "Telemetry": { - "CirconusAPIApp": "", - "CirconusAPIToken": "", - "CirconusAPIURL": "", - "CirconusBrokerID": "", - "CirconusBrokerSelectTag": "", - "CirconusCheckDisplayName": "", - "CirconusCheckForceMetricActivation": "", - "CirconusCheckID": "", - "CirconusCheckInstanceID": "", - "CirconusCheckSearchTag": "", - "CirconusCheckSubmissionURL": "", - "CirconusCheckTags": "", - "CirconusSubmissionInterval": "", - "CollectionInterval": "1s", - "DataDogAddr": "", - "DataDogTags": null, - "DisableDispatchedJobSummaryMetrics": false, - "DisableHostname": false, - "FilterDefault": null, - "PrefixFilter": null, - "PrometheusMetrics": false, - "PublishAllocationMetrics": false, - "PublishNodeMetrics": false, - "StatsdAddr": "", - "StatsiteAddr": "", - "UseNodeName": false - }, - "UI": { - "Consul": { - "BaseUIURL": "" - }, - "Enabled": true, - "Vault": { - "BaseUIURL": "" - } - }, - "Vault": { - "Addr": "https://vault.service.consul:8200", - "AllowUnauthenticated": true, - "ConnectionRetryIntv": 30000000000, - "Enabled": null, - "Namespace": "", - "Role": "", - "TLSCaFile": "", - "TLSCaPath": "", - "TLSCertFile": "", - "TLSKeyFile": "", - "TLSServerName": "", - "TLSSkipVerify": null, - "TaskTokenTTL": "", - "Token": "" - }, - "Version": { - "Revision": "f464aca721d222ae9c1f3df643b3c3aaa20e2da7", - "Version": "1.4.3", - "VersionMetadata": "", - "VersionPrerelease": "" - } - }, - "member": { - "Addr": "192.168.29.76", - "DelegateCur": 4, - "DelegateMax": 5, - "DelegateMin": 2, - "Name": "pop-os.global", - "Port": 4648, - "ProtocolCur": 2, - "ProtocolMax": 5, - "ProtocolMin": 1, - "Status": "alive", - "Tags": { - "rpc_addr": "192.168.29.76", - "bootstrap": "1", - "expect": "1", - "role": "nomad", - "region": "global", - "build": "1.4.3", - "revision": "f464aca721d222ae9c1f3df643b3c3aaa20e2da7", - "port": "4647", - "vsn": "1", - "dc": "dc1", - "raft_vsn": "3", - "id": "d78cdda9-7e35-48d0-a0e0-36041f6df0ec" - } - }, - "stats": { - "nomad": { - "leader": "true", - "leader_addr": "192.168.29.76:4647", - "bootstrap": "true", - "known_regions": "1", - "server": "true" - }, - "raft": { - "term": "3", - "latest_configuration_index": "0", - "applied_index": "384", - "protocol_version_min": "0", - "protocol_version_max": "3", - "snapshot_version_min": "0", - "num_peers": "0", - "state": "Leader", - "last_log_term": "3", - "commit_index": "384", - "last_contact": "0", - "last_snapshot_index": "0", - "protocol_version": "3", - "snapshot_version_max": "1", - "latest_configuration": "[{Suffrage:Voter ID:d78cdda9-7e35-48d0-a0e0-36041f6df0ec Address:172.20.10.3:4647}]", - "last_log_index": "384", - "fsm_pending": "0", - "last_snapshot_term": "0" - }, - "serf": { - "coordinate_resets": "0", - "health_score": "0", - "member_time": "1", - "query_time": "1", - "intent_queue": "0", - "event_queue": "0", - "encrypted": "false", - "members": "1", - "failed": "0", - "left": "0", - "event_time": "1", - "query_queue": "0" - }, - "runtime": { - "arch": "amd64", - "version": "go1.19.3", - "max_procs": "8", - "goroutines": "294", - "cpu_count": "8", - "kernel.name": "linux" - }, - "vault": { - "tracked_for_revoked": "0", - "token_ttl": "0s", - "token_expire_time": "", - "token_last_renewal_time": "", - "token_next_renewal_time": "" - }, - "client": { - "node_id": "9e02c85b-db59-45f1-ddee-40d0317bd33d", - "known_servers": "192.168.29.76:4647", - "num_allocations": "3", - "last_heartbeat": "13.254411467s", - "heartbeat_ttl": "19.86001456s" - } - } - } -` - a, err := parseAgent([]byte(data)) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - aExpected := &Agent{ - Config: AgentConfig{ - Datacenter: "dc1", - }, - } - if !reflect.DeepEqual(a, aExpected) { - t.Fatalf("unexpected Agent parsed;\ngot\n%v\nwant\n%v", a, aExpected) - } -} diff --git a/lib/promscrape/discovery/nomad/api.go b/lib/promscrape/discovery/nomad/api.go index b93029deb..c5ca5d182 100644 --- a/lib/promscrape/discovery/nomad/api.go +++ b/lib/promscrape/discovery/nomad/api.go @@ -72,10 +72,6 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { if sdc.TagSeparator != nil { tagSeparator = *sdc.TagSeparator } - dc, err := getDatacenter(client, sdc.Datacenter) - if err != nil { - return nil, fmt.Errorf("cannot obtain Nomad datacenter: %w", err) - } namespace := sdc.Namespace // default namespace can be detected from env var. @@ -83,7 +79,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { namespace = os.Getenv("NOMAD_NAMESPACE") } - nw := newNomadWatcher(client, sdc, dc, namespace) + nw := newNomadWatcher(client, sdc, namespace) cfg := &apiConfig{ tagSeparator: tagSeparator, nomadWatcher: nw, @@ -100,22 +96,6 @@ func getToken(token *promauth.Secret) string { return t } -func getDatacenter(client *discoveryutils.Client, dc string) (string, error) { - if dc != "" { - return dc, nil - } - // See https://developer.hashicorp.com/nomad/api-docs/agent#query-self - data, err := client.GetAPIResponse("/v1/agent/self") - if err != nil { - return "", fmt.Errorf("cannot query nomad agent info: %w", err) - } - a, err := parseAgent(data) - if err != nil { - return "", err - } - return a.Config.Datacenter, nil -} - // maxWaitTime is duration for Nomad blocking request. func maxWaitTime() time.Duration { d := discoveryutils.BlockingClientReadTimeout diff --git a/lib/promscrape/discovery/nomad/nomad.go b/lib/promscrape/discovery/nomad/nomad.go index 799fc80ed..9ffc11068 100644 --- a/lib/promscrape/discovery/nomad/nomad.go +++ b/lib/promscrape/discovery/nomad/nomad.go @@ -12,10 +12,9 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#nomad_sd_config type SDConfig struct { - Server string `yaml:"server,omitempty"` - Token *promauth.Secret `yaml:"token,omitempty"` - Datacenter string `yaml:"datacenter,omitempty"` - Namespace string `yaml:"namespace,omitempty"` + Server string `yaml:"server,omitempty"` + Token *promauth.Secret `yaml:"token,omitempty"` + Namespace string `yaml:"namespace,omitempty"` // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.nomadSDCheckInterval` command-line option. Scheme string `yaml:"scheme,omitempty"` diff --git a/lib/promscrape/discovery/nomad/watch.go b/lib/promscrape/discovery/nomad/watch.go index 6534b236f..5a3610973 100644 --- a/lib/promscrape/discovery/nomad/watch.go +++ b/lib/promscrape/discovery/nomad/watch.go @@ -45,15 +45,14 @@ type serviceWatcher struct { } // newNomadWatcher creates new watcher and starts background service discovery for Nomad. -func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, datacenter, namespace string) *nomadWatcher { - baseQueryArgs := "?dc=" + url.QueryEscape(datacenter) +func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, namespace string) *nomadWatcher { + var serviceNodesQueryArgs string if sdc.AllowStale == nil || *sdc.AllowStale { - baseQueryArgs += "&stale" + serviceNodesQueryArgs += "&stale" } if namespace != "" { - baseQueryArgs += "&namespace=" + url.QueryEscape(namespace) + serviceNodesQueryArgs += "&namespace=" + url.QueryEscape(namespace) } - serviceNodesQueryArgs := baseQueryArgs cw := &nomadWatcher{ client: client,