lib/promscrape/discovery/nomad: sync nomad_sd_configs fields with the Prometheus implementation

See the list of configs supported by Prometheus at f88a0a7d83/discovery/nomad/nomad.go (L76-L84)

- Removed "token" option. In can be set either via NOMAD_TOKEN env var or via `bearer_token` config option.
- Removed "scheme" option. It is automatically detected depending on whether the `tls_config` is set.
- Removed "services" and "tags" options, since they aren't supported by Prometheus.
- Added "region" option. If it is missing, then the region is read from NOMAD_REGION env var.
  If this var is empty, then it is set to "global" in the same way as Nomad client does.
  See 865ee8d37c/api/api.go (L297)
  and 865ee8d37c/api/api.go (L555-L556)
- If the "server" option is missing, then it is read from NOMAD_ADDR in the same way
  as Nomad client does - see 865ee8d37c/api/api.go (L294-L296)

This is a follow-up for 8aee209c53

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3367

docs/sd_configs.md

@@ -881,29 +881,19 @@ scrape_configs:
 - job_name: nomad
   nomad_sd_configs:
 
-    # server is an optional Nomad server to connect to. By default localhost:4646 is used
+    # server is an optional Nomad server to connect to.
+    # If the server isn't specified, then it is read from NOMAD_ADDR environment var.
+    # If the NOMAD_ADDR environment var isn't set, then localhost:4646 is used.
   - server: "localhost:4646"
 
-    # token is an optional Nomad API token.
-    # If the token isn't specified, then it is read from the NOMAD_TOKEN environment var.
-    # token: "..."
-
     # namespace is an optional Nomad namespace.
     # If the namespace isn't specified, then it is read from NOMAD_NAMESPACE environment var.
     # namespace: "..."
 
-    # scheme is an optional scheme (http or https) to use for connecting to Nomad server.
+    # region is an optional Nomad region.
-    # By default http scheme is used.
+    # If the region isn't specified, then it is read from NOMAD_REGION environment var.
-    # scheme: "..."
+    # If NOMAD_REGION environment var isn't set, then "global" region is used
-
+    # region: "..."
-    # services is an optional list of services for which targets are retrieved.
-    # If omitted, all services are scraped.
-    # See https://developer.hashicorp.com/nomad/api-docs/services#list-services .
-    # services: ["...", "..."]
-
-    # tags is an optional list of tags used to filter nodes for a given service.
-    # Services must contain all tags in the list.
-    # tags: ["...", "..."]
 
     # tag_separator is an optional string by which Nomad tags are joined into the __meta_nomad_tags label.
     # By default "," is used as a tag separator.

lib/promscrape/discovery/consul/api.go

@@ -69,6 +69,9 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		scheme := sdc.Scheme
 		if scheme == "" {
 			scheme = "http"
+			if hcc.TLSConfig != nil {
+				scheme = "https"
+			}
 		}
 		apiServer = scheme + "://" + apiServer
 	}

lib/promscrape/discovery/nomad/api.go

@@ -38,10 +38,10 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 
 func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	hcc := sdc.HTTPClientConfig
-	token := getToken(sdc.Token)
+	token := os.Getenv("NOMAD_TOKEN")
 	if token != "" {
 		if hcc.BearerToken != nil {
-			return nil, fmt.Errorf("cannot set both token and bearer_token configs")
+			return nil, fmt.Errorf("cannot set both NOMAD_TOKEN and bearer_token")
 		}
 		hcc.BearerToken = promauth.NewSecret(token)
 	}
@@ -51,12 +51,15 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	}
 	apiServer := sdc.Server
 	if apiServer == "" {
-		apiServer = "localhost:4646"
+		apiServer = os.Getenv("NOMAD_ADDR")
+		if apiServer == "" {
+			apiServer = "localhost:4646"
+		}
 	}
 	if !strings.Contains(apiServer, "://") {
-		scheme := sdc.Scheme
+		scheme := "http"
-		if scheme == "" {
+		if hcc.TLSConfig != nil {
-			scheme = "http"
+			scheme = "https"
 		}
 		apiServer = scheme + "://" + apiServer
 	}
@@ -74,12 +77,19 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	}
 
 	namespace := sdc.Namespace
-	// default namespace can be detected from env var.
 	if namespace == "" {
 		namespace = os.Getenv("NOMAD_NAMESPACE")
 	}
 
-	nw := newNomadWatcher(client, sdc, namespace)
+	region := sdc.Region
+	if region == "" {
+		region = os.Getenv("NOMAD_REGION")
+		if region == "" {
+			region = "global"
+		}
+	}
+
+	nw := newNomadWatcher(client, sdc, namespace, region)
 	cfg := &apiConfig{
 		tagSeparator: tagSeparator,
 		nomadWatcher: nw,
@@ -87,15 +97,6 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	return cfg, nil
 }
 
-func getToken(token *promauth.Secret) string {
-	if token != nil {
-		return token.String()
-	}
-	t := os.Getenv("NOMAD_TOKEN")
-	// Allow empty token - it should work if ACL is disabled in Nomad.
-	return t
-}
-
 // maxWaitTime is duration for Nomad blocking request.
 func maxWaitTime() time.Duration {
 	d := discoveryutils.BlockingClientReadTimeout

lib/promscrape/discovery/nomad/nomad.go

@@ -12,16 +12,13 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#nomad_sd_config
 type SDConfig struct {
-	Server    string           `yaml:"server,omitempty"`
+	Server    string `yaml:"server,omitempty"`
-	Token     *promauth.Secret `yaml:"token,omitempty"`
+	Namespace string `yaml:"namespace,omitempty"`
-	Namespace string           `yaml:"namespace,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.nomadSDCheckInterval` command-line option.
-	Scheme       string   `yaml:"scheme,omitempty"`
+	Region       string  `yaml:"region,omitempty"`
-	Services     []string `yaml:"services,omitempty"`
+	TagSeparator *string `yaml:"tag_separator,omitempty"`
-	Tags         []string `yaml:"tags,omitempty"`
+	AllowStale   *bool   `yaml:"allow_stale,omitempty"`
-	TagSeparator *string  `yaml:"tag_separator,omitempty"`
-	AllowStale   *bool    `yaml:"allow_stale,omitempty"`
 
 	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
 	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`

lib/promscrape/discovery/nomad/watch.go

@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
-	"strings"
 	"sync"
 	"time"
 
@@ -26,8 +25,6 @@ type nomadWatcher struct {
 	client *discoveryutils.Client
 
 	serviceNamesQueryArgs string
-	watchServices         []string
-	watchTags             []string
 
 	// servicesLock protects services
 	servicesLock sync.Mutex
@@ -45,20 +42,25 @@ type serviceWatcher struct {
 }
 
 // newNomadWatcher creates new watcher and starts background service discovery for Nomad.
-func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, namespace string) *nomadWatcher {
+func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, namespace, region string) *nomadWatcher {
-	var serviceNodesQueryArgs string
+	var qa url.Values
 	if sdc.AllowStale == nil || *sdc.AllowStale {
-		serviceNodesQueryArgs += "&stale"
+		qa.Set("stale", "")
 	}
 	if namespace != "" {
-		serviceNodesQueryArgs += "&namespace=" + url.QueryEscape(namespace)
+		qa.Set("namespace", namespace)
+	}
+	if region != "" {
+		qa.Set("region", region)
+	}
+	queryArgs := qa.Encode()
+	if queryArgs != "" {
+		queryArgs = "?" + queryArgs
 	}
 
 	cw := &nomadWatcher{
 		client:                client,
-		serviceNamesQueryArgs: serviceNodesQueryArgs,
+		serviceNamesQueryArgs: queryArgs,
-		watchServices:         sdc.Services,
-		watchTags:             sdc.Tags,
 		services:              make(map[string]*serviceWatcher),
 		stopCh:                make(chan struct{}),
 		stoppedCh:             make(chan struct{}),
@@ -215,12 +217,6 @@ func (cw *nomadWatcher) getBlockingServiceNames(index int64) ([]string, int64, e
 	serviceNames := make([]string, 0, len(svcs))
 	for _, svc := range svcs {
 		for _, s := range svc.Services {
-			if !shouldCollectServiceByName(cw.watchServices, s.ServiceName) {
-				continue
-			}
-			if !shouldCollectServiceByTags(cw.watchTags, s.Tags) {
-				continue
-			}
 			serviceNames = append(serviceNames, s.ServiceName)
 		}
 	}
@@ -289,38 +285,6 @@ func (sw *serviceWatcher) watchForServiceAddressUpdates(nw *nomadWatcher, initWG
 	}
 }
 
-func shouldCollectServiceByName(filterServices []string, serviceName string) bool {
-	if len(filterServices) == 0 {
-		return true
-	}
-	for _, filterService := range filterServices {
-		// Use case-insensitive comparison for service names according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1422
-		if strings.EqualFold(filterService, serviceName) {
-			return true
-		}
-	}
-	return false
-}
-
-func shouldCollectServiceByTags(filterTags, tags []string) bool {
-	if len(filterTags) == 0 {
-		return true
-	}
-	for _, filterTag := range filterTags {
-		hasTag := false
-		for _, tag := range tags {
-			if tag == filterTag {
-				hasTag = true
-				break
-			}
-		}
-		if !hasTag {
-			return false
-		}
-	}
-	return true
-}
-
 func getCheckInterval() time.Duration {
 	d := *SDCheckInterval
 	if d <= time.Second {