app/{vminsert,vmagent}: hide passwords and auth tokens by default at /config page

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764

app/vmagent/remotewrite/client.go

@@ -165,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if username != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
-			Password:     password,
+			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
 	}
@@ -179,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if clientSecretFile != "" || clientSecret != "" {
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     clientSecret,
+			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),

app/vmalert/datasource/vm_test.go

@@ -20,7 +20,7 @@ var (
 	basicAuthPass = "bar"
 	baCfg         = &promauth.BasicAuthConfig{
 		Username: basicAuthName,
-		Password: basicAuthPass,
+		Password: promauth.NewSecret(basicAuthPass),
 	}
 	query       = "vm_rows"
 	queryRender = "constantLine(10)"

app/vmalert/utils/auth.go

@@ -10,7 +10,7 @@ func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*p
 	if baUser != "" || baPass != "" || baFile != "" {
 		baCfg = &promauth.BasicAuthConfig{
 			Username:     baUser,
-			Password:     baPass,
+			Password:     promauth.NewSecret(baPass),
 			PasswordFile: baFile,
 		}
 	}

docs/CHANGELOG.md

@@ -9,7 +9,8 @@ sort: 15
 * FEATURE: vmalert: allow groups with empty rules list like Prometheus does. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1742).
 * FEATURE: vmagent: add `collapse` and `expand` buttons per each group of targets with the same `job_name` at `http://vmagent:8429/targets` page.
 * FEATURE: automatically detect timestamp precision (ns, us, ms or s) for the data ingested into VictoriaMetrics via [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
-* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive information such as passwords, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
+* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive config information, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
+* FEATURE: vmagent: hide passwords and auth tokens at `/config` page like Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
 * FEATURE: vmagent: add `-promscrape.maxResponseHeadersSize` command-line flag for tuning the maximum HTTP response headers size for Prometheus scrape targets.
 * FEATURE: vmagent: send data to multiple configured remote storage systems in parallel (e.g. when multiple `-remoteWrite.url` flag values are specified). This should improve data ingestion speed.
 * FEATURE: vmagent: add `-remoteWrite.maxRowsPerBlock` command-line flag for tuning the number of samples to send to remote storage per each block. Bigger values may improve data ingestion performance at the cost of higher memory usage.

lib/promauth/config.go

@@ -17,6 +17,51 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 )
 
+// Secret represents a string secret such as password or auth token.
+//
+// It is marshaled to "<secret>" string in yaml.
+//
+// This is needed for hiding secret strings in /config page output.
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764
+type Secret struct {
+	s string
+}
+
+// NewSecret returns new secret for s.
+func NewSecret(s string) *Secret {
+	if s == "" {
+		return nil
+	}
+	return &Secret{
+		s: s,
+	}
+}
+
+// MarshalYAML implements yaml.Marshaler interface.
+//
+// It substitutes the secret with "<secret>" string.
+func (s *Secret) MarshalYAML() (interface{}, error) {
+	return "<secret>", nil
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler interface.
+func (s *Secret) UnmarshalYAML(f func(interface{}) error) error {
+	var secret string
+	if err := f(&secret); err != nil {
+		return fmt.Errorf("cannot parse secret: %w", err)
+	}
+	s.s = secret
+	return nil
+}
+
+// String returns the secret in plaintext.
+func (s *Secret) String() string {
+	if s == nil {
+		return ""
+	}
+	return s.s
+}
+
 // TLSConfig represents TLS config.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
@@ -32,23 +77,23 @@ type TLSConfig struct {
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/
 type Authorization struct {
-	Type            string `yaml:"type,omitempty"`
+	Type            string  `yaml:"type,omitempty"`
-	Credentials     string `yaml:"credentials,omitempty"`
+	Credentials     *Secret `yaml:"credentials,omitempty"`
-	CredentialsFile string `yaml:"credentials_file,omitempty"`
+	CredentialsFile string  `yaml:"credentials_file,omitempty"`
 }
 
 // BasicAuthConfig represents basic auth config.
 type BasicAuthConfig struct {
-	Username     string `yaml:"username"`
+	Username     string  `yaml:"username"`
-	Password     string `yaml:"password,omitempty"`
+	Password     *Secret `yaml:"password,omitempty"`
-	PasswordFile string `yaml:"password_file,omitempty"`
+	PasswordFile string  `yaml:"password_file,omitempty"`
 }
 
 // HTTPClientConfig represents http client config.
 type HTTPClientConfig struct {
 	Authorization   *Authorization   `yaml:"authorization,omitempty"`
 	BasicAuth       *BasicAuthConfig `yaml:"basic_auth,omitempty"`
-	BearerToken     string           `yaml:"bearer_token,omitempty"`
+	BearerToken     *Secret          `yaml:"bearer_token,omitempty"`
 	BearerTokenFile string           `yaml:"bearer_token_file,omitempty"`
 	OAuth2          *OAuth2Config    `yaml:"oauth2,omitempty"`
 	TLSConfig       *TLSConfig       `yaml:"tls_config,omitempty"`
@@ -58,7 +103,7 @@ type HTTPClientConfig struct {
 type ProxyClientConfig struct {
 	Authorization   *Authorization   `yaml:"proxy_authorization,omitempty"`
 	BasicAuth       *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"`
-	BearerToken     string           `yaml:"proxy_bearer_token,omitempty"`
+	BearerToken     *Secret          `yaml:"proxy_bearer_token,omitempty"`
 	BearerTokenFile string           `yaml:"proxy_bearer_token_file,omitempty"`
 	TLSConfig       *TLSConfig       `yaml:"proxy_tls_config,omitempty"`
 }
@@ -66,7 +111,7 @@ type ProxyClientConfig struct {
 // OAuth2Config represent OAuth2 configuration
 type OAuth2Config struct {
 	ClientID         string            `yaml:"client_id"`
-	ClientSecret     string            `yaml:"client_secret,omitempty"`
+	ClientSecret     *Secret           `yaml:"client_secret,omitempty"`
 	ClientSecretFile string            `yaml:"client_secret_file,omitempty"`
 	Scopes           []string          `yaml:"scopes,omitempty"`
 	TokenURL         string            `yaml:"token_url"`
@@ -83,10 +128,10 @@ func (o *OAuth2Config) validate() error {
 	if o.ClientID == "" {
 		return fmt.Errorf("client_id cannot be empty")
 	}
-	if o.ClientSecret == "" && o.ClientSecretFile == "" {
+	if o.ClientSecret == nil && o.ClientSecretFile == "" {
 		return fmt.Errorf("ClientSecret or ClientSecretFile must be set")
 	}
-	if o.ClientSecret != "" && o.ClientSecretFile != "" {
+	if o.ClientSecret != nil && o.ClientSecretFile != "" {
 		return fmt.Errorf("ClientSecret and ClientSecretFile cannot be set simultaneously")
 	}
 	if o.TokenURL == "" {
@@ -109,7 +154,7 @@ func newOAuth2ConfigInternal(baseDir string, o *OAuth2Config) (*oauth2ConfigInte
 	oi := &oauth2ConfigInternal{
 		cfg: &clientcredentials.Config{
 			ClientID:       o.ClientID,
-			ClientSecret:   o.ClientSecret,
+			ClientSecret:   o.ClientSecret.String(),
 			TokenURL:       o.TokenURL,
 			Scopes:         o.Scopes,
 			EndpointParams: urlValuesFromMap(o.EndpointParams),
@@ -238,12 +283,12 @@ func (ac *Config) NewTLSConfig() *tls.Config {
 
 // NewConfig creates auth config for the given hcc.
 func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) {
-	return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken, hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
+	return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
 }
 
 // NewConfig creates auth config for the given pcc.
 func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) {
-	return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken, pcc.BearerTokenFile, nil, pcc.TLSConfig)
+	return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig)
 }
 
 // NewConfig creates auth config from the given args.
@@ -256,7 +301,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 			azType = az.Type
 		}
 		if az.CredentialsFile != "" {
-			if az.Credentials != "" {
+			if az.Credentials != nil {
 				return nil, fmt.Errorf("both `credentials`=%q and `credentials_file`=%q are set", az.Credentials, az.CredentialsFile)
 			}
 			filePath := getFilepath(baseDir, az.CredentialsFile)
@@ -271,7 +316,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 			authDigest = fmt.Sprintf("custom(type=%q, credsFile=%q)", az.Type, filePath)
 		} else {
 			getAuthHeader = func() string {
-				return azType + " " + az.Credentials
+				return azType + " " + az.Credentials.String()
 			}
 			authDigest = fmt.Sprintf("custom(type=%q, creds=%q)", az.Type, az.Credentials)
 		}
@@ -284,7 +329,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 			return nil, fmt.Errorf("missing `username` in `basic_auth` section")
 		}
 		if basicAuth.PasswordFile != "" {
-			if basicAuth.Password != "" {
+			if basicAuth.Password != nil {
 				return nil, fmt.Errorf("both `password`=%q and `password_file`=%q are set in `basic_auth` section", basicAuth.Password, basicAuth.PasswordFile)
 			}
 			filePath := getFilepath(baseDir, basicAuth.PasswordFile)
@@ -303,7 +348,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 		} else {
 			getAuthHeader = func() string {
 				// See https://en.wikipedia.org/wiki/Basic_access_authentication
-				token := basicAuth.Username + ":" + basicAuth.Password
+				token := basicAuth.Username + ":" + basicAuth.Password.String()
 				token64 := base64.StdEncoding.EncodeToString([]byte(token))
 				return "Basic " + token64
 			}

lib/promauth/config_test.go

@@ -27,7 +27,7 @@ func TestNewConfig(t *testing.T) {
 			args: args{
 				oauth: &OAuth2Config{
 					ClientID:     "some-id",
-					ClientSecret: "some-secret",
+					ClientSecret: NewSecret("some-secret"),
 					TokenURL:     "http://localhost:8511",
 				},
 			},
@@ -49,7 +49,7 @@ func TestNewConfig(t *testing.T) {
 			args: args{
 				oauth: &OAuth2Config{
 					ClientID:         "some-id",
-					ClientSecret:     "some-secret",
+					ClientSecret:     NewSecret("some-secret"),
 					ClientSecretFile: "testdata/test_secretfile.txt",
 					TokenURL:         "http://localhost:8511",
 				},
@@ -61,7 +61,7 @@ func TestNewConfig(t *testing.T) {
 			args: args{
 				basicAuth: &BasicAuthConfig{
 					Username: "user",
-					Password: "password",
+					Password: NewSecret("password"),
 				},
 			},
 			expectHeader: "Basic dXNlcjpwYXNzd29yZA==",
@@ -81,7 +81,7 @@ func TestNewConfig(t *testing.T) {
 			args: args{
 				az: &Authorization{
 					Type:        "Bearer",
-					Credentials: "Value",
+					Credentials: NewSecret("Value"),
 				},
 			},
 			expectHeader: "Bearer Value",

lib/promscrape/config.go

@@ -240,27 +240,29 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
 }
 
 // loadConfig loads Prometheus config from the given path.
-func loadConfig(path string) (*Config, error) {
+func loadConfig(path string) (*Config, []byte, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
+		return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
 	}
 	var c Config
-	if err := c.parseData(data, path); err != nil {
+	dataNew, err := c.parseData(data, path)
-		return nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
 	}
-	return &c, nil
+	return &c, dataNew, nil
 }
 
-func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, error) {
+func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, []byte, error) {
 	var scrapeConfigs []ScrapeConfig
+	var scsData []byte
 	for _, filePath := range scrapeConfigFiles {
 		filePath := getFilepath(baseDir, filePath)
 		paths := []string{filePath}
 		if strings.Contains(filePath, "*") {
 			ps, err := filepath.Glob(filePath)
 			if err != nil {
-				return nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err)
+				return nil, nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err)
 			}
 			sort.Strings(ps)
 			paths = ps
@@ -268,17 +270,19 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]Scrape
 		for _, path := range paths {
 			data, err := ioutil.ReadFile(path)
 			if err != nil {
-				return nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err)
+				return nil, nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err)
 			}
 			data = envtemplate.Replace(data)
 			var scs []ScrapeConfig
 			if err = yaml.UnmarshalStrict(data, &scs); err != nil {
-				return nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err)
+				return nil, nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err)
 			}
 			scrapeConfigs = append(scrapeConfigs, scs...)
+			scsData = append(scsData, '\n')
+			scsData = append(scsData, data...)
 		}
 	}
-	return scrapeConfigs, nil
+	return scrapeConfigs, scsData, nil
 }
 
 // IsDryRun returns true if -promscrape.config.dryRun command-line flag is set
@@ -286,30 +290,31 @@ func IsDryRun() bool {
 	return *dryRun
 }
 
-func (cfg *Config) parseData(data []byte, path string) error {
+func (cfg *Config) parseData(data []byte, path string) ([]byte, error) {
 	if err := unmarshalMaybeStrict(data, cfg); err != nil {
-		return fmt.Errorf("cannot unmarshal data: %w", err)
+		return nil, fmt.Errorf("cannot unmarshal data: %w", err)
 	}
 	absPath, err := filepath.Abs(path)
 	if err != nil {
-		return fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
+		return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
 	}
 	cfg.baseDir = filepath.Dir(absPath)
 
 	// Load cfg.ScrapeConfigFiles into c.ScrapeConfigs
-	scs, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles)
+	scs, scsData, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles)
 	if err != nil {
-		return fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err)
+		return nil, fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err)
 	}
 	cfg.ScrapeConfigFiles = nil
 	cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scs...)
+	dataNew := append(data, scsData...)
 
 	// Check that all the scrape configs have unique JobName
 	m := make(map[string]struct{}, len(cfg.ScrapeConfigs))
 	for i := range cfg.ScrapeConfigs {
 		jobName := cfg.ScrapeConfigs[i].JobName
 		if _, ok := m[jobName]; ok {
-			return fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName)
+			return nil, fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName)
 		}
 		m[jobName] = struct{}{}
 	}
@@ -319,11 +324,11 @@ func (cfg *Config) parseData(data []byte, path string) error {
 		sc := &cfg.ScrapeConfigs[i]
 		swc, err := getScrapeWorkConfig(sc, cfg.baseDir, &cfg.Global)
 		if err != nil {
-			return fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err)
+			return nil, fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err)
 		}
 		sc.swc = swc
 	}
-	return nil
+	return dataNew, nil
 }
 
 func unmarshalMaybeStrict(data []byte, dst interface{}) error {

lib/promscrape/config_test.go

@@ -1,6 +1,7 @@
 package promscrape
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -67,39 +68,51 @@ func TestLoadStaticConfigs(t *testing.T) {
 }
 
 func TestLoadConfig(t *testing.T) {
-	cfg, err := loadConfig("testdata/prometheus.yml")
+	cfg, data, err := loadConfig("testdata/prometheus.yml")
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
 	if cfg == nil {
 		t.Fatalf("expecting non-nil config")
 	}
+	if data == nil {
+		t.Fatalf("expecting non-nil data")
+	}
 
-	cfg, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
+	cfg, data, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
 	if cfg == nil {
 		t.Fatalf("expecting non-nil config")
 	}
+	if data == nil {
+		t.Fatalf("expecting non-nil data")
+	}
 
 	// Try loading non-existing file
-	cfg, err = loadConfig("testdata/non-existing-file")
+	cfg, data, err = loadConfig("testdata/non-existing-file")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}
 	if cfg != nil {
 		t.Fatalf("unexpected non-nil config: %#v", cfg)
 	}
+	if data != nil {
+		t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
+	}
 
 	// Try loading invalid file
-	cfg, err = loadConfig("testdata/file_sd_1.yml")
+	cfg, data, err = loadConfig("testdata/file_sd_1.yml")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}
 	if cfg != nil {
 		t.Fatalf("unexpected non-nil config: %#v", cfg)
 	}
+	if data != nil {
+		t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
+	}
 }
 
 func TestBlackboxExporter(t *testing.T) {
@@ -122,9 +135,13 @@ scrape_configs:
         replacement: black:9115  # The blackbox exporter's real hostname:port.%
 `
 	var cfg Config
-	if err := cfg.parseData([]byte(data), "sss"); err != nil {
+	allData, err := cfg.parseData([]byte(data), "sss")
+	if err != nil {
 		t.Fatalf("cannot parase data: %s", err)
 	}
+	if string(allData) != data {
+		t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	sws := cfg.getStaticScrapeWork()
 	resetNonEssentialFields(sws)
 	swsExpected := []*ScrapeWork{{
@@ -187,9 +204,13 @@ scrape_configs:
   - files: [testdata/file_sd.json]
 `
 	var cfg Config
-	if err := cfg.parseData([]byte(data), "sss"); err != nil {
+	allData, err := cfg.parseData([]byte(data), "sss")
+	if err != nil {
 		t.Fatalf("cannot parase data: %s", err)
 	}
+	if string(allData) != data {
+		t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	sws := cfg.getFileSDScrapeWork(nil)
 	if !equalStaticConfigForScrapeWorks(sws, sws) {
 		t.Fatalf("unexpected non-equal static configs;\nsws:\n%#v", sws)
@@ -203,9 +224,13 @@ scrape_configs:
   - files: [testdata/file_sd_1.yml]
 `
 	var cfgNew Config
-	if err := cfgNew.parseData([]byte(dataNew), "sss"); err != nil {
+	allData, err = cfgNew.parseData([]byte(dataNew), "sss")
+	if err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
+	if string(allData) != dataNew {
+		t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, dataNew)
+	}
 	swsNew := cfgNew.getFileSDScrapeWork(sws)
 	if equalStaticConfigForScrapeWorks(swsNew, sws) {
 		t.Fatalf("unexpected equal static configs;\nswsNew:\n%#v\nsws:\n%#v", swsNew, sws)
@@ -218,9 +243,13 @@ scrape_configs:
   file_sd_configs:
   - files: [testdata/prometheus.yml]
 `
-	if err := cfg.parseData([]byte(data), "sss"); err != nil {
+	allData, err = cfg.parseData([]byte(data), "sss")
+	if err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
+	if string(allData) != data {
+		t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	sws = cfg.getFileSDScrapeWork(swsNew)
 	if len(sws) != 0 {
 		t.Fatalf("unexpected non-empty sws:\n%#v", sws)
@@ -233,9 +262,13 @@ scrape_configs:
   file_sd_configs:
   - files: [testdata/empty_target_file_sd.yml]
 `
-	if err := cfg.parseData([]byte(data), "sss"); err != nil {
+	allData, err = cfg.parseData([]byte(data), "sss")
+	if err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
+	if string(allData) != data {
+		t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	sws = cfg.getFileSDScrapeWork(swsNew)
 	if len(sws) != 0 {
 		t.Fatalf("unexpected non-empty sws:\n%#v", sws)
@@ -244,17 +277,25 @@ scrape_configs:
 
 func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
 	var cfg Config
-	if err := cfg.parseData(data, path); err != nil {
+	allData, err := cfg.parseData(data, path)
+	if err != nil {
 		return nil, fmt.Errorf("cannot parse data: %w", err)
 	}
+	if !bytes.Equal(allData, data) {
+		return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	return cfg.getFileSDScrapeWork(nil), nil
 }
 
 func getStaticScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
 	var cfg Config
-	if err := cfg.parseData(data, path); err != nil {
+	allData, err := cfg.parseData(data, path)
+	if err != nil {
 		return nil, fmt.Errorf("cannot parse data: %w", err)
 	}
+	if !bytes.Equal(allData, data) {
+		return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
+	}
 	return cfg.getStaticScrapeWork(), nil
 }
 

lib/promscrape/discovery/consul/api.go

@@ -44,10 +44,10 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		return nil, err
 	}
 	if token != "" {
-		if hcc.BearerToken != "" {
+		if hcc.BearerToken != nil {
 			return nil, fmt.Errorf("cannot set both token and bearer_token configs")
 		}
-		hcc.BearerToken = token
+		hcc.BearerToken = promauth.NewSecret(token)
 	}
 	if len(sdc.Username) > 0 {
 		if hcc.BasicAuth != nil {
@@ -104,9 +104,9 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	return cfg, nil
 }
 
-func getToken(token *string) (string, error) {
+func getToken(token *promauth.Secret) (string, error) {
 	if token != nil {
-		return *token, nil
+		return token.String(), nil
 	}
 	if tokenFile := os.Getenv("CONSUL_HTTP_TOKEN_FILE"); tokenFile != "" {
 		data, err := ioutil.ReadFile(tokenFile)

lib/promscrape/discovery/consul/consul.go

@@ -11,15 +11,15 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 type SDConfig struct {
-	Server     string  `yaml:"server,omitempty"`
+	Server     string           `yaml:"server,omitempty"`
-	Token      *string `yaml:"token"`
+	Token      *promauth.Secret `yaml:"token"`
-	Datacenter string  `yaml:"datacenter"`
+	Datacenter string           `yaml:"datacenter"`
 	// Namespace only supported at enterprise consul.
 	// https://www.consul.io/docs/enterprise/namespaces
 	Namespace         string                     `yaml:"namespace,omitempty"`
 	Scheme            string                     `yaml:"scheme,omitempty"`
 	Username          string                     `yaml:"username"`
-	Password          string                     `yaml:"password"`
+	Password          *promauth.Secret           `yaml:"password"`
 	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
 	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`
 	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`

lib/promscrape/discovery/ec2/api.go

@@ -93,8 +93,8 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	if len(sdc.AccessKey) > 0 {
 		cfg.defaultAccessKey = sdc.AccessKey
 	}
-	if len(sdc.SecretKey) > 0 {
+	if sdc.SecretKey != nil {
-		cfg.defaultSecretKey = sdc.SecretKey
+		cfg.defaultSecretKey = sdc.SecretKey.String()
 	}
 	cfg.creds = &apiCredentials{
 		AccessKeyID:     cfg.defaultAccessKey,

lib/promscrape/discovery/ec2/ec2.go

@@ -4,6 +4,8 @@ import (
 	"flag"
 	"fmt"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // SDCheckInterval defines interval for targets refresh.
@@ -15,10 +17,10 @@ var SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config
 type SDConfig struct {
-	Region    string `yaml:"region,omitempty"`
+	Region    string           `yaml:"region,omitempty"`
-	Endpoint  string `yaml:"endpoint,omitempty"`
+	Endpoint  string           `yaml:"endpoint,omitempty"`
-	AccessKey string `yaml:"access_key,omitempty"`
+	AccessKey string           `yaml:"access_key,omitempty"`
-	SecretKey string `yaml:"secret_key,omitempty"`
+	SecretKey *promauth.Secret `yaml:"secret_key,omitempty"`
 	// TODO add support for Profile, not working atm
 	Profile string `yaml:"profile,omitempty"`
 	RoleARN string `yaml:"role_arn,omitempty"`

lib/promscrape/discovery/openstack/auth.go

@@ -6,6 +6,8 @@ import (
 	"net/url"
 	"os"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // authResponse represents identity api response
@@ -53,7 +55,7 @@ func getComputeEndpointURL(catalog []catalogItem, availability, region string) (
 
 // buildAuthRequestBody builds request for authentication
 func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
-	if len(sdc.Password) == 0 && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 {
+	if sdc.Password == nil && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 {
 		return nil, fmt.Errorf("password and application credentials are missing")
 	}
 	type domainReq struct {
@@ -97,24 +99,25 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
 	// if insufficient or incompatible information is present.
 	var req request
 
-	if len(sdc.Password) == 0 {
+	if sdc.Password == nil {
 		// There are three kinds of possible application_credential requests
 		// 1. application_credential id + secret
 		// 2. application_credential name + secret + user_id
 		// 3. application_credential name + secret + username + domain_id / domain_name
 		if len(sdc.ApplicationCredentialID) > 0 {
-			if len(sdc.ApplicationCredentialSecret) == 0 {
+			if sdc.ApplicationCredentialSecret == nil {
 				return nil, fmt.Errorf("ApplicationCredentialSecret is empty")
 			}
 			req.Auth.Identity.Methods = []string{"application_credential"}
+			secret := sdc.ApplicationCredentialSecret.String()
 			req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
 				ID:     &sdc.ApplicationCredentialID,
-				Secret: &sdc.ApplicationCredentialSecret,
+				Secret: &secret,
 			}
 			return json.Marshal(req)
 		}
 
-		if len(sdc.ApplicationCredentialSecret) == 0 {
+		if sdc.ApplicationCredentialSecret == nil {
 			return nil, fmt.Errorf("missing application_credential_secret when application_credential_name is set")
 		}
 		var userRequest *userReq
@@ -143,10 +146,11 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
 			return nil, fmt.Errorf("domain_id and domain_name cannot be empty for application_credential_name auth")
 		}
 		req.Auth.Identity.Methods = []string{"application_credential"}
+		secret := sdc.ApplicationCredentialSecret.String()
 		req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
 			Name:   &sdc.ApplicationCredentialName,
 			User:   userRequest,
-			Secret: &sdc.ApplicationCredentialSecret,
+			Secret: &secret,
 		}
 		return json.Marshal(req)
 	}
@@ -168,11 +172,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
 				return nil, fmt.Errorf("both domain_id and domain_name is present")
 			}
 			// Configure the request for Username and Password authentication with a DomainID.
-			if len(sdc.Password) > 0 {
+			if sdc.Password != nil {
+				password := sdc.Password.String()
 				req.Auth.Identity.Password = &passwordReq{
 					User: userReq{
 						Name:     &sdc.Username,
-						Password: &sdc.Password,
+						Password: &password,
 						Domain:   &domainReq{ID: &sdc.DomainID},
 					},
 				}
@@ -180,11 +185,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
 		}
 		if len(sdc.DomainName) > 0 {
 			// Configure the request for Username and Password authentication with a DomainName.
-			if len(sdc.Password) > 0 {
+			if sdc.Password != nil {
+				password := sdc.Password.String()
 				req.Auth.Identity.Password = &passwordReq{
 					User: userReq{
 						Name:     &sdc.Username,
-						Password: &sdc.Password,
+						Password: &password,
 						Domain:   &domainReq{Name: &sdc.DomainName},
 					},
 				}
@@ -199,11 +205,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
 			return nil, fmt.Errorf("both user_id and domain_name is present")
 		}
 		// Configure the request for UserID and Password authentication.
-		if len(sdc.Password) > 0 {
+		if sdc.Password != nil {
+			password := sdc.Password.String()
 			req.Auth.Identity.Password = &passwordReq{
 				User: userReq{
 					ID:       &sdc.UserID,
-					Password: &sdc.Password,
+					Password: &password,
 				},
 			}
 		}
@@ -300,13 +307,13 @@ func readCredentialsFromEnv() SDConfig {
 		IdentityEndpoint:            authURL,
 		Username:                    username,
 		UserID:                      userID,
-		Password:                    password,
+		Password:                    promauth.NewSecret(password),
 		ProjectName:                 tenantName,
 		ProjectID:                   tenantID,
 		DomainName:                  domainName,
 		DomainID:                    domainID,
 		ApplicationCredentialName:   applicationCredentialName,
 		ApplicationCredentialID:     applicationCredentialID,
-		ApplicationCredentialSecret: applicationCredentialSecret,
+		ApplicationCredentialSecret: promauth.NewSecret(applicationCredentialSecret),
 	}
 }

lib/promscrape/discovery/openstack/auth_test.go

@@ -3,6 +3,8 @@ package openstack
 import (
 	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 func Test_buildAuthRequestBody1(t *testing.T) {
@@ -27,7 +29,7 @@ func Test_buildAuthRequestBody1(t *testing.T) {
 			args: args{
 				sdc: &SDConfig{
 					Username:   "some-user",
-					Password:   "some-password",
+					Password:   promauth.NewSecret("some-password"),
 					DomainName: "some-domain",
 				},
 			},
@@ -38,7 +40,7 @@ func Test_buildAuthRequestBody1(t *testing.T) {
 			args: args{
 				sdc: &SDConfig{
 					ApplicationCredentialID:     "some-id",
-					ApplicationCredentialSecret: "some-secret",
+					ApplicationCredentialSecret: promauth.NewSecret("some-secret"),
 				},
 			},
 			want: []byte(`{"auth":{"identity":{"methods":["application_credential"],"application_credential":{"id":"some-id","secret":"some-secret"}}}}`),

lib/promscrape/discovery/openstack/openstack.go

@@ -17,19 +17,19 @@ var SDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*ti
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config
 type SDConfig struct {
-	IdentityEndpoint            string `yaml:"identity_endpoint,omitempty"`
+	IdentityEndpoint            string           `yaml:"identity_endpoint,omitempty"`
-	Username                    string `yaml:"username,omitempty"`
+	Username                    string           `yaml:"username,omitempty"`
-	UserID                      string `yaml:"userid,omitempty"`
+	UserID                      string           `yaml:"userid,omitempty"`
-	Password                    string `yaml:"password,omitempty"`
+	Password                    *promauth.Secret `yaml:"password,omitempty"`
-	ProjectName                 string `yaml:"project_name,omitempty"`
+	ProjectName                 string           `yaml:"project_name,omitempty"`
-	ProjectID                   string `yaml:"project_id,omitempty"`
+	ProjectID                   string           `yaml:"project_id,omitempty"`
-	DomainName                  string `yaml:"domain_name,omitempty"`
+	DomainName                  string           `yaml:"domain_name,omitempty"`
-	DomainID                    string `yaml:"domain_id,omitempty"`
+	DomainID                    string           `yaml:"domain_id,omitempty"`
-	ApplicationCredentialName   string `yaml:"application_credential_name,omitempty"`
+	ApplicationCredentialName   string           `yaml:"application_credential_name,omitempty"`
-	ApplicationCredentialID     string `yaml:"application_credential_id,omitempty"`
+	ApplicationCredentialID     string           `yaml:"application_credential_id,omitempty"`
-	ApplicationCredentialSecret string `yaml:"application_credential_secret,omitempty"`
+	ApplicationCredentialSecret *promauth.Secret `yaml:"application_credential_secret,omitempty"`
-	Role                        string `yaml:"role"`
+	Role                        string           `yaml:"role"`
-	Region                      string `yaml:"region"`
+	Region                      string           `yaml:"region"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.openstackSDCheckInterval` command-line option.
 	Port         int                 `yaml:"port,omitempty"`

lib/promscrape/scraper.go

@@ -43,7 +43,7 @@ func CheckConfig() error {
 	if *promscrapeConfigFile == "" {
 		return fmt.Errorf("missing -promscrape.config option")
 	}
-	_, err := loadConfig(*promscrapeConfigFile)
+	_, _, err := loadConfig(*promscrapeConfigFile)
 	return err
 }
 
@@ -99,12 +99,12 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	sighupCh := procutil.NewSighupChan()
 
 	logger.Infof("reading Prometheus configs from %q", configFile)
-	cfg, err := loadConfig(configFile)
+	cfg, data, err := loadConfig(configFile)
 	if err != nil {
 		logger.Fatalf("cannot read %q: %s", configFile, err)
 	}
-	data := cfg.marshal()
+	marshaledData := cfg.marshal()
-	configData.Store(&data)
+	configData.Store(&marshaledData)
 	cfg.mustStart()
 
 	scs := newScrapeConfigs(pushData)
@@ -134,12 +134,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 		select {
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile)
-			cfgNew, err := loadConfig(configFile)
+			cfgNew, dataNew, err := loadConfig(configFile)
 			if err != nil {
 				logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err)
 				goto waitForChans
 			}
-			dataNew := cfgNew.marshal()
 			if bytes.Equal(data, dataNew) {
 				logger.Infof("nothing changed in %q", configFile)
 				goto waitForChans
@@ -148,14 +147,14 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 			cfgNew.mustStart()
 			cfg = cfgNew
 			data = dataNew
-			configData.Store(&data)
+			marshaledData = cfgNew.marshal()
+			configData.Store(&marshaledData)
 		case <-tickerCh:
-			cfgNew, err := loadConfig(configFile)
+			cfgNew, dataNew, err := loadConfig(configFile)
 			if err != nil {
 				logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err)
 				goto waitForChans
 			}
-			dataNew := cfgNew.marshal()
 			if bytes.Equal(data, dataNew) {
 				// Nothing changed since the previous loadConfig
 				goto waitForChans
@@ -164,7 +163,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 			cfgNew.mustStart()
 			cfg = cfgNew
 			data = dataNew
-			configData.Store(&data)
+			configData.Store(&marshaledData)
 		case <-globalStopCh:
 			cfg.mustStop()
 			logger.Infof("stopping Prometheus scrapers")