diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index fc67835d8..479a21080 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -165,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) { if username != "" || password != "" || passwordFile != "" { basicAuthCfg = &promauth.BasicAuthConfig{ Username: username, - Password: password, + Password: promauth.NewSecret(password), PasswordFile: passwordFile, } } @@ -179,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) { if clientSecretFile != "" || clientSecret != "" { oauth2Cfg = &promauth.OAuth2Config{ ClientID: oauth2ClientID.GetOptionalArg(argIdx), - ClientSecret: clientSecret, + ClientSecret: promauth.NewSecret(clientSecret), ClientSecretFile: clientSecretFile, TokenURL: oauth2TokenURL.GetOptionalArg(argIdx), Scopes: strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"), diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go index ba406a46c..7dd584662 100644 --- a/app/vmalert/datasource/vm_test.go +++ b/app/vmalert/datasource/vm_test.go @@ -20,7 +20,7 @@ var ( basicAuthPass = "bar" baCfg = &promauth.BasicAuthConfig{ Username: basicAuthName, - Password: basicAuthPass, + Password: promauth.NewSecret(basicAuthPass), } query = "vm_rows" queryRender = "constantLine(10)" diff --git a/app/vmalert/utils/auth.go b/app/vmalert/utils/auth.go index ed9c54e70..20ff5715b 100644 --- a/app/vmalert/utils/auth.go +++ b/app/vmalert/utils/auth.go @@ -10,7 +10,7 @@ func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*p if baUser != "" || baPass != "" || baFile != "" { baCfg = &promauth.BasicAuthConfig{ Username: baUser, - Password: baPass, + Password: promauth.NewSecret(baPass), PasswordFile: baFile, } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7ae067888..0c4cd1983 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,7 +9,8 @@ sort: 15 * FEATURE: vmalert: allow groups with empty rules list like Prometheus does. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1742). * FEATURE: vmagent: add `collapse` and `expand` buttons per each group of targets with the same `job_name` at `http://vmagent:8429/targets` page. * FEATURE: automatically detect timestamp precision (ns, us, ms or s) for the data ingested into VictoriaMetrics via [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf). -* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive information such as passwords, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764). +* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive config information, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764). +* FEATURE: vmagent: hide passwords and auth tokens at `/config` page like Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764). * FEATURE: vmagent: add `-promscrape.maxResponseHeadersSize` command-line flag for tuning the maximum HTTP response headers size for Prometheus scrape targets. * FEATURE: vmagent: send data to multiple configured remote storage systems in parallel (e.g. when multiple `-remoteWrite.url` flag values are specified). This should improve data ingestion speed. * FEATURE: vmagent: add `-remoteWrite.maxRowsPerBlock` command-line flag for tuning the number of samples to send to remote storage per each block. Bigger values may improve data ingestion performance at the cost of higher memory usage. diff --git a/lib/promauth/config.go b/lib/promauth/config.go index ee669b028..9ec856faa 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -17,6 +17,51 @@ import ( "golang.org/x/oauth2/clientcredentials" ) +// Secret represents a string secret such as password or auth token. +// +// It is marshaled to "" string in yaml. +// +// This is needed for hiding secret strings in /config page output. +// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764 +type Secret struct { + s string +} + +// NewSecret returns new secret for s. +func NewSecret(s string) *Secret { + if s == "" { + return nil + } + return &Secret{ + s: s, + } +} + +// MarshalYAML implements yaml.Marshaler interface. +// +// It substitutes the secret with "" string. +func (s *Secret) MarshalYAML() (interface{}, error) { + return "", nil +} + +// UnmarshalYAML implements yaml.Unmarshaler interface. +func (s *Secret) UnmarshalYAML(f func(interface{}) error) error { + var secret string + if err := f(&secret); err != nil { + return fmt.Errorf("cannot parse secret: %w", err) + } + s.s = secret + return nil +} + +// String returns the secret in plaintext. +func (s *Secret) String() string { + if s == nil { + return "" + } + return s.s +} + // TLSConfig represents TLS config. // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config @@ -32,23 +77,23 @@ type TLSConfig struct { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/ type Authorization struct { - Type string `yaml:"type,omitempty"` - Credentials string `yaml:"credentials,omitempty"` - CredentialsFile string `yaml:"credentials_file,omitempty"` + Type string `yaml:"type,omitempty"` + Credentials *Secret `yaml:"credentials,omitempty"` + CredentialsFile string `yaml:"credentials_file,omitempty"` } // BasicAuthConfig represents basic auth config. type BasicAuthConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password,omitempty"` - PasswordFile string `yaml:"password_file,omitempty"` + Username string `yaml:"username"` + Password *Secret `yaml:"password,omitempty"` + PasswordFile string `yaml:"password_file,omitempty"` } // HTTPClientConfig represents http client config. type HTTPClientConfig struct { Authorization *Authorization `yaml:"authorization,omitempty"` BasicAuth *BasicAuthConfig `yaml:"basic_auth,omitempty"` - BearerToken string `yaml:"bearer_token,omitempty"` + BearerToken *Secret `yaml:"bearer_token,omitempty"` BearerTokenFile string `yaml:"bearer_token_file,omitempty"` OAuth2 *OAuth2Config `yaml:"oauth2,omitempty"` TLSConfig *TLSConfig `yaml:"tls_config,omitempty"` @@ -58,7 +103,7 @@ type HTTPClientConfig struct { type ProxyClientConfig struct { Authorization *Authorization `yaml:"proxy_authorization,omitempty"` BasicAuth *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"` - BearerToken string `yaml:"proxy_bearer_token,omitempty"` + BearerToken *Secret `yaml:"proxy_bearer_token,omitempty"` BearerTokenFile string `yaml:"proxy_bearer_token_file,omitempty"` TLSConfig *TLSConfig `yaml:"proxy_tls_config,omitempty"` } @@ -66,7 +111,7 @@ type ProxyClientConfig struct { // OAuth2Config represent OAuth2 configuration type OAuth2Config struct { ClientID string `yaml:"client_id"` - ClientSecret string `yaml:"client_secret,omitempty"` + ClientSecret *Secret `yaml:"client_secret,omitempty"` ClientSecretFile string `yaml:"client_secret_file,omitempty"` Scopes []string `yaml:"scopes,omitempty"` TokenURL string `yaml:"token_url"` @@ -83,10 +128,10 @@ func (o *OAuth2Config) validate() error { if o.ClientID == "" { return fmt.Errorf("client_id cannot be empty") } - if o.ClientSecret == "" && o.ClientSecretFile == "" { + if o.ClientSecret == nil && o.ClientSecretFile == "" { return fmt.Errorf("ClientSecret or ClientSecretFile must be set") } - if o.ClientSecret != "" && o.ClientSecretFile != "" { + if o.ClientSecret != nil && o.ClientSecretFile != "" { return fmt.Errorf("ClientSecret and ClientSecretFile cannot be set simultaneously") } if o.TokenURL == "" { @@ -109,7 +154,7 @@ func newOAuth2ConfigInternal(baseDir string, o *OAuth2Config) (*oauth2ConfigInte oi := &oauth2ConfigInternal{ cfg: &clientcredentials.Config{ ClientID: o.ClientID, - ClientSecret: o.ClientSecret, + ClientSecret: o.ClientSecret.String(), TokenURL: o.TokenURL, Scopes: o.Scopes, EndpointParams: urlValuesFromMap(o.EndpointParams), @@ -238,12 +283,12 @@ func (ac *Config) NewTLSConfig() *tls.Config { // NewConfig creates auth config for the given hcc. func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) { - return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken, hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig) + return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig) } // NewConfig creates auth config for the given pcc. func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) { - return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken, pcc.BearerTokenFile, nil, pcc.TLSConfig) + return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig) } // NewConfig creates auth config from the given args. @@ -256,7 +301,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be azType = az.Type } if az.CredentialsFile != "" { - if az.Credentials != "" { + if az.Credentials != nil { return nil, fmt.Errorf("both `credentials`=%q and `credentials_file`=%q are set", az.Credentials, az.CredentialsFile) } filePath := getFilepath(baseDir, az.CredentialsFile) @@ -271,7 +316,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be authDigest = fmt.Sprintf("custom(type=%q, credsFile=%q)", az.Type, filePath) } else { getAuthHeader = func() string { - return azType + " " + az.Credentials + return azType + " " + az.Credentials.String() } authDigest = fmt.Sprintf("custom(type=%q, creds=%q)", az.Type, az.Credentials) } @@ -284,7 +329,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be return nil, fmt.Errorf("missing `username` in `basic_auth` section") } if basicAuth.PasswordFile != "" { - if basicAuth.Password != "" { + if basicAuth.Password != nil { return nil, fmt.Errorf("both `password`=%q and `password_file`=%q are set in `basic_auth` section", basicAuth.Password, basicAuth.PasswordFile) } filePath := getFilepath(baseDir, basicAuth.PasswordFile) @@ -303,7 +348,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be } else { getAuthHeader = func() string { // See https://en.wikipedia.org/wiki/Basic_access_authentication - token := basicAuth.Username + ":" + basicAuth.Password + token := basicAuth.Username + ":" + basicAuth.Password.String() token64 := base64.StdEncoding.EncodeToString([]byte(token)) return "Basic " + token64 } diff --git a/lib/promauth/config_test.go b/lib/promauth/config_test.go index 12bd9f427..04795710a 100644 --- a/lib/promauth/config_test.go +++ b/lib/promauth/config_test.go @@ -27,7 +27,7 @@ func TestNewConfig(t *testing.T) { args: args{ oauth: &OAuth2Config{ ClientID: "some-id", - ClientSecret: "some-secret", + ClientSecret: NewSecret("some-secret"), TokenURL: "http://localhost:8511", }, }, @@ -49,7 +49,7 @@ func TestNewConfig(t *testing.T) { args: args{ oauth: &OAuth2Config{ ClientID: "some-id", - ClientSecret: "some-secret", + ClientSecret: NewSecret("some-secret"), ClientSecretFile: "testdata/test_secretfile.txt", TokenURL: "http://localhost:8511", }, @@ -61,7 +61,7 @@ func TestNewConfig(t *testing.T) { args: args{ basicAuth: &BasicAuthConfig{ Username: "user", - Password: "password", + Password: NewSecret("password"), }, }, expectHeader: "Basic dXNlcjpwYXNzd29yZA==", @@ -81,7 +81,7 @@ func TestNewConfig(t *testing.T) { args: args{ az: &Authorization{ Type: "Bearer", - Credentials: "Value", + Credentials: NewSecret("Value"), }, }, expectHeader: "Bearer Value", diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index ea8f92e84..de2b5eebf 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -240,27 +240,29 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) { } // loadConfig loads Prometheus config from the given path. -func loadConfig(path string) (*Config, error) { +func loadConfig(path string) (*Config, []byte, error) { data, err := ioutil.ReadFile(path) if err != nil { - return nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err) + return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err) } var c Config - if err := c.parseData(data, path); err != nil { - return nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err) + dataNew, err := c.parseData(data, path) + if err != nil { + return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err) } - return &c, nil + return &c, dataNew, nil } -func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, error) { +func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, []byte, error) { var scrapeConfigs []ScrapeConfig + var scsData []byte for _, filePath := range scrapeConfigFiles { filePath := getFilepath(baseDir, filePath) paths := []string{filePath} if strings.Contains(filePath, "*") { ps, err := filepath.Glob(filePath) if err != nil { - return nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err) + return nil, nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err) } sort.Strings(ps) paths = ps @@ -268,17 +270,19 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]Scrape for _, path := range paths { data, err := ioutil.ReadFile(path) if err != nil { - return nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err) + return nil, nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err) } data = envtemplate.Replace(data) var scs []ScrapeConfig if err = yaml.UnmarshalStrict(data, &scs); err != nil { - return nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err) + return nil, nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err) } scrapeConfigs = append(scrapeConfigs, scs...) + scsData = append(scsData, '\n') + scsData = append(scsData, data...) } } - return scrapeConfigs, nil + return scrapeConfigs, scsData, nil } // IsDryRun returns true if -promscrape.config.dryRun command-line flag is set @@ -286,30 +290,31 @@ func IsDryRun() bool { return *dryRun } -func (cfg *Config) parseData(data []byte, path string) error { +func (cfg *Config) parseData(data []byte, path string) ([]byte, error) { if err := unmarshalMaybeStrict(data, cfg); err != nil { - return fmt.Errorf("cannot unmarshal data: %w", err) + return nil, fmt.Errorf("cannot unmarshal data: %w", err) } absPath, err := filepath.Abs(path) if err != nil { - return fmt.Errorf("cannot obtain abs path for %q: %w", path, err) + return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err) } cfg.baseDir = filepath.Dir(absPath) // Load cfg.ScrapeConfigFiles into c.ScrapeConfigs - scs, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles) + scs, scsData, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles) if err != nil { - return fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err) + return nil, fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err) } cfg.ScrapeConfigFiles = nil cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scs...) + dataNew := append(data, scsData...) // Check that all the scrape configs have unique JobName m := make(map[string]struct{}, len(cfg.ScrapeConfigs)) for i := range cfg.ScrapeConfigs { jobName := cfg.ScrapeConfigs[i].JobName if _, ok := m[jobName]; ok { - return fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName) + return nil, fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName) } m[jobName] = struct{}{} } @@ -319,11 +324,11 @@ func (cfg *Config) parseData(data []byte, path string) error { sc := &cfg.ScrapeConfigs[i] swc, err := getScrapeWorkConfig(sc, cfg.baseDir, &cfg.Global) if err != nil { - return fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err) + return nil, fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err) } sc.swc = swc } - return nil + return dataNew, nil } func unmarshalMaybeStrict(data []byte, dst interface{}) error { diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 8ed36a49c..47657b061 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -1,6 +1,7 @@ package promscrape import ( + "bytes" "fmt" "reflect" "strconv" @@ -67,39 +68,51 @@ func TestLoadStaticConfigs(t *testing.T) { } func TestLoadConfig(t *testing.T) { - cfg, err := loadConfig("testdata/prometheus.yml") + cfg, data, err := loadConfig("testdata/prometheus.yml") if err != nil { t.Fatalf("unexpected error: %s", err) } if cfg == nil { t.Fatalf("expecting non-nil config") } + if data == nil { + t.Fatalf("expecting non-nil data") + } - cfg, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml") + cfg, data, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml") if err != nil { t.Fatalf("unexpected error: %s", err) } if cfg == nil { t.Fatalf("expecting non-nil config") } + if data == nil { + t.Fatalf("expecting non-nil data") + } // Try loading non-existing file - cfg, err = loadConfig("testdata/non-existing-file") + cfg, data, err = loadConfig("testdata/non-existing-file") if err == nil { t.Fatalf("expecting non-nil error") } if cfg != nil { t.Fatalf("unexpected non-nil config: %#v", cfg) } + if data != nil { + t.Fatalf("unexpected data wit length=%d: %q", len(data), data) + } // Try loading invalid file - cfg, err = loadConfig("testdata/file_sd_1.yml") + cfg, data, err = loadConfig("testdata/file_sd_1.yml") if err == nil { t.Fatalf("expecting non-nil error") } if cfg != nil { t.Fatalf("unexpected non-nil config: %#v", cfg) } + if data != nil { + t.Fatalf("unexpected data wit length=%d: %q", len(data), data) + } } func TestBlackboxExporter(t *testing.T) { @@ -122,9 +135,13 @@ scrape_configs: replacement: black:9115 # The blackbox exporter's real hostname:port.% ` var cfg Config - if err := cfg.parseData([]byte(data), "sss"); err != nil { + allData, err := cfg.parseData([]byte(data), "sss") + if err != nil { t.Fatalf("cannot parase data: %s", err) } + if string(allData) != data { + t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } sws := cfg.getStaticScrapeWork() resetNonEssentialFields(sws) swsExpected := []*ScrapeWork{{ @@ -187,9 +204,13 @@ scrape_configs: - files: [testdata/file_sd.json] ` var cfg Config - if err := cfg.parseData([]byte(data), "sss"); err != nil { + allData, err := cfg.parseData([]byte(data), "sss") + if err != nil { t.Fatalf("cannot parase data: %s", err) } + if string(allData) != data { + t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } sws := cfg.getFileSDScrapeWork(nil) if !equalStaticConfigForScrapeWorks(sws, sws) { t.Fatalf("unexpected non-equal static configs;\nsws:\n%#v", sws) @@ -203,9 +224,13 @@ scrape_configs: - files: [testdata/file_sd_1.yml] ` var cfgNew Config - if err := cfgNew.parseData([]byte(dataNew), "sss"); err != nil { + allData, err = cfgNew.parseData([]byte(dataNew), "sss") + if err != nil { t.Fatalf("cannot parse data: %s", err) } + if string(allData) != dataNew { + t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, dataNew) + } swsNew := cfgNew.getFileSDScrapeWork(sws) if equalStaticConfigForScrapeWorks(swsNew, sws) { t.Fatalf("unexpected equal static configs;\nswsNew:\n%#v\nsws:\n%#v", swsNew, sws) @@ -218,9 +243,13 @@ scrape_configs: file_sd_configs: - files: [testdata/prometheus.yml] ` - if err := cfg.parseData([]byte(data), "sss"); err != nil { + allData, err = cfg.parseData([]byte(data), "sss") + if err != nil { t.Fatalf("cannot parse data: %s", err) } + if string(allData) != data { + t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } sws = cfg.getFileSDScrapeWork(swsNew) if len(sws) != 0 { t.Fatalf("unexpected non-empty sws:\n%#v", sws) @@ -233,9 +262,13 @@ scrape_configs: file_sd_configs: - files: [testdata/empty_target_file_sd.yml] ` - if err := cfg.parseData([]byte(data), "sss"); err != nil { + allData, err = cfg.parseData([]byte(data), "sss") + if err != nil { t.Fatalf("cannot parse data: %s", err) } + if string(allData) != data { + t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } sws = cfg.getFileSDScrapeWork(swsNew) if len(sws) != 0 { t.Fatalf("unexpected non-empty sws:\n%#v", sws) @@ -244,17 +277,25 @@ scrape_configs: func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) { var cfg Config - if err := cfg.parseData(data, path); err != nil { + allData, err := cfg.parseData(data, path) + if err != nil { return nil, fmt.Errorf("cannot parse data: %w", err) } + if !bytes.Equal(allData, data) { + return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } return cfg.getFileSDScrapeWork(nil), nil } func getStaticScrapeWork(data []byte, path string) ([]*ScrapeWork, error) { var cfg Config - if err := cfg.parseData(data, path); err != nil { + allData, err := cfg.parseData(data, path) + if err != nil { return nil, fmt.Errorf("cannot parse data: %w", err) } + if !bytes.Equal(allData, data) { + return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data) + } return cfg.getStaticScrapeWork(), nil } diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go index 87cf265a6..479654e77 100644 --- a/lib/promscrape/discovery/consul/api.go +++ b/lib/promscrape/discovery/consul/api.go @@ -44,10 +44,10 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { return nil, err } if token != "" { - if hcc.BearerToken != "" { + if hcc.BearerToken != nil { return nil, fmt.Errorf("cannot set both token and bearer_token configs") } - hcc.BearerToken = token + hcc.BearerToken = promauth.NewSecret(token) } if len(sdc.Username) > 0 { if hcc.BasicAuth != nil { @@ -104,9 +104,9 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { return cfg, nil } -func getToken(token *string) (string, error) { +func getToken(token *promauth.Secret) (string, error) { if token != nil { - return *token, nil + return token.String(), nil } if tokenFile := os.Getenv("CONSUL_HTTP_TOKEN_FILE"); tokenFile != "" { data, err := ioutil.ReadFile(tokenFile) diff --git a/lib/promscrape/discovery/consul/consul.go b/lib/promscrape/discovery/consul/consul.go index 671540f8c..01134e6f4 100644 --- a/lib/promscrape/discovery/consul/consul.go +++ b/lib/promscrape/discovery/consul/consul.go @@ -11,15 +11,15 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config type SDConfig struct { - Server string `yaml:"server,omitempty"` - Token *string `yaml:"token"` - Datacenter string `yaml:"datacenter"` + Server string `yaml:"server,omitempty"` + Token *promauth.Secret `yaml:"token"` + Datacenter string `yaml:"datacenter"` // Namespace only supported at enterprise consul. // https://www.consul.io/docs/enterprise/namespaces Namespace string `yaml:"namespace,omitempty"` Scheme string `yaml:"scheme,omitempty"` Username string `yaml:"username"` - Password string `yaml:"password"` + Password *promauth.Secret `yaml:"password"` HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` ProxyURL *proxy.URL `yaml:"proxy_url,omitempty"` ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go index 954bfc345..17fd04046 100644 --- a/lib/promscrape/discovery/ec2/api.go +++ b/lib/promscrape/discovery/ec2/api.go @@ -93,8 +93,8 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { if len(sdc.AccessKey) > 0 { cfg.defaultAccessKey = sdc.AccessKey } - if len(sdc.SecretKey) > 0 { - cfg.defaultSecretKey = sdc.SecretKey + if sdc.SecretKey != nil { + cfg.defaultSecretKey = sdc.SecretKey.String() } cfg.creds = &apiCredentials{ AccessKeyID: cfg.defaultAccessKey, diff --git a/lib/promscrape/discovery/ec2/ec2.go b/lib/promscrape/discovery/ec2/ec2.go index 015909f40..9f8c61059 100644 --- a/lib/promscrape/discovery/ec2/ec2.go +++ b/lib/promscrape/discovery/ec2/ec2.go @@ -4,6 +4,8 @@ import ( "flag" "fmt" "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) // SDCheckInterval defines interval for targets refresh. @@ -15,10 +17,10 @@ var SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config type SDConfig struct { - Region string `yaml:"region,omitempty"` - Endpoint string `yaml:"endpoint,omitempty"` - AccessKey string `yaml:"access_key,omitempty"` - SecretKey string `yaml:"secret_key,omitempty"` + Region string `yaml:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey *promauth.Secret `yaml:"secret_key,omitempty"` // TODO add support for Profile, not working atm Profile string `yaml:"profile,omitempty"` RoleARN string `yaml:"role_arn,omitempty"` diff --git a/lib/promscrape/discovery/openstack/auth.go b/lib/promscrape/discovery/openstack/auth.go index 3208677b6..e80745c49 100644 --- a/lib/promscrape/discovery/openstack/auth.go +++ b/lib/promscrape/discovery/openstack/auth.go @@ -6,6 +6,8 @@ import ( "net/url" "os" "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) // authResponse represents identity api response @@ -53,7 +55,7 @@ func getComputeEndpointURL(catalog []catalogItem, availability, region string) ( // buildAuthRequestBody builds request for authentication func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { - if len(sdc.Password) == 0 && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 { + if sdc.Password == nil && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 { return nil, fmt.Errorf("password and application credentials are missing") } type domainReq struct { @@ -97,24 +99,25 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { // if insufficient or incompatible information is present. var req request - if len(sdc.Password) == 0 { + if sdc.Password == nil { // There are three kinds of possible application_credential requests // 1. application_credential id + secret // 2. application_credential name + secret + user_id // 3. application_credential name + secret + username + domain_id / domain_name if len(sdc.ApplicationCredentialID) > 0 { - if len(sdc.ApplicationCredentialSecret) == 0 { + if sdc.ApplicationCredentialSecret == nil { return nil, fmt.Errorf("ApplicationCredentialSecret is empty") } req.Auth.Identity.Methods = []string{"application_credential"} + secret := sdc.ApplicationCredentialSecret.String() req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{ ID: &sdc.ApplicationCredentialID, - Secret: &sdc.ApplicationCredentialSecret, + Secret: &secret, } return json.Marshal(req) } - if len(sdc.ApplicationCredentialSecret) == 0 { + if sdc.ApplicationCredentialSecret == nil { return nil, fmt.Errorf("missing application_credential_secret when application_credential_name is set") } var userRequest *userReq @@ -143,10 +146,11 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { return nil, fmt.Errorf("domain_id and domain_name cannot be empty for application_credential_name auth") } req.Auth.Identity.Methods = []string{"application_credential"} + secret := sdc.ApplicationCredentialSecret.String() req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{ Name: &sdc.ApplicationCredentialName, User: userRequest, - Secret: &sdc.ApplicationCredentialSecret, + Secret: &secret, } return json.Marshal(req) } @@ -168,11 +172,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { return nil, fmt.Errorf("both domain_id and domain_name is present") } // Configure the request for Username and Password authentication with a DomainID. - if len(sdc.Password) > 0 { + if sdc.Password != nil { + password := sdc.Password.String() req.Auth.Identity.Password = &passwordReq{ User: userReq{ Name: &sdc.Username, - Password: &sdc.Password, + Password: &password, Domain: &domainReq{ID: &sdc.DomainID}, }, } @@ -180,11 +185,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { } if len(sdc.DomainName) > 0 { // Configure the request for Username and Password authentication with a DomainName. - if len(sdc.Password) > 0 { + if sdc.Password != nil { + password := sdc.Password.String() req.Auth.Identity.Password = &passwordReq{ User: userReq{ Name: &sdc.Username, - Password: &sdc.Password, + Password: &password, Domain: &domainReq{Name: &sdc.DomainName}, }, } @@ -199,11 +205,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) { return nil, fmt.Errorf("both user_id and domain_name is present") } // Configure the request for UserID and Password authentication. - if len(sdc.Password) > 0 { + if sdc.Password != nil { + password := sdc.Password.String() req.Auth.Identity.Password = &passwordReq{ User: userReq{ ID: &sdc.UserID, - Password: &sdc.Password, + Password: &password, }, } } @@ -300,13 +307,13 @@ func readCredentialsFromEnv() SDConfig { IdentityEndpoint: authURL, Username: username, UserID: userID, - Password: password, + Password: promauth.NewSecret(password), ProjectName: tenantName, ProjectID: tenantID, DomainName: domainName, DomainID: domainID, ApplicationCredentialName: applicationCredentialName, ApplicationCredentialID: applicationCredentialID, - ApplicationCredentialSecret: applicationCredentialSecret, + ApplicationCredentialSecret: promauth.NewSecret(applicationCredentialSecret), } } diff --git a/lib/promscrape/discovery/openstack/auth_test.go b/lib/promscrape/discovery/openstack/auth_test.go index 33a95800d..9657bd36e 100644 --- a/lib/promscrape/discovery/openstack/auth_test.go +++ b/lib/promscrape/discovery/openstack/auth_test.go @@ -3,6 +3,8 @@ package openstack import ( "reflect" "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) func Test_buildAuthRequestBody1(t *testing.T) { @@ -27,7 +29,7 @@ func Test_buildAuthRequestBody1(t *testing.T) { args: args{ sdc: &SDConfig{ Username: "some-user", - Password: "some-password", + Password: promauth.NewSecret("some-password"), DomainName: "some-domain", }, }, @@ -38,7 +40,7 @@ func Test_buildAuthRequestBody1(t *testing.T) { args: args{ sdc: &SDConfig{ ApplicationCredentialID: "some-id", - ApplicationCredentialSecret: "some-secret", + ApplicationCredentialSecret: promauth.NewSecret("some-secret"), }, }, want: []byte(`{"auth":{"identity":{"methods":["application_credential"],"application_credential":{"id":"some-id","secret":"some-secret"}}}}`), diff --git a/lib/promscrape/discovery/openstack/openstack.go b/lib/promscrape/discovery/openstack/openstack.go index 5867afd2c..d9186f4c8 100644 --- a/lib/promscrape/discovery/openstack/openstack.go +++ b/lib/promscrape/discovery/openstack/openstack.go @@ -17,19 +17,19 @@ var SDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*ti // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config type SDConfig struct { - IdentityEndpoint string `yaml:"identity_endpoint,omitempty"` - Username string `yaml:"username,omitempty"` - UserID string `yaml:"userid,omitempty"` - Password string `yaml:"password,omitempty"` - ProjectName string `yaml:"project_name,omitempty"` - ProjectID string `yaml:"project_id,omitempty"` - DomainName string `yaml:"domain_name,omitempty"` - DomainID string `yaml:"domain_id,omitempty"` - ApplicationCredentialName string `yaml:"application_credential_name,omitempty"` - ApplicationCredentialID string `yaml:"application_credential_id,omitempty"` - ApplicationCredentialSecret string `yaml:"application_credential_secret,omitempty"` - Role string `yaml:"role"` - Region string `yaml:"region"` + IdentityEndpoint string `yaml:"identity_endpoint,omitempty"` + Username string `yaml:"username,omitempty"` + UserID string `yaml:"userid,omitempty"` + Password *promauth.Secret `yaml:"password,omitempty"` + ProjectName string `yaml:"project_name,omitempty"` + ProjectID string `yaml:"project_id,omitempty"` + DomainName string `yaml:"domain_name,omitempty"` + DomainID string `yaml:"domain_id,omitempty"` + ApplicationCredentialName string `yaml:"application_credential_name,omitempty"` + ApplicationCredentialID string `yaml:"application_credential_id,omitempty"` + ApplicationCredentialSecret *promauth.Secret `yaml:"application_credential_secret,omitempty"` + Role string `yaml:"role"` + Region string `yaml:"region"` // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.openstackSDCheckInterval` command-line option. Port int `yaml:"port,omitempty"` diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index 80a3067f3..f462ae41d 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -43,7 +43,7 @@ func CheckConfig() error { if *promscrapeConfigFile == "" { return fmt.Errorf("missing -promscrape.config option") } - _, err := loadConfig(*promscrapeConfigFile) + _, _, err := loadConfig(*promscrapeConfigFile) return err } @@ -99,12 +99,12 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) sighupCh := procutil.NewSighupChan() logger.Infof("reading Prometheus configs from %q", configFile) - cfg, err := loadConfig(configFile) + cfg, data, err := loadConfig(configFile) if err != nil { logger.Fatalf("cannot read %q: %s", configFile, err) } - data := cfg.marshal() - configData.Store(&data) + marshaledData := cfg.marshal() + configData.Store(&marshaledData) cfg.mustStart() scs := newScrapeConfigs(pushData) @@ -134,12 +134,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) select { case <-sighupCh: logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile) - cfgNew, err := loadConfig(configFile) + cfgNew, dataNew, err := loadConfig(configFile) if err != nil { logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err) goto waitForChans } - dataNew := cfgNew.marshal() if bytes.Equal(data, dataNew) { logger.Infof("nothing changed in %q", configFile) goto waitForChans @@ -148,14 +147,14 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) cfgNew.mustStart() cfg = cfgNew data = dataNew - configData.Store(&data) + marshaledData = cfgNew.marshal() + configData.Store(&marshaledData) case <-tickerCh: - cfgNew, err := loadConfig(configFile) + cfgNew, dataNew, err := loadConfig(configFile) if err != nil { logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err) goto waitForChans } - dataNew := cfgNew.marshal() if bytes.Equal(data, dataNew) { // Nothing changed since the previous loadConfig goto waitForChans @@ -164,7 +163,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) cfgNew.mustStart() cfg = cfgNew data = dataNew - configData.Store(&data) + configData.Store(&marshaledData) case <-globalStopCh: cfg.mustStop() logger.Infof("stopping Prometheus scrapers")