github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/{vminsert,vmagent}: hide passwords and auth tokens by default at /config page

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764
This commit is contained in:
Aliaksandr Valialkin 2021-11-05 14:41:14 +02:00
parent bc72b83102
commit 847004fa77
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
16 changed files with 211 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/vmagent/remotewrite/client.go
View file

@ -165,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
if username != "" || password != "" || passwordFile != "" {
basicAuthCfg = &promauth.BasicAuthConfig{
Username: username,
Password: password,
Password: promauth.NewSecret(password),
PasswordFile: passwordFile,
}
}
@ -179,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
if clientSecretFile != "" || clientSecret != "" {
oauth2Cfg = &promauth.OAuth2Config{
ClientID: oauth2ClientID.GetOptionalArg(argIdx),
ClientSecret: clientSecret,
ClientSecret: promauth.NewSecret(clientSecret),
ClientSecretFile: clientSecretFile,
TokenURL: oauth2TokenURL.GetOptionalArg(argIdx),
Scopes: strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),

2
app/vmalert/datasource/vm_test.go
View file

@ -20,7 +20,7 @@ var (
basicAuthPass = "bar"
baCfg = &promauth.BasicAuthConfig{
Username: basicAuthName,
Password: basicAuthPass,
Password: promauth.NewSecret(basicAuthPass),
}
query = "vm_rows"
queryRender = "constantLine(10)"

2
app/vmalert/utils/auth.go
View file

@ -10,7 +10,7 @@ func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*p
if baUser != "" || baPass != "" || baFile != "" {
baCfg = &promauth.BasicAuthConfig{
Username: baUser,
Password: baPass,
Password: promauth.NewSecret(baPass),
PasswordFile: baFile,
}
}

3
docs/CHANGELOG.md
View file

@ -9,7 +9,8 @@ sort: 15
* FEATURE: vmalert: allow groups with empty rules list like Prometheus does. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1742).
* FEATURE: vmagent: add `collapse` and `expand` buttons per each group of targets with the same `job_name` at `http://vmagent:8429/targets` page.
* FEATURE: automatically detect timestamp precision (ns, us, ms or s) for the data ingested into VictoriaMetrics via [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive information such as passwords, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
* FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive config information, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
* FEATURE: vmagent: hide passwords and auth tokens at `/config` page like Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764).
* FEATURE: vmagent: add `-promscrape.maxResponseHeadersSize` command-line flag for tuning the maximum HTTP response headers size for Prometheus scrape targets.
* FEATURE: vmagent: send data to multiple configured remote storage systems in parallel (e.g. when multiple `-remoteWrite.url` flag values are specified). This should improve data ingestion speed.
* FEATURE: vmagent: add `-remoteWrite.maxRowsPerBlock` command-line flag for tuning the number of samples to send to remote storage per each block. Bigger values may improve data ingestion performance at the cost of higher memory usage.

81
lib/promauth/config.go
View file

@ -17,6 +17,51 @@ import (
"golang.org/x/oauth2/clientcredentials"
)
// Secret represents a string secret such as password or auth token.
//
// It is marshaled to "<secret>" string in yaml.
//
// This is needed for hiding secret strings in /config page output.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764
type Secret struct {
s string
}
// NewSecret returns new secret for s.
func NewSecret(s string) *Secret {
if s == "" {
return nil
}
return &Secret{
s: s,
}
}
// MarshalYAML implements yaml.Marshaler interface.
//
// It substitutes the secret with "<secret>" string.
func (s *Secret) MarshalYAML() (interface{}, error) {
return "<secret>", nil
}
// UnmarshalYAML implements yaml.Unmarshaler interface.
func (s *Secret) UnmarshalYAML(f func(interface{}) error) error {
var secret string
if err := f(&secret); err != nil {
return fmt.Errorf("cannot parse secret: %w", err)
}
s.s = secret
return nil
}
// String returns the secret in plaintext.
func (s *Secret) String() string {
if s == nil {
return ""
}
return s.s
}
// TLSConfig represents TLS config.
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
@ -32,23 +77,23 @@ type TLSConfig struct {
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/
type Authorization struct {
Type string `yaml:"type,omitempty"`
Credentials string `yaml:"credentials,omitempty"`
CredentialsFile string `yaml:"credentials_file,omitempty"`
Type string `yaml:"type,omitempty"`
Credentials *Secret `yaml:"credentials,omitempty"`
CredentialsFile string `yaml:"credentials_file,omitempty"`
}
// BasicAuthConfig represents basic auth config.
type BasicAuthConfig struct {
Username string `yaml:"username"`
Password string `yaml:"password,omitempty"`
PasswordFile string `yaml:"password_file,omitempty"`
Username string `yaml:"username"`
Password *Secret `yaml:"password,omitempty"`
PasswordFile string `yaml:"password_file,omitempty"`
}
// HTTPClientConfig represents http client config.
type HTTPClientConfig struct {
Authorization *Authorization `yaml:"authorization,omitempty"`
BasicAuth *BasicAuthConfig `yaml:"basic_auth,omitempty"`
BearerToken string `yaml:"bearer_token,omitempty"`
BearerToken *Secret `yaml:"bearer_token,omitempty"`
BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
OAuth2 *OAuth2Config `yaml:"oauth2,omitempty"`
TLSConfig *TLSConfig `yaml:"tls_config,omitempty"`
@ -58,7 +103,7 @@ type HTTPClientConfig struct {
type ProxyClientConfig struct {
Authorization *Authorization `yaml:"proxy_authorization,omitempty"`
BasicAuth *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"`
BearerToken string `yaml:"proxy_bearer_token,omitempty"`
BearerToken *Secret `yaml:"proxy_bearer_token,omitempty"`
BearerTokenFile string `yaml:"proxy_bearer_token_file,omitempty"`
TLSConfig *TLSConfig `yaml:"proxy_tls_config,omitempty"`
}
@ -66,7 +111,7 @@ type ProxyClientConfig struct {
// OAuth2Config represent OAuth2 configuration
type OAuth2Config struct {
ClientID string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret,omitempty"`
ClientSecret *Secret `yaml:"client_secret,omitempty"`
ClientSecretFile string `yaml:"client_secret_file,omitempty"`
Scopes []string `yaml:"scopes,omitempty"`
TokenURL string `yaml:"token_url"`
@ -83,10 +128,10 @@ func (o *OAuth2Config) validate() error {
if o.ClientID == "" {
return fmt.Errorf("client_id cannot be empty")
}
if o.ClientSecret == "" && o.ClientSecretFile == "" {
if o.ClientSecret == nil && o.ClientSecretFile == "" {
return fmt.Errorf("ClientSecret or ClientSecretFile must be set")
}
if o.ClientSecret != "" && o.ClientSecretFile != "" {
if o.ClientSecret != nil && o.ClientSecretFile != "" {
return fmt.Errorf("ClientSecret and ClientSecretFile cannot be set simultaneously")
}
if o.TokenURL == "" {
@ -109,7 +154,7 @@ func newOAuth2ConfigInternal(baseDir string, o *OAuth2Config) (*oauth2ConfigInte
oi := &oauth2ConfigInternal{
cfg: &clientcredentials.Config{
ClientID: o.ClientID,
ClientSecret: o.ClientSecret,
ClientSecret: o.ClientSecret.String(),
TokenURL: o.TokenURL,
Scopes: o.Scopes,
EndpointParams: urlValuesFromMap(o.EndpointParams),
@ -238,12 +283,12 @@ func (ac *Config) NewTLSConfig() *tls.Config {
// NewConfig creates auth config for the given hcc.
func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken, hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
}
// NewConfig creates auth config for the given pcc.
func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken, pcc.BearerTokenFile, nil, pcc.TLSConfig)
return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig)
}
// NewConfig creates auth config from the given args.
@ -256,7 +301,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
azType = az.Type
}
if az.CredentialsFile != "" {
if az.Credentials != "" {
if az.Credentials != nil {
return nil, fmt.Errorf("both `credentials`=%q and `credentials_file`=%q are set", az.Credentials, az.CredentialsFile)
}
filePath := getFilepath(baseDir, az.CredentialsFile)
@ -271,7 +316,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
authDigest = fmt.Sprintf("custom(type=%q, credsFile=%q)", az.Type, filePath)
} else {
getAuthHeader = func() string {
return azType + " " + az.Credentials
return azType + " " + az.Credentials.String()
}
authDigest = fmt.Sprintf("custom(type=%q, creds=%q)", az.Type, az.Credentials)
}
@ -284,7 +329,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
return nil, fmt.Errorf("missing `username` in `basic_auth` section")
}
if basicAuth.PasswordFile != "" {
if basicAuth.Password != "" {
if basicAuth.Password != nil {
return nil, fmt.Errorf("both `password`=%q and `password_file`=%q are set in `basic_auth` section", basicAuth.Password, basicAuth.PasswordFile)
}
filePath := getFilepath(baseDir, basicAuth.PasswordFile)
@ -303,7 +348,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
} else {
getAuthHeader = func() string {
// See https://en.wikipedia.org/wiki/Basic_access_authentication
token := basicAuth.Username + ":" + basicAuth.Password
token := basicAuth.Username + ":" + basicAuth.Password.String()
token64 := base64.StdEncoding.EncodeToString([]byte(token))
return "Basic " + token64
}

8
lib/promauth/config_test.go
View file

@ -27,7 +27,7 @@ func TestNewConfig(t *testing.T) {
args: args{
oauth: &OAuth2Config{
ClientID: "some-id",
ClientSecret: "some-secret",
ClientSecret: NewSecret("some-secret"),
TokenURL: "http://localhost:8511",
},
},
@ -49,7 +49,7 @@ func TestNewConfig(t *testing.T) {
args: args{
oauth: &OAuth2Config{
ClientID: "some-id",
ClientSecret: "some-secret",
ClientSecret: NewSecret("some-secret"),
ClientSecretFile: "testdata/test_secretfile.txt",
TokenURL: "http://localhost:8511",
},
@ -61,7 +61,7 @@ func TestNewConfig(t *testing.T) {
args: args{
basicAuth: &BasicAuthConfig{
Username: "user",
Password: "password",
Password: NewSecret("password"),
},
},
expectHeader: "Basic dXNlcjpwYXNzd29yZA==",
@ -81,7 +81,7 @@ func TestNewConfig(t *testing.T) {
args: args{
az: &Authorization{
Type: "Bearer",
Credentials: "Value",
Credentials: NewSecret("Value"),
},
},
expectHeader: "Bearer Value",

41
lib/promscrape/config.go
View file

@ -240,27 +240,29 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
}
// loadConfig loads Prometheus config from the given path.
func loadConfig(path string) (*Config, error) {
func loadConfig(path string) (*Config, []byte, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
}
var c Config
if err := c.parseData(data, path); err != nil {
return nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
dataNew, err := c.parseData(data, path)
if err != nil {
return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
}
return &c, nil
return &c, dataNew, nil
}
func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, error) {
func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, []byte, error) {
var scrapeConfigs []ScrapeConfig
var scsData []byte
for _, filePath := range scrapeConfigFiles {
filePath := getFilepath(baseDir, filePath)
paths := []string{filePath}
if strings.Contains(filePath, "*") {
ps, err := filepath.Glob(filePath)
if err != nil {
return nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err)
return nil, nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err)
}
sort.Strings(ps)
paths = ps
@ -268,17 +270,19 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]Scrape
for _, path := range paths {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err)
return nil, nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err)
}
data = envtemplate.Replace(data)
var scs []ScrapeConfig
if err = yaml.UnmarshalStrict(data, &scs); err != nil {
return nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err)
return nil, nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err)
}
scrapeConfigs = append(scrapeConfigs, scs...)
scsData = append(scsData, '\n')
scsData = append(scsData, data...)
}
}
return scrapeConfigs, nil
return scrapeConfigs, scsData, nil
}
// IsDryRun returns true if -promscrape.config.dryRun command-line flag is set
@ -286,30 +290,31 @@ func IsDryRun() bool {
return *dryRun
}
func (cfg *Config) parseData(data []byte, path string) error {
func (cfg *Config) parseData(data []byte, path string) ([]byte, error) {
if err := unmarshalMaybeStrict(data, cfg); err != nil {
return fmt.Errorf("cannot unmarshal data: %w", err)
return nil, fmt.Errorf("cannot unmarshal data: %w", err)
}
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
}
cfg.baseDir = filepath.Dir(absPath)
// Load cfg.ScrapeConfigFiles into c.ScrapeConfigs
scs, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles)
scs, scsData, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles)
if err != nil {
return fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err)
return nil, fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err)
}
cfg.ScrapeConfigFiles = nil
cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scs...)
dataNew := append(data, scsData...)
// Check that all the scrape configs have unique JobName
m := make(map[string]struct{}, len(cfg.ScrapeConfigs))
for i := range cfg.ScrapeConfigs {
jobName := cfg.ScrapeConfigs[i].JobName
if _, ok := m[jobName]; ok {
return fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName)
return nil, fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName)
}
m[jobName] = struct{}{}
}
@ -319,11 +324,11 @@ func (cfg *Config) parseData(data []byte, path string) error {
sc := &cfg.ScrapeConfigs[i]
swc, err := getScrapeWorkConfig(sc, cfg.baseDir, &cfg.Global)
if err != nil {
return fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err)
return nil, fmt.Errorf("cannot parse `scrape_config` #%d: %w", i+1, err)
}
sc.swc = swc
}
return nil
return dataNew, nil
}
func unmarshalMaybeStrict(data []byte, dst interface{}) error {

63
lib/promscrape/config_test.go
View file

@ -1,6 +1,7 @@
package promscrape
import (
"bytes"
"fmt"
"reflect"
"strconv"
@ -67,39 +68,51 @@ func TestLoadStaticConfigs(t *testing.T) {
}
func TestLoadConfig(t *testing.T) {
cfg, err := loadConfig("testdata/prometheus.yml")
cfg, data, err := loadConfig("testdata/prometheus.yml")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if cfg == nil {
t.Fatalf("expecting non-nil config")
}
if data == nil {
t.Fatalf("expecting non-nil data")
}
cfg, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
cfg, data, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if cfg == nil {
t.Fatalf("expecting non-nil config")
}
if data == nil {
t.Fatalf("expecting non-nil data")
}
// Try loading non-existing file
cfg, err = loadConfig("testdata/non-existing-file")
cfg, data, err = loadConfig("testdata/non-existing-file")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if cfg != nil {
t.Fatalf("unexpected non-nil config: %#v", cfg)
}
if data != nil {
t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
}
// Try loading invalid file
cfg, err = loadConfig("testdata/file_sd_1.yml")
cfg, data, err = loadConfig("testdata/file_sd_1.yml")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if cfg != nil {
t.Fatalf("unexpected non-nil config: %#v", cfg)
}
if data != nil {
t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
}
}
func TestBlackboxExporter(t *testing.T) {
@ -122,9 +135,13 @@ scrape_configs:
replacement: black:9115 # The blackbox exporter's real hostname:port.%
`
var cfg Config
if err := cfg.parseData([]byte(data), "sss"); err != nil {
allData, err := cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parase data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws := cfg.getStaticScrapeWork()
resetNonEssentialFields(sws)
swsExpected := []*ScrapeWork{{
@ -187,9 +204,13 @@ scrape_configs:
- files: [testdata/file_sd.json]
`
var cfg Config
if err := cfg.parseData([]byte(data), "sss"); err != nil {
allData, err := cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parase data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws := cfg.getFileSDScrapeWork(nil)
if !equalStaticConfigForScrapeWorks(sws, sws) {
t.Fatalf("unexpected non-equal static configs;\nsws:\n%#v", sws)
@ -203,9 +224,13 @@ scrape_configs:
- files: [testdata/file_sd_1.yml]
`
var cfgNew Config
if err := cfgNew.parseData([]byte(dataNew), "sss"); err != nil {
allData, err = cfgNew.parseData([]byte(dataNew), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != dataNew {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, dataNew)
}
swsNew := cfgNew.getFileSDScrapeWork(sws)
if equalStaticConfigForScrapeWorks(swsNew, sws) {
t.Fatalf("unexpected equal static configs;\nswsNew:\n%#v\nsws:\n%#v", swsNew, sws)
@ -218,9 +243,13 @@ scrape_configs:
file_sd_configs:
- files: [testdata/prometheus.yml]
`
if err := cfg.parseData([]byte(data), "sss"); err != nil {
allData, err = cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws = cfg.getFileSDScrapeWork(swsNew)
if len(sws) != 0 {
t.Fatalf("unexpected non-empty sws:\n%#v", sws)
@ -233,9 +262,13 @@ scrape_configs:
file_sd_configs:
- files: [testdata/empty_target_file_sd.yml]
`
if err := cfg.parseData([]byte(data), "sss"); err != nil {
allData, err = cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws = cfg.getFileSDScrapeWork(swsNew)
if len(sws) != 0 {
t.Fatalf("unexpected non-empty sws:\n%#v", sws)
@ -244,17 +277,25 @@ scrape_configs:
func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
var cfg Config
if err := cfg.parseData(data, path); err != nil {
allData, err := cfg.parseData(data, path)
if err != nil {
return nil, fmt.Errorf("cannot parse data: %w", err)
}
if !bytes.Equal(allData, data) {
return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
return cfg.getFileSDScrapeWork(nil), nil
}
func getStaticScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
var cfg Config
if err := cfg.parseData(data, path); err != nil {
allData, err := cfg.parseData(data, path)
if err != nil {
return nil, fmt.Errorf("cannot parse data: %w", err)
}
if !bytes.Equal(allData, data) {
return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
return cfg.getStaticScrapeWork(), nil
}

8
lib/promscrape/discovery/consul/api.go
View file

@ -44,10 +44,10 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
return nil, err
}
if token != "" {
if hcc.BearerToken != "" {
if hcc.BearerToken != nil {
return nil, fmt.Errorf("cannot set both token and bearer_token configs")
}
hcc.BearerToken = token
hcc.BearerToken = promauth.NewSecret(token)
}
if len(sdc.Username) > 0 {
if hcc.BasicAuth != nil {
@ -104,9 +104,9 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
return cfg, nil
}
func getToken(token *string) (string, error) {
func getToken(token *promauth.Secret) (string, error) {
if token != nil {
return *token, nil
return token.String(), nil
}
if tokenFile := os.Getenv("CONSUL_HTTP_TOKEN_FILE"); tokenFile != "" {
data, err := ioutil.ReadFile(tokenFile)

8
lib/promscrape/discovery/consul/consul.go
View file

@ -11,15 +11,15 @@ import (
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
type SDConfig struct {
Server string `yaml:"server,omitempty"`
Token *string `yaml:"token"`
Datacenter string `yaml:"datacenter"`
Server string `yaml:"server,omitempty"`
Token *promauth.Secret `yaml:"token"`
Datacenter string `yaml:"datacenter"`
// Namespace only supported at enterprise consul.
// https://www.consul.io/docs/enterprise/namespaces
Namespace string `yaml:"namespace,omitempty"`
Scheme string `yaml:"scheme,omitempty"`
Username string `yaml:"username"`
Password string `yaml:"password"`
Password *promauth.Secret `yaml:"password"`
HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
ProxyURL *proxy.URL `yaml:"proxy_url,omitempty"`
ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`

4
lib/promscrape/discovery/ec2/api.go
View file

@ -93,8 +93,8 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
if len(sdc.AccessKey) > 0 {
cfg.defaultAccessKey = sdc.AccessKey
}
if len(sdc.SecretKey) > 0 {
cfg.defaultSecretKey = sdc.SecretKey
if sdc.SecretKey != nil {
cfg.defaultSecretKey = sdc.SecretKey.String()
}
cfg.creds = &apiCredentials{
AccessKeyID: cfg.defaultAccessKey,

10
lib/promscrape/discovery/ec2/ec2.go
View file

@ -4,6 +4,8 @@ import (
"flag"
"fmt"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
// SDCheckInterval defines interval for targets refresh.
@ -15,10 +17,10 @@ var SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config
type SDConfig struct {
Region string `yaml:"region,omitempty"`
Endpoint string `yaml:"endpoint,omitempty"`
AccessKey string `yaml:"access_key,omitempty"`
SecretKey string `yaml:"secret_key,omitempty"`
Region string `yaml:"region,omitempty"`
Endpoint string `yaml:"endpoint,omitempty"`
AccessKey string `yaml:"access_key,omitempty"`
SecretKey *promauth.Secret `yaml:"secret_key,omitempty"`
// TODO add support for Profile, not working atm
Profile string `yaml:"profile,omitempty"`
RoleARN string `yaml:"role_arn,omitempty"`

35
lib/promscrape/discovery/openstack/auth.go
View file

@ -6,6 +6,8 @@ import (
"net/url"
"os"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
// authResponse represents identity api response
@ -53,7 +55,7 @@ func getComputeEndpointURL(catalog []catalogItem, availability, region string) (
// buildAuthRequestBody builds request for authentication
func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
if len(sdc.Password) == 0 && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 {
if sdc.Password == nil && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 {
return nil, fmt.Errorf("password and application credentials are missing")
}
type domainReq struct {
@ -97,24 +99,25 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
// if insufficient or incompatible information is present.
var req request
if len(sdc.Password) == 0 {
if sdc.Password == nil {
// There are three kinds of possible application_credential requests
// 1. application_credential id + secret
// 2. application_credential name + secret + user_id
// 3. application_credential name + secret + username + domain_id / domain_name
if len(sdc.ApplicationCredentialID) > 0 {
if len(sdc.ApplicationCredentialSecret) == 0 {
if sdc.ApplicationCredentialSecret == nil {
return nil, fmt.Errorf("ApplicationCredentialSecret is empty")
}
req.Auth.Identity.Methods = []string{"application_credential"}
secret := sdc.ApplicationCredentialSecret.String()
req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
ID: &sdc.ApplicationCredentialID,
Secret: &sdc.ApplicationCredentialSecret,
Secret: &secret,
}
return json.Marshal(req)
}
if len(sdc.ApplicationCredentialSecret) == 0 {
if sdc.ApplicationCredentialSecret == nil {
return nil, fmt.Errorf("missing application_credential_secret when application_credential_name is set")
}
var userRequest *userReq
@ -143,10 +146,11 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
return nil, fmt.Errorf("domain_id and domain_name cannot be empty for application_credential_name auth")
}
req.Auth.Identity.Methods = []string{"application_credential"}
secret := sdc.ApplicationCredentialSecret.String()
req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
Name: &sdc.ApplicationCredentialName,
User: userRequest,
Secret: &sdc.ApplicationCredentialSecret,
Secret: &secret,
}
return json.Marshal(req)
}
@ -168,11 +172,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
return nil, fmt.Errorf("both domain_id and domain_name is present")
}
// Configure the request for Username and Password authentication with a DomainID.
if len(sdc.Password) > 0 {
if sdc.Password != nil {
password := sdc.Password.String()
req.Auth.Identity.Password = &passwordReq{
User: userReq{
Name: &sdc.Username,
Password: &sdc.Password,
Password: &password,
Domain: &domainReq{ID: &sdc.DomainID},
},
}
@ -180,11 +185,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
}
if len(sdc.DomainName) > 0 {
// Configure the request for Username and Password authentication with a DomainName.
if len(sdc.Password) > 0 {
if sdc.Password != nil {
password := sdc.Password.String()
req.Auth.Identity.Password = &passwordReq{
User: userReq{
Name: &sdc.Username,
Password: &sdc.Password,
Password: &password,
Domain: &domainReq{Name: &sdc.DomainName},
},
}
@ -199,11 +205,12 @@ func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
return nil, fmt.Errorf("both user_id and domain_name is present")
}
// Configure the request for UserID and Password authentication.
if len(sdc.Password) > 0 {
if sdc.Password != nil {
password := sdc.Password.String()
req.Auth.Identity.Password = &passwordReq{
User: userReq{
ID: &sdc.UserID,
Password: &sdc.Password,
Password: &password,
},
}
}
@ -300,13 +307,13 @@ func readCredentialsFromEnv() SDConfig {
IdentityEndpoint: authURL,
Username: username,
UserID: userID,
Password: password,
Password: promauth.NewSecret(password),
ProjectName: tenantName,
ProjectID: tenantID,
DomainName: domainName,
DomainID: domainID,
ApplicationCredentialName: applicationCredentialName,
ApplicationCredentialID: applicationCredentialID,
ApplicationCredentialSecret: applicationCredentialSecret,
ApplicationCredentialSecret: promauth.NewSecret(applicationCredentialSecret),
}
}

6
lib/promscrape/discovery/openstack/auth_test.go
View file

@ -3,6 +3,8 @@ package openstack
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
func Test_buildAuthRequestBody1(t *testing.T) {
@ -27,7 +29,7 @@ func Test_buildAuthRequestBody1(t *testing.T) {
args: args{
sdc: &SDConfig{
Username: "some-user",
Password: "some-password",
Password: promauth.NewSecret("some-password"),
DomainName: "some-domain",
},
},
@ -38,7 +40,7 @@ func Test_buildAuthRequestBody1(t *testing.T) {
args: args{
sdc: &SDConfig{
ApplicationCredentialID: "some-id",
ApplicationCredentialSecret: "some-secret",
ApplicationCredentialSecret: promauth.NewSecret("some-secret"),
},
},
want: []byte(`{"auth":{"identity":{"methods":["application_credential"],"application_credential":{"id":"some-id","secret":"some-secret"}}}}`),

26
lib/promscrape/discovery/openstack/openstack.go
View file

@ -17,19 +17,19 @@ var SDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*ti
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config
type SDConfig struct {
IdentityEndpoint string `yaml:"identity_endpoint,omitempty"`
Username string `yaml:"username,omitempty"`
UserID string `yaml:"userid,omitempty"`
Password string `yaml:"password,omitempty"`
ProjectName string `yaml:"project_name,omitempty"`
ProjectID string `yaml:"project_id,omitempty"`
DomainName string `yaml:"domain_name,omitempty"`
DomainID string `yaml:"domain_id,omitempty"`
ApplicationCredentialName string `yaml:"application_credential_name,omitempty"`
ApplicationCredentialID string `yaml:"application_credential_id,omitempty"`
ApplicationCredentialSecret string `yaml:"application_credential_secret,omitempty"`
Role string `yaml:"role"`
Region string `yaml:"region"`
IdentityEndpoint string `yaml:"identity_endpoint,omitempty"`
Username string `yaml:"username,omitempty"`
UserID string `yaml:"userid,omitempty"`
Password *promauth.Secret `yaml:"password,omitempty"`
ProjectName string `yaml:"project_name,omitempty"`
ProjectID string `yaml:"project_id,omitempty"`
DomainName string `yaml:"domain_name,omitempty"`
DomainID string `yaml:"domain_id,omitempty"`
ApplicationCredentialName string `yaml:"application_credential_name,omitempty"`
ApplicationCredentialID string `yaml:"application_credential_id,omitempty"`
ApplicationCredentialSecret *promauth.Secret `yaml:"application_credential_secret,omitempty"`
Role string `yaml:"role"`
Region string `yaml:"region"`
// RefreshInterval time.Duration `yaml:"refresh_interval"`
// refresh_interval is obtained from `-promscrape.openstackSDCheckInterval` command-line option.
Port int `yaml:"port,omitempty"`

19
lib/promscrape/scraper.go
View file

@ -43,7 +43,7 @@ func CheckConfig() error {
if *promscrapeConfigFile == "" {
return fmt.Errorf("missing -promscrape.config option")
}
_, err := loadConfig(*promscrapeConfigFile)
_, _, err := loadConfig(*promscrapeConfigFile)
return err
}
@ -99,12 +99,12 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
sighupCh := procutil.NewSighupChan()
logger.Infof("reading Prometheus configs from %q", configFile)
cfg, err := loadConfig(configFile)
cfg, data, err := loadConfig(configFile)
if err != nil {
logger.Fatalf("cannot read %q: %s", configFile, err)
}
data := cfg.marshal()
configData.Store(&data)
marshaledData := cfg.marshal()
configData.Store(&marshaledData)
cfg.mustStart()
scs := newScrapeConfigs(pushData)
@ -134,12 +134,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
select {
case <-sighupCh:
logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile)
cfgNew, err := loadConfig(configFile)
cfgNew, dataNew, err := loadConfig(configFile)
if err != nil {
logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err)
goto waitForChans
}
dataNew := cfgNew.marshal()
if bytes.Equal(data, dataNew) {
logger.Infof("nothing changed in %q", configFile)
goto waitForChans
@ -148,14 +147,14 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
cfgNew.mustStart()
cfg = cfgNew
data = dataNew
configData.Store(&data)
marshaledData = cfgNew.marshal()
configData.Store(&marshaledData)
case <-tickerCh:
cfgNew, err := loadConfig(configFile)
cfgNew, dataNew, err := loadConfig(configFile)
if err != nil {
logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err)
goto waitForChans
}
dataNew := cfgNew.marshal()
if bytes.Equal(data, dataNew) {
// Nothing changed since the previous loadConfig
goto waitForChans
@ -164,7 +163,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
cfgNew.mustStart()
cfg = cfgNew
data = dataNew
configData.Store(&data)
configData.Store(&marshaledData)
case <-globalStopCh:
cfg.mustStop()
logger.Infof("stopping Prometheus scrapers")