From fca3b14b7b8ea8367603e02d497f5ae208a3fba2 Mon Sep 17 00:00:00 2001 From: Nihal <38865967+wasim-nihal@users.noreply.github.com> Date: Wed, 31 Jan 2024 23:11:16 +0530 Subject: [PATCH] Support for username_file in scrape config (basic_auth) similar to Prometheus for having config compatibility (#5720) * adding support for username_file in basic_auth of scrape config Signed-off-by: Syed Nihal * adding support for username_file in basic_auth of scrape config. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5511 Signed-off-by: Syed Nihal * adding support for username_file in basic_auth of scrape config Signed-off-by: Syed Nihal * adding support for username_file in basic_auth of scrape config Signed-off-by: Syed Nihal * adding support for username_file in basic_auth of scrape config Signed-off-by: Syed Nihal --------- Signed-off-by: Syed Nihal --- docs/CHANGELOG.md | 1 + docs/sd_configs.md | 7 ++- lib/promauth/config.go | 87 +++++++++++++++++++++++++++---------- lib/promauth/config_test.go | 31 ++++++++++++- 4 files changed, 100 insertions(+), 26 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a7b8bdc56..85ec7455b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -39,6 +39,7 @@ Released at 2024-01-30 * SECURITY: upgrade base docker image (Alpine) from 3.19.0 to 3.19.1. See [alpine 3.19.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.19.1-released.html). * SECURITY: upgrade Go builder from Go1.21.5 to Go1.21.6. See [the list of issues addressed in Go1.21.6](https://github.com/golang/go/issues?q=milestone%3AGo1.21.6+label%3ACherryPickApproved). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html) added support of `username_file` for `basic_auth` in scrape_config to have config compatibility with Prometheus. See [these docs](https://docs.victoriametrics.com/sd_configs/#http-api-client-options) and [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5511). Thanks to @wasim-nihal for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5720). * FEATURE: improve new [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series) registration speed on systems with high number of CPU cores. Thanks to @misutoth for the initial idea and [implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5212). * FEATURE: make [background merge](https://docs.victoriametrics.com/#storage) more responsive and scalable. This should help the following issues: [5190](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5190), [3425](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3425), [648](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/648). * FEATURE: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): add support for negative index in `groupByNode` and `aliasByNode` functions. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5581). diff --git a/docs/sd_configs.md b/docs/sd_configs.md index cd6001c7f..ba879a776 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -821,6 +821,7 @@ scrape_configs: # # basic_auth: # username: "..." + # username_file: "..." # is mutually-exclusive with username # password: "..." # password_file: "..." # is mutually-exclusive with password @@ -1625,8 +1626,9 @@ and in the majority of [supported service discovery configs](#supported-service- # basic_auth is an optional HTTP basic authentication configuration. # basic_auth: # username: "..." + # username_file: "..." # is mutually-exclusive with username # password: "..." - # password_file: "..." + # password_file: "..." # is mutually-exclusive with password # bearer_token is an optional Bearer token to send in every HTTP API request during service discovery. # bearer_token: "..." @@ -1663,8 +1665,9 @@ and in the majority of [supported service discovery configs](#supported-service- # proxy_basic_auth is an optional HTTP basic authentication configuration for the proxy_url. # proxy_basic_auth: # username: "..." + # username_file: "..." # is mutually-exclusive with username # password: "..." - # password_file: "..." + # password_file: "..." # is mutually-exclusive with password # proxy_bearer_token is an optional Bearer token to send to proxy_url. # proxy_bearer_token: "..." diff --git a/lib/promauth/config.go b/lib/promauth/config.go index 6b95a8017..e7084e59b 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -93,6 +93,7 @@ type Authorization struct { // BasicAuthConfig represents basic auth config. type BasicAuthConfig struct { Username string `yaml:"username"` + UsernameFile string `yaml:"username_file"` Password *Secret `yaml:"password,omitempty"` PasswordFile string `yaml:"password_file,omitempty"` } @@ -643,36 +644,76 @@ func (actx *authContext) initFromAuthorization(baseDir string, az *Authorization } func (actx *authContext) initFromBasicAuthConfig(baseDir string, ba *BasicAuthConfig) error { - if ba.Username == "" { - return fmt.Errorf("missing `username` in `basic_auth` section") + if ba.Username == "" && ba.UsernameFile == "" { + return fmt.Errorf("missing `username` and `username_file` in `basic_auth` section. either one should be configured") } - if ba.PasswordFile == "" { - // See https://en.wikipedia.org/wiki/Basic_access_authentication - token := ba.Username + ":" + ba.Password.String() - token64 := base64.StdEncoding.EncodeToString([]byte(token)) - ah := "Basic " + token64 - actx.getAuthHeader = func() (string, error) { - return ah, nil - } - actx.authHeaderDigest = fmt.Sprintf("basic(username=%q, password=%q)", ba.Username, ba.Password) - return nil + if ba.Username != "" && ba.UsernameFile != "" { + return fmt.Errorf("both `username`=%q and `username_file`=%q are set in `basic_auth` section", ba.Username, ba.UsernameFile) } - if ba.Password != nil { + if ba.Password != nil && ba.PasswordFile != "" { return fmt.Errorf("both `password`=%q and `password_file`=%q are set in `basic_auth` section", ba.Password, ba.PasswordFile) } - filePath := fscore.GetFilepath(baseDir, ba.PasswordFile) - actx.getAuthHeader = func() (string, error) { - password, err := fscore.ReadPasswordFromFileOrHTTP(filePath) - if err != nil { - return "", fmt.Errorf("cannot read password from `password_file`=%q set in `basic_auth` section: %w", ba.PasswordFile, err) + actx.getAuthHeader, actx.authHeaderDigest = fetchBasicAuthHeaderAndDigest(baseDir, ba) + return nil +} + +func fetchBasicAuthHeaderAndDigest(baseDir string, ba *BasicAuthConfig) (func() (string, error), string) { + var getUsername, getPassword func() (string, error) + var usernameDigest, passwordDigest string + + if ba.Username != "" { + usernameDigest = fmt.Sprintf("username=%q", ba.Username) + getUsername = func() (string, error) { + return ba.Username, nil } - // See https://en.wikipedia.org/wiki/Basic_access_authentication - token := ba.Username + ":" + password - token64 := base64.StdEncoding.EncodeToString([]byte(token)) + } + if ba.UsernameFile != "" { + usernameDigest = fmt.Sprintf("usernameFile=%q", ba.UsernameFile) + getUsername = func() (string, error) { + username, err := fscore.ReadPasswordFromFileOrHTTP(fscore.GetFilepath(baseDir, ba.UsernameFile)) + if err != nil { + return "", fmt.Errorf("cannot read username from `username_file`=%q set in `basic_auth` section: %w", ba.UsernameFile, err) + } + return username, nil + } + } + if ba.Password != nil { + passwordDigest = fmt.Sprintf("password=%q", ba.Password.String()) + getPassword = func() (string, error) { + return ba.Password.String(), nil + } + } + if ba.PasswordFile != "" { + passwordDigest = fmt.Sprintf("passwordFile=%q", ba.PasswordFile) + getPassword = func() (string, error) { + password, err := fscore.ReadPasswordFromFileOrHTTP(fscore.GetFilepath(baseDir, ba.PasswordFile)) + if err != nil { + return "", fmt.Errorf("cannot read password from `password_file`=%q set in `basic_auth` section: %w", ba.PasswordFile, err) + } + return password, nil + } + } + authHeader := func() (string, error) { + username, err := getUsername() + if err != nil { + return "", err + } + password, err := getPassword() + if err != nil { + return "", err + } + token64 := generateBasicAuthEncodedToken(username, password) return "Basic " + token64, nil } - actx.authHeaderDigest = fmt.Sprintf("basic(username=%q, passwordFile=%q)", ba.Username, filePath) - return nil + authHeaderDigest := fmt.Sprintf("basic(%q, %q)", usernameDigest, passwordDigest) + return authHeader, authHeaderDigest +} + +func generateBasicAuthEncodedToken(username string, password string) string { + // See https://en.wikipedia.org/wiki/Basic_access_authentication + token := username + ":" + password + token64 := base64.StdEncoding.EncodeToString([]byte(token)) + return token64 } func (actx *authContext) mustInitFromBearerTokenFile(baseDir string, bearerTokenFile string) { diff --git a/lib/promauth/config_test.go b/lib/promauth/config_test.go index 861ab3265..c3e12ddc1 100644 --- a/lib/promauth/config_test.go +++ b/lib/promauth/config_test.go @@ -55,6 +55,14 @@ basic_auth: password_file: testdata/test_secretfile.txt `) + // basic_auth: username and username_file are set + f(` +basic_auth: + username: user + username_file: testdata/test_secretfile.txt + password: pass +`) + // bearer_token: both authorization and bearer_token are set f(` authorization: @@ -345,13 +353,20 @@ oauth2: ca_file: non-existing-file `) - // basic auth via non-existing file + // basic auth via non-existing password_file f(` basic_auth: username: user password_file: non-existing-file `) + // basic auth via non-existing username_file + f(` +basic_auth: + username_file: non-existing-file + password: pass +`) + // bearer token via non-existing file f(` bearer_token_file: non-existing-file @@ -471,6 +486,20 @@ basic_auth: password_file: testdata/test_secretfile.txt `, "Basic dXNlcjpzZWNyZXQtY29udGVudA==") + // basic auth username via file + f(` + basic_auth: + username_file: testdata/test_secretfile.txt + password: password + `, "Basic c2VjcmV0LWNvbnRlbnQ6cGFzc3dvcmQ=") + + // basic auth username and password via file + f(` + basic_auth: + username_file: testdata/test_secretfile.txt + password_file: testdata/test_secretfile.txt +`, "Basic c2VjcmV0LWNvbnRlbnQ6c2VjcmV0LWNvbnRlbnQ=") + // inline authorization config f(` authorization: