github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promauth: add ability to send additional http headers in requests to scrape targets

Browse source 
This solves https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header
This commit is contained in:
Aliaksandr Valialkin 2022-06-22 20:38:43 +03:00
parent 51362f9333
commit 1c4f67c5d2
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
20 changed files with 329 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
README.md
View file

@ -1134,6 +1134,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome
The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
Example contents for `-relabelConfig` file:
```yml
@ -1147,8 +1149,7 @@ Example contents for `-relabelConfig` file:
regex: true
```
VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
## Federation

18
app/vmagent/README.md
View file

@ -183,6 +183,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
`vmagent` also support the following additional options in `scrape_configs` section:
* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example:
```yaml
scrape_configs:
- job_name: custom_headers
headers:
- "TenantID: abc"
- "My-Auth: TopSecret"
```
* `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets
to save network bandwidth.
* `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
@ -297,6 +307,8 @@ The relabeling can be defined in the following places:
* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
You can read more about relabeling in the following articles:
* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
@ -424,9 +436,11 @@ scrape_configs:
Proxy can be configured with the following optional settings:
* `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)
* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
* `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2).
* `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config).
* `proxy_headers` for passing additional HTTP headers in requests to proxy.
For example:
@ -443,6 +457,8 @@ scrape_configs:
key_file: /path/to/key
ca_file: /path/to/ca
server_name: real-server-name
proxy_headers:
- "Proxy-Auth: top-secret"
```
## Cardinality limiter

6
app/vmagent/remotewrite/client.go
View file

@ -219,7 +219,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
}
authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg, nil)
if err != nil {
return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
}
@ -306,9 +306,7 @@ again:
h.Set("Content-Type", "application/x-protobuf")
h.Set("Content-Encoding", "snappy")
h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
if ah := c.authCfg.GetAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
c.authCfg.SetHeaders(req, true)
if c.awsCfg != nil {
if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
// there is no need in retry, request will be rejected by client.Do and retried by code below

4
app/vmalert/datasource/vm.go
View file

@ -146,9 +146,7 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
}
req.Header.Set("Content-Type", "application/json")
if s.authCfg != nil {
if auth := s.authCfg.GetAuthHeader(); auth != "" {
req.Header.Set("Authorization", auth)
}
s.authCfg.SetHeaders(req, true)
}
return req, nil
}

6
app/vmalert/datasource/vm_test.go
View file

@ -83,7 +83,7 @@ func TestVMInstantQuery(t *testing.T) {
srv := httptest.NewServer(mux)
defer srv.Close()
authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
authCfg, err := baCfg.NewConfig(".")
if err != nil {
t.Fatalf("unexpected: %s", err)
}
@ -206,7 +206,7 @@ func TestVMRangeQuery(t *testing.T) {
srv := httptest.NewServer(mux)
defer srv.Close()
authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
authCfg, err := baCfg.NewConfig(".")
if err != nil {
t.Fatalf("unexpected: %s", err)
}
@ -247,7 +247,7 @@ func TestVMRangeQuery(t *testing.T) {
}
func TestRequestParams(t *testing.T) {
authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
authCfg, err := baCfg.NewConfig(".")
if err != nil {
t.Fatalf("unexpected: %s", err)
}

4
app/vmalert/notifier/alertmanager.go
View file

@ -79,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
req = req.WithContext(ctx)
if am.authCfg != nil {
if auth := am.authCfg.GetAuthHeader(); auth != "" {
req.Header.Set("Authorization", auth)
}
am.authCfg.SetHeaders(req, true)
}
resp, err := am.client.Do(req)
if err != nil {

4
app/vmalert/remotewrite/remotewrite.go
View file

@ -245,9 +245,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
if c.authCfg != nil {
if auth := c.authCfg.GetAuthHeader(); auth != "" {
req.Header.Set("Authorization", auth)
}
c.authCfg.SetHeaders(req, true)
}
if !*disablePathAppend {
req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")

10
docs/CHANGELOG.md
View file

@ -16,7 +16,17 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* FEATURE: add `-search.setLookbackToStep` command-line flag, which enables InfluxDB-like gap filling during querying. See [these docs](https://docs.victoriametrics.com/guides/migrate-from-influx.html) for details.
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`:
```yaml
scrape_configs:
- job_name: foo
headers:
- "My-Auth: top-secret"
- "TenantID: FooBar"
static_configs:
- targets: ["host123:8080"]
```
## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)

5
docs/README.md
View file

@ -1134,6 +1134,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome
The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
Example contents for `-relabelConfig` file:
```yml
@ -1147,8 +1149,7 @@ Example contents for `-relabelConfig` file:
regex: true
```
VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
## Federation

5
docs/Single-server-VictoriaMetrics.md
View file

@ -1138,6 +1138,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome
The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
Example contents for `-relabelConfig` file:
```yml
@ -1151,8 +1153,7 @@ Example contents for `-relabelConfig` file:
regex: true
```
VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
## Federation

18
docs/vmagent.md
View file

@ -187,6 +187,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
`vmagent` also support the following additional options in `scrape_configs` section:
* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example:
```yaml
scrape_configs:
- job_name: custom_headers
headers:
- "TenantID: abc"
- "My-Auth: TopSecret"
```
* `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets
to save network bandwidth.
* `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
@ -301,6 +311,8 @@ The relabeling can be defined in the following places:
* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
You can read more about relabeling in the following articles:
* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
@ -428,9 +440,11 @@ scrape_configs:
Proxy can be configured with the following optional settings:
* `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)
* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
* `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2).
* `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config).
* `proxy_headers` for passing additional HTTP headers in requests to proxy.
For example:
@ -447,6 +461,8 @@ scrape_configs:
key_file: /path/to/key
ca_file: /path/to/ca
server_name: real-server-name
proxy_headers:
- "Proxy-Auth: top-secret"
```
## Cardinality limiter

92
lib/promauth/config.go
View file

@ -15,6 +15,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/fasthttp"
"github.com/cespare/xxhash/v2"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
@ -116,6 +117,9 @@ type HTTPClientConfig struct {
BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
OAuth2 *OAuth2Config `yaml:"oauth2,omitempty"`
TLSConfig *TLSConfig `yaml:"tls_config,omitempty"`
// Headers contains optional HTTP headers, which must be sent in the request to the server
Headers []string `yaml:"headers,omitempty"`
}
// ProxyClientConfig represents proxy client config.
@ -124,7 +128,11 @@ type ProxyClientConfig struct {
BasicAuth *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"`
BearerToken *Secret `yaml:"proxy_bearer_token,omitempty"`
BearerTokenFile string `yaml:"proxy_bearer_token_file,omitempty"`
OAuth2 *OAuth2Config `yaml:"proxy_oauth2,omitempty"`
TLSConfig *TLSConfig `yaml:"proxy_tls_config,omitempty"`
// Headers contains optional HTTP headers, which must be sent in the request to the proxy
Headers []string `yaml:"proxy_headers,omitempty"`
}
// OAuth2Config represent OAuth2 configuration
@ -257,9 +265,71 @@ type Config struct {
authHeader string
authHeaderDeadline uint64
headers []keyValue
authDigest string
}
type keyValue struct {
key string
value string
}
func parseHeaders(headers []string) ([]keyValue, error) {
if len(headers) == 0 {
return nil, nil
}
kvs := make([]keyValue, len(headers))
for i, h := range headers {
n := strings.IndexByte(h, ':')
if n < 0 {
return nil, fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, h)
}
kv := &kvs[i]
kv.key = strings.TrimSpace(h[:n])
kv.value = strings.TrimSpace(h[n+1:])
}
return kvs, nil
}
// HeadersNoAuthString returns string representation of ac headers
func (ac *Config) HeadersNoAuthString() string {
if len(ac.headers) == 0 {
return ""
}
a := make([]string, len(ac.headers))
for i, h := range ac.headers {
a[i] = h.key + ": " + h.value + "\r\n"
}
return strings.Join(a, "")
}
// SetHeaders sets the configuted ac headers to req.
func (ac *Config) SetHeaders(req *http.Request, setAuthHeader bool) {
reqHeaders := req.Header
for _, h := range ac.headers {
reqHeaders.Set(h.key, h.value)
}
if setAuthHeader {
if ah := ac.GetAuthHeader(); ah != "" {
reqHeaders.Set("Authorization", ah)
}
}
}
// SetFasthttpHeaders sets the configured ac headers to req.
func (ac *Config) SetFasthttpHeaders(req *fasthttp.Request, setAuthHeader bool) {
reqHeaders := &req.Header
for _, h := range ac.headers {
reqHeaders.Set(h.key, h.value)
}
if setAuthHeader {
if ah := ac.GetAuthHeader(); ah != "" {
reqHeaders.Set("Authorization", ah)
}
}
}
// GetAuthHeader returns optional `Authorization: ...` http header.
func (ac *Config) GetAuthHeader() string {
f := ac.getAuthHeader
@ -281,8 +351,8 @@ func (ac *Config) GetAuthHeader() string {
// It is also used for comparing Config objects for equality. If two Config
// objects have the same string representation, then they are considered equal.
func (ac *Config) String() string {
return fmt.Sprintf("AuthDigest=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d",
ac.authDigest, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion)
return fmt.Sprintf("AuthDigest=%s, Headers=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d",
ac.authDigest, ac.headers, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion)
}
func (ac *Config) tlsRootCAString() string {
@ -330,21 +400,26 @@ func (ac *Config) NewTLSConfig() *tls.Config {
// NewConfig creates auth config for the given hcc.
func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig, hcc.Headers)
}
// NewConfig creates auth config for the given pcc.
func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig)
return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, pcc.OAuth2, pcc.TLSConfig, pcc.Headers)
}
// NewConfig creates auth config for the given o.
func (o *OAuth2Config) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig)
return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig, nil)
}
// NewConfig creates auth config for the given ba.
func (ba *BasicAuthConfig) NewConfig(baseDir string) (*Config, error) {
return NewConfig(baseDir, nil, ba, "", "", nil, nil, nil)
}
// NewConfig creates auth config from the given args.
func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig) (*Config, error) {
func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig, headers []string) (*Config, error) {
var getAuthHeader func() string
authDigest := ""
if az != nil {
@ -517,6 +592,10 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
tlsMinVersion = v
}
}
parsedHeaders, err := parseHeaders(headers)
if err != nil {
return nil, err
}
ac := &Config{
TLSRootCA: tlsRootCA,
TLSServerName: tlsServerName,
@ -527,6 +606,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
tlsCertDigest: tlsCertDigest,
getAuthHeader: getAuthHeader,
headers: parsedHeaders,
authDigest: authDigest,
}
return ac, nil

93
lib/promauth/config_test.go
View file

@ -4,6 +4,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"github.com/VictoriaMetrics/fasthttp"
)
func TestNewConfig(t *testing.T) {
@ -116,18 +118,103 @@ func TestNewConfig(t *testing.T) {
mock := httptest.NewServer(r)
tt.args.oauth.TokenURL = mock.URL
}
got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig)
got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig, nil)
if (err != nil) != tt.wantErr {
t.Errorf("NewConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != nil {
ah := got.GetAuthHeader()
req, err := http.NewRequest("GET", "http://foo", nil)
if err != nil {
t.Fatalf("unexpected error in http.NewRequest: %s", err)
}
got.SetHeaders(req, true)
ah := req.Header.Get("Authorization")
if ah != tt.expectHeader {
t.Fatalf("unexpected auth header; got %q; want %q", ah, tt.expectHeader)
t.Fatalf("unexpected auth header from net/http request; got %q; want %q", ah, tt.expectHeader)
}
var fhreq fasthttp.Request
got.SetFasthttpHeaders(&fhreq, true)
ahb := fhreq.Header.Peek("Authorization")
if string(ahb) != tt.expectHeader {
t.Fatalf("unexpected auth header from fasthttp request; got %q; want %q", ahb, tt.expectHeader)
}
}
})
}
}
func TestParseHeadersSuccess(t *testing.T) {
f := func(headers []string) {
t.Helper()
headersParsed, err := parseHeaders(headers)
if err != nil {
t.Fatalf("unexpected error when parsing %s: %s", headers, err)
}
for i, h := range headersParsed {
s := h.key + ": " + h.value
if s != headers[i] {
t.Fatalf("unexpected header parsed; got %q; want %q", s, headers[i])
}
}
}
f(nil)
f([]string{"foo: bar"})
f([]string{"Foo: bar", "A-b-c: d-e-f"})
}
func TestParseHeadersFailure(t *testing.T) {
f := func(headers []string) {
t.Helper()
headersParsed, err := parseHeaders(headers)
if err == nil {
t.Fatalf("expecting non-nil error from parseHeaders(%s)", headers)
}
if headersParsed != nil {
t.Fatalf("expecting nil result from parseHeaders(%s)", headers)
}
}
f([]string{"foo"})
f([]string{"foo bar baz"})
}
func TestConfigHeaders(t *testing.T) {
f := func(headers []string, resultExpected string) {
t.Helper()
headersParsed, err := parseHeaders(headers)
if err != nil {
t.Fatalf("cannot parse headers: %s", err)
}
c, err := NewConfig("", nil, nil, "", "", nil, nil, headers)
if err != nil {
t.Fatalf("cannot create config: %s", err)
}
req, err := http.NewRequest("GET", "http://foo", nil)
if err != nil {
t.Fatalf("unexpected error in http.NewRequest: %s", err)
}
result := c.HeadersNoAuthString()
if result != resultExpected {
t.Fatalf("unexpected result from HeadersNoAuthString; got\n%s\nwant\n%s", result, resultExpected)
}
c.SetHeaders(req, false)
for _, h := range headersParsed {
v := req.Header.Get(h.key)
if v != h.value {
t.Fatalf("unexpected value for net/http header %q; got %q; want %q", h.key, v, h.value)
}
}
var fhreq fasthttp.Request
c.SetFasthttpHeaders(&fhreq, false)
for _, h := range headersParsed {
v := fhreq.Header.Peek(h.key)
if string(v) != h.value {
t.Fatalf("unexpected value for fasthttp header %q; got %q; want %q", h.key, v, h.value)
}
}
}
f(nil, "")
f([]string{"foo: bar"}, "foo: bar\r\n")
f([]string{"Foo-Bar: Baz s:sdf", "A:b", "X-Forwarded-For: A-B:c"}, "Foo-Bar: Baz s:sdf\r\nA: b\r\nX-Forwarded-For: A-B:c\r\n")
}

38
lib/promscrape/client.go
View file

@ -48,8 +48,10 @@ type client struct {
scrapeTimeoutSecondsStr string
host string
requestURI string
getAuthHeader func() string
getProxyAuthHeader func() string
setHeaders func(req *http.Request)
setProxyHeaders func(req *http.Request)
setFasthttpHeaders func(req *fasthttp.Request)
setFasthttpProxyHeaders func(req *fasthttp.Request)
denyRedirects bool
disableCompression bool
disableKeepAlive bool
@ -65,7 +67,8 @@ func newClient(sw *ScrapeWork) *client {
if isTLS {
tlsCfg = sw.AuthConfig.NewTLSConfig()
}
getProxyAuthHeader := func() string { return "" }
setProxyHeaders := func(req *http.Request) {}
setFasthttpProxyHeaders := func(req *fasthttp.Request) {}
proxyURL := sw.ProxyURL
if !isTLS && proxyURL.IsHTTPOrHTTPS() {
// Send full sw.ScrapeURL in requests to a proxy host for non-TLS scrape targets
@ -79,8 +82,11 @@ func newClient(sw *ScrapeWork) *client {
tlsCfg = sw.ProxyAuthConfig.NewTLSConfig()
}
proxyURLOrig := proxyURL
getProxyAuthHeader = func() string {
return proxyURLOrig.GetAuthHeader(sw.ProxyAuthConfig)
setProxyHeaders = func(req *http.Request) {
proxyURLOrig.SetHeaders(sw.ProxyAuthConfig, req)
}
setFasthttpProxyHeaders = func(req *fasthttp.Request) {
proxyURLOrig.SetFasthttpHeaders(sw.ProxyAuthConfig, req)
}
proxyURL = &proxy.URL{}
}
@ -148,8 +154,10 @@ func newClient(sw *ScrapeWork) *client {
scrapeTimeoutSecondsStr: fmt.Sprintf("%.3f", sw.ScrapeTimeout.Seconds()),
host: host,
requestURI: requestURI,
getAuthHeader: sw.AuthConfig.GetAuthHeader,
getProxyAuthHeader: getProxyAuthHeader,
setHeaders: func(req *http.Request) { sw.AuthConfig.SetHeaders(req, true) },
setProxyHeaders: setProxyHeaders,
setFasthttpHeaders: func(req *fasthttp.Request) { sw.AuthConfig.SetFasthttpHeaders(req, true) },
setFasthttpProxyHeaders: setFasthttpProxyHeaders,
denyRedirects: sw.DenyRedirects,
disableCompression: sw.DisableCompression,
disableKeepAlive: sw.DisableKeepAlive,
@ -173,12 +181,8 @@ func (c *client) GetStreamReader() (*streamReader, error) {
// Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162
req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr)
if ah := c.getAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
if ah := c.getProxyAuthHeader(); ah != "" {
req.Header.Set("Proxy-Authorization", ah)
}
c.setHeaders(req)
c.setProxyHeaders(req)
resp, err := c.sc.Do(req)
if err != nil {
cancel()
@ -224,12 +228,8 @@ func (c *client) ReadData(dst []byte) ([]byte, error) {
// Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162
req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr)
if ah := c.getAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
if ah := c.getProxyAuthHeader(); ah != "" {
req.Header.Set("Proxy-Authorization", ah)
}
c.setFasthttpHeaders(req)
c.setFasthttpProxyHeaders(req)
if !*disableCompression && !c.disableCompression {
req.Header.Set("Accept-Encoding", "gzip")
}

24
lib/promscrape/config_test.go
View file

@ -141,6 +141,9 @@ scrape_configs:
- x
authorization:
type: foobar
headers:
- 'TenantID: fooBar'
- 'X: y:z'
relabel_configs:
- source_labels: [abc]
static_configs:
@ -149,6 +152,8 @@ scrape_configs:
relabel_debug: true
scrape_align_interval: 1h30m0s
proxy_bearer_token_file: file.txt
proxy_headers:
- 'My-Auth-Header: top-secret'
`)
}
@ -332,7 +337,7 @@ scrape_configs:
jobNameOriginal: "blackbox",
}}
if !reflect.DeepEqual(sws, swsExpected) {
t.Fatalf("unexpected scrapeWork;\ngot\n%+v\nwant\n%+v", sws, swsExpected)
t.Fatalf("unexpected scrapeWork;\ngot\n%#v\nwant\n%#v", sws, swsExpected)
}
}
@ -1650,12 +1655,25 @@ scrape_configs:
jobNameOriginal: "aaa",
},
})
ac, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"My-Auth: foo-Bar"})
if err != nil {
t.Fatalf("unexpected error when creating promauth.Config: %s", err)
}
proxyAC, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"Foo:bar"})
if err != nil {
t.Fatalf("unexpected error when creating promauth.Config for proxy: %s", err)
}
f(`
scrape_configs:
- job_name: 'snmp'
sample_limit: 100
disable_keepalive: true
disable_compression: true
headers:
- "My-Auth: foo-Bar"
proxy_headers:
- "Foo: bar"
scrape_align_interval: 1s
scrape_offset: 0.5s
static_configs:
@ -1727,8 +1745,8 @@ scrape_configs:
Value: "snmp",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
AuthConfig: ac,
ProxyAuthConfig: proxyAC,
SampleLimit: 100,
DisableKeepAlive: true,
DisableCompression: true,

7
lib/promscrape/discovery/kubernetes/api.go
View file

@ -16,7 +16,8 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslice` or `ingress`", role)
}
ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
cc := &sdc.HTTPClientConfig
ac, err := cc.NewConfig(baseDir)
if err != nil {
return nil, fmt.Errorf("cannot parse auth config: %w", err)
}
@ -30,7 +31,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
if err != nil {
return nil, fmt.Errorf("cannot build kube config from the specified `kubeconfig_file` config option: %w", err)
}
acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, nil, kc.tlsConfig)
acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, cc.OAuth2, kc.tlsConfig, cc.Headers)
if err != nil {
return nil, fmt.Errorf("cannot initialize auth config from `kubeconfig_file: %q`: %w", sdc.KubeConfigFile, err)
}
@ -57,7 +58,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
tlsConfig := promauth.TLSConfig{
CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
}
acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", nil, &tlsConfig)
acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", cc.OAuth2, &tlsConfig, cc.Headers)
if err != nil {
return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
}

14
lib/promscrape/discovery/kubernetes/api_watcher.go
View file

@ -207,8 +207,8 @@ type groupWatcher struct {
selectors []Selector
attachNodeMetadata bool
getAuthHeader func() string
client *http.Client
setHeaders func(req *http.Request)
client *http.Client
mu sync.Mutex
m map[string]*urlWatcher
@ -235,9 +235,9 @@ func newGroupWatcher(apiServer string, ac *promauth.Config, namespaces []string,
selectors: selectors,
attachNodeMetadata: attachNodeMetadata,
getAuthHeader: ac.GetAuthHeader,
client: client,
m: make(map[string]*urlWatcher),
setHeaders: func(req *http.Request) { ac.SetHeaders(req, true) },
client: client,
m: make(map[string]*urlWatcher),
}
}
@ -407,9 +407,7 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
if err != nil {
logger.Fatalf("cannot create a request for %q: %s", requestURL, err)
}
if ah := gw.getAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
gw.setHeaders(req)
resp, err := gw.client.Do(req)
if err != nil {
return nil, err

2
lib/promscrape/discovery/openstack/api.go
View file

@ -81,7 +81,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
port: sdc.Port,
}
if sdc.TLSConfig != nil {
ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig)
ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig, nil)
if err != nil {
return nil, err
}

40
lib/promscrape/discoveryutils/client.go
View file

@ -42,10 +42,10 @@ type Client struct {
apiServer string
hostPort string
getAuthHeader func() string
getProxyAuthHeader func() string
sendFullURL bool
hostPort string
setFasthttpHeaders func(req *fasthttp.Request)
setFasthttpProxyHeaders func(req *fasthttp.Request)
sendFullURL bool
}
// NewClient returns new Client for the given args.
@ -70,7 +70,7 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
tlsCfg = ac.NewTLSConfig()
}
sendFullURL := !isTLS && proxyURL.IsHTTPOrHTTPS()
getProxyAuthHeader := func() string { return "" }
setFasthttpProxyHeaders := func(req *fasthttp.Request) {}
if sendFullURL {
// Send full urls in requests to a proxy host for non-TLS apiServer
// like net/http package from Go does.
@ -82,8 +82,8 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
tlsCfg = proxyAC.NewTLSConfig()
}
proxyURLOrig := proxyURL
getProxyAuthHeader = func() string {
return proxyURLOrig.GetAuthHeader(proxyAC)
setFasthttpProxyHeaders = func(req *fasthttp.Request) {
proxyURLOrig.SetFasthttpHeaders(proxyAC, req)
}
proxyURL = &proxy.URL{}
}
@ -123,18 +123,18 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
MaxConns: 64 * 1024,
Dial: dialFunc,
}
getAuthHeader := func() string { return "" }
setFasthttpHeaders := func(req *fasthttp.Request) {}
if ac != nil {
getAuthHeader = ac.GetAuthHeader
setFasthttpHeaders = func(req *fasthttp.Request) { ac.SetFasthttpHeaders(req, true) }
}
return &Client{
hc: hc,
blockingClient: blockingClient,
apiServer: apiServer,
hostPort: hostPort,
getAuthHeader: getAuthHeader,
getProxyAuthHeader: getProxyAuthHeader,
sendFullURL: sendFullURL,
hc: hc,
blockingClient: blockingClient,
apiServer: apiServer,
hostPort: hostPort,
setFasthttpHeaders: setFasthttpHeaders,
setFasthttpProxyHeaders: setFasthttpProxyHeaders,
sendFullURL: sendFullURL,
}, nil
}
@ -202,12 +202,8 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient,
}
req.Header.SetHost(c.hostPort)
req.Header.Set("Accept-Encoding", "gzip")
if ah := c.getAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
if ah := c.getProxyAuthHeader(); ah != "" {
req.Header.Set("Proxy-Authorization", ah)
}
c.setFasthttpHeaders(&req)
c.setFasthttpProxyHeaders(&req)
if modifyRequest != nil {
modifyRequest(&req)
}

26
lib/proxy/proxy.go
View file

@ -6,6 +6,7 @@ import (
"encoding/base64"
"fmt"
"net"
"net/http"
"net/url"
"strings"
"time"
@ -60,8 +61,26 @@ func (u *URL) String() string {
return pu.String()
}
// GetAuthHeader returns Proxy-Authorization auth header for the given u and ac.
func (u *URL) GetAuthHeader(ac *promauth.Config) string {
// SetHeaders sets headers to req according to u and ac configs.
func (u *URL) SetHeaders(ac *promauth.Config, req *http.Request) {
ah := u.getAuthHeader(ac)
if ah != "" {
req.Header.Set("Proxy-Authorization", ah)
}
ac.SetHeaders(req, false)
}
// SetFasthttpHeaders sets headers to req according to u and ac configs.
func (u *URL) SetFasthttpHeaders(ac *promauth.Config, req *fasthttp.Request) {
ah := u.getAuthHeader(ac)
if ah != "" {
req.Header.Set("Proxy-Authorization", ah)
}
ac.SetFasthttpHeaders(req, false)
}
// getAuthHeader returns Proxy-Authorization auth header for the given u and ac.
func (u *URL) getAuthHeader(ac *promauth.Config) string {
authHeader := ""
if ac != nil {
authHeader = ac.GetAuthHeader()
@ -130,9 +149,10 @@ func (u *URL) NewDialFunc(ac *promauth.Config) (fasthttp.DialFunc, error) {
if isTLS {
proxyConn = tls.Client(proxyConn, tlsCfg)
}
authHeader := u.GetAuthHeader(ac)
authHeader := u.getAuthHeader(ac)
if authHeader != "" {
authHeader = "Proxy-Authorization: " + authHeader + "\r\n"
authHeader += ac.HeadersNoAuthString()
}
conn, err := sendConnectRequest(proxyConn, proxyAddr, addr, authHeader)
if err != nil {