app/vmauth: add ability to specify http headers to send in requests to backends

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1736
2024-11-21 14:44:00 +00:00 · 2021-10-22 19:08:06 +03:00 · 2021-10-22 19:08:06 +03:00 · 013d626889
commit 013d626889
parent 7fa15f7f86
8 changed files with 160 additions and 41 deletions
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@ -37,9 +37,8 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl
 `-auth.config` is represented in the following simple `yml` format:

 ```yml
-
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.

 users:
  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
@ -47,6 +46,14 @@ users:
 - bearer_token: "XXXX"
  url_prefix: "http://localhost:8428"

+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
  # The user for querying local single-node VictoriaMetrics.
  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
  # will be proxied to http://localhost:8428 .
@ -89,7 +96,6 @@ users:
  - "http://vminsert1:8480/insert/42/prometheus"
  - "http://vminsert2:8480/insert/42/prometheus"

-
  # A single user for querying and inserting data:
  # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@ -97,7 +103,8 @@ users:
  #     - http://vmselect2:8481/select/42/prometheus
  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
  url_map:
  - src_paths:
@ -109,7 +116,8 @@ users:
    - "http://vmselect2:8481/select/42/prometheus"
  - src_paths: ["/api/v1/write"]
    url_prefix: "http://vminsert:8480/insert/42/prometheus"
-```
+    headers:
+    - "X-Scope-OrgID: abc"```

 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@ -27,24 +27,53 @@ var (

 // AuthConfig represents auth config.
 type AuthConfig struct {
-	Users []UserInfo `yaml:"users"`
+	Users []UserInfo `yaml:"users,omitempty"`
 }

 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	BearerToken string     `yaml:"bearer_token"`
-	Username    string     `yaml:"username"`
-	Password    string     `yaml:"password"`
-	URLPrefix   *URLPrefix `yaml:"url_prefix"`
-	URLMap      []URLMap   `yaml:"url_map"`
+	BearerToken string     `yaml:"bearer_token,omitempty"`
+	Username    string     `yaml:"username,omitempty"`
+	Password    string     `yaml:"password,omitempty"`
+	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMap      []URLMap   `yaml:"url_map,omitempty"`
+	Headers     []Header   `yaml:"headers,omitempty"`

 	requests *metrics.Counter
 }

+// Header is `Name: Value` http header, which must be added to the proxied request.
+type Header struct {
+	Name  string
+	Value string
+}
+
+// UnmarshalYAML unmarshals h from f.
+func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
+	var s string
+	if err := f(&s); err != nil {
+		return err
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
+	}
+	h.Name = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
+}
+
+// MarshalYAML marshals h to yaml.
+func (h *Header) MarshalYAML() (interface{}, error) {
+	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
+	return s, nil
+}
+
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths"`
-	URLPrefix *URLPrefix `yaml:"url_prefix"`
+	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+	Headers   []Header   `yaml:"headers,omitempty"`
 }

 // SrcPath represents an src path
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@ -69,6 +69,14 @@ users:
  - [foo]
 `)

+	// Invalid headers
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  headers: foobar
+`)
+
 	// empty url_prefix
 	f(`
 users:
@ -156,6 +164,27 @@ users:
  - src_paths: ['fo[obar']
    url_prefix: http://foobar
 `)
+
+	// Invalid headers in url_map (missing ':')
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+    - foobar
+`)
+	// Invalid headers in url_map (dictionary instead of array)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+      aaa: bbb
+`)
 }

 func TestParseAuthConfigSuccess(t *testing.T) {
@ -231,6 +260,9 @@ users:
    url_prefix: http://vmselect/select/0/prometheus
  - src_paths: ["/api/v1/write"]
    url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
+    headers:
+    - "foo: bar"
+    - "xxx: y"
 `, map[string]*UserInfo{
 		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
@ -245,6 +277,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
 				},
 			},
 		},
--- a/app/vmauth/example_config.yml
+++ b/app/vmauth/example_config.yml
@ -1,5 +1,5 @@
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.

 users:
  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
@ -7,6 +7,14 @@ users:
 - bearer_token: "XXXX"
  url_prefix: "http://localhost:8428"

+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
  # The user for querying local single-node VictoriaMetrics.
  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
  # will be proxied to http://localhost:8428 .
@ -49,7 +57,6 @@ users:
  - "http://vminsert1:8480/insert/42/prometheus"
  - "http://vminsert2:8480/insert/42/prometheus"

-
  # A single user for querying and inserting data:
  # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@ -57,7 +64,8 @@ users:
  #     - http://vmselect2:8481/select/42/prometheus
  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
  url_map:
  - src_paths:
@ -69,3 +77,5 @@ users:
    - "http://vmselect2:8481/select/42/prometheus"
  - src_paths: ["/api/v1/write"]
    url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@ -84,12 +84,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	ui.requests.Inc()
-	targetURL, err := createTargetURL(ui, r.URL)
+	targetURL, headers, err := createTargetURL(ui, r.URL)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
 		return true
 	}
 	r.Header.Set("vm-target-url", targetURL.String())
+	for _, h := range headers {
+		r.Header.Set(h.Name, h.Value)
+	}
 	proxyRequest(w, r)
 	return true
 }
--- a/app/vmauth/target_url.go
+++ b/app/vmauth/target_url.go
@ -35,7 +35,7 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }

-func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, error) {
+func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
 	u := *uOrig
 	// Prevent from attacks with using `..` in r.URL.Path
 	u.Path = path.Clean(u.Path)
@ -46,13 +46,13 @@ func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, error) {
 	for _, e := range ui.URLMap {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
-				return e.URLPrefix.mergeURLs(&u), nil
+				return e.URLPrefix.mergeURLs(&u), e.Headers, nil
 			}
 		}
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix.mergeURLs(&u), nil
+		return ui.URLPrefix.mergeURLs(&u), ui.Headers, nil
 	}
 	missingRouteRequests.Inc()
-	return nil, fmt.Errorf("missing route for %q", u.String())
+	return nil, nil, fmt.Errorf("missing route for %q", u.String())
 }
--- a/app/vmauth/target_url_test.go
+++ b/app/vmauth/target_url_test.go
@ -1,47 +1,56 @@
 package main

 import (
+	"fmt"
 	"net/url"
 	"testing"
 )

 func TestCreateTargetURLSuccess(t *testing.T) {
-	f := func(ui *UserInfo, requestURI, expectedTarget string) {
+	f := func(ui *UserInfo, requestURI, expectedTarget, expectedHeaders string) {
 		t.Helper()
 		u, err := url.Parse(requestURI)
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, err := createTargetURL(ui, u)
+		target, headers, err := createTargetURL(ui, u)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
+		headersStr := fmt.Sprintf("%q", headers)
+		if headersStr != expectedHeaders {
+			t.Fatalf("unexpected headers; got %s; want %s", headersStr, expectedHeaders)
+		}
 	}
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "", "http://foo.bar/.")
+	}, "", "http://foo.bar/.", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "/", "http://foo.bar")
+		Headers: []Header{{
+			Name:  "bb",
+			Value: "aaa",
+		}},
+	}, "/", "http://foo.bar", `[{"bb" "aaa"}]`)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar/federate"),
-	}, "/", "http://foo.bar/federate")
+	}, "/", "http://foo.bar/federate", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "a/b?c=d", "http://foo.bar/a/b?c=d")
+	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/z", "https://sss:3894/x/y/z")
+	}, "/z", "https://sss:3894/x/y/z", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/../../aaa", "https://sss:3894/x/y/aaa")
+	}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd")
+	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]")

 	// Complex routing with `url_map`
 	ui := &UserInfo{
@ -49,6 +58,16 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
+				Headers: []Header{
+					{
+						Name:  "xx",
+						Value: "aa",
+					},
+					{
+						Name:  "yy",
+						Value: "asdf",
+					},
+				},
 			},
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
@ -56,10 +75,14 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
+		Headers: []Header{{
+			Name:  "bb",
+			Value: "aaa",
+		}},
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up")
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write")
-	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range")
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`)
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
+	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`)

 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
@ -75,17 +98,17 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up")
-	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up")
-	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values")
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write")
-	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar")
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]")
+	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]")
+	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]")
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
+	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
-	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev")
+	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
-	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile")
+	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]")

 }

@ -96,13 +119,16 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, err := createTargetURL(ui, u)
+		target, headers, err := createTargetURL(ui, u)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
 		if target != nil {
 			t.Fatalf("unexpected target=%q; want empty string", target)
 		}
+		if headers != nil {
+			t.Fatalf("unexpected headers=%q; want empty string", headers)
+		}
 	}
 	f(&UserInfo{}, "/foo/bar")
 	f(&UserInfo{
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@ -18,6 +18,7 @@ sort: 15
 * FEATURE: add trigonometric functions, which are going to be added in [Prometheus 2.31](https://github.com/prometheus/prometheus/pull/9239): [acosh](https://docs.victoriametrics.com/MetricsQL.html#acosh), [asinh](https://docs.victoriametrics.com/MetricsQL.html#asinh), [atan](https://docs.victoriametrics.com/MetricsQL.html#atan), [atanh](https://docs.victoriametrics.com/MetricsQL.html#atanh), [cosh](https://docs.victoriametrics.com/MetricsQL.html#cosh), [deg](https://docs.victoriametrics.com/MetricsQL.html#deg), [rad](https://docs.victoriametrics.com/MetricsQL.html#rad), [sinh](https://docs.victoriametrics.com/MetricsQL.html#sinh), [tan](https://docs.victoriametrics.com/MetricsQL.html#tan), [tanh](https://docs.victoriametrics.com/MetricsQL.html#tanh). Also add `atan2` binary operator. See [this pull request](https://github.com/prometheus/prometheus/pull/9248).
 * FEATURE: consistently return the same set of time series from [limitk](https://docs.victoriametrics.com/MetricsQL.html#limitk) function. This improves the usability of periodically refreshed graphs.
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): varios UX improvements. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1711) and [these docs](https://docs.victoriametrics.com/#vmui).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify HTTP headers, which will be sent in requests to backends. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1736).
 * FEATURE: add `/flags` page to all the VictoriaMetrics components. This page contains command-line flags passed to the component.
 * FEATURE: allow using tab separators additionally to whitespace separators when [ingesting data in Graphite plaintext protocol](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd). Such separators are [supported by Carbon-c-relay](https://github.com/grobian/carbon-c-relay/commit/f3ffe6cc2b52b07d14acbda649ad3fd6babdd528).