app/vmauth: add ability to route requests from a single users to multiple targets depending on the requested path

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1064

app/vmauth/README.md

@@ -56,12 +56,25 @@ users:
 
   # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
   # See https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format
-  # All the reuqests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be routed to http://vminsert:8480/insert/42/prometheus .
   # For example, http://vmauth:8427/api/v1/write is routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
 - username: "cluster-insert-account-42"
   password: "***"
   url_prefix: "http://vminsert:8480/insert/42/prometheus"
+
+
+  # A single user for querying and inserting data:
+  # - Requests to http://vmauth:8427/api/v1/query or http://vmauth:8427/api/v1/query_range
+  #   are routed to http://vmselect:8481/select/42/prometheus.
+  #   For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8480/select/42/prometheus/api/v1/query
+  # - Requests to http://vmauth:8427/api/v1/write are routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
+- username: "foobar"
+  url_map:
+  - src_paths: ["/api/v1/query", "/api/v1/query_range"]
+    url_prefix: "http://vmselect:8481/select/42/prometheus"
+  - src_paths: ["/api/v1/write"]
+    url_prefix: "http://vminsert:8480/insert/42/prometheus"
 ```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.

app/vmauth/auth_config.go

@@ -28,13 +28,20 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Username  string `yaml:"username"`
-	Password  string `yaml:"password"`
-	URLPrefix string `yaml:"url_prefix"`
+	Username  string   `yaml:"username"`
+	Password  string   `yaml:"password"`
+	URLPrefix string   `yaml:"url_prefix"`
+	URLMap    []URLMap `yaml:"url_map"`
 
 	requests *metrics.Counter
 }
 
+// URLMap is a mapping from source paths to target urls.
+type URLMap struct {
+	SrcPaths  []string `yaml:"src_paths"`
+	URLPrefix string   `yaml:"url_prefix"`
+}
+
 func initAuthConfig() {
 	if len(*authConfigPath) == 0 {
 		logger.Fatalf("missing required `-auth.config` command-line flag")
@@ -109,23 +116,52 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if m[ui.Username] != nil {
 			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
 		}
-		urlPrefix := ui.URLPrefix
-		// Remove trailing '/' from urlPrefix
-		for strings.HasSuffix(urlPrefix, "/") {
-			urlPrefix = urlPrefix[:len(urlPrefix)-1]
+		if len(ui.URLPrefix) > 0 {
+			urlPrefix, err := sanitizeURLPrefix(ui.URLPrefix)
+			if err != nil {
+				return nil, err
+			}
+			ui.URLPrefix = urlPrefix
 		}
-		// Validate urlPrefix
-		target, err := url.Parse(urlPrefix)
-		if err != nil {
-			return nil, fmt.Errorf("invalid `url_prefix: %q`: %w", urlPrefix, err)
+		for _, e := range ui.URLMap {
+			if len(e.SrcPaths) == 0 {
+				return nil, fmt.Errorf("missing `src_paths`")
+			}
+			for _, path := range e.SrcPaths {
+				if !strings.HasPrefix(path, "/") {
+					return nil, fmt.Errorf("`src_path`=%q must start with `/`", path)
+				}
+			}
+			urlPrefix, err := sanitizeURLPrefix(e.URLPrefix)
+			if err != nil {
+				return nil, err
+			}
+			e.URLPrefix = urlPrefix
 		}
-		if target.Scheme != "http" && target.Scheme != "https" {
-			return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)
+		if len(ui.URLMap) == 0 && len(ui.URLPrefix) == 0 {
+			return nil, fmt.Errorf("missing `url_prefix`")
 		}
-
-		ui.URLPrefix = urlPrefix
 		ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
 		m[ui.Username] = ui
 	}
 	return m, nil
 }
+
+func sanitizeURLPrefix(urlPrefix string) (string, error) {
+	// Remove trailing '/' from urlPrefix
+	for strings.HasSuffix(urlPrefix, "/") {
+		urlPrefix = urlPrefix[:len(urlPrefix)-1]
+	}
+	// Validate urlPrefix
+	target, err := url.Parse(urlPrefix)
+	if err != nil {
+		return "", fmt.Errorf("invalid `url_prefix: %q`: %w", urlPrefix, err)
+	}
+	if target.Scheme != "http" && target.Scheme != "https" {
+		return "", fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)
+	}
+	if target.Host == "" {
+		return "", fmt.Errorf("missing hostname in `url_prefix %q`", urlPrefix)
+	}
+	return urlPrefix, nil
+}

app/vmauth/auth_config_test.go

@@ -46,6 +46,11 @@ users:
 - username: foo
   url_prefix: //bar
 `)
+	f(`
+users:
+- username: foo
+  url_prefix: http:///bar
+`)
 
 	// Duplicate users
 	f(`
@@ -57,6 +62,31 @@ users:
 - username: foo
   url_prefix: https://sss.sss
 `)
+
+	// Missing url_prefix in url_map
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ["/foo/bar"]
+`)
+
+	// Missing src_paths in url_map
+	f(`
+users:
+- username: a
+  url_map:
+  - url_prefix: http://foobar
+`)
+
+	// src_path not starting with `/`
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: [foobar]
+    url_prefix: http://foobar
+`)
 }
 
 func TestParseAuthConfigSuccess(t *testing.T) {
@@ -103,6 +133,31 @@ users:
 			URLPrefix: "https://bar/x",
 		},
 	})
+
+	// non-empty URLMap
+	f(`
+users:
+- username: foo
+  url_map:
+  - src_paths: ["/api/v1/query","/api/v1/query_range"]
+    url_prefix: http://vmselect/select/0/prometheus
+  - src_paths: ["/api/v1/write"]
+    url_prefix: http://vminsert/insert/0/prometheus
+`, map[string]*UserInfo{
+		"foo": {
+			Username: "foo",
+			URLMap: []URLMap{
+				{
+					SrcPaths:  []string{"/api/v1/query", "/api/v1/query_range"},
+					URLPrefix: "http://vmselect/select/0/prometheus",
+				},
+				{
+					SrcPaths:  []string{"/api/v1/write"},
+					URLPrefix: "http://vminsert/insert/0/prometheus",
+				},
+			},
+		},
+	})
 }
 
 func removeMetrics(m map[string]*UserInfo) {

app/vmauth/main.go

@@ -54,14 +54,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	ac := authConfig.Load().(map[string]*UserInfo)
-	info := ac[username]
-	if info == nil || info.Password != password {
+	ui := ac[username]
+	if ui == nil || ui.Password != password {
 		httpserver.Errorf(w, r, "cannot find the provided username %q or password in config", username)
 		return true
 	}
-	info.requests.Inc()
-
-	targetURL := createTargetURL(info.URLPrefix, r.URL)
+	ui.requests.Inc()
+	targetURL, err := createTargetURL(ui, r.URL)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
+		return true
+	}
 	if _, err := url.Parse(targetURL); err != nil {
 		httpserver.Errorf(w, r, "invalid targetURL=%q: %s", targetURL, err)
 		return true

app/vmauth/target_url.go

@@ -1,16 +1,31 @@
 package main
 
 import (
+	"fmt"
 	"net/url"
 	"path"
 	"strings"
 )
 
-func createTargetURL(prefix string, u *url.URL) string {
+func createTargetURL(ui *UserInfo, uOrig *url.URL) (string, error) {
+	u, err := url.Parse(uOrig.String())
+	if err != nil {
+		return "", fmt.Errorf("cannot make a copy of %q: %w", u, err)
+	}
 	// Prevent from attacks with using `..` in r.URL.Path
 	u.Path = path.Clean(u.Path)
 	if !strings.HasPrefix(u.Path, "/") {
 		u.Path = "/" + u.Path
 	}
-	return prefix + u.RequestURI()
+	for _, e := range ui.URLMap {
+		for _, path := range e.SrcPaths {
+			if u.Path == path {
+				return e.URLPrefix + u.RequestURI(), nil
+			}
+		}
+	}
+	if len(ui.URLPrefix) > 0 {
+		return ui.URLPrefix + u.RequestURI(), nil
+	}
+	return "", fmt.Errorf("missing route for %q", u)
 }

app/vmauth/target_url_test.go

@@ -5,22 +5,82 @@ import (
 	"testing"
 )
 
-func TestCreateTargetURL(t *testing.T) {
-	f := func(prefix, requestURI, expectedTarget string) {
+func TestCreateTargetURLSuccess(t *testing.T) {
+	f := func(ui *UserInfo, requestURI, expectedTarget string) {
 		t.Helper()
 		u, err := url.Parse(requestURI)
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target := createTargetURL(prefix, u)
+		target, err := createTargetURL(ui, u)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
 		if target != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
 	}
-	f("http://foo.bar", "", "http://foo.bar/.")
-	f("http://foo.bar", "/", "http://foo.bar/")
-	f("http://foo.bar", "a/b?c=d", "http://foo.bar/a/b?c=d")
-	f("https://sss:3894/x/y", "/z", "https://sss:3894/x/y/z")
-	f("https://sss:3894/x/y", "/../../aaa", "https://sss:3894/x/y/aaa")
-	f("https://sss:3894/x/y", "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s/../d")
+	// Simple routing with `url_prefix`
+	f(&UserInfo{
+		URLPrefix: "http://foo.bar",
+	}, "", "http://foo.bar/.")
+	f(&UserInfo{
+		URLPrefix: "http://foo.bar",
+	}, "/", "http://foo.bar/")
+	f(&UserInfo{
+		URLPrefix: "http://foo.bar",
+	}, "a/b?c=d", "http://foo.bar/a/b?c=d")
+	f(&UserInfo{
+		URLPrefix: "https://sss:3894/x/y",
+	}, "/z", "https://sss:3894/x/y/z")
+	f(&UserInfo{
+		URLPrefix: "https://sss:3894/x/y",
+	}, "/../../aaa", "https://sss:3894/x/y/aaa")
+	f(&UserInfo{
+		URLPrefix: "https://sss:3894/x/y",
+	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s/../d")
+
+	// Complex routing with `url_map`
+	ui := &UserInfo{
+		URLMap: []URLMap{
+			{
+				SrcPaths:  []string{"/api/v1/query"},
+				URLPrefix: "http://vmselect/0/prometheus",
+			},
+			{
+				SrcPaths:  []string{"/api/v1/write"},
+				URLPrefix: "http://vminsert/0/prometheus",
+			},
+		},
+		URLPrefix: "http://default-server",
+	}
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up")
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write")
+	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range")
+}
+
+func TestCreateTargetURLFailure(t *testing.T) {
+	f := func(ui *UserInfo, requestURI string) {
+		t.Helper()
+		u, err := url.Parse(requestURI)
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", requestURI, err)
+		}
+		target, err := createTargetURL(ui, u)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if target != "" {
+			t.Fatalf("unexpected target=%q; want empty string", target)
+		}
+	}
+	f(&UserInfo{}, "/foo/bar")
+	f(&UserInfo{
+		URLMap: []URLMap{
+			{
+				SrcPaths:  []string{"/api/v1/query"},
+				URLPrefix: "http://foobar/baz",
+			},
+		},
+	}, "/api/v1/write")
 }

docs/CHANGELOG.md

@@ -7,6 +7,7 @@
 * FEATURE: single-node VictoriaMetrics now accepts requests to handlers with `/prometheus` and `/graphite` prefixes such as `/prometheus/api/v1/query`. This improves compatibility with [handlers from VictoriaMetrics cluster](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format).
 * FEATURE: expose `process_open_fds` and `process_max_fds` metrics. These metrics can be used for alerting when `process_open_fds` reaches `process_max_fds`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/402 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1037
 * FEATURE: vmalert: add `-datasource.appendTypePrefix` command-line option for querying both Prometheus and Graphite datasource in cluster version of VictoriaMetrics. See [these docs](https://victoriametrics.github.io/vmalert.html#graphite) for details.
+* FEATURE: vmauth: add ability to route requests from a single user to multiple destinations depending on the requested paths. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1064
 * FEATURE: remove dependency on external programs such as `cat`, `grep` and `cut` when detecting cpu and memory limits inside Docker or LXC container.
 
 * BUGFIX: do not spam error logs when discovering Docker Swarm targets without dedicated IP. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1028 .

docs/vmauth.md

@@ -56,12 +56,25 @@ users:
 
   # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
   # See https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format
-  # All the reuqests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be routed to http://vminsert:8480/insert/42/prometheus .
   # For example, http://vmauth:8427/api/v1/write is routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
 - username: "cluster-insert-account-42"
   password: "***"
   url_prefix: "http://vminsert:8480/insert/42/prometheus"
+
+
+  # A single user for querying and inserting data:
+  # - Requests to http://vmauth:8427/api/v1/query or http://vmauth:8427/api/v1/query_range
+  #   are routed to http://vmselect:8481/select/42/prometheus.
+  #   For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8480/select/42/prometheus/api/v1/query
+  # - Requests to http://vmauth:8427/api/v1/write are routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
+- username: "foobar"
+  url_map:
+  - src_paths: ["/api/v1/query", "/api/v1/query_range"]
+    url_prefix: "http://vmselect:8481/select/42/prometheus"
+  - src_paths: ["/api/v1/write"]
+    url_prefix: "http://vminsert:8480/insert/42/prometheus"
 ```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.