app/vmauth: add support for mTLS-based routing of incoming requests to different backends depending on the subject field in the TLS certificate provided by the user

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1547

app/vmauth/auth_config.go

@@ -570,18 +570,23 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		if ui.BearerToken == "" && ui.Username == "" {
+		if ui.Username != "" && ui.Password == "" {
-			return nil, fmt.Errorf("either bearer_token or username must be set")
+			// Do not allow setting username without password if there are other auth configs exist.
+			// This should prevent from typical mis-configuration when access by username without password
+			// remains open if other authorization schemes are defined.
+			if ui.BearerToken != "" {
+				return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+			}
 		}
-		if ui.BearerToken != "" && ui.Username != "" {
+		ats := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
-			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+		if len(ats) == 0 {
+			return nil, fmt.Errorf("one of bearer_token, username or mtls must be set")
 		}
-		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		for _, at := range ats {
-		if byAuthToken[at1] != nil {
+			if uiOld := byAuthToken[at]; uiOld != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
-		}
+					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
-		if byAuthToken[at2] != nil {
+			}
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
 		}
 
 		if err := ui.initURLs(); err != nil {
@@ -615,8 +620,9 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		}
 		ui.httpTransport = tr
 
-		byAuthToken[at1] = ui
+		for _, at := range ats {
-		byAuthToken[at2] = ui
+			byAuthToken[at] = ui
+		}
 	}
 	return byAuthToken, nil
 }
@@ -720,24 +726,45 @@ func (ui *UserInfo) name() string {
 	return ""
 }
 
-func getAuthTokens(bearerToken, username, password string) (string, string) {
+func getAuthTokens(bearerToken, username, password string) []string {
+	var ats []string
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getAuthToken(bearerToken, "", "")
+		at1 := getHTTPAuthBearerToken(bearerToken)
-		at2 := getAuthToken("", bearerToken, "")
+		at2 := getHTTPAuthBasicToken(bearerToken, "")
-		return at1, at2
+		ats = append(ats, at1, at2)
+	} else if username != "" {
+		at := getHTTPAuthBasicToken(username, password)
+		ats = append(ats, at)
 	}
-	at := getAuthToken("", username, password)
+	return ats
-	return at, at
 }
 
-func getAuthToken(bearerToken, username, password string) string {
+func getHTTPAuthBearerToken(bearerToken string) string {
-	if bearerToken != "" {
+	return "http_auth:Bearer " + bearerToken
-		return "Bearer " + bearerToken
+}
-	}
+
+func getHTTPAuthBasicToken(username, password string) string {
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "Basic " + token64
+	return "http_auth:Basic " + token64
+}
+
+func getAuthTokensFromRequest(r *http.Request) []string {
+	var ats []string
+
+	ah := r.Header.Get("Authorization")
+	if ah == "" {
+		return ats
+	}
+	if strings.HasPrefix(ah, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		ah = strings.Replace(ah, "Token", "Bearer", 1)
+	}
+	at := "http_auth:" + ah
+	ats = append(ats, at)
+	return ats
 }
 
 func (up *URLPrefix) sanitize() error {

app/vmauth/auth_config_test.go

@@ -267,7 +267,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -290,7 +290,7 @@ users:
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
@@ -312,11 +312,11 @@ users:
 - username: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "bar", ""): {
+		getHTTPAuthBasicToken("bar", ""): {
 			Username:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
@@ -336,7 +336,7 @@ users:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -365,7 +365,7 @@ users:
 				},
 			},
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -395,7 +395,7 @@ users:
 			},
 		},
 	})
-	// Multiple users with the same name
+	// Multiple users with the same name - this should work, since these users have different passwords
 	f(`
 users:
 - username: foo-same
@@ -405,17 +405,18 @@ users:
   password: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo-same", "baz"): {
+		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "foo-same", "bar"): {
+		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
+
 	// with default url
 	f(`
 users:
@@ -432,7 +433,7 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -464,7 +465,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -513,7 +514,7 @@ users:
     backend_env: test
     team: accounting
 `, map[string]*UserInfo{
-		getAuthToken("", "foo-same", "baz"): {
+		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
@@ -522,7 +523,7 @@ users:
 				"team": "dev",
 			},
 		},
-		getAuthToken("", "foo-same", "bar"): {
+		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
@@ -558,7 +559,7 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getAuthToken("", "foo", "bar")]
+	ui := m[getHTTPAuthBasicToken("foo", "bar")]
 	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}

app/vmauth/main.go

@@ -101,8 +101,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
-	authToken := r.Header.Get("Authorization")
+
-	if authToken == "" {
+	ats := getAuthTokensFromRequest(r)
+	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
@@ -114,18 +115,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
-	if strings.HasPrefix(authToken, "Token ") {
-		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-		// See https://docs.influxdata.com/influxdb/v2.0/api/
-		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
-	}
 
-	ac := *authUsers.Load()
+	ui := getUserInfoByAuthTokens(ats)
-	ui := ac[authToken]
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
+			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
@@ -141,6 +136,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+func getUserInfoByAuthTokens(ats []string) *UserInfo {
+	ac := *authUsers.Load()
+	for _, at := range ats {
+		ui := ac[at]
+		if ui != nil {
+			return ui
+		}
+	}
+	return nil
+}
+
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)

docs/CHANGELOG.md

@@ -35,6 +35,7 @@ The sandbox cluster installation is running under the constant load generated by
 * FEATURE: all VictoriaMetrics components: add support for accepting http requests over multiple distinct TCP addresses by starting VictoriaMetrics component with multiple `-httpListenAddr` command-line flags. For example, `./victoria-metrics -httpListenAddr=some-host:12345 -httpListenAddr=localhost:8428` starts VictoriaMetrics, which accepts incoming http requests at both `some-host:12345` and `localhost:8428`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1470).
 * FEATURE: all VictoriaMetrics components: add support for empty command flag values in short array notation. For example, `-remoteWrite.sendTimeout=',20s,'` specifies three `-remoteWrite.sendTimeout` values - the first and the last ones are default values (`30s` in this case), while the second one is `20s`.
 * FEATURE: all VictoriaMetrics components: do not close connections to `-httpListenAddr` every 2 minutes. This behavior didn't help spreading load among multiple backend servers behind load-balancing TCP proxy. Instead, it could lead to hard-to-debug issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1304#issuecomment-1636997037). If you still need periodically closing client connections because of some reason, then pass the desired timeout to `-http.connTimeout` command-line flag.
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add support for [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication)-based request routing to different backends depending on the subject of the TLS certificate provided by the client. See [these docs](https://docs.victoriametrics.com/vmauth.html#mtls-based-request-routing). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1547).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html) and [single-node VictoriaMetrics](https://docs.victoriametrics.com): add support for data ingestion via [DataDog lambda extension](https://docs.datadoghq.com/serverless/libraries_integrations/extension/) aka `/api/beta/sketches` endpoint. See [these docs](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3091). Thanks to @AndrewChubatiuk for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5584).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add `-disableReroutingOnUnavailable` command-line flag to `vminsert`, which can be used for reducing resource usage spikes at `vmstorage` nodes during rolling restart. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#improving-re-routing-performance-during-restart). Thanks to @Muxa1L for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5713).
 * FEATURE: add `-search.resetRollupResultCacheOnStartup` command-line flag for resetting [query cache](https://docs.victoriametrics.com/#rollup-result-cache) on startup. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/834).

docs/enterprise.md

@@ -60,6 +60,7 @@ On top of this, Enterprise package of VictoriaMetrics includes the following fea
 - [Advanced auth and rate limiter](https://docs.victoriametrics.com/vmgateway.html).
 - [mTLS for all the VictoriaMetrics components](https://docs.victoriametrics.com/#mtls-protection).
 - [mTLS for communications between cluster components](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection).
+- [mTLS-based request routing](https://docs.victoriametrics.com/vmauth.html#mtls-based-request-routing).
 - [Kafka integration](https://docs.victoriametrics.com/vmagent.html#kafka-integration).
 - [Google PubSub integration](https://docs.victoriametrics.com/vmagent.html#google-pubsub-integration).
 - [Multitenant support in vmalert](https://docs.victoriametrics.com/vmalert.html#multitenancy).

docs/vmauth.md

@@ -58,6 +58,7 @@ accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.co
 * [Basic Auth proxy](#basic-auth-proxy)
 * [Bearer Token auth proxy](#bearer-token-auth-proxy)
 * [Per-tenant authorization](#per-tenant-authorization)
+* [mTLS-based request routing](#mtls-based-request-routing)
 * [Enforcing query args](#enforcing-query-args)
 
 ### Simple HTTP proxy
@@ -274,6 +275,28 @@ users:
     url_prefix: "http://vmselect-backend:8481/select/2/prometheus/"
 ```
 
+### mTLS-based request routing
+
+[Enterprise version of `vmauth`](https://docs.victoriametrics.com/enterprise.html) can be configured for routing requests
+to different backends depending on the following [subject fields](https://en.wikipedia.org/wiki/Public_key_certificate#Common_fields) in the TLS certificate provided by client:
+
+* `organizational_unit` aka `OU`
+* `organization` aka `O`
+* `common_name` aka `CN`
+
+For example, the following [`-auth.config`](#auth-config) routes requests from clients with `organizational_unit: finance` TLS certificates
+to `http://victoriametrics-finance:8428` backend:
+
+```yaml
+users:
+- mtls:
+    organizational_unit: finance
+  url_prefix: "http://victoriametrics-finance:8428"
+```
+
+[mTLS protection](#mtls-protection) must be enabled for mTLS-based routing.
+
+
 ### Enforcing query args
 
 `vmauth` can be configured for adding some mandatory query args before proxying requests to backends.
@@ -678,6 +701,8 @@ requests at this port, by specifying `-tls` and `-mtls` command-line flags. For
 By default system-wide [TLS Root CA](https://en.wikipedia.org/wiki/Root_certificate) is used for verifying client certificates if `-mtls` command-line flag is specified.
 It is possible to specify custom TLS Root CA via `-mtlsCAFile` command-line flag.
 
+See also [mTLS-based request routing](#mtls-based-request-routing)
+
 ## Security
 
 It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.