From 256c561005ca4845011fdfee136aaf39ebcb79b5 Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <roman@victoriametrics.com>
Date: Wed, 17 Apr 2024 09:54:43 +0200
Subject: [PATCH] app/vmauth: support regex matching in `src_query_args`
(#6115)
Support regex matching when routing incoming requests based on HTTP query args
via `src_query_args` option at `url_map`.
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6070
Signed-off-by: hagen1778 <roman@victoriametrics.com>
---
app/vmauth/auth_config.go | 15 ++++++---
app/vmauth/auth_config_test.go | 7 +++--
app/vmauth/target_url.go | 10 ++++--
app/vmauth/target_url_test.go | 57 ++++++++++++++++++++++++++++++----
docs/CHANGELOG.md | 2 ++
docs/vmauth.md | 16 +++++++++-
6 files changed, 92 insertions(+), 15 deletions(-)
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index 69ec0b1a94..751c858256 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -189,7 +189,7 @@ type Regex struct {
// QueryArg represents HTTP query arg
type QueryArg struct {
Name string
- Value string
+ Value *Regex
sOriginal string
}
@@ -203,10 +203,17 @@ func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
qa.sOriginal = s
n := strings.IndexByte(s, '=')
- if n >= 0 {
- qa.Name = s[:n]
- qa.Value = s[n+1:]
+ if n < 0 {
+ return nil
}
+
+ qa.Name = s[:n]
+ expr := []byte(s[n+1:])
+ var re Regex
+ if err := yaml.Unmarshal(expr, &re); err != nil {
+ return fmt.Errorf("failed to unmarshal regex %q: %s", expr, err)
+ }
+ qa.Value = &re
return nil
}
diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go
index f6ae94ba9c..210fdafbe7 100644
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@@ -386,8 +386,11 @@ users:
SrcPaths: getRegexs([]string{"/api/v1/write"}),
SrcQueryArgs: []QueryArg{
{
- Name: "foo",
- Value: "bar",
+ Name: "foo",
+ Value: &Regex{
+ sOriginal: "bar",
+ re: regexp.MustCompile("^(?:bar)$"),
+ },
},
},
SrcHeaders: []Header{
diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go
index 9d43ef828b..086c2e080c 100644
--- a/app/vmauth/target_url.go
+++ b/app/vmauth/target_url.go
@@ -91,8 +91,14 @@ func matchAnyQueryArg(qas []QueryArg, args url.Values) bool {
return true
}
for _, qa := range qas {
- if slices.Contains(args[qa.Name], qa.Value) {
- return true
+ vs, ok := args[qa.Name]
+ if !ok {
+ continue
+ }
+ for _, v := range vs {
+ if qa.Value.match(v) {
+ return true
+ }
}
}
return false
diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go
index 17e2983086..1a2d22a403 100644
--- a/app/vmauth/target_url_test.go
+++ b/app/vmauth/target_url_test.go
@@ -4,6 +4,7 @@ import (
"fmt"
"net/url"
"reflect"
+ "regexp"
"strings"
"testing"
)
@@ -97,8 +98,13 @@ func TestCreateTargetURLSuccess(t *testing.T) {
bu := up.getBackendURL()
target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
bu.put()
- if target.String() != expectedTarget {
- t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
+
+ gotTarget, err := url.QueryUnescape(target.String())
+ if err != nil {
+ t.Fatalf("failed to unescape query %q: %s", target, err)
+ }
+ if gotTarget != expectedTarget {
+ t.Fatalf("unexpected target; \ngot:\n%q;\nwant:\n%q", gotTarget, expectedTarget)
}
if s := headersToString(hc.RequestHeaders); s != expectedRequestHeaders {
t.Fatalf("unexpected request headers; got %q; want %q", s, expectedRequestHeaders)
@@ -154,7 +160,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
}, "/../../aaa", "https://sss:3894/x/y/aaa", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("https://sss:3894/x/y"),
- }, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "", "", nil, "least_loaded", 0)
+ }, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s/../d", "", "", nil, "least_loaded", 0)
// Complex routing with `url_map`
ui := &UserInfo{
@@ -164,8 +170,11 @@ func TestCreateTargetURLSuccess(t *testing.T) {
SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}),
SrcQueryArgs: []QueryArg{
{
- Name: "db",
- Value: "foo",
+ Name: "db",
+ Value: &Regex{
+ sOriginal: "foo",
+ re: regexp.MustCompile("^(?:foo)$"),
+ },
},
},
URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
@@ -249,7 +258,43 @@ func TestCreateTargetURLSuccess(t *testing.T) {
}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
- }, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "", "", nil, "least_loaded", 0)
+ }, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team=mobile", "", "", nil, "least_loaded", 0)
+
+ // Complex routing regexp query args in `url_map`
+ ui = &UserInfo{
+ URLMaps: []URLMap{
+ {
+ SrcPaths: getRegexs([]string{"/api/v1/query"}),
+ SrcQueryArgs: []QueryArg{
+ {
+ Name: "query",
+ Value: &Regex{
+ sOriginal: "foo",
+ re: regexp.MustCompile(`^(?:.*env="dev".*)$`),
+ },
+ },
+ },
+ URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
+ },
+ {
+ SrcPaths: getRegexs([]string{"/api/v1/query"}),
+ SrcQueryArgs: []QueryArg{
+ {
+ Name: "query",
+ Value: &Regex{
+ sOriginal: "foo",
+ re: regexp.MustCompile(`^(?:.*env="prod".*)$`),
+ },
+ },
+ },
+ URLPrefix: mustParseURL("http://vmselect/1/prometheus"),
+ },
+ },
+ URLPrefix: mustParseURL("http://default-server"),
+ }
+ f(ui, `/api/v1/query?query=up{env="prod"}`, `http://vmselect/1/prometheus/api/v1/query?query=up{env="prod"}`, "", "", nil, "least_loaded", 0)
+ f(ui, `/api/v1/query?query=up{foo="bar", env="dev", pod!=""}`, `http://vmselect/0/prometheus/api/v1/query?query=up{foo="bar", env="dev", pod!=""}`, "", "", nil, "least_loaded", 0)
+ f(ui, `/api/v1/query?query=up{foo="bar"}`, `http://default-server/api/v1/query?query=up{foo="bar"}`, "", "", nil, "least_loaded", 0)
}
func TestCreateTargetURLFailure(t *testing.T) {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c383aa416b..dea5ebcbb1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -30,6 +30,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
## tip
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): support regex matching when routing incoming requests based on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6070).
+
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): supported any status codes from the range 200-299 from alertmanager. Previously, only 200 status code considered a successful action. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6110).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): don't treat concurrency limit hit as an error of the backend. Previously, hitting the concurrency limit would increment both `vmauth_concurrent_requests_limit_reached_total` and `vmauth_user_request_backend_errors_total` counters. Now, only concurrency limit counter is incremented. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
diff --git a/docs/vmauth.md b/docs/vmauth.md
index 314f2cba07..d560fe9ed5 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -117,7 +117,7 @@ if the whole request path matches at least one `src_paths` entry. The incoming r
If both `src_paths` and `src_hosts` lists are specified, then the request is routed to the given `url_prefix` when both request path and request host match at least one entry
in the corresponding lists.
-An optional `src_query_args` can be used for routing requests based on [HTTP query args](https://en.wikipedia.org/wiki/Query_string) additionaly to hostname and path.
+An optional `src_query_args` can be used for routing requests based on [HTTP query args](https://en.wikipedia.org/wiki/Query_string) additionally to hostname and path.
For example, the following config routes requests to `http://app1-backend/` if `db=foo` query arg is present in the request,
while routing requests with `db=bar` query arg to `http://app2-backend`:
@@ -135,6 +135,20 @@ If `src_query_args` contains multiple entries, then it is enough to match only a
If `src_query_args` are specified together with `src_hosts`, `src_paths` or `src_headers`, then the request is routed to the given `url_prefix`
if its query args, host, path and headers match the given lists simultaneously.
+`src_query_args` supports regex matching:
+```yaml
+unauthorized_user:
+ url_map:
+ - src_query_args: [ "query=.*env=\"prod\".*" ]
+ url_prefix: "http://prod-backend/"
+ - src_query_args: [ "query=.*env=\"dev\".*" ]
+ url_prefix: "http://dev-backend/"
+```
+The config above will route requests like `/api/v1/query?query=up{env="prod"}` to `http://prod-backend/`.
+And queries matching `.*env=\"dev\".*` will be routed to `http://dev-backend/`.
+_Please note, by default Grafana sends `query` param in request's body and vmauth won't be able to read it.
+You need to manually switch datasource settings in Grafana to use GET method for sending queries._
+
An optional `src_headers` can be used for routing requests based on HTTP request headers additionally to hostname, path and [HTTP query args](https://en.wikipedia.org/wiki/Query_string).
For example, the following config routes requests to `http://app1-backend` if `TenantID` request header equals to `42`, while routing requests to `http://app2-backend`
if `TenantID` request header equals to `123:456`: