From 61d1af805008276739960df59887b6c3b540d9eb Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 6 Mar 2024 20:52:23 +0200 Subject: [PATCH] app/vmauth: add ability to route requests based on HTTP query args via src_query_args option See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5878 --- app/vmauth/auth_config.go | 18 ++++++++++++----- app/vmauth/auth_config_test.go | 35 +++++++++++++++++++++++++++++++++- app/vmauth/target_url.go | 15 ++++++++++++++- app/vmauth/target_url_test.go | 12 +++++++++--- docs/CHANGELOG.md | 1 + docs/vmauth.md | 22 +++++++++++++++++++++ 6 files changed, 93 insertions(+), 10 deletions(-) diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 2ce3c4354..7d1e1db8a 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -136,11 +136,14 @@ func (h *Header) MarshalYAML() (interface{}, error) { // URLMap is a mapping from source paths to target urls. type URLMap struct { - // SrcHosts is the list of regular expressions, which match the request hostname. + // SrcPaths is the list of regular expressions, which must match the request path. + SrcPaths []*Regex `yaml:"src_paths,omitempty"` + + // SrcHosts is an optional list of regular expressions, which must match the request hostname. SrcHosts []*Regex `yaml:"src_hosts,omitempty"` - // SrcPaths is the list of regular expressions, which match the request path. - SrcPaths []*Regex `yaml:"src_paths,omitempty"` + // SrcQueryArgs is an optional list of query args, which must match request URL query args. + SrcQueryArgs []QueryArg `yaml:"src_query_args,omitempty"` // UrlPrefix contains backend url prefixes for the proxied request url. URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"` @@ -164,6 +167,11 @@ type Regex struct { re *regexp.Regexp } +type QueryArg struct { + Name string `yaml:"name"` + Value string `yaml:"value,omitempty"` +} + // URLPrefix represents passed `url_prefix` type URLPrefix struct { n atomic.Uint32 @@ -680,8 +688,8 @@ func (ui *UserInfo) initURLs() error { } } for _, e := range ui.URLMaps { - if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 { - return fmt.Errorf("missing `src_paths` and `src_hosts` in `url_map`") + if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 && len(e.SrcQueryArgs) == 0 { + return fmt.Errorf("missing `src_paths`, `src_hosts` and `src_query_args` in `url_map`") } if e.URLPrefix == nil { return fmt.Errorf("missing `url_prefix` in `url_map`") diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go index 115712d06..7586b6315 100644 --- a/app/vmauth/auth_config_test.go +++ b/app/vmauth/auth_config_test.go @@ -192,7 +192,7 @@ users: - url_prefix: http://foobar `) - // Invalid regexp in src_path. + // Invalid regexp in src_paths f(` users: - username: a @@ -210,6 +210,24 @@ users: url_prefix: http://foobar `) + // Invalid src_query_args + f(` +users: +- username: a + url_map: + - src_query_args: abc + url_prefix: http://foobar +`) + f(` +users: +- username: a + url_map: + - src_query_args: + - name: foo + incorrect_value: bar + url_prefix: http://foobar +`) + // Invalid headers in url_map (missing ':') f(` users: @@ -331,6 +349,9 @@ users: url_prefix: http://vmselect/select/0/prometheus - src_paths: ["/api/v1/write"] src_hosts: ["foo\\.bar", "baz:1234"] + src_query_args: + - name: foo + value: bar url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"] headers: - "foo: bar" @@ -346,6 +367,12 @@ users: { SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}), SrcPaths: getRegexs([]string{"/api/v1/write"}), + SrcQueryArgs: []QueryArg{ + { + Name: "foo", + Value: "bar", + }, + }, URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", @@ -375,6 +402,12 @@ users: { SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}), SrcPaths: getRegexs([]string{"/api/v1/write"}), + SrcQueryArgs: []QueryArg{ + { + Name: "foo", + Value: "bar", + }, + }, URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go index 9fa229be4..73bf07a2c 100644 --- a/app/vmauth/target_url.go +++ b/app/vmauth/target_url.go @@ -3,6 +3,7 @@ package main import ( "net/url" "path" + "slices" "strings" ) @@ -51,7 +52,7 @@ func dropPrefixParts(path string, parts int) string { func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf) { for _, e := range ui.URLMaps { - if matchAnyRegex(e.SrcHosts, u.Host) && matchAnyRegex(e.SrcPaths, u.Path) { + if matchAnyRegex(e.SrcHosts, u.Host) && matchAnyRegex(e.SrcPaths, u.Path) && matchAnyQueryArg(e.SrcQueryArgs, u.Query()) { return e.URLPrefix, e.HeadersConf } } @@ -73,6 +74,18 @@ func matchAnyRegex(rs []*Regex, s string) bool { return false } +func matchAnyQueryArg(qas []QueryArg, args url.Values) bool { + if len(qas) == 0 { + return true + } + for _, qa := range qas { + if slices.Contains(args[qa.Name], qa.Value) { + return true + } + } + return false +} + func normalizeURL(uOrig *url.URL) *url.URL { u := *uOrig // Prevent from attacks with using `..` in r.URL.Path diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go index 2f2f271c2..c2491bd60 100644 --- a/app/vmauth/target_url_test.go +++ b/app/vmauth/target_url_test.go @@ -149,8 +149,14 @@ func TestCreateTargetURLSuccess(t *testing.T) { ui := &UserInfo{ URLMaps: []URLMap{ { - SrcHosts: getRegexs([]string{"host42"}), - SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}), + SrcHosts: getRegexs([]string{"host42"}), + SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}), + SrcQueryArgs: []QueryArg{ + { + Name: "db", + Value: "foo", + }, + }, URLPrefix: mustParseURL("http://vmselect/0/prometheus"), HeadersConf: HeadersConf{ RequestHeaders: []Header{ @@ -195,7 +201,7 @@ func TestCreateTargetURLSuccess(t *testing.T) { RetryStatusCodes: []int{502}, DropSrcPathPrefixParts: intp(2), } - f(ui, "http://host42/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", + f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1) f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2ae996c64..81eeeaaff 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow routing incoming requests bassed on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5878). * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): reduce memory usage by up to 5x when aggregating over big number of unique [time series](https://docs.victoriametrics.com/keyconcepts/#time-series). The memory usage reduction is most visible when [stream deduplication](https://docs.victoriametrics.com/stream-aggregation/#deduplication) is enabled. The downside is increased CPU usage by up to 30%. * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): allow using `-streamAggr.dedupInterval` and `-remoteWrite.streamAggr.dedupInterval` command-line flags without the need to specify `-streamAggr.config` and `-remoteWrite.streamAggr.config`. See [these docs](https://docs.victoriametrics.com/stream-aggregation/#deduplication). * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): add `-streamAggr.dropInputLabels` command-line flag, which can be used for dropping the listed labels from input samples before applying stream [de-duplication](https://docs.victoriametrics.com/stream-aggregation/#deduplication) and aggregation. This is faster and easier to use alternative to [input_relabel_configs](https://docs.victoriametrics.com/stream-aggregation/#relabeling). See [these docs](https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels). diff --git a/docs/vmauth.md b/docs/vmauth.md index 6878f1711..40416fa63 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -117,6 +117,28 @@ if the whole request path matches at least one `src_paths` entry. The incoming r If both `src_paths` and `src_hosts` lists are specified, then the request is routed to the given `url_prefix` when both request path and request host match at least one entry in the corresponding lists. +An optional `src_query_args` can be used for routing requests based on [HTTP query args](https://en.wikipedia.org/wiki/Query_string) additionaly to hostname and path. +For example, the following config routes requests to `http://app1-backend/` if `db=foo` query arg is present in the request, +while routing requests with `db=bar` query arg to `http://app2-backend`: + +```yaml +unauthorized_user: + url_map: + - src_query_args: + - name: db + value: foo + url_prefix: "http://app1-backend/" + - src_query_args: + - name: db + value: bar + url_prefix: "http://app2-backend/" +``` + +If `src_query_args` contains multiple entries, then it is enough to match only a single entry in order to route the request to the given `url_prefix`. + +If `src_hosts` and/or `src_paths` are specified together with `src_query_args`, then the request is routed to the given `url_prefix` if its host, path and query args +match the given lists simultaneously. + ### Generic HTTP load balancer `vmauth` can balance load among multiple HTTP backends in least-loaded round-robin mode.