github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmauth: add ability to route requests to different backends depending on the request host

Browse source
This commit is contained in:
Aliaksandr Valialkin 2023-12-14 00:46:36 +02:00
parent 6f15ca4a16
commit 68be182075
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
7 changed files with 132 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
app/vmauth/auth_config.go
View file

@ -126,16 +126,30 @@ func (h *Header) MarshalYAML() (interface{}, error) {
// URLMap is a mapping from source paths to target urls.
type URLMap struct {
SrcPaths []*SrcPath `yaml:"src_paths,omitempty"`
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
HeadersConf HeadersConf `yaml:",inline"`
RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"`
LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"`
DropSrcPathPrefixParts int `yaml:"drop_src_path_prefix_parts,omitempty"`
// SrcHosts is the list of regular expressions, which match the request hostname.
SrcHosts []*Regex `yaml:"src_hosts,omitempty"`
// SrcPaths is the list of regular expressions, which match the request path.
SrcPaths []*Regex `yaml:"src_paths,omitempty"`
// UrlPrefix contains backend url prefixes for the proxied request url.
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
// HeadersConf is the config for augumenting request and response headers.
HeadersConf HeadersConf `yaml:",inline"`
// RetryStatusCodes is the list of response status codes used for retrying requests.
RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"`
// LoadBalancingPolicy is load balancing policy among UrlPrefix backends.
LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"`
// DropSrcPathPrefixParts is the number of `/`-delimited request path prefix parts to drop before proxying the request to backend.
DropSrcPathPrefixParts int `yaml:"drop_src_path_prefix_parts,omitempty"`
}
// SrcPath represents an src path
type SrcPath struct {
// Regex represents a regex
type Regex struct {
sOriginal string
re *regexp.Regexp
}
@ -333,8 +347,8 @@ func (up *URLPrefix) MarshalYAML() (interface{}, error) {
return string(b), nil
}
func (sp *SrcPath) match(s string) bool {
prefix, ok := sp.re.LiteralPrefix()
func (r *Regex) match(s string) bool {
prefix, ok := r.re.LiteralPrefix()
if ok {
// Fast path - literal match
return s == prefix
@ -342,11 +356,11 @@ func (sp *SrcPath) match(s string) bool {
if !strings.HasPrefix(s, prefix) {
return false
}
return sp.re.MatchString(s)
return r.re.MatchString(s)
}
// UnmarshalYAML implements yaml.Unmarshaler
func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error {
func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
var s string
if err := f(&s); err != nil {
return err
@ -356,14 +370,14 @@ func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error {
if err != nil {
return fmt.Errorf("cannot build regexp from %q: %w", s, err)
}
sp.sOriginal = s
sp.re = re
r.sOriginal = s
r.re = re
return nil
}
// MarshalYAML implements yaml.Marshaler.
func (sp *SrcPath) MarshalYAML() (interface{}, error) {
return sp.sOriginal, nil
func (r *Regex) MarshalYAML() (interface{}, error) {
return r.sOriginal, nil
}
var (
@ -613,8 +627,8 @@ func (ui *UserInfo) initURLs() error {
}
}
for _, e := range ui.URLMaps {
if len(e.SrcPaths) == 0 {
return fmt.Errorf("missing `src_paths` in `url_map`")
if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 {
return fmt.Errorf("missing `src_paths` and `src_hosts` in `url_map`")
}
if e.URLPrefix == nil {
return fmt.Errorf("missing `url_prefix` in `url_map`")

56
app/vmauth/auth_config_test.go
View file

@ -145,6 +145,12 @@ users:
url_map:
- src_paths: ["/foo/bar"]
`)
f(`
users:
- username: a
url_map:
- src_hosts: ["foobar"]
`)
// Invalid url_prefix in url_map
f(`
@ -154,6 +160,13 @@ users:
- src_paths: ["/foo/bar"]
url_prefix: foo.bar
`)
f(`
users:
- username: a
url_map:
- src_hosts: ["foobar"]
url_prefix: foo.bar
`)
// empty url_prefix in url_map
f(`
@ -163,8 +176,15 @@ users:
- src_paths: ['/foo/bar']
url_prefix: []
`)
f(`
users:
- username: a
url_map:
- src_phosts: ['foobar']
url_prefix: []
`)
// Missing src_paths in url_map
// Missing src_paths and src_hosts in url_map
f(`
users:
- username: a
@ -181,6 +201,15 @@ users:
url_prefix: http://foobar
`)
// Invalid regexp in src_hosts
f(`
users:
- username: a
url_map:
- src_hosts: ['fo[obar']
url_prefix: http://foobar
`)
// Invalid headers in url_map (missing ':')
f(`
users:
@ -293,6 +322,7 @@ users:
- src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"]
url_prefix: http://vmselect/select/0/prometheus
- src_paths: ["/api/v1/write"]
src_hosts: ["foo\\.bar", "baz:1234"]
url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
headers:
- "foo: bar"
@ -302,11 +332,12 @@ users:
BearerToken: "foo",
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURLs([]string{
"http://vminsert1/insert/0/prometheus",
"http://vminsert2/insert/0/prometheus",
@ -330,11 +361,12 @@ users:
BearerToken: "foo",
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURLs([]string{
"http://vminsert1/insert/0/prometheus",
"http://vminsert2/insert/0/prometheus",
@ -396,11 +428,11 @@ users:
BearerToken: "foo",
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURLs([]string{
"http://vminsert1/insert/0/prometheus",
"http://vminsert2/insert/0/prometheus",
@ -428,11 +460,11 @@ users:
BearerToken: "foo",
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURLs([]string{
"http://vminsert1/insert/0/prometheus",
"http://vminsert2/insert/0/prometheus",
@ -501,10 +533,10 @@ func isSetBool(boolP *bool, expectedValue bool) bool {
return *boolP == expectedValue
}
func getSrcPaths(paths []string) []*SrcPath {
var sps []*SrcPath
func getRegexs(paths []string) []*Regex {
var sps []*Regex
for _, path := range paths {
sps = append(sps, &SrcPath{
sps = append(sps, &Regex{
sOriginal: path,
re: regexp.MustCompile("^(?:" + path + ")$"),
})

2
app/vmauth/example_config.yml
View file

@ -92,7 +92,7 @@ users:
# - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
# - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
#
# Regular expressions are allowed in `src_paths` entries.
# Regular expressions are allowed in `src_paths` and `src_hosts` entries.
- username: "foobar"
url_map:
- src_paths:

18
app/vmauth/target_url.go
View file

@ -51,10 +51,8 @@ func dropPrefixParts(path string, parts int) string {
func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, int) {
for _, e := range ui.URLMaps {
for _, sp := range e.SrcPaths {
if sp.match(u.Path) {
return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts
}
if matchAnyRegex(e.SrcHosts, u.Host) && matchAnyRegex(e.SrcPaths, u.Path) {
return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts
}
}
if ui.URLPrefix != nil {
@ -63,6 +61,18 @@ func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf,
return nil, HeadersConf{}, 0
}
func matchAnyRegex(rs []*Regex, s string) bool {
if len(rs) == 0 {
return true
}
for _, r := range rs {
if r.match(s) {
return true
}
}
return false
}
func normalizeURL(uOrig *url.URL) *url.URL {
u := *uOrig
// Prevent from attacks with using `..` in r.URL.Path

26
app/vmauth/target_url_test.go
View file

@ -149,7 +149,8 @@ func TestCreateTargetURLSuccess(t *testing.T) {
ui := &UserInfo{
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/vmsingle/api/v1/query"}),
SrcHosts: getRegexs([]string{"host42"}),
SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}),
URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
HeadersConf: HeadersConf{
RequestHeaders: []Header{
@ -174,7 +175,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
DropSrcPathPrefixParts: 1,
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
},
},
@ -192,21 +193,28 @@ func TestCreateTargetURLSuccess(t *testing.T) {
RetryStatusCodes: []int{502},
DropSrcPathPrefixParts: 2,
}
f(ui, "/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0)
f(ui, "/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
f(ui, "http://host42/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up",
`[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up",
`[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0)
f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
// Complex routing regexp paths in `url_map`
ui = &UserInfo{
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
SrcPaths: getRegexs([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
},
{
SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
SrcPaths: getRegexs([]string{"/api/v1/write"}),
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
},
{
SrcHosts: getRegexs([]string{"vmui\\..+"}),
URLPrefix: mustParseURL("http://vmui.host:1234/vmui/"),
},
},
URLPrefix: mustParseURL("http://default-server"),
}
@ -215,6 +223,8 @@ func TestCreateTargetURLSuccess(t *testing.T) {
f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0)
f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "[]", "[]", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0)
@ -249,7 +259,7 @@ func TestCreateTargetURLFailure(t *testing.T) {
f(&UserInfo{
URLMaps: []URLMap{
{
SrcPaths: getSrcPaths([]string{"/api/v1/query"}),
SrcPaths: getRegexs([]string{"/api/v1/query"}),
URLPrefix: mustParseURL("http://foobar/baz"),
},
},

1
docs/CHANGELOG.md
View file

@ -28,6 +28,7 @@ The sandbox cluster installation is running under the constant load generated by
## tip
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)

29
docs/vmauth.md
View file

@ -74,13 +74,13 @@ unauthorized_user:
### Generic HTTP proxy for different backends
`vmauth` can proxy requests to different backends depending on the requested path.
`vmauth` can proxy requests to different backends depending on the requested host and/or path.
For example, the following [`-auth.config`](#auth-config) instructs `vmauth` to make the following:
- Requests starting with `/app1/` are proxied to `http://app1-backend/`. For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe`
is proxied to `http://app1-backend/foo/bar?baz=qwe`.
- Requests starting with `/app2/` are proxied to `http://app2-backend/`. For example, the request to `http://vmauth:8427/app2/index.html`
is proxied to `http://app2-backend/index.html`.
- Requests starting with `/app1/` are proxied to `http://app1-backend/`, while the `/app1/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix).
For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe` is proxied to `http://app1-backend/foo/bar?baz=qwe`.
- Requests starting with `/app2/` are proxied to `http://app2-backend/`, while the `/app2/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix).
For example, the request to `http://vmauth:8427/app2/index.html` is proxied to `http://app2-backend/index.html`.
- Other requests are proxied to `http://some-backend/404-page.html`, while the requested path is passed via `request_path` query arg.
For example, the request to `http://vmauth:8427/foo/bar?baz=qwe` is proxied to `http://some-backend/404-page.html?request_path=%2Ffoo%2Fbar%3Fbaz%3Dqwe`.
@ -98,8 +98,23 @@ unauthorized_user:
default_url: http://some-backend/404-page.html
```
See [these docs](#dropping-request-path-prefix) for more details.
The following config routes requests to host `app1.my-host.com` to `http://app1-backend`, while routing requests to `app2.my-host.com` to `http://app2-backend`:
```yml
unauthorized_user:
url_map:
- src_hosts:
- "app1\\.my-host\\.com"
url_prefix: "http://app1-backend/"
- src_paths:
- "app2\\.my-host\\.com"
url_prefix: "http://app2-backend/"
```
`src_paths` and `src_hosts` accept a list of [regular expressions](https://github.com/google/re2/wiki/Syntax). The incoming request is routed to the given `url_prefix`
if the whole request path matches at least one `src_paths` entry. The incoming request is routed to the given `url_prefix` if the whole request host matches at least one `src_hosts` entry.
If both `src_paths` and `src_hosts` lists are specified, then the request is routed to the given `url_prefix` when both request path and request host match at least one entry
in the corresponding lists.
### Generic HTTP load balancer
@ -603,7 +618,7 @@ users:
# - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
# - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
#
# Regular expressions are allowed in `src_paths` entries.
# Regular expressions are allowed in `src_paths` and `src_hosts` entries.
- username: "foobar"
url_map:
- src_paths: