From 26cb6f886129a32329de1786360ee1f39ae309f4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 5 Mar 2021 18:21:11 +0200 Subject: [PATCH] app/vmauth: allow using regexps in `url_map` paths See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112 --- app/vmauth/README.md | 6 ++--- app/vmauth/auth_config.go | 49 +++++++++++++++++++++++++++++----- app/vmauth/auth_config_test.go | 46 +++++++++++++++++++++++++------ app/vmauth/target_url.go | 4 +-- app/vmauth/target_url_test.go | 26 +++++++++++++++--- docs/CHANGELOG.md | 1 + docs/vmauth.md | 6 ++--- 7 files changed, 112 insertions(+), 26 deletions(-) diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 0c241aeb3..133718787 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -65,13 +65,13 @@ users: # A single user for querying and inserting data: - # - Requests to http://vmauth:8427/api/v1/query or http://vmauth:8427/api/v1/query_range - # are routed to http://vmselect:8481/select/42/prometheus. + # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range + # and http://vmauth:8427/api/v1/label//values are routed to http://vmselect:8481/select/42/prometheus. # For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8480/select/42/prometheus/api/v1/query # - Requests to http://vmauth:8427/api/v1/write are routed to http://vminsert:8480/insert/42/prometheus/api/v1/write - username: "foobar" url_map: - - src_paths: ["/api/v1/query", "/api/v1/query_range"] + - src_paths: ["/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^/]+/values"] url_prefix: "http://vmselect:8481/select/42/prometheus" - src_paths: ["/api/v1/write"] url_prefix: "http://vminsert:8480/insert/42/prometheus" diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index fca24d25f..dc8c1da2f 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/url" + "regexp" "strings" "sync" "sync/atomic" @@ -38,8 +39,47 @@ type UserInfo struct { // URLMap is a mapping from source paths to target urls. type URLMap struct { - SrcPaths []string `yaml:"src_paths"` - URLPrefix string `yaml:"url_prefix"` + SrcPaths []*SrcPath `yaml:"src_paths"` + URLPrefix string `yaml:"url_prefix"` +} + +// SrcPath represents an src path +type SrcPath struct { + sOriginal string + re *regexp.Regexp +} + +func (sp *SrcPath) match(s string) bool { + prefix, ok := sp.re.LiteralPrefix() + if ok { + // Fast path - literal match + return s == prefix + } + if !strings.HasPrefix(s, prefix) { + return false + } + return sp.re.MatchString(s) +} + +// UnmarshalYAML implements yaml.Unmarshaler +func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error { + var s string + if err := f(&s); err != nil { + return err + } + sAnchored := "^(?:" + s + ")$" + re, err := regexp.Compile(sAnchored) + if err != nil { + return fmt.Errorf("cannot build regexp from %q: %w", s, err) + } + sp.sOriginal = s + sp.re = re + return nil +} + +// MarshalYAML implements yaml.Marshaler. +func (sp *SrcPath) MarshalYAML() (interface{}, error) { + return sp.sOriginal, nil } func initAuthConfig() { @@ -127,11 +167,6 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) { if len(e.SrcPaths) == 0 { return nil, fmt.Errorf("missing `src_paths`") } - for _, path := range e.SrcPaths { - if !strings.HasPrefix(path, "/") { - return nil, fmt.Errorf("`src_path`=%q must start with `/`", path) - } - } urlPrefix, err := sanitizeURLPrefix(e.URLPrefix) if err != nil { return nil, err diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go index dbd8299c8..0033b2b42 100644 --- a/app/vmauth/auth_config_test.go +++ b/app/vmauth/auth_config_test.go @@ -1,8 +1,12 @@ package main import ( - "reflect" + "bytes" + "fmt" + "regexp" "testing" + + "gopkg.in/yaml.v2" ) func TestParseAuthConfigFailure(t *testing.T) { @@ -79,12 +83,12 @@ users: - url_prefix: http://foobar `) - // src_path not starting with `/` + // Invalid regexp in src_path. f(` users: - username: a url_map: - - src_paths: [foobar] + - src_paths: ['fo[obar'] url_prefix: http://foobar `) } @@ -97,8 +101,8 @@ func TestParseAuthConfigSuccess(t *testing.T) { t.Fatalf("unexpected error: %s", err) } removeMetrics(m) - if !reflect.DeepEqual(m, expectedAuthConfig) { - t.Fatalf("unexpected auth config\ngot\n%v\nwant\n%v", m, expectedAuthConfig) + if err := areEqualConfigs(m, expectedAuthConfig); err != nil { + t.Fatal(err) } } @@ -139,7 +143,7 @@ users: users: - username: foo url_map: - - src_paths: ["/api/v1/query","/api/v1/query_range"] + - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"] url_prefix: http://vmselect/select/0/prometheus - src_paths: ["/api/v1/write"] url_prefix: http://vminsert/insert/0/prometheus @@ -148,11 +152,11 @@ users: Username: "foo", URLMap: []URLMap{ { - SrcPaths: []string{"/api/v1/query", "/api/v1/query_range"}, + SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), URLPrefix: "http://vmselect/select/0/prometheus", }, { - SrcPaths: []string{"/api/v1/write"}, + SrcPaths: getSrcPaths([]string{"/api/v1/write"}), URLPrefix: "http://vminsert/insert/0/prometheus", }, }, @@ -160,8 +164,34 @@ users: }) } +func getSrcPaths(paths []string) []*SrcPath { + var sps []*SrcPath + for _, path := range paths { + sps = append(sps, &SrcPath{ + sOriginal: path, + re: regexp.MustCompile("^(?:" + path + ")$"), + }) + } + return sps +} + func removeMetrics(m map[string]*UserInfo) { for _, info := range m { info.requests = nil } } + +func areEqualConfigs(a, b map[string]*UserInfo) error { + aData, err := yaml.Marshal(a) + if err != nil { + return fmt.Errorf("cannot marshal a: %w", err) + } + bData, err := yaml.Marshal(b) + if err != nil { + return fmt.Errorf("cannot marshal b: %w", err) + } + if !bytes.Equal(aData, bData) { + return fmt.Errorf("unexpected configs;\ngot\n%s\nwant\n%s", aData, bData) + } + return nil +} diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go index f086227f0..9f4db4dcf 100644 --- a/app/vmauth/target_url.go +++ b/app/vmauth/target_url.go @@ -18,8 +18,8 @@ func createTargetURL(ui *UserInfo, uOrig *url.URL) (string, error) { u.Path = "/" + u.Path } for _, e := range ui.URLMap { - for _, path := range e.SrcPaths { - if u.Path == path { + for _, sp := range e.SrcPaths { + if sp.match(u.Path) { return e.URLPrefix + u.RequestURI(), nil } } diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go index 5dcfd07cf..34d2e6c33 100644 --- a/app/vmauth/target_url_test.go +++ b/app/vmauth/target_url_test.go @@ -44,11 +44,11 @@ func TestCreateTargetURLSuccess(t *testing.T) { ui := &UserInfo{ URLMap: []URLMap{ { - SrcPaths: []string{"/api/v1/query"}, + SrcPaths: getSrcPaths([]string{"/api/v1/query"}), URLPrefix: "http://vmselect/0/prometheus", }, { - SrcPaths: []string{"/api/v1/write"}, + SrcPaths: getSrcPaths([]string{"/api/v1/write"}), URLPrefix: "http://vminsert/0/prometheus", }, }, @@ -57,6 +57,26 @@ func TestCreateTargetURLSuccess(t *testing.T) { f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up") f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write") f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range") + + // Complex routing regexp paths in `url_map` + ui = &UserInfo{ + URLMap: []URLMap{ + { + SrcPaths: getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}), + URLPrefix: "http://vmselect/0/prometheus", + }, + { + SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + URLPrefix: "http://vminsert/0/prometheus", + }, + }, + URLPrefix: "http://default-server", + } + f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up") + f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up") + f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values") + f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write") + f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar") } func TestCreateTargetURLFailure(t *testing.T) { @@ -78,7 +98,7 @@ func TestCreateTargetURLFailure(t *testing.T) { f(&UserInfo{ URLMap: []URLMap{ { - SrcPaths: []string{"/api/v1/query"}, + SrcPaths: getSrcPaths([]string{"/api/v1/query"}), URLPrefix: "http://foobar/baz", }, }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fd73b30bc..86372ae43 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ - `histogram_stdvar(buckets)` - returns standard variance for the given buckets. - `histogram_stddev(buckets)` - returns standard deviation for the given buckets. * FEATURE: vmagent: add ability to replicate scrape targets among `vmagent` instances in the cluster with `-promscrape.cluster.replicationFactor` command-line flag. See [these docs](https://victoriametrics.github.io/vmagent.html#scraping-big-number-of-targets). +* FEATURE: vmauth: allow using regexp paths in `url_map`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112) for details. * BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct jobs by sharing the cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113 diff --git a/docs/vmauth.md b/docs/vmauth.md index 0c241aeb3..133718787 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -65,13 +65,13 @@ users: # A single user for querying and inserting data: - # - Requests to http://vmauth:8427/api/v1/query or http://vmauth:8427/api/v1/query_range - # are routed to http://vmselect:8481/select/42/prometheus. + # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range + # and http://vmauth:8427/api/v1/label//values are routed to http://vmselect:8481/select/42/prometheus. # For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8480/select/42/prometheus/api/v1/query # - Requests to http://vmauth:8427/api/v1/write are routed to http://vminsert:8480/insert/42/prometheus/api/v1/write - username: "foobar" url_map: - - src_paths: ["/api/v1/query", "/api/v1/query_range"] + - src_paths: ["/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^/]+/values"] url_prefix: "http://vmselect:8481/select/42/prometheus" - src_paths: ["/api/v1/write"] url_prefix: "http://vminsert:8480/insert/42/prometheus"