From cae87da4bbb54e901bfc5b355958b413488a6840 Mon Sep 17 00:00:00 2001 From: Haleygo Date: Mon, 8 May 2023 15:52:57 +0800 Subject: [PATCH] vmalert: support reading rule from http url (#4212) vmalert: support reading rule's config from HTTP URL --- app/vmalert/config/config.go | 33 ++++++++++++++++++++++--- app/vmalert/config/config_test.go | 40 +++++++++++++++++++++---------- app/vmalert/config/fs.go | 12 +++++++--- app/vmalert/config/url.go | 34 ++++++++++++++++++++++++++ app/vmalert/main.go | 3 ++- 5 files changed, 103 insertions(+), 19 deletions(-) create mode 100644 app/vmalert/config/url.go diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 0fc201ef90..438847d2e2 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -209,8 +209,7 @@ var cLogger = &log.Logger{} func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { cLogger.Suppress(true) defer cLogger.Suppress(false) - - files, err := readFromFS(pathPatterns) + files, err := readFromFSOrHTTP(pathPatterns) if err != nil { return nil, fmt.Errorf("failed to read from the config: %s", err) } @@ -219,7 +218,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp // Parse parses rule configs from given file patterns func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { - files, err := readFromFS(pathPatterns) + files, err := readFromFSOrHTTP(pathPatterns) if err != nil { return nil, fmt.Errorf("failed to read from the config: %s", err) } @@ -233,6 +232,34 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio return groups, nil } +// readFromFSOrHTTP reads path either from filesystem or from http if path starts with http or https. +func readFromFSOrHTTP(paths []string) (map[string][]byte, error) { + var httpPaths []string + var fsPaths []string + for _, path := range paths { + if isHTTPURL(path) { + httpPaths = append(httpPaths, path) + continue + } + fsPaths = append(fsPaths, path) + } + result, err := readFromFS(fsPaths) + if err != nil { + return nil, err + } + httpResult, err := readFromHTTP(httpPaths) + if err != nil { + return nil, err + } + for k, v := range httpResult { + if _, ok := result[k]; ok { + return nil, fmt.Errorf("duplicate found for config name %q: config names must be unique", k) + } + result[k] = v + } + return result, nil +} + func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { errGroup := new(utils.ErrGroup) var groups []Group diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 71eee2822e..4d7ed3ec45 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -64,6 +64,10 @@ func TestParseBad(t *testing.T) { []string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header", }, + { + []string{"http://unreachable-url"}, + "failed to read from the config: cannot fetch \"http://unreachable-url\"", + }, } for _, tc := range testCases { _, err := Parse(tc.path, notifier.ValidateTemplates, true) @@ -102,7 +106,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "group name must be set", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ { Record: "record", @@ -113,7 +118,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ { Record: "record", @@ -125,7 +131,8 @@ func TestGroup_Validate(t *testing.T) { validateExpressions: true, }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ { Alert: "alert", @@ -139,7 +146,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ { Alert: "alert", @@ -156,7 +164,8 @@ func TestGroup_Validate(t *testing.T) { validateAnnotations: true, }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ { Alert: "alert", @@ -171,7 +180,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "duplicate", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ "summary": "{{ value|query }}", @@ -184,7 +194,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "duplicate", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ {Record: "record", Expr: "up == 1", Labels: map[string]string{ "summary": "{{ value|query }}", @@ -197,7 +208,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "duplicate", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ "summary": "{{ value|query }}", @@ -210,7 +222,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "", }, { - group: &Group{Name: "test", + group: &Group{ + Name: "test", Rules: []Rule{ {Record: "alert", Expr: "up == 1", Labels: map[string]string{ "summary": "{{ value|query }}", @@ -223,7 +236,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "", }, { - group: &Group{Name: "test thanos", + group: &Group{ + Name: "test thanos", Type: NewRawType("thanos"), Rules: []Rule{ {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ @@ -235,7 +249,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "unknown datasource type", }, { - group: &Group{Name: "test graphite", + group: &Group{ + Name: "test graphite", Type: NewGraphiteType(), Rules: []Rule{ {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ @@ -247,7 +262,8 @@ func TestGroup_Validate(t *testing.T) { expErr: "", }, { - group: &Group{Name: "test prometheus", + group: &Group{ + Name: "test prometheus", Type: NewPrometheusType(), Rules: []Rule{ {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ diff --git a/app/vmalert/config/fs.go b/app/vmalert/config/fs.go index 372d8066fa..609d6b8a62 100644 --- a/app/vmalert/config/fs.go +++ b/app/vmalert/config/fs.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "net/url" "strings" "sync" "time" @@ -32,17 +33,16 @@ var ( ) // readFromFS parses the given path list and inits FS for each item. -// Once inited, readFromFS will try to read and return files from each FS. +// Once initialed, readFromFS will try to read and return files from each FS. // readFromFS returns an error if at least one FS failed to init. // The function can be called multiple times but each unique path -// will be inited only once. +// will be initialed only once. // // It is allowed to mix different FS types in path list. func readFromFS(paths []string) (map[string][]byte, error) { var err error result := make(map[string][]byte) for _, path := range paths { - fsRegistryMu.Lock() fs, ok := fsRegistry[path] if !ok { @@ -106,3 +106,9 @@ func newFS(path string) (FS, error) { return nil, fmt.Errorf("unsupported scheme %q", scheme) } } + +// isHTTPURL checks if a given targetURL is valid and contains a valid http scheme +func isHTTPURL(targetURL string) bool { + parsed, err := url.Parse(targetURL) + return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" +} diff --git a/app/vmalert/config/url.go b/app/vmalert/config/url.go new file mode 100644 index 0000000000..a89fdf3131 --- /dev/null +++ b/app/vmalert/config/url.go @@ -0,0 +1,34 @@ +package config + +import ( + "fmt" + "io" + "net/http" +) + +// readFromHTTP reads config from http path. +func readFromHTTP(paths []string) (map[string][]byte, error) { + result := make(map[string][]byte) + for _, path := range paths { + if _, ok := result[path]; ok { + return nil, fmt.Errorf("duplicate found for url path %q: url path must be unique", path) + } + resp, err := http.Get(path) + if err != nil { + return nil, fmt.Errorf("cannot fetch %q: %w", path, err) + } + data, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + if len(data) > 4*1024 { + data = data[:4*1024] + } + return nil, fmt.Errorf("unexpected status code when fetching %q: %d, expecting %d; response: %q", path, resp.StatusCode, http.StatusOK, data) + } + if err != nil { + return nil, fmt.Errorf("cannot read %q: %s", path, err) + } + result[path] = data + } + return result, nil +} diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 8b49029e26..cea9be80fa 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -29,10 +29,11 @@ import ( ) var ( - rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules. + rulePath = flagutil.NewArrayString("rule", `Path to the files or http url with alerting and/or recording rules. Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules + -rule="http://:path/to/rules". HTTP URL to alerting rules. -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". -rule="dir/**/*.yaml". Includes to all .yaml files in "dir" folder and it's subfolders recursively. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.