vmalert: support reading rule from http url (#4212)

vmalert: support reading rule's config from HTTP URL

app/vmalert/config/config.go

@@ -209,8 +209,7 @@ var cLogger = &log.Logger{}
 func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	cLogger.Suppress(true)
 	defer cLogger.Suppress(false)
-
-	files, err := readFromFS(pathPatterns)
+	files, err := readFromFSOrHTTP(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
@@ -219,7 +218,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
+	files, err := readFromFSOrHTTP(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
@@ -233,6 +232,34 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
 	return groups, nil
 }
 
+// readFromFSOrHTTP reads path either from filesystem or from http if path starts with http or https.
+func readFromFSOrHTTP(paths []string) (map[string][]byte, error) {
+	var httpPaths []string
+	var fsPaths []string
+	for _, path := range paths {
+		if isHTTPURL(path) {
+			httpPaths = append(httpPaths, path)
+			continue
+		}
+		fsPaths = append(fsPaths, path)
+	}
+	result, err := readFromFS(fsPaths)
+	if err != nil {
+		return nil, err
+	}
+	httpResult, err := readFromHTTP(httpPaths)
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range httpResult {
+		if _, ok := result[k]; ok {
+			return nil, fmt.Errorf("duplicate found for config name %q: config names must be unique", k)
+		}
+		result[k] = v
+	}
+	return result, nil
+}
+
 func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	errGroup := new(utils.ErrGroup)
 	var groups []Group

app/vmalert/config/config_test.go

@@ -64,6 +64,10 @@ func TestParseBad(t *testing.T) {
 			[]string{"testdata/dir/rules6-bad.rules"},
 			"missing ':' in header",
 		},
+		{
+			[]string{"http://unreachable-url"},
+			"failed to read from the config: cannot fetch \"http://unreachable-url\"",
+		},
 	}
 	for _, tc := range testCases {
 		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
@@ -102,7 +106,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "group name must be set",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{
 						Record: "record",
@@ -113,7 +118,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{
 						Record: "record",
@@ -125,7 +131,8 @@ func TestGroup_Validate(t *testing.T) {
 			validateExpressions: true,
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -139,7 +146,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -156,7 +164,8 @@ func TestGroup_Validate(t *testing.T) {
 			validateAnnotations: true,
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -171,7 +180,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -184,7 +194,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{Record: "record", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -197,7 +208,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -210,7 +222,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{Name: "test",
+			group: &Group{
+				Name: "test",
 				Rules: []Rule{
 					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -223,7 +236,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{Name: "test thanos",
+			group: &Group{
+				Name: "test thanos",
 				Type: NewRawType("thanos"),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
@@ -235,7 +249,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr:              "unknown datasource type",
 		},
 		{
-			group: &Group{Name: "test graphite",
+			group: &Group{
+				Name: "test graphite",
 				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
@@ -247,7 +262,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr:              "",
 		},
 		{
-			group: &Group{Name: "test prometheus",
+			group: &Group{
+				Name: "test prometheus",
 				Type: NewPrometheusType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{

app/vmalert/config/fs.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -32,17 +33,16 @@ var (
 )
 
 // readFromFS parses the given path list and inits FS for each item.
-// Once inited, readFromFS will try to read and return files from each FS.
+// Once initialed, readFromFS will try to read and return files from each FS.
 // readFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
-// will be inited only once.
+// will be initialed only once.
 //
 // It is allowed to mix different FS types in path list.
 func readFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {
-
 		fsRegistryMu.Lock()
 		fs, ok := fsRegistry[path]
 		if !ok {
@@ -106,3 +106,9 @@ func newFS(path string) (FS, error) {
 		return nil, fmt.Errorf("unsupported scheme %q", scheme)
 	}
 }
+
+// isHTTPURL checks if a given targetURL is valid and contains a valid http scheme
+func isHTTPURL(targetURL string) bool {
+	parsed, err := url.Parse(targetURL)
+	return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != ""
+}

app/vmalert/config/url.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// readFromHTTP reads config from http path.
+func readFromHTTP(paths []string) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	for _, path := range paths {
+		if _, ok := result[path]; ok {
+			return nil, fmt.Errorf("duplicate found for url path %q: url path must be unique", path)
+		}
+		resp, err := http.Get(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot fetch %q: %w", path, err)
+		}
+		data, err := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			if len(data) > 4*1024 {
+				data = data[:4*1024]
+			}
+			return nil, fmt.Errorf("unexpected status code when fetching %q: %d, expecting %d; response: %q", path, resp.StatusCode, http.StatusOK, data)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("cannot read %q: %s", path, err)
+		}
+		result[path] = data
+	}
+	return result, nil
+}

app/vmalert/main.go

@@ -29,10 +29,11 @@ import (
 )
 
 var (
-	rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules.
+	rulePath = flagutil.NewArrayString("rule", `Path to the files or http url with alerting and/or recording rules.
 Supports hierarchical patterns and regexpes.
 Examples:
  -rule="/path/to/file". Path to a single file with alerting rules
+ -rule="http://<some-server-addr>:path/to/rules". HTTP URL to alerting rules.
  -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
  -rule="dir/**/*.yaml". Includes to all .yaml files in "dir" folder and it's subfolders recursively. 
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.