app/vmalert: log number of configration files found for each specified -rule (#3936)

The change also introduces `List` method to `FS` interface.
The `List` method can be used for wildcard support in object storage FS.

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Nikolay <nik@victoriametrics.com>

app/vmalert/config/config.go

@@ -199,12 +199,32 @@ func (r *Rule) Validate() error {
 // ValidateTplFn must validate the given annotations
 type ValidateTplFn func(annotations map[string]string) error
 
-// Parse parses rule configs from given file patterns
+// ParseSilent parses rule configs from given file patterns without emitting logs
-func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
+	files, err := readFromFS(pathPatterns, true)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
+	return parse(files, validateTplFn, validateExpressions)
+}
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	files, err := readFromFS(pathPatterns, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
+	}
+	groups, err := parse(files, validateTplFn, validateExpressions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
+	}
+	if len(groups) < 1 {
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	errGroup := new(utils.ErrGroup)
 	var groups []Group
 	for file, data := range files {
@@ -231,9 +251,12 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
 	if err := errGroup.Err(); err != nil {
 		return nil, err
 	}
-	if len(groups) < 1 {
+	sort.SliceStable(groups, func(i, j int) bool {
-		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+		if groups[i].File != groups[j].File {
-	}
+			return groups[i].File < groups[j].File
+		}
+		return groups[i].Name < groups[j].Name
+	})
 	return groups, nil
 }
 

app/vmalert/config/fs.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
 	"strings"
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 // FS represent a file system abstract for reading files.
@@ -15,10 +17,13 @@ type FS interface {
 	// String must return human-readable representation of FS.
 	String() string
 
+	// List returns the list of file names which will be read via Read fn
+	List() ([]string, error)
+
 	// Read returns a list of read files in form of a map
 	// where key is a file name and value is a content of read file.
 	// Read must be called only after the successful Init call.
-	Read() (map[string][]byte, error)
+	Read(files []string) (map[string][]byte, error)
 }
 
 var (
@@ -31,9 +36,10 @@ var (
 // readFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be inited only once.
+// If silent == true, readFromFS will not emit any logs.
 //
 // It is allowed to mix different FS types in path list.
-func readFromFS(paths []string) (map[string][]byte, error) {
+func readFromFS(paths []string, silent bool) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {
@@ -54,7 +60,20 @@ func readFromFS(paths []string) (map[string][]byte, error) {
 		}
 		fsRegistryMu.Unlock()
 
-		files, err := fs.Read()
+		list, err := fs.List()
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files from %q", fs)
+		}
+
+		if !silent {
+			logger.Infof("found %d files to read from %q", len(list), fs)
+		}
+
+		if len(list) < 1 {
+			continue
+		}
+
+		files, err := fs.Read(list)
 		if err != nil {
 			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
 		}

app/vmalert/config/fslocal/fslocal.go

@@ -25,15 +25,20 @@ func (fs *FS) String() string {
 	return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern)
 }
 
-// Read returns a map of read files where
+// List returns the list of file names which will be read via Read fn
-// key is the file name and value is file's content.
+func (fs *FS) List() ([]string, error) {
-func (fs *FS) Read() (map[string][]byte, error) {
 	matches, err := filepath.Glob(fs.Pattern)
 	if err != nil {
 		return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err)
 	}
+	return matches, nil
+}
+
+// Read returns a map of read files where
+// key is the file name and value is file's content.
+func (fs *FS) Read(files []string) (map[string][]byte, error) {
 	result := make(map[string][]byte)
-	for _, path := range matches {
+	for _, path := range files {
 		data, err := os.ReadFile(path)
 		if err != nil {
 			return nil, fmt.Errorf("error while reading file %q: %w", path, err)

app/vmalert/main.go

@@ -345,7 +345,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
+		newGroupsCfg, err := config.ParseSilent(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)

docs/CHANGELOG.md

@@ -20,6 +20,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): increase the default value for `--remote-read-http-timeout` command-line option from 30s (30 seconds) to 5m (5 minutes). This reduces the probability of timeout errors when migrating big number of time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3879).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): migrate series one-by-one in [vm-native mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). This allows better tracking the migration progress and resuming the migration process from the last migrated time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3859) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3600).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `--vm-native-src-headers` and `--vm-native-dst-headers` command-line flags, which can be used for setting custom HTTP headers during [vm-native migration mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3906).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): log number of configration files found for each specified `-rule` command-line flag. 
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `--vm-native-disable-http-keep-alive` command-line flags to allow `vmctl` to use non-persistent HTTP connections in [vm-native migration mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3909).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix panic when [writing data to Kafka](https://docs.victoriametrics.com/vmagent.html#writing-metrics-to-kafka). The panic has been introduced in [v1.88.0](https://docs.victoriametrics.com/CHANGELOG.html#v1880).