From d66bae212b4c5d7ad93ff534b572dd77e82ca09d Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 9 Mar 2023 14:46:19 +0100 Subject: [PATCH] app/vmalert: log number of configration files found for each specified `-rule` (#3936) The change also introduces `List` method to `FS` interface. The `List` method can be used for wildcard support in object storage FS. Signed-off-by: hagen1778 Co-authored-by: Nikolay --- app/vmalert/config/config.go | 35 ++++++++++++++++++++++----- app/vmalert/config/fs.go | 27 ++++++++++++++++++--- app/vmalert/config/fslocal/fslocal.go | 13 +++++++--- app/vmalert/main.go | 2 +- docs/CHANGELOG.md | 1 + 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index a50f15fdf..2de93fc55 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -199,12 +199,32 @@ func (r *Rule) Validate() error { // ValidateTplFn must validate the given annotations type ValidateTplFn func(annotations map[string]string) error -// Parse parses rule configs from given file patterns -func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { - files, err := readFromFS(pathPatterns) +// ParseSilent parses rule configs from given file patterns without emitting logs +func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { + files, err := readFromFS(pathPatterns, true) if err != nil { return nil, fmt.Errorf("failed to read from the config: %s", err) } + return parse(files, validateTplFn, validateExpressions) +} + +// Parse parses rule configs from given file patterns +func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { + files, err := readFromFS(pathPatterns, false) + if err != nil { + return nil, fmt.Errorf("failed to read from the config: %s", err) + } + groups, err := parse(files, validateTplFn, validateExpressions) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err) + } + if len(groups) < 1 { + logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";")) + } + return groups, nil +} + +func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { errGroup := new(utils.ErrGroup) var groups []Group for file, data := range files { @@ -231,9 +251,12 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio if err := errGroup.Err(); err != nil { return nil, err } - if len(groups) < 1 { - logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";")) - } + sort.SliceStable(groups, func(i, j int) bool { + if groups[i].File != groups[j].File { + return groups[i].File < groups[j].File + } + return groups[i].Name < groups[j].Name + }) return groups, nil } diff --git a/app/vmalert/config/fs.go b/app/vmalert/config/fs.go index 35107d4f4..f1aa8f511 100644 --- a/app/vmalert/config/fs.go +++ b/app/vmalert/config/fs.go @@ -2,9 +2,11 @@ package config import ( "fmt" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal" "strings" "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) // FS represent a file system abstract for reading files. @@ -15,10 +17,13 @@ type FS interface { // String must return human-readable representation of FS. String() string + // List returns the list of file names which will be read via Read fn + List() ([]string, error) + // Read returns a list of read files in form of a map // where key is a file name and value is a content of read file. // Read must be called only after the successful Init call. - Read() (map[string][]byte, error) + Read(files []string) (map[string][]byte, error) } var ( @@ -31,9 +36,10 @@ var ( // readFromFS returns an error if at least one FS failed to init. // The function can be called multiple times but each unique path // will be inited only once. +// If silent == true, readFromFS will not emit any logs. // // It is allowed to mix different FS types in path list. -func readFromFS(paths []string) (map[string][]byte, error) { +func readFromFS(paths []string, silent bool) (map[string][]byte, error) { var err error result := make(map[string][]byte) for _, path := range paths { @@ -54,7 +60,20 @@ func readFromFS(paths []string) (map[string][]byte, error) { } fsRegistryMu.Unlock() - files, err := fs.Read() + list, err := fs.List() + if err != nil { + return nil, fmt.Errorf("failed to list files from %q", fs) + } + + if !silent { + logger.Infof("found %d files to read from %q", len(list), fs) + } + + if len(list) < 1 { + continue + } + + files, err := fs.Read(list) if err != nil { return nil, fmt.Errorf("error while reading files from %q: %w", fs, err) } diff --git a/app/vmalert/config/fslocal/fslocal.go b/app/vmalert/config/fslocal/fslocal.go index 23366adc4..6ebfaac17 100644 --- a/app/vmalert/config/fslocal/fslocal.go +++ b/app/vmalert/config/fslocal/fslocal.go @@ -25,15 +25,20 @@ func (fs *FS) String() string { return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern) } -// Read returns a map of read files where -// key is the file name and value is file's content. -func (fs *FS) Read() (map[string][]byte, error) { +// List returns the list of file names which will be read via Read fn +func (fs *FS) List() ([]string, error) { matches, err := filepath.Glob(fs.Pattern) if err != nil { return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err) } + return matches, nil +} + +// Read returns a map of read files where +// key is the file name and value is file's content. +func (fs *FS) Read(files []string) (map[string][]byte, error) { result := make(map[string][]byte) - for _, path := range matches { + for _, path := range files { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("error while reading file %q: %w", path, err) diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 8f382a23f..2d32a5d3b 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -345,7 +345,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig logger.Errorf("failed to load new templates: %s", err) continue } - newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions) + newGroupsCfg, err := config.ParseSilent(*rulePath, validateTplFn, *validateExpressions) if err != nil { configReloadErrors.Inc() configSuccess.Set(0) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0f6dfd776..63dab58b1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,6 +20,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): increase the default value for `--remote-read-http-timeout` command-line option from 30s (30 seconds) to 5m (5 minutes). This reduces the probability of timeout errors when migrating big number of time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3879). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): migrate series one-by-one in [vm-native mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). This allows better tracking the migration progress and resuming the migration process from the last migrated time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3859) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3600). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `--vm-native-src-headers` and `--vm-native-dst-headers` command-line flags, which can be used for setting custom HTTP headers during [vm-native migration mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3906). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): log number of configration files found for each specified `-rule` command-line flag. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `--vm-native-disable-http-keep-alive` command-line flags to allow `vmctl` to use non-persistent HTTP connections in [vm-native migration mode](https://docs.victoriametrics.com/vmctl.html#native-protocol). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3909). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix panic when [writing data to Kafka](https://docs.victoriametrics.com/vmagent.html#writing-metrics-to-kafka). The panic has been introduced in [v1.88.0](https://docs.victoriametrics.com/CHANGELOG.html#v1880).