From c12eb2cc7f31a49c7878a9cba603b5053a7f32d1 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 9 Feb 2023 15:53:08 -0800 Subject: [PATCH 1/3] deployment/docker: update VictoriaMetrics Docker images from v1.87.0 to v1.87.1 --- deployment/docker/docker-compose-cluster.yml | 12 ++++++------ deployment/docker/docker-compose.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index daff5d8c2..075d90d92 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -2,7 +2,7 @@ version: '3.5' services: vmagent: container_name: vmagent - image: victoriametrics/vmagent:v1.87.0 + image: victoriametrics/vmagent:v1.87.1 depends_on: - "vminsert" ports: @@ -32,7 +32,7 @@ services: vmstorage-1: container_name: vmstorage-1 - image: victoriametrics/vmstorage:v1.87.0-cluster + image: victoriametrics/vmstorage:v1.87.1-cluster ports: - 8482 - 8400 @@ -44,7 +44,7 @@ services: restart: always vmstorage-2: container_name: vmstorage-2 - image: victoriametrics/vmstorage:v1.87.0-cluster + image: victoriametrics/vmstorage:v1.87.1-cluster ports: - 8482 - 8400 @@ -56,7 +56,7 @@ services: restart: always vminsert: container_name: vminsert - image: victoriametrics/vminsert:v1.87.0-cluster + image: victoriametrics/vminsert:v1.87.1-cluster depends_on: - "vmstorage-1" - "vmstorage-2" @@ -68,7 +68,7 @@ services: restart: always vmselect: container_name: vmselect - image: victoriametrics/vmselect:v1.87.0-cluster + image: victoriametrics/vmselect:v1.87.1-cluster depends_on: - "vmstorage-1" - "vmstorage-2" @@ -82,7 +82,7 @@ services: vmalert: container_name: vmalert - image: victoriametrics/vmalert:v1.87.0 + image: victoriametrics/vmalert:v1.87.1 depends_on: - "vmselect" ports: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index e821222a6..453b40fe7 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.5" services: vmagent: container_name: vmagent - image: victoriametrics/vmagent:v1.87.0 + image: victoriametrics/vmagent:v1.87.1 depends_on: - "victoriametrics" ports: @@ -18,7 +18,7 @@ services: restart: always victoriametrics: container_name: victoriametrics - image: victoriametrics/victoria-metrics:v1.87.0 + image: victoriametrics/victoria-metrics:v1.87.1 ports: - 8428:8428 - 8089:8089 @@ -56,7 +56,7 @@ services: restart: always vmalert: container_name: vmalert - image: victoriametrics/vmalert:v1.87.0 + image: victoriametrics/vmalert:v1.87.1 depends_on: - "victoriametrics" - "alertmanager" From 14c20c18436001bf09d38ee2e9525cd100321885 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Fri, 10 Feb 2023 02:18:27 +0100 Subject: [PATCH 2/3] vmalert: support object storage for rules (#519) * vmalert: support object storage for rules Support loading of alerting and recording rules from object storages `gcs://`, `gs://`, `s3://`. * review fixes --- app/vmalert/config/config.go | 26 +++----- app/vmalert/config/fs.go | 89 +++++++++++++++++++++++++++ app/vmalert/config/fs_test.go | 39 ++++++++++++ app/vmalert/config/fslocal/fslocal.go | 44 +++++++++++++ app/vmalert/main.go | 15 +++-- 5 files changed, 190 insertions(+), 23 deletions(-) create mode 100644 app/vmalert/config/fs.go create mode 100644 app/vmalert/config/fs_test.go create mode 100644 app/vmalert/config/fslocal/fslocal.go diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 9be2edb55..a50f15fdf 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -5,8 +5,6 @@ import ( "fmt" "hash/fnv" "net/url" - "os" - "path/filepath" "sort" "strings" @@ -203,19 +201,15 @@ type ValidateTplFn func(annotations map[string]string) error // Parse parses rule configs from given file patterns func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) { - var fp []string - for _, pattern := range pathPatterns { - matches, err := filepath.Glob(pattern) - if err != nil { - return nil, fmt.Errorf("error reading file pattern %s: %w", pattern, err) - } - fp = append(fp, matches...) + files, err := readFromFS(pathPatterns) + if err != nil { + return nil, fmt.Errorf("failed to read from the config: %s", err) } errGroup := new(utils.ErrGroup) var groups []Group - for _, file := range fp { + for file, data := range files { uniqueGroups := map[string]struct{}{} - gr, err := parseFile(file) + gr, err := parseConfig(data) if err != nil { errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err)) continue @@ -243,14 +237,10 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio return groups, nil } -func parseFile(path string) ([]Group, error) { - data, err := os.ReadFile(path) +func parseConfig(data []byte) ([]Group, error) { + data, err := envtemplate.ReplaceBytes(data) if err != nil { - return nil, fmt.Errorf("error reading alert rule file %q: %w", path, err) - } - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err) + return nil, fmt.Errorf("cannot expand environment vars: %w", err) } g := struct { Groups []Group `yaml:"groups"` diff --git a/app/vmalert/config/fs.go b/app/vmalert/config/fs.go new file mode 100644 index 000000000..35107d4f4 --- /dev/null +++ b/app/vmalert/config/fs.go @@ -0,0 +1,89 @@ +package config + +import ( + "fmt" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal" + "strings" + "sync" +) + +// FS represent a file system abstract for reading files. +type FS interface { + // Init initializes FS. + Init() error + + // String must return human-readable representation of FS. + String() string + + // Read returns a list of read files in form of a map + // where key is a file name and value is a content of read file. + // Read must be called only after the successful Init call. + Read() (map[string][]byte, error) +} + +var ( + fsRegistryMu sync.Mutex + fsRegistry = make(map[string]FS) +) + +// readFromFS parses the given path list and inits FS for each item. +// Once inited, readFromFS will try to read and return files from each FS. +// readFromFS returns an error if at least one FS failed to init. +// The function can be called multiple times but each unique path +// will be inited only once. +// +// It is allowed to mix different FS types in path list. +func readFromFS(paths []string) (map[string][]byte, error) { + var err error + result := make(map[string][]byte) + for _, path := range paths { + + fsRegistryMu.Lock() + fs, ok := fsRegistry[path] + if !ok { + fs, err = newFS(path) + if err != nil { + fsRegistryMu.Unlock() + return nil, fmt.Errorf("error while parsing path %q: %w", path, err) + } + if err := fs.Init(); err != nil { + fsRegistryMu.Unlock() + return nil, fmt.Errorf("error while initializing path %q: %w", path, err) + } + fsRegistry[path] = fs + } + fsRegistryMu.Unlock() + + files, err := fs.Read() + if err != nil { + return nil, fmt.Errorf("error while reading files from %q: %w", fs, err) + } + for k, v := range files { + if _, ok := result[k]; ok { + return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k) + } + result[k] = v + } + } + return result, nil +} + +// newFS creates FS based on the give path. +// Supported file systems are: fs +func newFS(path string) (FS, error) { + scheme := "fs" + n := strings.Index(path, "://") + if n >= 0 { + scheme = path[:n] + path = path[n+len("://"):] + } + if len(path) == 0 { + return nil, fmt.Errorf("path cannot be empty") + } + switch scheme { + case "fs": + return &fslocal.FS{Pattern: path}, nil + default: + return nil, fmt.Errorf("unsupported scheme %q", scheme) + } +} diff --git a/app/vmalert/config/fs_test.go b/app/vmalert/config/fs_test.go new file mode 100644 index 000000000..103c0cc3d --- /dev/null +++ b/app/vmalert/config/fs_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "strings" + "testing" +) + +func TestNewFS(t *testing.T) { + f := func(path, expStr string) { + t.Helper() + fs, err := newFS(path) + if err != nil { + t.Fatalf("unexpected err: %s", err) + } + if fs.String() != expStr { + t.Fatalf("expected FS %q; got %q", expStr, fs.String()) + } + } + + f("/foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}") + f("fs:///foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}") +} + +func TestNewFSNegative(t *testing.T) { + f := func(path, expErr string) { + t.Helper() + _, err := newFS(path) + if err == nil { + t.Fatalf("expected to have err: %s", expErr) + } + if !strings.Contains(err.Error(), expErr) { + t.Fatalf("expected to have err %q; got %q instead", expErr, err) + } + } + + f("", "path cannot be empty") + f("fs://", "path cannot be empty") + f("foobar://baz", `unsupported scheme "foobar"`) +} diff --git a/app/vmalert/config/fslocal/fslocal.go b/app/vmalert/config/fslocal/fslocal.go new file mode 100644 index 000000000..23366adc4 --- /dev/null +++ b/app/vmalert/config/fslocal/fslocal.go @@ -0,0 +1,44 @@ +package fslocal + +import ( + "fmt" + "os" + "path/filepath" +) + +// FS represents a local file system +type FS struct { + // Pattern is used for matching one or multiple files. + // The pattern may describe hierarchical names such as + // /usr/*/bin/ed (assuming the Separator is '/'). + Pattern string +} + +// Init verifies that configured Pattern is correct +func (fs *FS) Init() error { + _, err := filepath.Glob(fs.Pattern) + return err +} + +// String implements Stringer interface +func (fs *FS) String() string { + return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern) +} + +// Read returns a map of read files where +// key is the file name and value is file's content. +func (fs *FS) Read() (map[string][]byte, error) { + matches, err := filepath.Glob(fs.Pattern) + if err != nil { + return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err) + } + result := make(map[string][]byte) + for _, path := range matches { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error while reading file %q: %w", path, err) + } + result[path] = data + } + return result, nil +} diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 5278c51c8..f669027e3 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -28,13 +28,18 @@ import ( ) var ( - rulePath = flagutil.NewArrayString("rule", `Path to the file with alert rules. -Supports patterns. Flag can be specified multiple times. + rulePath = flagutil.NewArrayString("rule", `Path to the files with alert rules. +Example: gs://bucket/path/to/rules, s3://bucket/path/to/rules, or fs:///path/to/local/rules/dir +If scheme remote storage scheme is omitted, local file system is used. +Local file system supports hierarchical patterns and regexes. +Remote file system supports only matching by prefix, e.g. s3://bucket/dir/rule_ will match all files with prefix +rule_ in folder dir. +This flag can be specified multiple times. Examples: -rule="/path/to/file". Path to a single file with alerting rules - -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, -absolute path to all .yaml files in root. -Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`) + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". +Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars +`) ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions for rules annotations templating. Flag can be specified multiple times. From 73358571eec983b9e604368f8536e4ba9aebe84f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 9 Feb 2023 17:51:00 -0800 Subject: [PATCH 3/3] app/vmalert: follow-up after d3c64aae8768d58781ee7e358bd7f3d8e0eb836d - Document the change at docs/CHANGELOG.md - Add `Reading rules from object storage` section to docs/vmalert.md - Add `s3` prefix to command-line flags related to the configuration of s3 and gcs clients - Explicitly mention that reading rules from object storage is supported only in enterprise version --- app/vmalert/README.md | 61 ++++++++++++++++++++++++++++++++++++++----- app/vmalert/main.go | 17 ++++++------ docs/CHANGELOG.md | 2 ++ docs/enterprise.md | 3 ++- docs/vmalert.md | 61 ++++++++++++++++++++++++++++++++++++++----- 5 files changed, 123 insertions(+), 21 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 5bd4335a6..b2db420f9 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -99,6 +99,26 @@ groups: [ - ] ``` +### Reading rules from object storage + +[Enterprise version](https://docs.victoriametrics.com/enterprise.html) of `vmalert` may read alerting and recording rules +from object storage: + +- `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket +- `./bin/vmalert -rule=gs://bucket/bir/alert.rules` would read rules from the given path at GCS bucket + +S3 and GCS paths support only matching by prefix, e.g. `s3://bucket/dir/rule_` matches +all files with prefix `rule_` in the folder `dir`. + +The following [command-line flags](#flags) can be used for fine-tuning access to S3 and GCS: + +- `-s3.credsFilePath` - path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. +- `-s3.configFilePath` - path to file with S3 configs. Configs are loaded from default location if not set. +- `-s3.configProfile` - profile name for S3 configs. If no set, the value of the environment variable will be loaded (`AWS_PROFILE` or `AWS_DEFAULT_PROFILE`). +- `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. +- `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default. + + ### Groups Each group has the following attributes: @@ -907,6 +927,10 @@ The shortlist of configuration flags is the following: Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880") -httpListenAddr.useProxyProtocol Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + -insert.maxQueueDuration duration + The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s) + -internStringMaxLen int + The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500) -loggerDisableTimestamps Whether to disable writing timestamps in logs -loggerErrorsPerSecondLimit int @@ -923,6 +947,13 @@ The shortlist of configuration flags is the following: Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC") -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit + -maxConcurrentInserts int + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) + -memory.allowedBytes size + Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage + Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) + -memory.allowedPercent float + Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60) -metricsAuthKey string Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -notifier.basicAuth.password array @@ -1089,8 +1120,8 @@ The shortlist of configuration flags is the following: Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'. -replay.disableProgressBar Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode. - -replay.maxDatapointsPerQuery int - Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000) + -replay.maxDatapointsPerQuery /query_range + Max number of data points expected in one request. It affects the max time range for every /query_range request during the replay. The higher the value, the less requests will be made during replay. (default 1000) -replay.ruleRetryAttempts int Defines how many retries to make before giving up on rule if request for it returns an error. (default 5) -replay.rulesDelay duration @@ -1100,13 +1131,19 @@ The shortlist of configuration flags is the following: -replay.timeTo string The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z' -rule array - Path to the file with alert rules. - Supports patterns. Flag can be specified multiple times. + Path to the files with alerting and/or recording rules. + Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules - -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, - absolute path to all .yaml files in root. + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. + + Enterprise version of vmalert supports S3 and GCS paths to rules. + For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules + S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches + all files with prefix rule_ in folder dir. + See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage + Supports an array of values separated by comma or specified via multiple flags. -rule.configCheckInterval duration Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead @@ -1128,6 +1165,18 @@ The shortlist of configuration flags is the following: Whether to validate rules expressions via MetricsQL engine (default true) -rule.validateTemplates Whether to validate annotation and label templates (default true) + -s3.configFilePath string + Path to file with S3 configs. Configs are loaded from default location if not set. + See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.configProfile string + Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.credsFilePath string + Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. + See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.customEndpoint string + Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.forcePathStyle + Prefixing endpoint with bucket name when set false, true by default. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default true) -tls Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set -tlsCertFile string diff --git a/app/vmalert/main.go b/app/vmalert/main.go index f669027e3..b8203d71a 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -28,17 +28,18 @@ import ( ) var ( - rulePath = flagutil.NewArrayString("rule", `Path to the files with alert rules. -Example: gs://bucket/path/to/rules, s3://bucket/path/to/rules, or fs:///path/to/local/rules/dir -If scheme remote storage scheme is omitted, local file system is used. -Local file system supports hierarchical patterns and regexes. -Remote file system supports only matching by prefix, e.g. s3://bucket/dir/rule_ will match all files with prefix -rule_ in folder dir. -This flag can be specified multiple times. + rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules. +Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". -Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars +Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. + +Enterprise version of vmalert supports S3 and GCS paths to rules. +For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules +S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches +all files with prefix rule_ in folder dir. +See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage `) ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e25129f16..2b61499ce 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage). + ## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1) Released at 2023-02-09 diff --git a/docs/enterprise.md b/docs/enterprise.md index 85892de5e..fb63b3b75 100644 --- a/docs/enterprise.md +++ b/docs/enterprise.md @@ -44,7 +44,8 @@ plus the following additional features: - [mTLS for cluster components](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection). - [Kafka integration](https://docs.victoriametrics.com/vmagent.html#kafka-integration). - [Multitenant support in vmalert](https://docs.victoriametrics.com/vmalert.html#multitenancy). -- [Anomaly Detection Service](https://docs.victoriametrics.com/vmanomaly.html) +- [Ability to read alerting and recording rules from object storage](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage). +- [Anomaly Detection Service](https://docs.victoriametrics.com/vmanomaly.html). On top of this, enterprise package of VictoriaMetrics includes the following important Enterprise features: diff --git a/docs/vmalert.md b/docs/vmalert.md index 50b2f10f9..7f2ba38ff 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -103,6 +103,26 @@ groups: [ - ] ``` +### Reading rules from object storage + +[Enterprise version](https://docs.victoriametrics.com/enterprise.html) of `vmalert` may read alerting and recording rules +from object storage: + +- `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket +- `./bin/vmalert -rule=gs://bucket/bir/alert.rules` would read rules from the given path at GCS bucket + +S3 and GCS paths support only matching by prefix, e.g. `s3://bucket/dir/rule_` matches +all files with prefix `rule_` in the folder `dir`. + +The following [command-line flags](#flags) can be used for fine-tuning access to S3 and GCS: + +- `-s3.credsFilePath` - path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. +- `-s3.configFilePath` - path to file with S3 configs. Configs are loaded from default location if not set. +- `-s3.configProfile` - profile name for S3 configs. If no set, the value of the environment variable will be loaded (`AWS_PROFILE` or `AWS_DEFAULT_PROFILE`). +- `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. +- `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default. + + ### Groups Each group has the following attributes: @@ -911,6 +931,10 @@ The shortlist of configuration flags is the following: Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880") -httpListenAddr.useProxyProtocol Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + -insert.maxQueueDuration duration + The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s) + -internStringMaxLen int + The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500) -loggerDisableTimestamps Whether to disable writing timestamps in logs -loggerErrorsPerSecondLimit int @@ -927,6 +951,13 @@ The shortlist of configuration flags is the following: Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC") -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit + -maxConcurrentInserts int + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) + -memory.allowedBytes size + Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage + Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) + -memory.allowedPercent float + Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60) -metricsAuthKey string Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -notifier.basicAuth.password array @@ -1093,8 +1124,8 @@ The shortlist of configuration flags is the following: Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'. -replay.disableProgressBar Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode. - -replay.maxDatapointsPerQuery int - Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000) + -replay.maxDatapointsPerQuery /query_range + Max number of data points expected in one request. It affects the max time range for every /query_range request during the replay. The higher the value, the less requests will be made during replay. (default 1000) -replay.ruleRetryAttempts int Defines how many retries to make before giving up on rule if request for it returns an error. (default 5) -replay.rulesDelay duration @@ -1104,13 +1135,19 @@ The shortlist of configuration flags is the following: -replay.timeTo string The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z' -rule array - Path to the file with alert rules. - Supports patterns. Flag can be specified multiple times. + Path to the files with alerting and/or recording rules. + Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules - -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, - absolute path to all .yaml files in root. + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. + + Enterprise version of vmalert supports S3 and GCS paths to rules. + For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules + S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches + all files with prefix rule_ in folder dir. + See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage + Supports an array of values separated by comma or specified via multiple flags. -rule.configCheckInterval duration Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead @@ -1132,6 +1169,18 @@ The shortlist of configuration flags is the following: Whether to validate rules expressions via MetricsQL engine (default true) -rule.validateTemplates Whether to validate annotation and label templates (default true) + -s3.configFilePath string + Path to file with S3 configs. Configs are loaded from default location if not set. + See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.configProfile string + Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.credsFilePath string + Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. + See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.customEndpoint string + Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -s3.forcePathStyle + Prefixing endpoint with bucket name when set false, true by default. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default true) -tls Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set -tlsCertFile string