From 9658c134c24167b375a626214f2a27d1c506d0ed Mon Sep 17 00:00:00 2001 From: Jiekun Date: Sat, 7 Sep 2024 21:55:45 +0800 Subject: [PATCH] feature: [vmalert] support multi-doc yaml parsing --- app/vmalert/config/config.go | 34 ++++++++++++---- app/vmalert/config/config_test.go | 1 + .../testdata/rules/rules-multi-doc-bad.rules | 30 ++++++++++++++ .../testdata/rules/rules-multi-doc-good.rules | 39 +++++++++++++++++++ docs/changelog/CHANGELOG.md | 1 + 5 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules create mode 100644 app/vmalert/config/testdata/rules/rules-multi-doc-good.rules diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 0e7ba7d6a..27b37628a 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -1,9 +1,11 @@ package config import ( + "bytes" "crypto/md5" "fmt" "hash/fnv" + "maps" "net/url" "sort" "strings" @@ -48,6 +50,14 @@ type Group struct { XXX map[string]any `yaml:",inline"` } +// parseGroup +// It is used for parseConfig only +type parseGroup struct { + Groups []Group `yaml:"groups"` + // Catches all undefined fields and must be empty after parsing. + XXX map[string]any `yaml:",inline"` +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (g *Group) UnmarshalYAML(unmarshal func(any) error) error { type group Group @@ -298,15 +308,23 @@ func parseConfig(data []byte) ([]Group, error) { if err != nil { return nil, fmt.Errorf("cannot expand environment vars: %w", err) } - g := struct { - Groups []Group `yaml:"groups"` - // Catches all undefined fields and must be empty after parsing. - XXX map[string]any `yaml:",inline"` - }{} - err = yaml.Unmarshal(data, &g) - if err != nil { - return nil, err + + // https://gettaurus.org/docs/YAMLTutorial/#YAML-Multi-Documents + multiDocs := bytes.Split(data, []byte("\n---\n")) + g := parseGroup{ + XXX: make(map[string]any), } + + for _, doc := range multiDocs { + var pg parseGroup + err = yaml.Unmarshal(doc, &pg) + if err != nil { + return nil, err + } + g.Groups = append(g.Groups, pg.Groups...) + maps.Copy(g.XXX, pg.XXX) + } + return g.Groups, checkOverflow(g.XXX, "config") } diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 68184002c..2a78fca63 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -79,6 +79,7 @@ func TestParse_Failure(t *testing.T) { f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval") f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token") + f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields in config: invalid-field-1, invalid-field-2, invalid-field-3") f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation") f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file") f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined") diff --git a/app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules b/app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules new file mode 100644 index 000000000..de81717bd --- /dev/null +++ b/app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules @@ -0,0 +1,30 @@ +groups: + - name: groupTest + rules: + - alert: VMRows + for: 1ms + expr: vm_rows > 0 + labels: + label: bar + host: "{{ $labels.instance }}" + annotations: + summary: "{{ $value }}" +invalid-field-1: invalid-value-1 +invalid-field-2: invalid-value-2 +--- +groups: + - name: TestGroup + interval: 2s + concurrency: 2 + type: graphite + rules: + - alert: Conns + expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500) + for: 3m + + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" +invalid-field-2: invalid-value-2 +invalid-field-3: invalid-value-3 + diff --git a/app/vmalert/config/testdata/rules/rules-multi-doc-good.rules b/app/vmalert/config/testdata/rules/rules-multi-doc-good.rules new file mode 100644 index 000000000..4b380a95d --- /dev/null +++ b/app/vmalert/config/testdata/rules/rules-multi-doc-good.rules @@ -0,0 +1,39 @@ + +--- +groups: + - name: groupTest + rules: + - alert: VMRows + for: 1ms + expr: vm_rows > 0 + labels: + label: bar + host: "{{ $labels.instance }}" + annotations: + summary: "{{ $value }}" +--- +groups: + - name: TestGroup + interval: 2s + concurrency: 2 + type: graphite + rules: + - alert: Conns + expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500) + for: 3m + + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" + - name: TestGroupPromMixed + interval: 2s + concurrency: 2 + type: prometheus + eval_delay: 30s + rules: + - alert: Conns + expr: sum(vm_tcplistener_conns) by (instance) > 1 + for: 3m + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 2df6d30fa..00db973f3 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -24,6 +24,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flags are set. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details. * FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag. +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support multi-doc yaml parsing. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): allow disabling `Bearer` prefix enforcement for authentication header. This is useful for cases when identity token is used instead of access token. * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parting `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format. * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Adds new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details.