feature: [vmalert] support multi-doc yaml parsing

This commit is contained in:
Jiekun 2024-09-07 21:55:45 +08:00
parent 0a40064a6f
commit 9658c134c2
No known key found for this signature in database
GPG key ID: 4674A8E5B0AAF6CE
5 changed files with 97 additions and 8 deletions

View file

@ -1,9 +1,11 @@
package config package config
import ( import (
"bytes"
"crypto/md5" "crypto/md5"
"fmt" "fmt"
"hash/fnv" "hash/fnv"
"maps"
"net/url" "net/url"
"sort" "sort"
"strings" "strings"
@ -48,6 +50,14 @@ type Group struct {
XXX map[string]any `yaml:",inline"` XXX map[string]any `yaml:",inline"`
} }
// parseGroup
// It is used for parseConfig only
type parseGroup struct {
Groups []Group `yaml:"groups"`
// Catches all undefined fields and must be empty after parsing.
XXX map[string]any `yaml:",inline"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface. // UnmarshalYAML implements the yaml.Unmarshaler interface.
func (g *Group) UnmarshalYAML(unmarshal func(any) error) error { func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
type group Group type group Group
@ -298,15 +308,23 @@ func parseConfig(data []byte) ([]Group, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot expand environment vars: %w", err) return nil, fmt.Errorf("cannot expand environment vars: %w", err)
} }
g := struct {
Groups []Group `yaml:"groups"` // https://gettaurus.org/docs/YAMLTutorial/#YAML-Multi-Documents
// Catches all undefined fields and must be empty after parsing. multiDocs := bytes.Split(data, []byte("\n---\n"))
XXX map[string]any `yaml:",inline"` g := parseGroup{
}{} XXX: make(map[string]any),
err = yaml.Unmarshal(data, &g) }
for _, doc := range multiDocs {
var pg parseGroup
err = yaml.Unmarshal(doc, &pg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.Groups = append(g.Groups, pg.Groups...)
maps.Copy(g.XXX, pg.XXX)
}
return g.Groups, checkOverflow(g.XXX, "config") return g.Groups, checkOverflow(g.XXX, "config")
} }

View file

@ -79,6 +79,7 @@ func TestParse_Failure(t *testing.T) {
f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval") f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token") f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields in config: invalid-field-1, invalid-field-2, invalid-field-3")
f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation") f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file") f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined") f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")

View file

@ -0,0 +1,30 @@
groups:
- name: groupTest
rules:
- alert: VMRows
for: 1ms
expr: vm_rows > 0
labels:
label: bar
host: "{{ $labels.instance }}"
annotations:
summary: "{{ $value }}"
invalid-field-1: invalid-value-1
invalid-field-2: invalid-value-2
---
groups:
- name: TestGroup
interval: 2s
concurrency: 2
type: graphite
rules:
- alert: Conns
expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
for: 3m
annotations:
summary: Too high connection number for {{$labels.instance}}
description: "It is {{ $value }} connections for {{$labels.instance}}"
invalid-field-2: invalid-value-2
invalid-field-3: invalid-value-3

View file

@ -0,0 +1,39 @@
---
groups:
- name: groupTest
rules:
- alert: VMRows
for: 1ms
expr: vm_rows > 0
labels:
label: bar
host: "{{ $labels.instance }}"
annotations:
summary: "{{ $value }}"
---
groups:
- name: TestGroup
interval: 2s
concurrency: 2
type: graphite
rules:
- alert: Conns
expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
for: 3m
annotations:
summary: Too high connection number for {{$labels.instance}}
description: "It is {{ $value }} connections for {{$labels.instance}}"
- name: TestGroupPromMixed
interval: 2s
concurrency: 2
type: prometheus
eval_delay: 30s
rules:
- alert: Conns
expr: sum(vm_tcplistener_conns) by (instance) > 1
for: 3m
annotations:
summary: Too high connection number for {{$labels.instance}}
description: "It is {{ $value }} connections for {{$labels.instance}}"

View file

@ -24,6 +24,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flags are set. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details. * FEATURE [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flags are set. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details.
* FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag. * FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support multi-doc yaml parsing. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753).
* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): allow disabling `Bearer` prefix enforcement for authentication header. This is useful for cases when identity token is used instead of access token. * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): allow disabling `Bearer` prefix enforcement for authentication header. This is useful for cases when identity token is used instead of access token.
* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parting `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format. * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parting `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format.
* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Adds new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details. * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Adds new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details.