From 88425bb2854b2f186af4074b24100e0f18f5e5b8 Mon Sep 17 00:00:00 2001 From: laixintao Date: Mon, 22 Aug 2022 19:32:36 +0800 Subject: [PATCH 1/3] vmalert: add $activeAt into template variables. (#3000) vmalert: add `$activeAt` template variable for annotations https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2999 --- app/vmalert/README.md | 19 ++++++------ app/vmalert/notifier/alert.go | 14 +++++---- app/vmalert/notifier/alert_test.go | 47 ++++++++++++++++++++++++++++++ app/vmalert/templates/template.go | 9 ++++++ docs/vmalert.md | 19 ++++++------ 5 files changed, 84 insertions(+), 24 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index b37fe3035..a5738f31c 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -199,15 +199,16 @@ It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in a or execute expressions. The following variables are available in templating: -| Variable | Description | Example | -|------------------------------------|-----------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| -| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}{% endraw %} | -| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | -| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | -| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | -| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | +| Variable | Description | Example | +|------------------------------------|-----------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| +| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}"{% endraw %} | +| $activeAt or .ActiveAt | The alert fire time. It a `time.Time` type so you can use its methods like {% raw %}`{{$activeAt.Unix}}`{% endraw %} | {% raw %}"http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}"{% endraw %} | +| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | +| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | +| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | +| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | +| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | +| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | Additionally, `vmalert` provides some extra templating functions listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go) diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index 88fc59544..a137b48e0 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -74,11 +74,12 @@ func (as AlertState) String() string { // AlertTplData is used to execute templating type AlertTplData struct { - Labels map[string]string - Value float64 - Expr string - AlertID uint64 - GroupID uint64 + Labels map[string]string + Value float64 + Expr string + AlertID uint64 + GroupID uint64 + ActiveAt time.Time } var tplHeaders = []string{ @@ -89,6 +90,7 @@ var tplHeaders = []string{ "{{ $externalURL := .ExternalURL }}", "{{ $alertID := .AlertID }}", "{{ $groupID := .GroupID }}", + "{{ $activeAt := .ActiveAt }}", } // ExecTemplate executes the Alert template for given @@ -96,7 +98,7 @@ var tplHeaders = []string{ // Every alert could have a different datasource, so function // requires a queryFunction as an argument. func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) { - tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr, AlertID: a.ID, GroupID: a.GroupID} + tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr, AlertID: a.ID, GroupID: a.GroupID, ActiveAt: a.ActiveAt} tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q)) if err != nil { return nil, fmt.Errorf("error getting a template: %w", err) diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index 47a4a11bb..d08f60730 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "testing" + "time" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -122,6 +123,52 @@ func TestAlert_ExecTemplate(t *testing.T) { "url": "/api/v1/alert?alertID=42&groupID=24", }, }, + { + name: "ActiveAt time", + alert: &Alert{ + ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC), + }, + annotations: map[string]string{ + "diagram": "![](http://example.com?render={{$activeAt.Unix}}", + }, + expTpl: map[string]string{ + "diagram": "![](http://example.com?render=1660941298", + }, + }, + { + name: "ActiveAt time is nil", + alert: &Alert{}, + annotations: map[string]string{ + "default_time": "{{$activeAt}}", + }, + expTpl: map[string]string{ + "default_time": "0001-01-01 00:00:00 +0000 UTC", + }, + }, + { + name: "ActiveAt custome format", + alert: &Alert{ + ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC), + }, + annotations: map[string]string{ + "fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`, + }, + expTpl: map[string]string{ + "fire_time": "2022/08/19 20:34:58", + }, + }, + { + name: "ActiveAt query range", + alert: &Alert{ + ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC), + }, + annotations: map[string]string{ + "grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`, + }, + expTpl: map[string]string{ + "grafana_url": "vm-grafana.com?from=1660944898&to=1660937698", + }, + }, } qFn := func(q string) ([]datasource.Metric, error) { diff --git a/app/vmalert/templates/template.go b/app/vmalert/templates/template.go index e1f88b5b1..7635f37c4 100644 --- a/app/vmalert/templates/template.go +++ b/app/vmalert/templates/template.go @@ -255,6 +255,15 @@ func templateFuncs() textTpl.FuncMap { return d.Seconds(), nil }, + // same with parseDuration but returns a time.Duration + "parseDurationTime": func(s string) (time.Duration, error) { + d, err := promutils.ParseDuration(s) + if err != nil { + return 0, err + } + return d, nil + }, + /* Numbers */ // humanize converts given number to a human readable format diff --git a/docs/vmalert.md b/docs/vmalert.md index ef2c3769d..46db8bab4 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -203,15 +203,16 @@ It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in a or execute expressions. The following variables are available in templating: -| Variable | Description | Example | -|------------------------------------|-----------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| -| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}{% endraw %} | -| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | -| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | -| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | -| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | +| Variable | Description | Example | +|------------------------------------|-----------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| +| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}"{% endraw %} | +| $activeAt or .ActiveAt | The alert fire time. It a `time.Time` type so you can use its methods like {% raw %}`{{$activeAt.Unix}}`{% endraw %} | {% raw %}"http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}"{% endraw %} | +| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | +| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | +| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | +| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | +| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | +| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | Additionally, `vmalert` provides some extra templating functions listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go) From 8d0f5b9e601f933c2cf6110d4efd89be81d3016c Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 22 Aug 2022 13:49:17 +0200 Subject: [PATCH 2/3] docs: follow-up after 88425bb2854b2f186af4074b24100e0f18f5e5b8 (#3007) Signed-off-by: hagen1778 Signed-off-by: hagen1778 --- app/vmalert/README.md | 20 ++++++++++---------- docs/CHANGELOG.md | 1 + docs/vmalert.md | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index a5738f31c..bd28a4cf5 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -199,16 +199,16 @@ It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in a or execute expressions. The following variables are available in templating: -| Variable | Description | Example | -|------------------------------------|-----------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| -| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}"{% endraw %} | -| $activeAt or .ActiveAt | The alert fire time. It a `time.Time` type so you can use its methods like {% raw %}`{{$activeAt.Unix}}`{% endraw %} | {% raw %}"http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}"{% endraw %} | -| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | -| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | -| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | -| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | +| Variable | Description | Example | +|------------------------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}Number of connections is {{ $value }}{% endraw %} | +| $activeAt or .ActiveAt | The moment of [time](https://pkg.go.dev/time) when alert became active (`pending` or `firing`). | {% raw %}http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}{% endraw %} | +| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}Too high number of connections for {{ .Labels.instance }}{% endraw %} | +| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %} | +| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %} | +| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}{% endraw %} | +| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }}){% endraw %} | +| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}Visit {{ $externalURL }} for more details{% endraw %} | Additionally, `vmalert` provides some extra templating functions listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6159f6254..b4fa60c18 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add a legend in the top right corner for shortcut keys. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2813). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `toTime()` template function in the same way as Prometheus 2.38 [does](https://github.com/prometheus/prometheus/pull/10993). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/template_reference/#numbers). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `$alertID` and `$groupID` template variables. These variables may be used for templating annotations or `-external.alert.source` command-line flag. See the full list of supported variables [here](https://docs.victoriametrics.com/vmalert.html#templating). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `$activeAt` template variable. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2999). See the full list of supported variables [here](https://docs.victoriametrics.com/vmalert.html#templating). Thanks to @laixintao. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): point alert source to [vmalert's UI](https://docs.victoriametrics.com/vmalert.html#web) at `/vmalert/alert?...` instead of JSON handler at `/vmalert/api/v1/alert?...`. This improves user experience. The old behavior can be achieved by setting {% raw %}`-external.alert.source=vmalert/api/v1/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}`{% endraw %} command-line flag. * BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). diff --git a/docs/vmalert.md b/docs/vmalert.md index 46db8bab4..c008e07b7 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -203,16 +203,16 @@ It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in a or execute expressions. The following variables are available in templating: -| Variable | Description | Example | -|------------------------------------|-----------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| -| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}"Number of connections is {{ $value }}"{% endraw %} | -| $activeAt or .ActiveAt | The alert fire time. It a `time.Time` type so you can use its methods like {% raw %}`{{$activeAt.Unix}}`{% endraw %} | {% raw %}"http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}"{% endraw %} | -| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}"Too high number of connections for {{ .Labels.instance }}"{% endraw %} | -| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}"Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}"{% endraw %} | -| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}"/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}"{% endraw %} | -| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}"Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})"{% endraw %} | -| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}"Visit {{ $externalURL }} for more details"{% endraw %} | +| Variable | Description | Example | +|------------------------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| $value or .Value | The current alert's value. Avoid using value in labels, it may cause unexpected issues. | {% raw %}Number of connections is {{ $value }}{% endraw %} | +| $activeAt or .ActiveAt | The moment of [time](https://pkg.go.dev/time) when alert became active (`pending` or `firing`). | {% raw %}http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}{% endraw %} | +| $labels or .Labels | The list of labels of the current alert. Use as ".Labels.". | {% raw %}Too high number of connections for {{ .Labels.instance }}{% endraw %} | +| $alertID or .AlertID | The current alert's ID generated by vmalert. | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %} | +| $groupID or .GroupID | The current alert's group ID generated by vmalert. | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %} | +| $expr or .Expr | Alert's expression. Can be used for generating links to Grafana or other systems. | {% raw %}/api/v1/query?query={{ $expr|quotesEscape|queryEscape }}{% endraw %} | +| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | {% raw %}Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }}){% endraw %} | +| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}Visit {{ $externalURL }} for more details{% endraw %} | Additionally, `vmalert` provides some extra templating functions listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go) From c011fb0f3082e210283a5a213e6697b175ed6232 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 24 Aug 2022 01:16:26 +0300 Subject: [PATCH 3/3] docs/vmagent.md: fix alerting query when scraped samples are dropped because of exceeded series limit This is a follow-up after 7d26414b2e98b0a949df207c1100293499c3db3b --- app/vmagent/README.md | 2 +- docs/vmagent.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index efff2fb4d..dd746a87d 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -604,7 +604,7 @@ These metrics are automatically sent to the configured `-remoteWrite.url` alongs These metrics allow building the following alerting rules: - `scrape_series_current / scrape_series_limit > 0.9` - alerts when the number of series exposed by the target reaches 90% of the limit. -- `rate(scrape_series_samples_dropped_total) > 0` - alerts when some samples are dropped because the series limit on a particular target is reached. +- `sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0` - alerts when some samples are dropped because the series limit on a particular target is reached. By default `vmagent` doesn't limit the number of time series written to remote storage systems specified at `-remoteWrite.url`. The limit can be enforced by setting the following command-line flags: diff --git a/docs/vmagent.md b/docs/vmagent.md index ad20d4cd2..7c708fb93 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -608,7 +608,7 @@ These metrics are automatically sent to the configured `-remoteWrite.url` alongs These metrics allow building the following alerting rules: - `scrape_series_current / scrape_series_limit > 0.9` - alerts when the number of series exposed by the target reaches 90% of the limit. -- `rate(scrape_series_samples_dropped_total) > 0` - alerts when some samples are dropped because the series limit on a particular target is reached. +- `sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0` - alerts when some samples are dropped because the series limit on a particular target is reached. By default `vmagent` doesn't limit the number of time series written to remote storage systems specified at `-remoteWrite.url`. The limit can be enforced by setting the following command-line flags: