diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml index 3913367323..127c009914 100644 --- a/deployment/docker/alerts.yml +++ b/deployment/docker/alerts.yml @@ -302,3 +302,23 @@ groups: is saturated by more than 90% and vmagent won't be able to keep up.\n This usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage." + + - alert: SeriesLimitHourReached + expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series) > 0.9 + labels: + severity: critical + annotations: + dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=88&var-instance={{ $labels.instance }}" + summary: "Instance {{ $labels.instance }} reached 90% of the limit" + description: "Max series limit set via -remoteWrite.maxHourlySeries flag is close to reaching the max value + then samples for new time series will be dropped instead of sending them to remote storage systems." + + - alert: SeriesLimitDayReached + expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series) > 0.9 + labels: + severity: critical + annotations: + dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=90&var-instance={{ $labels.instance }}" + summary: "Instance {{ $labels.instance }} reached 90% of the limit" + description: "Max series limit set via -remoteWrite.maxDailySeries flag is close to reaching the max value + then samples for new time series will be dropped instead of sending them to remote storage systems."