From 5e9f3777bfcab03a9f525cb8d793ef33eca63f30 Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <hagen1778@gmail.com>
Date: Wed, 9 Jun 2021 12:15:36 +0300
Subject: [PATCH] alerts: add new alert `LabelsLimitExceededOnIngestion`
 (#1359)

---
 deployment/docker/alerts.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml
index d99afdaa4..0c900ef27 100644
--- a/deployment/docker/alerts.yml
+++ b/deployment/docker/alerts.yml
@@ -177,6 +177,18 @@ groups:
           description: "Exhausting OS file descriptors limit can cause severe degradation of the process.
           Consider to increase the limit as fast as possible."
 
+      - alert: LabelsLimitExceededOnIngestion
+        expr: sum(increase(vm_metrics_with_dropped_labels_total[5m])) by (instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=74&var-instance={{ $labels.instance }}"
+          summary: "Metrics ingested in ({{ $labels.instance }}) are exceeding labels limit"
+          description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
+           This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
+           correctly or that clients which send these metrics aren't misbehaving."
+
   # Alerts group for vmagent assumes that Grafana dashboard
   # https://grafana.com/grafana/dashboards/12683 is installed.
   # Pls update the `dashboard` annotation according to your setup.