From d3608be313b49cc2217b07325a155b810119aada Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <roman@victoriametrics.com>
Date: Fri, 17 Mar 2023 09:46:51 +0100
Subject: [PATCH] alerts: add `TooManyTSIDMisses` alerting rule (#3959)

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502#issuecomment-1358374954

Signed-off-by: hagen1778 <roman@victoriametrics.com>
---
 deployment/docker/alerts-health.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/alerts-health.yml
index 4668718e6..8686c1063 100644
--- a/deployment/docker/alerts-health.yml
+++ b/deployment/docker/alerts-health.yml
@@ -62,3 +62,14 @@ groups:
           summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
           description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
          Worth to check logs for specific error messages."
+
+      - alert: TooManyTSIDMisses
+        expr: sum(rate(vm_missing_tsids_for_metric_id_total[5m])) by (job, instance) > 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Too many TSID misses for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          description: "The rate of TSID misses during query lookups is too high for \"{{ $labels.job }}\" ({{ $labels.instance }}).\n
+         Make sure you're running VictoriaMetrics of v1.85.3 or higher.\n
+         Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502"