VictoriaMetrics/deployment/docker/rules/alerts-vmalert.yml

# File contains default list of alerts for vmalert service.
# The alerts below are just recommendations and may require some updates
# and threshold calibration according to every specific setup.
groups:
  # Alerts group for vmalert assumes that Grafana dashboard
  # https://grafana.com/grafana/dashboards/14950/ is installed.
  # Pls update the `dashboard` annotation according to your setup.
  - name: vmalert
    interval: 30s
    rules:
      - alert: ConfigurationReloadFailure
        expr: vmalert_config_last_reload_successful != 1
        labels:
          severity: warning
        annotations:
          summary: "Configuration reload failed for vmalert instance {{ $labels.instance }}"
          description: "Configuration hot-reload failed for vmalert on instance {{ $labels.instance }}.
            Check vmalert's logs for detailed error message."

      - alert: AlertingRulesError
        expr: sum(increase(vmalert_alerting_rules_errors_total[5m])) without(alertname, id) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
          summary: "Alerting rules are failing for vmalert instance {{ $labels.instance }}"
          description: "Alerting rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
            Check vmalert's logs for detailed error message."

      - alert: RecordingRulesError
        expr: sum(increase(vmalert_recording_rules_errors_total[5m])) without(recording, id) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
          summary: "Recording rules are failing for vmalert instance {{ $labels.instance }}"
          description: "Recording rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
            Check vmalert's logs for detailed error message."

      - alert: RecordingRulesNoData
        expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1
        for: 30m
        labels:
          severity: info
        annotations:
          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=33&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
          summary: "Recording rule {{ $labels.recording }} ({{ $labels.group }}) produces no data"
          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\ in file \"{{ $labels.file }}\" 
            produces 0 samples over the last 30min. It might be caused by a misconfiguration 
            or incorrect query expression."

      - alert: TooManyMissedIterations
        expr: increase(vmalert_iteration_missed_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "vmalert instance {{ $labels.instance }} is missing rules evaluations"
          description: "vmalert instance {{ $labels.instance }} is missing rules evaluations for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
            The group evaluation time takes longer than the configured evaluation interval. This may result in missed 
            alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency of
            group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/vmalert/#groups. 
            If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries."

      - alert: RemoteWriteErrors
        expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "vmalert instance {{ $labels.instance }} is failing to push metrics to remote write URL"
          description: "vmalert instance {{ $labels.instance }} is failing to push metrics generated via alerting 
            or recording rules to the configured remote write URL. Check vmalert's logs for detailed error message."

      - alert: RemoteWriteDroppingData
        expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "vmalert instance {{ $labels.instance }} is dropping data sent to remote write URL"
          description: "vmalert instance {{ $labels.instance }} is failing to send results of alerting or recording rules 
            to the configured remote write URL. This may result into gaps in recording rules or alerts state.
            Check vmalert's logs for detailed error message."

      - alert: AlertmanagerErrors
        expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
          description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
            Check vmalert's logs for detailed error message."