mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
877940a131
vmalert: add experimental feature of storing Rule's evaluation state The new feature keeps last 20 state changes of each Rule in memory. The state are available for view on the Rule's view page. The page can be opened by clicking on `Details` link next to Rule's name on the `/groups` page. States change suppose to help in investigating cases when Rule doesn't generate alerts or records. Signed-off-by: hagen1778 <roman@victoriametrics.com>
482 lines
18 KiB
Text
482 lines
18 KiB
Text
{% package main %}
|
|
|
|
{% import (
|
|
"time"
|
|
"sort"
|
|
"net/http"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
|
) %}
|
|
|
|
|
|
{% func Welcome(r *http.Request) %}
|
|
{%= tpl.Header(r, navItems, "vmalert") %}
|
|
<p>
|
|
API:<br>
|
|
{% for _, p := range apiLinks %}
|
|
{%code
|
|
p, doc := p[0], p[1]
|
|
%}
|
|
<a href="{%s p %}">{%s p %}</a> - {%s doc %}<br/>
|
|
{% endfor %}
|
|
</p>
|
|
{%= tpl.Footer(r) %}
|
|
{% endfunc %}
|
|
|
|
{% func ListGroups(r *http.Request, groups []APIGroup) %}
|
|
{%code prefix := utils.Prefix(r.URL.Path) %}
|
|
{%= tpl.Header(r, navItems, "Groups") %}
|
|
{% if len(groups) > 0 %}
|
|
{%code
|
|
rOk := make(map[string]int)
|
|
rNotOk := make(map[string]int)
|
|
for _, g := range groups {
|
|
for _, r := range g.Rules {
|
|
if r.LastError != "" {
|
|
rNotOk[g.Name]++
|
|
} else {
|
|
rOk[g.Name]++
|
|
}
|
|
}
|
|
}
|
|
%}
|
|
<a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
|
|
<a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
|
|
{% for _, g := range groups %}
|
|
<div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}" data-bs-target="rules-{%s g.ID %}">
|
|
<span class="anchor" id="group-{%s g.ID %}"></span>
|
|
<a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
|
|
{% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
|
|
<span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
|
|
<p class="fs-6 fw-lighter">{%s g.File %}</p>
|
|
{% if len(g.Params) > 0 %}
|
|
<div class="fs-6 fw-lighter">Extra params
|
|
{% for _, param := range g.Params %}
|
|
<span class="float-left badge bg-primary">{%s param %}</span>
|
|
{% endfor %}
|
|
</div>
|
|
{% endif %}
|
|
{% if len(g.Headers) > 0 %}
|
|
<div class="fs-6 fw-lighter">Extra headers
|
|
{% for _, header := range g.Headers %}
|
|
<span class="float-left badge bg-primary">{%s header %}</span>
|
|
{% endfor %}
|
|
</div>
|
|
{% endif %}
|
|
</div>
|
|
<div class="collapse" id="rules-{%s g.ID %}">
|
|
<table class="table table-striped table-hover table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col" style="width: 60%">Rule</th>
|
|
<th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
|
|
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for _, r := range g.Rules %}
|
|
<tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
|
|
<td>
|
|
<div class="row">
|
|
<div class="col-12 mb-2">
|
|
{% if r.Type == "alerting" %}
|
|
<b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
|
|
{% else %}
|
|
<b>record:</b> {%s r.Name %}
|
|
{% endif %}
|
|
| <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
|
|
</div>
|
|
<div class="col-12">
|
|
<code><pre>{%s r.Query %}</pre></code>
|
|
</div>
|
|
<div class="col-12 mb-2">
|
|
{% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
|
|
{% for k, v := range r.Labels %}
|
|
<span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
|
|
{% endfor %}
|
|
</div>
|
|
{% if r.LastError != "" %}
|
|
<div class="col-12">
|
|
<b>Error:</b>
|
|
<div class="error-cell">
|
|
{%s r.LastError %}
|
|
</div>
|
|
</div>
|
|
{% endif %}
|
|
</div>
|
|
</td>
|
|
<td class="text-center">{%d r.LastSamples %}</td>
|
|
<td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
{% endfor %}
|
|
|
|
{% else %}
|
|
<div>
|
|
<p>No groups...</p>
|
|
</div>
|
|
{% endif %}
|
|
|
|
{%= tpl.Footer(r) %}
|
|
|
|
{% endfunc %}
|
|
|
|
|
|
{% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %}
|
|
{%code prefix := utils.Prefix(r.URL.Path) %}
|
|
{%= tpl.Header(r, navItems, "Alerts") %}
|
|
{% if len(groupAlerts) > 0 %}
|
|
<a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
|
|
<a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
|
|
{% for _, ga := range groupAlerts %}
|
|
{%code g := ga.Group %}
|
|
<div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
|
|
<span class="anchor" id="group-{%s g.ID %}"></span>
|
|
<a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
|
|
<span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
|
|
<br>
|
|
<p class="fs-6 fw-lighter">{%s g.File %}</p>
|
|
</div>
|
|
{%code
|
|
var keys []string
|
|
alertsByRule := make(map[string][]*APIAlert)
|
|
for _, alert := range ga.Alerts {
|
|
if len(alertsByRule[alert.RuleID]) < 1 {
|
|
keys = append(keys, alert.RuleID)
|
|
}
|
|
alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
|
|
}
|
|
sort.Strings(keys)
|
|
%}
|
|
<div class="collapse" id="rules-{%s g.ID %}">
|
|
{% for _, ruleID := range keys %}
|
|
{%code
|
|
defaultAR := alertsByRule[ruleID][0]
|
|
var labelKeys []string
|
|
for k := range defaultAR.Labels {
|
|
labelKeys = append(labelKeys, k)
|
|
}
|
|
sort.Strings(labelKeys)
|
|
%}
|
|
<br>
|
|
<b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
|
|
| <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
|
|
<br>
|
|
<b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
|
|
<table class="table table-striped table-hover table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col">Labels</th>
|
|
<th scope="col">State</th>
|
|
<th scope="col">Active at</th>
|
|
<th scope="col">Value</th>
|
|
<th scope="col">Link</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for _, ar := range alertsByRule[ruleID] %}
|
|
<tr>
|
|
<td>
|
|
{% for _, k := range labelKeys %}
|
|
<span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
|
|
{% endfor %}
|
|
</td>
|
|
<td>{%= badgeState(ar.State) %}</td>
|
|
<td>
|
|
{%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
|
|
{% if ar.Restored %}{%= badgeRestored() %}{% endif %}
|
|
</td>
|
|
<td>{%s ar.Value %}</td>
|
|
<td>
|
|
<a href="{%s prefix+ar.WebLink() %}">Details</a>
|
|
</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
{% endfor %}
|
|
</div>
|
|
<br>
|
|
{% endfor %}
|
|
|
|
{% else %}
|
|
<div>
|
|
<p>No active alerts...</p>
|
|
</div>
|
|
{% endif %}
|
|
|
|
{%= tpl.Footer(r) %}
|
|
|
|
{% endfunc %}
|
|
|
|
{% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %}
|
|
{%= tpl.Header(r, navItems, "Notifiers") %}
|
|
{% if len(targets) > 0 %}
|
|
<a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
|
|
<a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
|
|
|
|
{%code
|
|
var keys []string
|
|
for key := range targets {
|
|
keys = append(keys, string(key))
|
|
}
|
|
sort.Strings(keys)
|
|
%}
|
|
|
|
{% for i := range keys %}
|
|
{%code typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
|
|
count := len(ns)
|
|
%}
|
|
<div class="group-heading data-bs-target="rules-{%s typeK %}">
|
|
<span class="anchor" id="notifiers-{%s typeK %}"></span>
|
|
<a href="#notifiers-{%s typeK %}">{%s typeK %} ({%d count %})</a>
|
|
</div>
|
|
<div class="collapse show" id="notifiers-{%s typeK %}">
|
|
<table class="table table-striped table-hover table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col">Labels</th>
|
|
<th scope="col">Address</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for _, n := range ns %}
|
|
<tr>
|
|
<td>
|
|
{% for _, l := range n.Labels %}
|
|
<span class="ms-1 badge bg-primary">{%s l.Name %}={%s l.Value %}</span>
|
|
{% endfor %}
|
|
</td>
|
|
<td>{%s n.Notifier.Addr() %}</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
{% endfor %}
|
|
|
|
{% else %}
|
|
<div>
|
|
<p>No targets...</p>
|
|
</div>
|
|
{% endif %}
|
|
|
|
{%= tpl.Footer(r) %}
|
|
|
|
{% endfunc %}
|
|
|
|
{% func Alert(r *http.Request, alert *APIAlert) %}
|
|
{%code prefix := utils.Prefix(r.URL.Path) %}
|
|
{%= tpl.Header(r, navItems, "") %}
|
|
{%code
|
|
var labelKeys []string
|
|
for k := range alert.Labels {
|
|
labelKeys = append(labelKeys, k)
|
|
}
|
|
sort.Strings(labelKeys)
|
|
|
|
var annotationKeys []string
|
|
for k := range alert.Annotations {
|
|
annotationKeys = append(annotationKeys, k)
|
|
}
|
|
sort.Strings(annotationKeys)
|
|
%}
|
|
<div class="display-6 pb-3 mb-3">Alert: {%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Active at
|
|
</div>
|
|
<div class="col">
|
|
{%s alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Expr
|
|
</div>
|
|
<div class="col">
|
|
<code><pre>{%s alert.Expression %}</pre></code>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Labels
|
|
</div>
|
|
<div class="col">
|
|
{% for _, k := range labelKeys %}
|
|
<span class="m-1 badge bg-primary">{%s k %}={%s alert.Labels[k] %}</span>
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Annotations
|
|
</div>
|
|
<div class="col">
|
|
{% for _, k := range annotationKeys %}
|
|
<b>{%s k %}:</b><br>
|
|
<p>{%s alert.Annotations[k] %}</p>
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Group
|
|
</div>
|
|
<div class="col">
|
|
<a target="_blank" href="{%s prefix %}groups#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Source link
|
|
</div>
|
|
<div class="col">
|
|
<a target="_blank" href="{%s alert.SourceLink %}">Link</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{%= tpl.Footer(r) %}
|
|
|
|
{% endfunc %}
|
|
|
|
|
|
{% func RuleDetails(r *http.Request, rule APIRule) %}
|
|
{%code prefix := utils.Prefix(r.URL.Path) %}
|
|
{%= tpl.Header(r, navItems, "") %}
|
|
{%code
|
|
var labelKeys []string
|
|
for k := range rule.Labels {
|
|
labelKeys = append(labelKeys, k)
|
|
}
|
|
sort.Strings(labelKeys)
|
|
|
|
var annotationKeys []string
|
|
for k := range rule.Annotations {
|
|
annotationKeys = append(annotationKeys, k)
|
|
}
|
|
sort.Strings(annotationKeys)
|
|
%}
|
|
<div class="display-6 pb-3 mb-3">Rule: {%s rule.Name %}<span class="ms-2 badge {% if rule.Health!="ok" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s rule.Health %}</span></div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Expr
|
|
</div>
|
|
<div class="col">
|
|
<code><pre>{%s rule.Query %}</pre></code>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
For
|
|
</div>
|
|
<div class="col">
|
|
{%v rule.Duration %} seconds
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Labels
|
|
</div>
|
|
<div class="col">
|
|
{% for _, k := range labelKeys %}
|
|
<span class="m-1 badge bg-primary">{%s k %}={%s rule.Labels[k] %}</span>
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Annotations
|
|
</div>
|
|
<div class="col">
|
|
{% for _, k := range annotationKeys %}
|
|
<b>{%s k %}:</b><br>
|
|
<p>{%s rule.Annotations[k] %}</p>
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container border-bottom p-2">
|
|
<div class="row">
|
|
<div class="col-2">
|
|
Group
|
|
</div>
|
|
<div class="col">
|
|
<a target="_blank" href="{%s prefix %}groups#group-{%s rule.GroupID %}">{%s rule.GroupID %}</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<br>
|
|
<div class="display-6 pb-3">Last {%d len(rule.Updates) %} updates</span>:</div>
|
|
<table class="table table-striped table-hover table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col" style="width: 20%" title="The time when event was created">Updated at</th>
|
|
<th scope="col" style="width: 20%" class="text-center" title="How many samples were returned">Samples</th>
|
|
<th scope="col" style="width: 20%" class="text-center" title="How many seconds request took">Duration</th>
|
|
<th scope="col" style="width: 20%" class="text-center" title="Time used for rule execution">Executed at</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
|
|
{% for _, u := range rule.Updates %}
|
|
<tr{% if u.err != nil %} class="alert-danger"{% endif %}>
|
|
<td>
|
|
<span class="badge bg-primary rounded-pill me-3" title="Updated at">{%s u.time.Format(time.RFC3339) %}</span>
|
|
</td>
|
|
<td class="text-center">{%d u.samples %}</td>
|
|
<td class="text-center">{%f.3 u.duration.Seconds() %}s</td>
|
|
<td class="text-center">{%s u.at.Format(time.RFC3339) %}</td>
|
|
</tr>
|
|
</li>
|
|
{% if u.err != nil %}
|
|
<tr{% if u.err != nil %} class="alert-danger"{% endif %}>
|
|
<td colspan="4">
|
|
<span class="alert-danger">{%v u.err %}</span>
|
|
</td>
|
|
</tr>
|
|
{% endif %}
|
|
{% endfor %}
|
|
|
|
{%= tpl.Footer(r) %}
|
|
{% endfunc %}
|
|
|
|
|
|
|
|
{% func badgeState(state string) %}
|
|
{%code
|
|
badgeClass := "bg-warning text-dark"
|
|
if state == "firing" {
|
|
badgeClass = "bg-danger"
|
|
}
|
|
%}
|
|
<span class="badge {%s badgeClass %}">{%s state %}</span>
|
|
{% endfunc %}
|
|
|
|
{% func badgeRestored() %}
|
|
<span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
|
|
{% endfunc %}
|