2020-04-27 21:19:27 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2022-05-30 23:55:28 +00:00
|
|
|
"embed"
|
2020-04-27 21:19:27 +00:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"sort"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
|
2022-02-02 12:11:41 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
2023-10-13 11:54:33 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
|
2021-09-21 11:41:01 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
|
2024-01-21 19:58:26 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
2020-04-27 21:19:27 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
2020-05-10 16:58:17 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
2020-04-27 21:19:27 +00:00
|
|
|
)
|
|
|
|
|
2024-01-21 19:58:26 +00:00
|
|
|
var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
|
|
|
|
|
2021-09-21 11:41:01 +00:00
|
|
|
var (
|
|
|
|
apiLinks = [][2]string{
|
2022-07-08 08:26:13 +00:00
|
|
|
// api links are relative since they can be used by external clients,
|
|
|
|
// such as Grafana, and proxied via vmselect.
|
2022-07-06 08:46:01 +00:00
|
|
|
{"api/v1/rules", "list all loaded groups and rules"},
|
|
|
|
{"api/v1/alerts", "list all active alerts"},
|
2022-07-08 08:26:13 +00:00
|
|
|
{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
|
2022-12-09 19:43:54 +00:00
|
|
|
}
|
|
|
|
systemLinks = [][2]string{
|
2022-07-06 08:46:01 +00:00
|
|
|
{"/flags", "command-line flags"},
|
|
|
|
{"/metrics", "list of application metrics"},
|
|
|
|
{"/-/reload", "reload configuration"},
|
2021-09-21 11:41:01 +00:00
|
|
|
}
|
|
|
|
navItems = []tpl.NavItem{
|
2022-07-11 16:52:22 +00:00
|
|
|
{Name: "vmalert", Url: "."},
|
2022-07-06 08:46:01 +00:00
|
|
|
{Name: "Groups", Url: "groups"},
|
|
|
|
{Name: "Alerts", Url: "alerts"},
|
|
|
|
{Name: "Notifiers", Url: "notifiers"},
|
2021-09-21 11:41:01 +00:00
|
|
|
{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
|
|
|
|
}
|
2022-12-09 19:43:54 +00:00
|
|
|
)
|
2021-09-21 11:41:01 +00:00
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
type requestHandler struct {
|
2020-05-10 16:58:17 +00:00
|
|
|
m *manager
|
2020-04-27 21:19:27 +00:00
|
|
|
}
|
|
|
|
|
2022-07-06 08:46:01 +00:00
|
|
|
var (
|
|
|
|
//go:embed static
|
|
|
|
staticFiles embed.FS
|
|
|
|
staticHandler = http.FileServer(http.FS(staticFiles))
|
|
|
|
staticServer = http.StripPrefix("/vmalert", staticHandler)
|
|
|
|
)
|
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
2022-07-06 08:46:01 +00:00
|
|
|
if strings.HasPrefix(r.URL.Path, "/vmalert/static") {
|
|
|
|
staticServer.ServeHTTP(w, r)
|
|
|
|
return true
|
2022-02-08 19:03:42 +00:00
|
|
|
}
|
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
switch r.URL.Path {
|
2022-07-11 16:52:22 +00:00
|
|
|
case "/", "/vmalert", "/vmalert/":
|
2023-02-23 02:58:44 +00:00
|
|
|
if r.Method != http.MethodGet {
|
2022-07-11 16:52:22 +00:00
|
|
|
httpserver.Errorf(w, r, "path %q supports only GET method", r.URL.Path)
|
2021-04-02 19:54:06 +00:00
|
|
|
return false
|
|
|
|
}
|
2022-07-06 08:46:01 +00:00
|
|
|
WriteWelcome(w, r)
|
|
|
|
return true
|
|
|
|
case "/vmalert/alerts":
|
|
|
|
WriteListAlerts(w, r, rh.groupAlerts())
|
2020-04-27 21:19:27 +00:00
|
|
|
return true
|
2022-07-08 08:26:13 +00:00
|
|
|
case "/vmalert/alert":
|
|
|
|
alert, err := rh.getAlert(r)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
WriteAlert(w, r, alert)
|
|
|
|
return true
|
2022-09-14 12:04:24 +00:00
|
|
|
case "/vmalert/rule":
|
|
|
|
rule, err := rh.getRule(r)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
WriteRuleDetails(w, r, rule)
|
|
|
|
return true
|
2022-07-06 08:46:01 +00:00
|
|
|
case "/vmalert/groups":
|
|
|
|
WriteListGroups(w, r, rh.groups())
|
2021-09-07 19:39:22 +00:00
|
|
|
return true
|
2022-07-06 08:46:01 +00:00
|
|
|
case "/vmalert/notifiers":
|
|
|
|
WriteListTargets(w, r, notifier.GetTargets())
|
2021-09-07 19:39:22 +00:00
|
|
|
return true
|
2022-07-06 08:46:01 +00:00
|
|
|
|
|
|
|
// special cases for Grafana requests,
|
|
|
|
// served without `vmalert` prefix:
|
|
|
|
case "/rules":
|
|
|
|
// Grafana makes an extra request to `/rules`
|
|
|
|
// handler in addition to `/api/v1/rules` calls in alerts UI,
|
|
|
|
WriteListGroups(w, r, rh.groups())
|
2022-02-02 12:11:41 +00:00
|
|
|
return true
|
2022-07-06 08:46:01 +00:00
|
|
|
|
|
|
|
case "/vmalert/api/v1/rules", "/api/v1/rules":
|
|
|
|
// path used by Grafana for ng alerting
|
2020-07-20 11:00:33 +00:00
|
|
|
data, err := rh.listGroups()
|
|
|
|
if err != nil {
|
2021-07-07 09:59:03 +00:00
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
2020-07-20 11:00:33 +00:00
|
|
|
return true
|
|
|
|
}
|
2021-11-09 16:03:50 +00:00
|
|
|
w.Header().Set("Content-Type", "application/json")
|
2020-07-20 11:00:33 +00:00
|
|
|
w.Write(data)
|
2020-06-01 10:46:37 +00:00
|
|
|
return true
|
2022-07-06 08:46:01 +00:00
|
|
|
case "/vmalert/api/v1/alerts", "/api/v1/alerts":
|
|
|
|
// path used by Grafana for ng alerting
|
2020-07-20 11:00:33 +00:00
|
|
|
data, err := rh.listAlerts()
|
|
|
|
if err != nil {
|
2021-07-07 09:59:03 +00:00
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
2020-07-20 11:00:33 +00:00
|
|
|
return true
|
|
|
|
}
|
2021-11-09 16:03:50 +00:00
|
|
|
w.Header().Set("Content-Type", "application/json")
|
2020-07-20 11:00:33 +00:00
|
|
|
w.Write(data)
|
2020-04-27 21:19:27 +00:00
|
|
|
return true
|
2022-07-08 08:26:13 +00:00
|
|
|
case "/vmalert/api/v1/alert", "/api/v1/alert":
|
|
|
|
alert, err := rh.getAlert(r)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
data, err := json.Marshal(alert)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
w.Write(data)
|
|
|
|
return true
|
2023-12-04 15:40:33 +00:00
|
|
|
case "/vmalert/api/v1/rule", "/api/v1/rule":
|
|
|
|
rule, err := rh.getRule(r)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "%s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
rwu := apiRuleWithUpdates{
|
|
|
|
apiRule: rule,
|
|
|
|
StateUpdates: rule.Updates,
|
|
|
|
}
|
|
|
|
data, err := json.Marshal(rwu)
|
|
|
|
if err != nil {
|
|
|
|
httpserver.Errorf(w, r, "failed to marshal rule: %s", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
w.Write(data)
|
|
|
|
return true
|
2020-05-09 09:32:12 +00:00
|
|
|
case "/-/reload":
|
2024-01-21 19:58:26 +00:00
|
|
|
if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
|
|
|
|
return true
|
|
|
|
}
|
2020-05-09 09:32:12 +00:00
|
|
|
logger.Infof("api config reload was called, sending sighup")
|
|
|
|
procutil.SelfSIGHUP()
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
return true
|
2022-05-30 23:55:28 +00:00
|
|
|
|
2022-07-06 08:46:01 +00:00
|
|
|
default:
|
2023-07-31 14:51:41 +00:00
|
|
|
httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
|
|
|
|
return false
|
2020-04-27 21:19:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
func (rh *requestHandler) getRule(r *http.Request) (apiRule, error) {
|
|
|
|
groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 64)
|
2022-09-14 12:04:24 +00:00
|
|
|
if err != nil {
|
2023-10-25 19:24:01 +00:00
|
|
|
return apiRule{}, fmt.Errorf("failed to read %q param: %w", paramGroupID, err)
|
2022-09-14 12:04:24 +00:00
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
ruleID, err := strconv.ParseUint(r.FormValue(paramRuleID), 10, 64)
|
2022-09-14 12:04:24 +00:00
|
|
|
if err != nil {
|
2023-10-25 19:24:01 +00:00
|
|
|
return apiRule{}, fmt.Errorf("failed to read %q param: %w", paramRuleID, err)
|
2022-09-14 12:04:24 +00:00
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
obj, err := rh.m.ruleAPI(groupID, ruleID)
|
2022-09-14 12:04:24 +00:00
|
|
|
if err != nil {
|
2023-10-13 11:54:33 +00:00
|
|
|
return apiRule{}, errResponse(err, http.StatusNotFound)
|
2022-09-14 12:04:24 +00:00
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
return obj, nil
|
2022-09-14 12:04:24 +00:00
|
|
|
}
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
|
|
|
|
groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 64)
|
2022-07-08 08:26:13 +00:00
|
|
|
if err != nil {
|
2023-10-25 19:24:01 +00:00
|
|
|
return nil, fmt.Errorf("failed to read %q param: %w", paramGroupID, err)
|
2022-07-08 08:26:13 +00:00
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
alertID, err := strconv.ParseUint(r.FormValue(paramAlertID), 10, 64)
|
2022-07-08 08:26:13 +00:00
|
|
|
if err != nil {
|
2023-10-25 19:24:01 +00:00
|
|
|
return nil, fmt.Errorf("failed to read %q param: %w", paramAlertID, err)
|
2022-07-08 08:26:13 +00:00
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
a, err := rh.m.alertAPI(groupID, alertID)
|
2022-07-08 08:26:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, errResponse(err, http.StatusNotFound)
|
|
|
|
}
|
|
|
|
return a, nil
|
|
|
|
}
|
|
|
|
|
2020-06-01 10:46:37 +00:00
|
|
|
type listGroupsResponse struct {
|
2022-03-15 11:54:53 +00:00
|
|
|
Status string `json:"status"`
|
|
|
|
Data struct {
|
2023-10-13 11:54:33 +00:00
|
|
|
Groups []apiGroup `json:"groups"`
|
2020-06-01 10:46:37 +00:00
|
|
|
} `json:"data"`
|
|
|
|
}
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
func (rh *requestHandler) groups() []apiGroup {
|
2020-06-01 10:46:37 +00:00
|
|
|
rh.m.groupsMu.RLock()
|
|
|
|
defer rh.m.groupsMu.RUnlock()
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
groups := make([]apiGroup, 0)
|
2020-06-01 10:46:37 +00:00
|
|
|
for _, g := range rh.m.groups {
|
2023-10-13 11:54:33 +00:00
|
|
|
groups = append(groups, groupToAPI(g))
|
2020-06-01 10:46:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// sort list of alerts for deterministic output
|
2021-09-07 19:39:22 +00:00
|
|
|
sort.Slice(groups, func(i, j int) bool {
|
|
|
|
return groups[i].Name < groups[j].Name
|
2020-06-01 10:46:37 +00:00
|
|
|
})
|
|
|
|
|
2021-09-07 19:39:22 +00:00
|
|
|
return groups
|
|
|
|
}
|
2022-03-15 11:54:53 +00:00
|
|
|
|
2021-09-07 19:39:22 +00:00
|
|
|
func (rh *requestHandler) listGroups() ([]byte, error) {
|
|
|
|
lr := listGroupsResponse{Status: "success"}
|
|
|
|
lr.Data.Groups = rh.groups()
|
2020-06-01 10:46:37 +00:00
|
|
|
b, err := json.Marshal(lr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, &httpserver.ErrorWithStatusCode{
|
2020-06-30 19:58:18 +00:00
|
|
|
Err: fmt.Errorf(`error encoding list of active alerts: %w`, err),
|
2020-06-01 10:46:37 +00:00
|
|
|
StatusCode: http.StatusInternalServerError,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return b, nil
|
|
|
|
}
|
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
type listAlertsResponse struct {
|
2022-03-15 11:54:53 +00:00
|
|
|
Status string `json:"status"`
|
|
|
|
Data struct {
|
2023-10-13 11:54:33 +00:00
|
|
|
Alerts []*apiAlert `json:"alerts"`
|
2020-04-27 21:19:27 +00:00
|
|
|
} `json:"data"`
|
|
|
|
}
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
func (rh *requestHandler) groupAlerts() []groupAlerts {
|
2021-09-07 19:39:22 +00:00
|
|
|
rh.m.groupsMu.RLock()
|
|
|
|
defer rh.m.groupsMu.RUnlock()
|
|
|
|
|
2023-10-13 11:54:33 +00:00
|
|
|
var gAlerts []groupAlerts
|
2021-09-07 19:39:22 +00:00
|
|
|
for _, g := range rh.m.groups {
|
2023-10-13 11:54:33 +00:00
|
|
|
var alerts []*apiAlert
|
2021-09-07 19:39:22 +00:00
|
|
|
for _, r := range g.Rules {
|
2023-10-13 11:54:33 +00:00
|
|
|
a, ok := r.(*rule.AlertingRule)
|
2021-09-07 19:39:22 +00:00
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
alerts = append(alerts, ruleToAPIAlert(a)...)
|
2021-09-07 19:39:22 +00:00
|
|
|
}
|
|
|
|
if len(alerts) > 0 {
|
2023-10-13 11:54:33 +00:00
|
|
|
gAlerts = append(gAlerts, groupAlerts{
|
|
|
|
Group: groupToAPI(g),
|
2021-09-07 19:39:22 +00:00
|
|
|
Alerts: alerts,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
sort.Slice(gAlerts, func(i, j int) bool {
|
|
|
|
return gAlerts[i].Group.Name < gAlerts[j].Group.Name
|
2022-08-09 07:51:29 +00:00
|
|
|
})
|
2023-10-13 11:54:33 +00:00
|
|
|
return gAlerts
|
2021-09-07 19:39:22 +00:00
|
|
|
}
|
|
|
|
|
2020-06-01 10:46:37 +00:00
|
|
|
func (rh *requestHandler) listAlerts() ([]byte, error) {
|
2020-05-10 16:58:17 +00:00
|
|
|
rh.m.groupsMu.RLock()
|
|
|
|
defer rh.m.groupsMu.RUnlock()
|
2020-06-01 10:46:37 +00:00
|
|
|
|
2020-04-27 21:19:27 +00:00
|
|
|
lr := listAlertsResponse{Status: "success"}
|
2023-10-13 11:54:33 +00:00
|
|
|
lr.Data.Alerts = make([]*apiAlert, 0)
|
2020-05-10 16:58:17 +00:00
|
|
|
for _, g := range rh.m.groups {
|
2020-04-27 21:19:27 +00:00
|
|
|
for _, r := range g.Rules {
|
2023-10-13 11:54:33 +00:00
|
|
|
a, ok := r.(*rule.AlertingRule)
|
2020-06-01 10:46:37 +00:00
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
2023-10-13 11:54:33 +00:00
|
|
|
lr.Data.Alerts = append(lr.Data.Alerts, ruleToAPIAlert(a)...)
|
2020-04-27 21:19:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// sort list of alerts for deterministic output
|
|
|
|
sort.Slice(lr.Data.Alerts, func(i, j int) bool {
|
2020-05-04 21:51:22 +00:00
|
|
|
return lr.Data.Alerts[i].ID < lr.Data.Alerts[j].ID
|
2020-04-27 21:19:27 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
b, err := json.Marshal(lr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, &httpserver.ErrorWithStatusCode{
|
2020-06-30 19:58:18 +00:00
|
|
|
Err: fmt.Errorf(`error encoding list of active alerts: %w`, err),
|
2020-04-27 21:19:27 +00:00
|
|
|
StatusCode: http.StatusInternalServerError,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return b, nil
|
|
|
|
}
|
|
|
|
|
2020-05-10 16:58:17 +00:00
|
|
|
func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
|
|
|
|
return &httpserver.ErrorWithStatusCode{
|
|
|
|
Err: err,
|
|
|
|
StatusCode: sc,
|
|
|
|
}
|
|
|
|
}
|