2020-04-27 21:19:27 +00:00
package main
import (
2022-05-30 23:55:28 +00:00
"embed"
2020-04-27 21:19:27 +00:00
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
2022-02-02 12:11:41 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
2023-10-13 11:54:33 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
2021-09-21 11:41:01 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
2024-01-21 19:58:26 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2020-04-27 21:19:27 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
2024-02-09 13:35:31 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
2020-05-10 16:58:17 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
2020-04-27 21:19:27 +00:00
)
2024-06-10 10:09:47 +00:00
var reloadAuthKey = flagutil . NewPassword ( "reloadAuthKey" , "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings." )
2024-01-21 19:58:26 +00:00
2021-09-21 11:41:01 +00:00
var (
apiLinks = [ ] [ 2 ] string {
2022-07-08 08:26:13 +00:00
// api links are relative since they can be used by external clients,
// such as Grafana, and proxied via vmselect.
2022-07-06 08:46:01 +00:00
{ "api/v1/rules" , "list all loaded groups and rules" } ,
{ "api/v1/alerts" , "list all active alerts" } ,
2022-07-08 08:26:13 +00:00
{ fmt . Sprintf ( "api/v1/alert?%s=<int>&%s=<int>" , paramGroupID , paramAlertID ) , "get alert status by group and alert ID" } ,
2022-12-09 19:43:54 +00:00
}
systemLinks = [ ] [ 2 ] string {
2024-06-14 11:34:23 +00:00
{ "flags" , "command-line flags" } ,
{ "metrics" , "list of application metrics" } ,
{ "-/reload" , "reload configuration" } ,
2021-09-21 11:41:01 +00:00
}
navItems = [ ] tpl . NavItem {
2022-07-11 16:52:22 +00:00
{ Name : "vmalert" , Url : "." } ,
2022-07-06 08:46:01 +00:00
{ Name : "Groups" , Url : "groups" } ,
{ Name : "Alerts" , Url : "alerts" } ,
{ Name : "Notifiers" , Url : "notifiers" } ,
2024-04-17 23:44:12 +00:00
{ Name : "Docs" , Url : "https://docs.victoriametrics.com/vmalert/" } ,
2021-09-21 11:41:01 +00:00
}
2022-12-09 19:43:54 +00:00
)
2021-09-21 11:41:01 +00:00
2020-04-27 21:19:27 +00:00
type requestHandler struct {
2020-05-10 16:58:17 +00:00
m * manager
2020-04-27 21:19:27 +00:00
}
2022-07-06 08:46:01 +00:00
var (
//go:embed static
staticFiles embed . FS
staticHandler = http . FileServer ( http . FS ( staticFiles ) )
staticServer = http . StripPrefix ( "/vmalert" , staticHandler )
)
2020-04-27 21:19:27 +00:00
func ( rh * requestHandler ) handler ( w http . ResponseWriter , r * http . Request ) bool {
2022-07-06 08:46:01 +00:00
if strings . HasPrefix ( r . URL . Path , "/vmalert/static" ) {
staticServer . ServeHTTP ( w , r )
return true
2022-02-08 19:03:42 +00:00
}
2020-04-27 21:19:27 +00:00
switch r . URL . Path {
2022-07-11 16:52:22 +00:00
case "/" , "/vmalert" , "/vmalert/" :
2023-02-23 02:58:44 +00:00
if r . Method != http . MethodGet {
2022-07-11 16:52:22 +00:00
httpserver . Errorf ( w , r , "path %q supports only GET method" , r . URL . Path )
2021-04-02 19:54:06 +00:00
return false
}
2022-07-06 08:46:01 +00:00
WriteWelcome ( w , r )
return true
case "/vmalert/alerts" :
WriteListAlerts ( w , r , rh . groupAlerts ( ) )
2020-04-27 21:19:27 +00:00
return true
2022-07-08 08:26:13 +00:00
case "/vmalert/alert" :
alert , err := rh . getAlert ( r )
if err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
WriteAlert ( w , r , alert )
return true
2022-09-14 12:04:24 +00:00
case "/vmalert/rule" :
rule , err := rh . getRule ( r )
if err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
WriteRuleDetails ( w , r , rule )
return true
2022-07-06 08:46:01 +00:00
case "/vmalert/groups" :
2024-02-09 08:02:35 +00:00
var data [ ] apiGroup
2024-02-09 13:35:31 +00:00
rf := extractRulesFilter ( r )
data = rh . groups ( rf )
2024-02-09 08:02:35 +00:00
WriteListGroups ( w , r , data )
2021-09-07 19:39:22 +00:00
return true
2022-07-06 08:46:01 +00:00
case "/vmalert/notifiers" :
WriteListTargets ( w , r , notifier . GetTargets ( ) )
2021-09-07 19:39:22 +00:00
return true
2022-07-06 08:46:01 +00:00
// special cases for Grafana requests,
// served without `vmalert` prefix:
case "/rules" :
// Grafana makes an extra request to `/rules`
// handler in addition to `/api/v1/rules` calls in alerts UI,
2024-02-09 08:02:35 +00:00
var data [ ] apiGroup
2024-02-09 13:35:31 +00:00
rf := extractRulesFilter ( r )
data = rh . groups ( rf )
2024-02-09 08:02:35 +00:00
WriteListGroups ( w , r , data )
2022-02-02 12:11:41 +00:00
return true
2022-07-06 08:46:01 +00:00
case "/vmalert/api/v1/rules" , "/api/v1/rules" :
// path used by Grafana for ng alerting
2024-02-09 08:02:35 +00:00
var data [ ] byte
var err error
2024-02-09 13:35:31 +00:00
rf := extractRulesFilter ( r )
data , err = rh . listGroups ( rf )
2024-02-09 08:02:35 +00:00
2020-07-20 11:00:33 +00:00
if err != nil {
2021-07-07 09:59:03 +00:00
httpserver . Errorf ( w , r , "%s" , err )
2020-07-20 11:00:33 +00:00
return true
}
2021-11-09 16:03:50 +00:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2020-07-20 11:00:33 +00:00
w . Write ( data )
2020-06-01 10:46:37 +00:00
return true
2024-02-09 08:02:35 +00:00
2022-07-06 08:46:01 +00:00
case "/vmalert/api/v1/alerts" , "/api/v1/alerts" :
// path used by Grafana for ng alerting
2020-07-20 11:00:33 +00:00
data , err := rh . listAlerts ( )
if err != nil {
2021-07-07 09:59:03 +00:00
httpserver . Errorf ( w , r , "%s" , err )
2020-07-20 11:00:33 +00:00
return true
}
2021-11-09 16:03:50 +00:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2020-07-20 11:00:33 +00:00
w . Write ( data )
2020-04-27 21:19:27 +00:00
return true
2022-07-08 08:26:13 +00:00
case "/vmalert/api/v1/alert" , "/api/v1/alert" :
alert , err := rh . getAlert ( r )
if err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
data , err := json . Marshal ( alert )
if err != nil {
httpserver . Errorf ( w , r , "failed to marshal alert: %s" , err )
return true
}
w . Header ( ) . Set ( "Content-Type" , "application/json" )
w . Write ( data )
return true
2023-12-04 15:40:33 +00:00
case "/vmalert/api/v1/rule" , "/api/v1/rule" :
rule , err := rh . getRule ( r )
if err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
rwu := apiRuleWithUpdates {
apiRule : rule ,
StateUpdates : rule . Updates ,
}
data , err := json . Marshal ( rwu )
if err != nil {
httpserver . Errorf ( w , r , "failed to marshal rule: %s" , err )
return true
}
w . Header ( ) . Set ( "Content-Type" , "application/json" )
w . Write ( data )
return true
2020-05-09 09:32:12 +00:00
case "/-/reload" :
2024-01-21 19:58:26 +00:00
if ! httpserver . CheckAuthFlag ( w , r , reloadAuthKey . Get ( ) , "reloadAuthKey" ) {
return true
}
2020-05-09 09:32:12 +00:00
logger . Infof ( "api config reload was called, sending sighup" )
procutil . SelfSIGHUP ( )
w . WriteHeader ( http . StatusOK )
return true
2022-05-30 23:55:28 +00:00
2022-07-06 08:46:01 +00:00
default :
2023-07-31 14:51:41 +00:00
return false
2020-04-27 21:19:27 +00:00
}
}
2023-10-13 11:54:33 +00:00
func ( rh * requestHandler ) getRule ( r * http . Request ) ( apiRule , error ) {
groupID , err := strconv . ParseUint ( r . FormValue ( paramGroupID ) , 10 , 64 )
2022-09-14 12:04:24 +00:00
if err != nil {
2023-10-25 19:24:01 +00:00
return apiRule { } , fmt . Errorf ( "failed to read %q param: %w" , paramGroupID , err )
2022-09-14 12:04:24 +00:00
}
2023-10-13 11:54:33 +00:00
ruleID , err := strconv . ParseUint ( r . FormValue ( paramRuleID ) , 10 , 64 )
2022-09-14 12:04:24 +00:00
if err != nil {
2023-10-25 19:24:01 +00:00
return apiRule { } , fmt . Errorf ( "failed to read %q param: %w" , paramRuleID , err )
2022-09-14 12:04:24 +00:00
}
2023-10-13 11:54:33 +00:00
obj , err := rh . m . ruleAPI ( groupID , ruleID )
2022-09-14 12:04:24 +00:00
if err != nil {
2023-10-13 11:54:33 +00:00
return apiRule { } , errResponse ( err , http . StatusNotFound )
2022-09-14 12:04:24 +00:00
}
2023-10-13 11:54:33 +00:00
return obj , nil
2022-09-14 12:04:24 +00:00
}
2023-10-13 11:54:33 +00:00
func ( rh * requestHandler ) getAlert ( r * http . Request ) ( * apiAlert , error ) {
groupID , err := strconv . ParseUint ( r . FormValue ( paramGroupID ) , 10 , 64 )
2022-07-08 08:26:13 +00:00
if err != nil {
2023-10-25 19:24:01 +00:00
return nil , fmt . Errorf ( "failed to read %q param: %w" , paramGroupID , err )
2022-07-08 08:26:13 +00:00
}
2023-10-13 11:54:33 +00:00
alertID , err := strconv . ParseUint ( r . FormValue ( paramAlertID ) , 10 , 64 )
2022-07-08 08:26:13 +00:00
if err != nil {
2023-10-25 19:24:01 +00:00
return nil , fmt . Errorf ( "failed to read %q param: %w" , paramAlertID , err )
2022-07-08 08:26:13 +00:00
}
2023-10-13 11:54:33 +00:00
a , err := rh . m . alertAPI ( groupID , alertID )
2022-07-08 08:26:13 +00:00
if err != nil {
return nil , errResponse ( err , http . StatusNotFound )
}
return a , nil
}
2020-06-01 10:46:37 +00:00
type listGroupsResponse struct {
2022-03-15 11:54:53 +00:00
Status string ` json:"status" `
Data struct {
2023-10-13 11:54:33 +00:00
Groups [ ] apiGroup ` json:"groups" `
2020-06-01 10:46:37 +00:00
} ` json:"data" `
}
2024-02-09 13:35:31 +00:00
// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
type rulesFilter struct {
files [ ] string
groupNames [ ] string
ruleNames [ ] string
ruleType string
excludeAlerts bool
}
func extractRulesFilter ( r * http . Request ) rulesFilter {
rf := rulesFilter { }
var ruleType string
ruleTypeParam := r . URL . Query ( ) . Get ( "type" )
// for some reason, `type` in filter doesn't match `type` in response,
// so we use this matching here
if ruleTypeParam == "alert" {
ruleType = ruleTypeAlerting
} else if ruleTypeParam == "record" {
ruleType = ruleTypeRecording
}
rf . ruleType = ruleType
rf . excludeAlerts = httputils . GetBool ( r , "exclude_alerts" )
rf . ruleNames = append ( [ ] string { } , r . Form [ "rule_name[]" ] ... )
rf . groupNames = append ( [ ] string { } , r . Form [ "rule_group[]" ] ... )
rf . files = append ( [ ] string { } , r . Form [ "file[]" ] ... )
return rf
}
func ( rh * requestHandler ) groups ( rf rulesFilter ) [ ] apiGroup {
2020-06-01 10:46:37 +00:00
rh . m . groupsMu . RLock ( )
defer rh . m . groupsMu . RUnlock ( )
2024-02-09 13:35:31 +00:00
isInList := func ( list [ ] string , needle string ) bool {
if len ( list ) < 1 {
return true
}
for _ , i := range list {
if i == needle {
return true
}
}
return false
}
2023-10-13 11:54:33 +00:00
groups := make ( [ ] apiGroup , 0 )
2024-02-09 13:35:31 +00:00
for _ , group := range rh . m . groups {
if ! isInList ( rf . groupNames , group . Name ) {
continue
}
if ! isInList ( rf . files , group . File ) {
continue
}
g := groupToAPI ( group )
// the returned list should always be non-nil
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
filteredRules := make ( [ ] apiRule , 0 )
for _ , r := range g . Rules {
if rf . ruleType != "" && rf . ruleType != r . Type {
continue
2024-02-09 08:02:35 +00:00
}
2024-02-09 13:35:31 +00:00
if ! isInList ( rf . ruleNames , r . Name ) {
2024-02-09 08:02:35 +00:00
continue
}
2024-02-09 13:35:31 +00:00
if rf . excludeAlerts {
r . Alerts = nil
}
filteredRules = append ( filteredRules , r )
2024-02-09 08:02:35 +00:00
}
2024-02-09 13:35:31 +00:00
g . Rules = filteredRules
groups = append ( groups , g )
2020-06-01 10:46:37 +00:00
}
2024-02-09 13:35:31 +00:00
// sort list of groups for deterministic output
2021-09-07 19:39:22 +00:00
sort . Slice ( groups , func ( i , j int ) bool {
2024-02-20 12:50:57 +00:00
a , b := groups [ i ] , groups [ j ]
if a . Name != b . Name {
return a . Name < b . Name
}
return a . File < b . File
2020-06-01 10:46:37 +00:00
} )
2021-09-07 19:39:22 +00:00
return groups
}
2022-03-15 11:54:53 +00:00
2024-02-09 13:35:31 +00:00
func ( rh * requestHandler ) listGroups ( rf rulesFilter ) ( [ ] byte , error ) {
2021-09-07 19:39:22 +00:00
lr := listGroupsResponse { Status : "success" }
2024-02-09 13:35:31 +00:00
lr . Data . Groups = rh . groups ( rf )
2020-06-01 10:46:37 +00:00
b , err := json . Marshal ( lr )
if err != nil {
return nil , & httpserver . ErrorWithStatusCode {
2020-06-30 19:58:18 +00:00
Err : fmt . Errorf ( ` error encoding list of active alerts: %w ` , err ) ,
2020-06-01 10:46:37 +00:00
StatusCode : http . StatusInternalServerError ,
}
}
return b , nil
}
2020-04-27 21:19:27 +00:00
type listAlertsResponse struct {
2022-03-15 11:54:53 +00:00
Status string ` json:"status" `
Data struct {
2023-10-13 11:54:33 +00:00
Alerts [ ] * apiAlert ` json:"alerts" `
2020-04-27 21:19:27 +00:00
} ` json:"data" `
}
2023-10-13 11:54:33 +00:00
func ( rh * requestHandler ) groupAlerts ( ) [ ] groupAlerts {
2021-09-07 19:39:22 +00:00
rh . m . groupsMu . RLock ( )
defer rh . m . groupsMu . RUnlock ( )
2023-10-13 11:54:33 +00:00
var gAlerts [ ] groupAlerts
2021-09-07 19:39:22 +00:00
for _ , g := range rh . m . groups {
2023-10-13 11:54:33 +00:00
var alerts [ ] * apiAlert
2021-09-07 19:39:22 +00:00
for _ , r := range g . Rules {
2023-10-13 11:54:33 +00:00
a , ok := r . ( * rule . AlertingRule )
2021-09-07 19:39:22 +00:00
if ! ok {
continue
}
2023-10-13 11:54:33 +00:00
alerts = append ( alerts , ruleToAPIAlert ( a ) ... )
2021-09-07 19:39:22 +00:00
}
if len ( alerts ) > 0 {
2023-10-13 11:54:33 +00:00
gAlerts = append ( gAlerts , groupAlerts {
Group : groupToAPI ( g ) ,
2021-09-07 19:39:22 +00:00
Alerts : alerts ,
} )
}
}
2023-10-13 11:54:33 +00:00
sort . Slice ( gAlerts , func ( i , j int ) bool {
return gAlerts [ i ] . Group . Name < gAlerts [ j ] . Group . Name
2022-08-09 07:51:29 +00:00
} )
2023-10-13 11:54:33 +00:00
return gAlerts
2021-09-07 19:39:22 +00:00
}
2020-06-01 10:46:37 +00:00
func ( rh * requestHandler ) listAlerts ( ) ( [ ] byte , error ) {
2020-05-10 16:58:17 +00:00
rh . m . groupsMu . RLock ( )
defer rh . m . groupsMu . RUnlock ( )
2020-06-01 10:46:37 +00:00
2020-04-27 21:19:27 +00:00
lr := listAlertsResponse { Status : "success" }
2023-10-13 11:54:33 +00:00
lr . Data . Alerts = make ( [ ] * apiAlert , 0 )
2020-05-10 16:58:17 +00:00
for _ , g := range rh . m . groups {
2020-04-27 21:19:27 +00:00
for _ , r := range g . Rules {
2023-10-13 11:54:33 +00:00
a , ok := r . ( * rule . AlertingRule )
2020-06-01 10:46:37 +00:00
if ! ok {
continue
}
2023-10-13 11:54:33 +00:00
lr . Data . Alerts = append ( lr . Data . Alerts , ruleToAPIAlert ( a ) ... )
2020-04-27 21:19:27 +00:00
}
}
// sort list of alerts for deterministic output
sort . Slice ( lr . Data . Alerts , func ( i , j int ) bool {
2020-05-04 21:51:22 +00:00
return lr . Data . Alerts [ i ] . ID < lr . Data . Alerts [ j ] . ID
2020-04-27 21:19:27 +00:00
} )
b , err := json . Marshal ( lr )
if err != nil {
return nil , & httpserver . ErrorWithStatusCode {
2020-06-30 19:58:18 +00:00
Err : fmt . Errorf ( ` error encoding list of active alerts: %w ` , err ) ,
2020-04-27 21:19:27 +00:00
StatusCode : http . StatusInternalServerError ,
}
}
return b , nil
}
2020-05-10 16:58:17 +00:00
func errResponse ( err error , sc int ) * httpserver . ErrorWithStatusCode {
return & httpserver . ErrorWithStatusCode {
Err : err ,
StatusCode : sc ,
}
}