2020-02-16 18:59:02 +00:00
package main
import (
2020-03-13 10:19:31 +00:00
"context"
2020-02-16 18:59:02 +00:00
"flag"
2020-03-13 10:19:31 +00:00
"fmt"
2020-02-16 18:59:02 +00:00
"net/http"
2020-04-01 15:17:53 +00:00
"net/url"
"os"
2020-03-13 10:19:31 +00:00
"strings"
"time"
2020-02-16 18:59:02 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
2020-04-06 11:44:03 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
2020-04-27 21:18:02 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
2020-02-16 18:59:02 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
2020-03-28 23:48:30 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2020-02-16 18:59:02 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
2020-04-11 19:42:01 +00:00
"github.com/VictoriaMetrics/metrics"
2020-02-16 18:59:02 +00:00
)
var (
2020-04-12 11:47:26 +00:00
rulePath = flagutil . NewArray ( "rule" , ` Path to the file with alert rules .
Supports patterns . Flag can be specified multiple times .
2020-03-28 23:48:30 +00:00
Examples :
2020-04-12 11:47:26 +00:00
- rule / path / to / file . Path to a single file with alerting rules
- rule dir / * . yaml - rule / * . yaml . Relative path to all . yaml files in "dir" folder ,
absolute path to all . yaml files in root . ` )
2020-04-27 21:18:02 +00:00
validateTemplates = flag . Bool ( "rule.validateTemplates" , true , "Indicates to validate annotation and label templates" )
httpListenAddr = flag . String ( "httpListenAddr" , ":8880" , "Address to listen for http connections" )
2020-05-04 21:51:22 +00:00
datasourceURL = flag . String ( "datasource.url" , "" , "Victoria Metrics or VMSelect url. Required parameter." +
" E.g. http://127.0.0.1:8428" )
basicAuthUsername = flag . String ( "datasource.basicAuth.username" , "" , "Optional basic auth username for -datasource.url" )
basicAuthPassword = flag . String ( "datasource.basicAuth.password" , "" , "Optional basic auth password for -datasource.url" )
2020-05-13 17:58:56 +00:00
remoteWriteURL = flag . String ( "remoteWrite.url" , "" , "Optional URL to Victoria Metrics or VMInsert where to persist alerts state" +
2020-05-04 21:51:22 +00:00
" in form of timeseries. E.g. http://127.0.0.1:8428" )
2020-05-13 17:58:56 +00:00
remoteWriteUsername = flag . String ( "remoteWrite.basicAuth.username" , "" , "Optional basic auth username for -remotewrite.url" )
remoteWritePassword = flag . String ( "remoteWrite.basicAuth.password" , "" , "Optional basic auth password for -remotewrite.url" )
remoteWriteMaxQueueSize = flag . Int ( "remoteWrite.maxQueueSize" , 10000 , "Optional Defines the max number of pending datapoints to remote write endpoint" )
2020-05-04 21:51:22 +00:00
2020-05-13 17:58:56 +00:00
remoteReadURL = flag . String ( "remoteRead.url" , "" , "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts" +
2020-05-04 21:51:22 +00:00
" state. This configuration makes sense only if `vmalert` was configured with `remotewrite.url` before and has been successfully persisted its state." +
" E.g. http://127.0.0.1:8428" )
2020-05-13 17:58:56 +00:00
remoteReadUsername = flag . String ( "remoteRead.basicAuth.username" , "" , "Optional basic auth username for -remoteread.url" )
remoteReadPassword = flag . String ( "remoteRead.basicAuth.password" , "" , "Optional basic auth password for -remoteread.url" )
remoteReadLookBack = flag . Duration ( "remoteRead.lookback" , time . Hour , "Lookback defines how far to look into past for alerts timeseries." +
2020-05-04 21:51:22 +00:00
" For example, if lookback=1h then range from now() to now()-1h will be scanned." )
evaluationInterval = flag . Duration ( "evaluationInterval" , time . Minute , "How often to evaluate the rules. Default 1m" )
2020-04-26 11:15:04 +00:00
notifierURL = flag . String ( "notifier.url" , "" , "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093" )
externalURL = flag . String ( "external.url" , "" , "External URL is used as alert's source for sent alerts to the notifier" )
2020-02-16 18:59:02 +00:00
)
func main ( ) {
envflag . Parse ( )
buildinfo . Init ( )
logger . Init ( )
2020-03-28 23:48:30 +00:00
checkFlags ( )
2020-03-13 10:19:31 +00:00
ctx , cancel := context . WithCancel ( context . Background ( ) )
2020-04-01 19:29:11 +00:00
eu , err := getExternalURL ( * externalURL , * httpListenAddr , httpserver . IsTLS ( ) )
2020-04-01 15:17:53 +00:00
if err != nil {
2020-05-10 16:58:17 +00:00
logger . Fatalf ( "can not get external url: %s " , err )
2020-04-01 15:17:53 +00:00
}
2020-04-06 11:44:03 +00:00
notifier . InitTemplateFunc ( eu )
2020-02-16 18:59:02 +00:00
2020-05-10 16:58:17 +00:00
manager := & manager {
groups : make ( map [ uint64 ] * Group ) ,
2020-03-13 10:19:31 +00:00
storage : datasource . NewVMStorage ( * datasourceURL , * basicAuthUsername , * basicAuthPassword , & http . Client { } ) ,
2020-05-10 16:58:17 +00:00
notifier : notifier . NewAlertManager ( * notifierURL , func ( group , alert string ) string {
return fmt . Sprintf ( "%s/api/v1/%s/%s/status" , eu , group , alert )
2020-03-13 10:19:31 +00:00
} , & http . Client { } ) ,
}
2020-04-27 21:18:02 +00:00
if * remoteWriteURL != "" {
c , err := remotewrite . NewClient ( ctx , remotewrite . Config {
Addr : * remoteWriteURL ,
2020-05-13 17:58:56 +00:00
MaxQueueSize : * remoteWriteMaxQueueSize ,
2020-04-27 21:18:02 +00:00
FlushInterval : * evaluationInterval ,
2020-05-04 21:51:22 +00:00
BasicAuthUser : * remoteWriteUsername ,
BasicAuthPass : * remoteWritePassword ,
2020-04-27 21:18:02 +00:00
} )
if err != nil {
logger . Fatalf ( "failed to init remotewrite client: %s" , err )
}
2020-05-10 16:58:17 +00:00
manager . rw = c
2020-04-27 21:18:02 +00:00
}
2020-05-04 21:51:22 +00:00
if * remoteReadURL != "" {
2020-05-10 16:58:17 +00:00
manager . rr = datasource . NewVMStorage ( * remoteReadURL , * remoteReadUsername , * remoteReadPassword , & http . Client { } )
2020-05-04 21:51:22 +00:00
}
2020-05-10 16:58:17 +00:00
if err := manager . start ( ctx , * rulePath , * validateTemplates ) ; err != nil {
logger . Fatalf ( "failed to start: %s" , err )
}
2020-05-09 09:32:12 +00:00
2020-05-10 16:58:17 +00:00
go func ( ) {
// init reload metrics with positive values to improve alerting conditions
configSuccess . Set ( 1 )
configTimestamp . Set ( uint64 ( time . Now ( ) . UnixNano ( ) ) / 1e9 )
sigHup := procutil . NewSighupChan ( )
for {
<- sigHup
configReloads . Inc ( )
logger . Infof ( "SIGHUP received. Going to reload rules %q ..." , * rulePath )
if err := manager . update ( ctx , * rulePath , * validateTemplates , false ) ; err != nil {
configReloadErrors . Inc ( )
configSuccess . Set ( 0 )
logger . Errorf ( "error while reloading rules: %s" , err )
continue
}
configSuccess . Set ( 1 )
configTimestamp . Set ( uint64 ( time . Now ( ) . UnixNano ( ) ) / 1e9 )
logger . Infof ( "Rules reloaded successfully from %q" , * rulePath )
}
} ( )
2020-05-09 09:32:12 +00:00
2020-05-10 16:58:17 +00:00
rh := & requestHandler { m : manager }
2020-05-09 09:32:12 +00:00
go httpserver . Serve ( * httpListenAddr , ( rh ) . handler )
2020-04-06 11:44:03 +00:00
2020-02-16 18:59:02 +00:00
sig := procutil . WaitForSigterm ( )
logger . Infof ( "service received signal %s" , sig )
2020-02-21 21:15:05 +00:00
if err := httpserver . Stop ( * httpListenAddr ) ; err != nil {
logger . Fatalf ( "cannot stop the webservice: %s" , err )
}
2020-03-13 10:19:31 +00:00
cancel ( )
2020-05-10 16:58:17 +00:00
manager . close ( )
2020-03-13 10:19:31 +00:00
}
2020-04-11 19:42:01 +00:00
var (
2020-05-10 16:58:17 +00:00
configReloads = metrics . NewCounter ( ` vmalert_config_last_reload_total ` )
configReloadErrors = metrics . NewCounter ( ` vmalert_config_last_reload_errors_total ` )
configSuccess = metrics . NewCounter ( ` vmalert_config_last_reload_successful ` )
configTimestamp = metrics . NewCounter ( ` vmalert_config_last_reload_success_timestamp_seconds ` )
2020-04-11 19:42:01 +00:00
)
2020-04-01 15:17:53 +00:00
func getExternalURL ( externalURL , httpListenAddr string , isSecure bool ) ( * url . URL , error ) {
if externalURL != "" {
return url . Parse ( externalURL )
2020-03-13 10:19:31 +00:00
}
2020-04-01 15:17:53 +00:00
hname , err := os . Hostname ( )
2020-03-13 10:19:31 +00:00
if err != nil {
2020-04-01 15:17:53 +00:00
return nil , err
2020-03-13 10:19:31 +00:00
}
2020-04-01 15:17:53 +00:00
port := ""
if ipport := strings . Split ( httpListenAddr , ":" ) ; len ( ipport ) > 1 {
port = ":" + ipport [ 1 ]
}
schema := "http://"
if isSecure {
schema = "https://"
2020-03-13 10:19:31 +00:00
}
2020-04-01 15:17:53 +00:00
return url . Parse ( fmt . Sprintf ( "%s%s%s" , schema , hname , port ) )
2020-02-16 18:59:02 +00:00
}
2020-03-28 23:48:30 +00:00
func checkFlags ( ) {
2020-04-11 15:49:23 +00:00
if * notifierURL == "" {
2020-03-28 23:48:30 +00:00
flag . PrintDefaults ( )
2020-04-11 15:49:23 +00:00
logger . Fatalf ( "notifier.url is empty" )
2020-03-28 23:48:30 +00:00
}
if * datasourceURL == "" {
flag . PrintDefaults ( )
logger . Fatalf ( "datasource.url is empty" )
}
}