2020-04-06 11:44:03 +00:00
|
|
|
package notifier
|
2020-02-16 18:59:02 +00:00
|
|
|
|
2020-02-21 21:15:05 +00:00
|
|
|
import (
|
|
|
|
"bytes"
|
2020-05-10 16:58:17 +00:00
|
|
|
"context"
|
2020-02-21 21:15:05 +00:00
|
|
|
"fmt"
|
2022-08-21 21:13:44 +00:00
|
|
|
"io"
|
2020-02-21 21:15:05 +00:00
|
|
|
"net/http"
|
2022-03-10 11:09:12 +00:00
|
|
|
"strings"
|
2022-02-02 12:11:41 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
2022-04-09 06:21:16 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
2020-02-21 21:15:05 +00:00
|
|
|
)
|
|
|
|
|
2020-02-16 18:59:02 +00:00
|
|
|
// AlertManager represents integration provider with Prometheus alert manager
|
2020-04-06 11:44:03 +00:00
|
|
|
// https://github.com/prometheus/alertmanager
|
2020-02-21 21:15:05 +00:00
|
|
|
type AlertManager struct {
|
2022-02-02 12:11:41 +00:00
|
|
|
addr string
|
|
|
|
argFunc AlertURLGenerator
|
|
|
|
client *http.Client
|
|
|
|
timeout time.Duration
|
|
|
|
|
|
|
|
authCfg *promauth.Config
|
2022-04-09 06:21:16 +00:00
|
|
|
// stores already parsed RelabelConfigs object
|
|
|
|
relabelConfigs *promrelabel.ParsedConfigs
|
2022-02-02 12:11:41 +00:00
|
|
|
|
|
|
|
metrics *metrics
|
|
|
|
}
|
|
|
|
|
|
|
|
type metrics struct {
|
|
|
|
alertsSent *utils.Counter
|
|
|
|
alertsSendErrors *utils.Counter
|
|
|
|
}
|
|
|
|
|
|
|
|
func newMetrics(addr string) *metrics {
|
|
|
|
return &metrics{
|
|
|
|
alertsSent: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
|
|
|
|
alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close is a destructor method for AlertManager
|
|
|
|
func (am *AlertManager) Close() {
|
|
|
|
am.metrics.alertsSent.Unregister()
|
|
|
|
am.metrics.alertsSendErrors.Unregister()
|
2020-02-21 21:15:05 +00:00
|
|
|
}
|
|
|
|
|
2021-08-31 09:28:02 +00:00
|
|
|
// Addr returns address where alerts are sent.
|
|
|
|
func (am AlertManager) Addr() string { return am.addr }
|
|
|
|
|
2020-02-16 18:59:02 +00:00
|
|
|
// Send an alert or resolve message
|
2020-05-10 16:58:17 +00:00
|
|
|
func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
|
2022-02-02 12:11:41 +00:00
|
|
|
am.metrics.alertsSent.Add(len(alerts))
|
|
|
|
err := am.send(ctx, alerts)
|
|
|
|
if err != nil {
|
|
|
|
am.metrics.alertsSendErrors.Add(len(alerts))
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
|
2020-04-06 11:44:03 +00:00
|
|
|
b := &bytes.Buffer{}
|
2022-04-09 06:21:16 +00:00
|
|
|
writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
|
2020-05-10 16:58:17 +00:00
|
|
|
|
2022-02-02 12:11:41 +00:00
|
|
|
req, err := http.NewRequest("POST", am.addr, b)
|
2020-02-21 21:15:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-11-09 16:03:50 +00:00
|
|
|
req.Header.Set("Content-Type", "application/json")
|
2022-02-02 12:11:41 +00:00
|
|
|
|
|
|
|
if am.timeout > 0 {
|
|
|
|
var cancel context.CancelFunc
|
|
|
|
ctx, cancel = context.WithTimeout(ctx, am.timeout)
|
|
|
|
defer cancel()
|
|
|
|
}
|
|
|
|
|
2020-05-10 16:58:17 +00:00
|
|
|
req = req.WithContext(ctx)
|
2022-02-02 12:11:41 +00:00
|
|
|
|
|
|
|
if am.authCfg != nil {
|
2022-06-22 17:38:43 +00:00
|
|
|
am.authCfg.SetHeaders(req, true)
|
2020-06-29 19:21:03 +00:00
|
|
|
}
|
2020-05-10 16:58:17 +00:00
|
|
|
resp, err := am.client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-02-21 21:15:05 +00:00
|
|
|
defer func() { _ = resp.Body.Close() }()
|
2020-04-06 11:44:03 +00:00
|
|
|
|
2020-02-21 21:15:05 +00:00
|
|
|
if resp.StatusCode != http.StatusOK {
|
2022-08-21 21:13:44 +00:00
|
|
|
body, err := io.ReadAll(resp.Body)
|
2020-04-06 11:44:03 +00:00
|
|
|
if err != nil {
|
2022-02-02 12:11:41 +00:00
|
|
|
return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
|
2020-02-21 21:15:05 +00:00
|
|
|
}
|
2022-02-02 12:11:41 +00:00
|
|
|
return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body))
|
2020-02-21 21:15:05 +00:00
|
|
|
}
|
2020-02-16 18:59:02 +00:00
|
|
|
return nil
|
|
|
|
}
|
2020-04-06 11:44:03 +00:00
|
|
|
|
|
|
|
// AlertURLGenerator returns URL to single alert by given name
|
2020-06-21 10:32:46 +00:00
|
|
|
type AlertURLGenerator func(Alert) string
|
2020-04-06 11:44:03 +00:00
|
|
|
|
|
|
|
const alertManagerPath = "/api/v2/alerts"
|
|
|
|
|
|
|
|
// NewAlertManager is a constructor for AlertManager
|
2022-04-09 06:21:16 +00:00
|
|
|
func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
|
|
|
|
relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
|
2022-02-02 12:11:41 +00:00
|
|
|
tls := &promauth.TLSConfig{}
|
|
|
|
if authCfg.TLSConfig != nil {
|
|
|
|
tls = authCfg.TLSConfig
|
|
|
|
}
|
|
|
|
tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to create transport: %w", err)
|
|
|
|
}
|
|
|
|
|
2022-03-10 11:09:12 +00:00
|
|
|
ba := new(promauth.BasicAuthConfig)
|
|
|
|
oauth := new(promauth.OAuth2Config)
|
2022-02-02 12:11:41 +00:00
|
|
|
if authCfg.BasicAuth != nil {
|
|
|
|
ba = authCfg.BasicAuth
|
2020-04-06 11:44:03 +00:00
|
|
|
}
|
2022-03-10 11:09:12 +00:00
|
|
|
if authCfg.OAuth2 != nil {
|
|
|
|
oauth = authCfg.OAuth2
|
|
|
|
}
|
|
|
|
|
|
|
|
aCfg, err := utils.AuthConfig(
|
|
|
|
utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
|
|
|
|
utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
|
|
|
|
utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
|
2022-02-02 12:11:41 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to configure auth: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return &AlertManager{
|
2022-04-09 06:21:16 +00:00
|
|
|
addr: alertManagerURL,
|
|
|
|
argFunc: fn,
|
|
|
|
authCfg: aCfg,
|
|
|
|
relabelConfigs: relabelCfg,
|
|
|
|
client: &http.Client{Transport: tr},
|
|
|
|
timeout: timeout,
|
|
|
|
metrics: newMetrics(alertManagerURL),
|
2022-02-02 12:11:41 +00:00
|
|
|
}, nil
|
2020-04-06 11:44:03 +00:00
|
|
|
}
|