2020-02-23 11:35:47 +00:00
package remotewrite
import (
2020-07-20 16:27:25 +00:00
"bytes"
2020-02-23 11:35:47 +00:00
"fmt"
2020-07-20 16:27:25 +00:00
"io/ioutil"
"net/http"
"net/url"
2020-02-23 11:35:47 +00:00
"strings"
"sync"
"time"
2022-05-04 17:24:19 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
2020-05-06 13:51:32 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2020-02-23 11:35:47 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
2020-05-12 14:20:55 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
2021-01-26 22:23:10 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
2020-02-23 11:35:47 +00:00
"github.com/VictoriaMetrics/metrics"
)
var (
2022-06-30 17:15:56 +00:00
rateLimit = flagutil . NewArrayInt ( "remoteWrite.rateLimit" , "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. " +
2021-01-26 22:19:35 +00:00
"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data " +
"is sent after temporary unavailability of the remote storage" )
2022-06-30 17:15:56 +00:00
sendTimeout = flagutil . NewArrayDuration ( "remoteWrite.sendTimeout" , "Timeout for sending a single block of data to the corresponding -remoteWrite.url" )
proxyURL = flagutil . NewArray ( "remoteWrite.proxyURL" , "Optional proxy URL for writing data to the corresponding -remoteWrite.url. " +
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234" )
2020-02-23 11:35:47 +00:00
2022-06-30 17:15:56 +00:00
tlsInsecureSkipVerify = flagutil . NewArrayBool ( "remoteWrite.tlsInsecureSkipVerify" , "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url" )
tlsCertFile = flagutil . NewArray ( "remoteWrite.tlsCertFile" , "Optional path to client-side TLS certificate file to use when connecting " +
"to the corresponding -remoteWrite.url" )
tlsKeyFile = flagutil . NewArray ( "remoteWrite.tlsKeyFile" , "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url" )
tlsCAFile = flagutil . NewArray ( "remoteWrite.tlsCAFile" , "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. " +
"By default system CA is used" )
tlsServerName = flagutil . NewArray ( "remoteWrite.tlsServerName" , "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. " +
"By default the server name from -remoteWrite.url is used" )
2020-02-23 11:35:47 +00:00
2022-06-30 17:15:56 +00:00
headers = flagutil . NewArray ( "remoteWrite.headers" , "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. " +
"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. " +
2022-06-30 17:00:03 +00:00
"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'" )
2022-06-30 17:15:56 +00:00
basicAuthUsername = flagutil . NewArray ( "remoteWrite.basicAuth.username" , "Optional basic auth username to use for the corresponding -remoteWrite.url" )
basicAuthPassword = flagutil . NewArray ( "remoteWrite.basicAuth.password" , "Optional basic auth password to use for the corresponding -remoteWrite.url" )
basicAuthPasswordFile = flagutil . NewArray ( "remoteWrite.basicAuth.passwordFile" , "Optional path to basic auth password to use for the corresponding -remoteWrite.url. " +
"The file is re-read every second" )
bearerToken = flagutil . NewArray ( "remoteWrite.bearerToken" , "Optional bearer auth token to use for the corresponding -remoteWrite.url" )
bearerTokenFile = flagutil . NewArray ( "remoteWrite.bearerTokenFile" , "Optional path to bearer token file to use for the corresponding -remoteWrite.url. " +
"The token is re-read from the file every second" )
2021-05-22 13:20:18 +00:00
2022-06-30 17:15:56 +00:00
oauth2ClientID = flagutil . NewArray ( "remoteWrite.oauth2.clientID" , "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url" )
oauth2ClientSecret = flagutil . NewArray ( "remoteWrite.oauth2.clientSecret" , "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url" )
oauth2ClientSecretFile = flagutil . NewArray ( "remoteWrite.oauth2.clientSecretFile" , "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url" )
oauth2TokenURL = flagutil . NewArray ( "remoteWrite.oauth2.tokenUrl" , "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url" )
oauth2Scopes = flagutil . NewArray ( "remoteWrite.oauth2.scopes" , "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'" )
2022-05-04 17:24:19 +00:00
2022-06-30 17:15:56 +00:00
awsUseSigv4 = flagutil . NewArrayBool ( "remoteWrite.aws.useSigv4" , "Enables SigV4 request signing for the corresponding -remoteWrite.url. " +
"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled" )
2022-08-05 15:50:00 +00:00
awsEC2Endpoint = flagutil . NewArray ( "remoteWrite.aws.ec2Endpoint" , "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
awsSTSEndpoint = flagutil . NewArray ( "remoteWrite.aws.stsEndpoint" , "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
awsRegion = flagutil . NewArray ( "remoteWrite.aws.region" , "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
awsRoleARN = flagutil . NewArray ( "remoteWrite.aws.roleARN" , "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
awsAccessKey = flagutil . NewArray ( "remoteWrite.aws.accessKey" , "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
awsService = flagutil . NewArray ( "remoteWrite.aws.service" , "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. " +
2022-06-30 17:15:56 +00:00
"Defaults to \"aps\"" )
awsSecretKey = flagutil . NewArray ( "remoteWrite.aws.secretKey" , "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set" )
2020-02-23 11:35:47 +00:00
)
type client struct {
2020-09-16 19:34:01 +00:00
sanitizedURL string
2020-02-23 11:35:47 +00:00
remoteWriteURL string
fq * persistentqueue . FastQueue
2020-07-20 16:27:25 +00:00
hc * http . Client
2020-02-23 11:35:47 +00:00
2021-09-28 21:52:07 +00:00
sendBlock func ( block [ ] byte ) bool
authCfg * promauth . Config
2022-05-04 17:24:19 +00:00
awsCfg * awsapi . Config
2021-05-22 13:20:18 +00:00
2021-01-26 22:19:35 +00:00
rl rateLimiter
2020-12-15 18:39:12 +00:00
bytesSent * metrics . Counter
blocksSent * metrics . Counter
2020-02-23 11:35:47 +00:00
requestDuration * metrics . Histogram
requestsOKCount * metrics . Counter
errorsCount * metrics . Counter
2020-11-01 22:43:51 +00:00
packetsDropped * metrics . Counter
2022-05-02 19:20:05 +00:00
rateLimit * metrics . Gauge
2020-02-23 11:35:47 +00:00
retriesCount * metrics . Counter
2021-08-15 10:32:40 +00:00
sendDuration * metrics . FloatCounter
2020-02-23 11:35:47 +00:00
wg sync . WaitGroup
stopCh chan struct { }
}
2021-09-28 21:52:07 +00:00
func newHTTPClient ( argIdx int , remoteWriteURL , sanitizedURL string , fq * persistentqueue . FastQueue , concurrency int ) * client {
2021-05-22 14:59:23 +00:00
authCfg , err := getAuthConfig ( argIdx )
2020-07-20 16:27:25 +00:00
if err != nil {
2022-05-04 17:24:19 +00:00
logger . Panicf ( "FATAL: cannot initialize auth config for remoteWrite.url=%q: %s" , remoteWriteURL , err )
2020-07-20 16:27:25 +00:00
}
2021-05-22 14:59:23 +00:00
tlsCfg := authCfg . NewTLSConfig ( )
2022-05-04 17:24:19 +00:00
awsCfg , err := getAWSAPIConfig ( argIdx )
if err != nil {
logger . Fatalf ( "FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s" , remoteWriteURL , err )
}
2020-07-20 16:27:25 +00:00
tr := & http . Transport {
2022-04-06 09:28:54 +00:00
DialContext : statDial ,
2020-07-20 16:27:25 +00:00
TLSClientConfig : tlsCfg ,
2022-04-06 09:28:54 +00:00
TLSHandshakeTimeout : 10 * time . Second ,
2020-07-20 16:27:25 +00:00
MaxConnsPerHost : 2 * concurrency ,
2020-08-04 17:59:55 +00:00
MaxIdleConnsPerHost : 2 * concurrency ,
IdleConnTimeout : time . Minute ,
WriteBufferSize : 64 * 1024 ,
2020-07-20 16:27:25 +00:00
}
pURL := proxyURL . GetOptionalArg ( argIdx )
if len ( pURL ) > 0 {
if ! strings . Contains ( pURL , "://" ) {
logger . Fatalf ( "cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`" , pURL )
}
2021-10-26 18:21:08 +00:00
pu , err := url . Parse ( pURL )
2020-07-20 16:27:25 +00:00
if err != nil {
logger . Fatalf ( "cannot parse -remoteWrite.proxyURL=%q: %s" , pURL , err )
}
2021-10-26 18:21:08 +00:00
tr . Proxy = http . ProxyURL ( pu )
2020-07-20 16:27:25 +00:00
}
2020-02-23 11:35:47 +00:00
c := & client {
2020-09-16 19:34:01 +00:00
sanitizedURL : sanitizedURL ,
2020-02-23 11:35:47 +00:00
remoteWriteURL : remoteWriteURL ,
2021-05-22 13:20:18 +00:00
authCfg : authCfg ,
2022-05-04 17:24:19 +00:00
awsCfg : awsCfg ,
2020-02-23 11:35:47 +00:00
fq : fq ,
2020-07-20 16:27:25 +00:00
hc : & http . Client {
Transport : tr ,
2020-12-15 10:51:12 +00:00
Timeout : sendTimeout . GetOptionalArgOrDefault ( argIdx , time . Minute ) ,
2020-07-20 16:27:25 +00:00
} ,
stopCh : make ( chan struct { } ) ,
2020-02-23 11:35:47 +00:00
}
2021-09-28 21:52:07 +00:00
c . sendBlock = c . sendBlockHTTP
return c
}
func ( c * client ) init ( argIdx , concurrency int , sanitizedURL string ) {
2021-02-01 12:27:05 +00:00
if bytesPerSec := rateLimit . GetOptionalArgOrDefault ( argIdx , 0 ) ; bytesPerSec > 0 {
logger . Infof ( "applying %d bytes per second rate limit for -remoteWrite.url=%q" , bytesPerSec , sanitizedURL )
c . rl . perSecondLimit = int64 ( bytesPerSec )
2021-01-26 22:19:35 +00:00
}
2022-05-06 12:01:52 +00:00
c . rl . limitReached = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_rate_limit_reached_total { url=%q} ` , c . sanitizedURL ) )
2021-01-26 22:19:35 +00:00
2020-12-15 18:39:12 +00:00
c . bytesSent = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_bytes_sent_total { url=%q} ` , c . sanitizedURL ) )
c . blocksSent = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_blocks_sent_total { url=%q} ` , c . sanitizedURL ) )
2022-05-02 19:20:05 +00:00
c . rateLimit = metrics . GetOrCreateGauge ( fmt . Sprintf ( ` vmagent_remotewrite_rate_limit { url=%q} ` , c . sanitizedURL ) , func ( ) float64 {
return float64 ( rateLimit . GetOptionalArgOrDefault ( argIdx , 0 ) )
} )
2020-09-16 19:34:01 +00:00
c . requestDuration = metrics . GetOrCreateHistogram ( fmt . Sprintf ( ` vmagent_remotewrite_duration_seconds { url=%q} ` , c . sanitizedURL ) )
c . requestsOKCount = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_requests_total { url=%q, status_code="2XX"} ` , c . sanitizedURL ) )
c . errorsCount = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_errors_total { url=%q} ` , c . sanitizedURL ) )
2020-11-01 22:43:51 +00:00
c . packetsDropped = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_packets_dropped_total { url=%q} ` , c . sanitizedURL ) )
2020-09-16 19:34:01 +00:00
c . retriesCount = metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_retries_count_total { url=%q} ` , c . sanitizedURL ) )
2021-08-15 10:32:40 +00:00
c . sendDuration = metrics . GetOrCreateFloatCounter ( fmt . Sprintf ( ` vmagent_remotewrite_send_duration_seconds_total { url=%q} ` , c . sanitizedURL ) )
2022-09-08 17:53:51 +00:00
metrics . GetOrCreateGauge ( fmt . Sprintf ( ` vmagent_remotewrite_queues { url=%q} ` , c . sanitizedURL ) , func ( ) float64 {
return float64 ( * queues )
} )
2020-03-03 11:08:17 +00:00
for i := 0 ; i < concurrency ; i ++ {
2020-02-23 11:35:47 +00:00
c . wg . Add ( 1 )
go func ( ) {
defer c . wg . Done ( )
c . runWorker ( )
} ( )
}
2020-09-16 19:34:01 +00:00
logger . Infof ( "initialized client for -remoteWrite.url=%q" , c . sanitizedURL )
2020-02-23 11:35:47 +00:00
}
func ( c * client ) MustStop ( ) {
close ( c . stopCh )
c . wg . Wait ( )
2020-09-16 19:34:01 +00:00
logger . Infof ( "stopped client for -remoteWrite.url=%q" , c . sanitizedURL )
2020-02-23 11:35:47 +00:00
}
2021-05-22 14:59:23 +00:00
func getAuthConfig ( argIdx int ) ( * promauth . Config , error ) {
2022-06-30 17:00:03 +00:00
headersValue := headers . GetOptionalArg ( argIdx )
2022-06-30 17:17:30 +00:00
var hdrs [ ] string
2022-06-30 17:00:03 +00:00
if headersValue != "" {
2022-06-30 17:17:30 +00:00
hdrs = strings . Split ( headersValue , "^^" )
2022-06-30 17:00:03 +00:00
}
2021-05-22 14:59:23 +00:00
username := basicAuthUsername . GetOptionalArg ( argIdx )
password := basicAuthPassword . GetOptionalArg ( argIdx )
passwordFile := basicAuthPasswordFile . GetOptionalArg ( argIdx )
var basicAuthCfg * promauth . BasicAuthConfig
if username != "" || password != "" || passwordFile != "" {
basicAuthCfg = & promauth . BasicAuthConfig {
Username : username ,
2021-11-05 12:41:14 +00:00
Password : promauth . NewSecret ( password ) ,
2021-05-22 14:59:23 +00:00
PasswordFile : passwordFile ,
}
}
token := bearerToken . GetOptionalArg ( argIdx )
tokenFile := bearerTokenFile . GetOptionalArg ( argIdx )
var oauth2Cfg * promauth . OAuth2Config
clientSecret := oauth2ClientSecret . GetOptionalArg ( argIdx )
clientSecretFile := oauth2ClientSecretFile . GetOptionalArg ( argIdx )
if clientSecretFile != "" || clientSecret != "" {
oauth2Cfg = & promauth . OAuth2Config {
ClientID : oauth2ClientID . GetOptionalArg ( argIdx ) ,
2021-11-05 12:41:14 +00:00
ClientSecret : promauth . NewSecret ( clientSecret ) ,
2021-05-22 14:59:23 +00:00
ClientSecretFile : clientSecretFile ,
TokenURL : oauth2TokenURL . GetOptionalArg ( argIdx ) ,
Scopes : strings . Split ( oauth2Scopes . GetOptionalArg ( argIdx ) , ";" ) ,
}
}
tlsCfg := & promauth . TLSConfig {
2020-05-12 14:20:55 +00:00
CAFile : tlsCAFile . GetOptionalArg ( argIdx ) ,
CertFile : tlsCertFile . GetOptionalArg ( argIdx ) ,
KeyFile : tlsKeyFile . GetOptionalArg ( argIdx ) ,
ServerName : tlsServerName . GetOptionalArg ( argIdx ) ,
2020-12-15 10:51:12 +00:00
InsecureSkipVerify : tlsInsecureSkipVerify . GetOptionalArg ( argIdx ) ,
2020-02-23 11:35:47 +00:00
}
2021-05-22 13:20:18 +00:00
2022-07-04 11:27:48 +00:00
opts := & promauth . Options {
BasicAuth : basicAuthCfg ,
BearerToken : token ,
BearerTokenFile : tokenFile ,
OAuth2 : oauth2Cfg ,
TLSConfig : tlsCfg ,
Headers : hdrs ,
}
authCfg , err := opts . NewConfig ( )
2021-05-22 13:20:18 +00:00
if err != nil {
return nil , fmt . Errorf ( "cannot populate OAuth2 config for remoteWrite idx: %d, err: %w" , argIdx , err )
}
return authCfg , nil
}
2022-05-04 17:24:19 +00:00
func getAWSAPIConfig ( argIdx int ) ( * awsapi . Config , error ) {
2022-05-04 17:39:38 +00:00
if ! awsUseSigv4 . GetOptionalArg ( argIdx ) {
2022-05-04 17:24:19 +00:00
return nil , nil
}
2022-08-05 15:50:00 +00:00
ec2Endpoint := awsEC2Endpoint . GetOptionalArg ( argIdx )
stsEndpoint := awsSTSEndpoint . GetOptionalArg ( argIdx )
2022-05-04 17:24:19 +00:00
region := awsRegion . GetOptionalArg ( argIdx )
roleARN := awsRoleARN . GetOptionalArg ( argIdx )
accessKey := awsAccessKey . GetOptionalArg ( argIdx )
secretKey := awsSecretKey . GetOptionalArg ( argIdx )
2022-05-18 12:58:31 +00:00
service := awsService . GetOptionalArg ( argIdx )
2022-08-05 15:50:00 +00:00
cfg , err := awsapi . NewConfig ( ec2Endpoint , stsEndpoint , region , roleARN , accessKey , secretKey , service )
2022-05-04 17:24:19 +00:00
if err != nil {
return nil , err
}
return cfg , nil
}
2020-02-23 11:35:47 +00:00
func ( c * client ) runWorker ( ) {
var ok bool
var block [ ] byte
2021-02-17 19:42:45 +00:00
ch := make ( chan bool , 1 )
2020-02-23 11:35:47 +00:00
for {
block , ok = c . fq . MustReadBlock ( block [ : 0 ] )
if ! ok {
return
}
go func ( ) {
2021-08-15 10:32:40 +00:00
startTime := time . Now ( )
2021-02-17 19:42:45 +00:00
ch <- c . sendBlock ( block )
2021-08-15 10:32:40 +00:00
c . sendDuration . Add ( time . Since ( startTime ) . Seconds ( ) )
2020-02-23 11:35:47 +00:00
} ( )
select {
2021-02-17 19:42:45 +00:00
case ok := <- ch :
if ok {
// The block has been sent successfully
continue
}
// Return unsent block to the queue.
c . fq . MustWriteBlock ( block )
return
2020-02-23 11:35:47 +00:00
case <- c . stopCh :
// c must be stopped. Wait for a while in the hope the block will be sent.
graceDuration := 5 * time . Second
select {
2021-02-17 19:42:45 +00:00
case ok := <- ch :
if ! ok {
// Return unsent block to the queue.
c . fq . MustWriteBlock ( block )
}
2020-02-23 11:35:47 +00:00
case <- time . After ( graceDuration ) :
2021-02-17 19:42:45 +00:00
// Return unsent block to the queue.
2021-02-17 19:23:38 +00:00
c . fq . MustWriteBlock ( block )
2020-02-23 11:35:47 +00:00
}
return
}
}
}
2021-09-28 21:52:07 +00:00
// sendBlockHTTP returns false only if c.stopCh is closed.
2021-02-17 19:42:45 +00:00
// Otherwise it tries sending the block to remote storage indefinitely.
2021-09-28 21:52:07 +00:00
func ( c * client ) sendBlockHTTP ( block [ ] byte ) bool {
2021-01-26 22:19:35 +00:00
c . rl . register ( len ( block ) , c . stopCh )
2020-07-27 14:31:36 +00:00
retryDuration := time . Second
2020-08-30 18:39:45 +00:00
retriesCount := 0
2020-12-15 18:39:12 +00:00
c . bytesSent . Add ( len ( block ) )
c . blocksSent . Inc ( )
2022-05-04 17:24:19 +00:00
sigv4Hash := ""
if c . awsCfg != nil {
sigv4Hash = awsapi . HashHex ( block )
}
2020-07-27 14:31:36 +00:00
again :
2020-07-20 16:27:25 +00:00
req , err := http . NewRequest ( "POST" , c . remoteWriteURL , bytes . NewBuffer ( block ) )
if err != nil {
2021-12-20 15:39:43 +00:00
logger . Panicf ( "BUG: unexpected error from http.NewRequest(%q): %s" , c . sanitizedURL , err )
2020-07-20 16:27:25 +00:00
}
2022-06-30 17:00:03 +00:00
c . authCfg . SetHeaders ( req , true )
2020-07-20 16:27:25 +00:00
h := req . Header
h . Set ( "User-Agent" , "vmagent" )
h . Set ( "Content-Type" , "application/x-protobuf" )
h . Set ( "Content-Encoding" , "snappy" )
h . Set ( "X-Prometheus-Remote-Write-Version" , "0.1.0" )
2022-05-04 17:24:19 +00:00
if c . awsCfg != nil {
2022-05-18 12:58:31 +00:00
if err := c . awsCfg . SignRequest ( req , sigv4Hash ) ; err != nil {
2022-05-04 17:24:19 +00:00
// there is no need in retry, request will be rejected by client.Do and retried by code below
logger . Warnf ( "cannot sign remoteWrite request with AWS sigv4: %s" , err )
}
}
2020-02-23 11:35:47 +00:00
startTime := time . Now ( )
2020-07-20 16:27:25 +00:00
resp , err := c . hc . Do ( req )
2020-02-23 11:35:47 +00:00
c . requestDuration . UpdateDuration ( startTime )
if err != nil {
c . errorsCount . Inc ( )
retryDuration *= 2
if retryDuration > time . Minute {
retryDuration = time . Minute
}
2021-05-24 12:42:43 +00:00
logger . Warnf ( "couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds" ,
2020-09-16 19:34:01 +00:00
len ( block ) , c . sanitizedURL , err , retryDuration . Seconds ( ) )
2021-01-26 22:23:10 +00:00
t := timerpool . Get ( retryDuration )
2020-07-20 16:27:25 +00:00
select {
case <- c . stopCh :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2021-02-17 19:23:38 +00:00
return false
2020-07-20 16:27:25 +00:00
case <- t . C :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2020-07-20 16:27:25 +00:00
}
2020-02-23 11:35:47 +00:00
c . retriesCount . Inc ( )
goto again
}
2020-07-20 16:27:25 +00:00
statusCode := resp . StatusCode
2020-07-28 17:52:00 +00:00
if statusCode / 100 == 2 {
_ = resp . Body . Close ( )
c . requestsOKCount . Inc ( )
2021-02-17 19:23:38 +00:00
return true
2020-07-28 17:52:00 +00:00
}
2020-11-01 22:43:51 +00:00
metrics . GetOrCreateCounter ( fmt . Sprintf ( ` vmagent_remotewrite_requests_total { url=%q, status_code="%d"} ` , c . sanitizedURL , statusCode ) ) . Inc ( )
2021-03-26 11:17:59 +00:00
if statusCode == 409 || statusCode == 400 {
2021-12-21 14:36:09 +00:00
body , err := ioutil . ReadAll ( resp . Body )
_ = resp . Body . Close ( )
if err != nil {
2022-06-27 09:31:16 +00:00
remoteWriteRejectedLogger . Errorf ( "sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; " +
2021-12-21 14:36:09 +00:00
"failed to read response body: %s" ,
len ( block ) , c . sanitizedURL , statusCode , err )
} else {
2022-06-27 09:31:16 +00:00
remoteWriteRejectedLogger . Errorf ( "sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s" ,
2021-12-21 14:36:09 +00:00
len ( block ) , c . sanitizedURL , statusCode , string ( body ) )
}
2021-05-13 13:16:16 +00:00
// Just drop block on 409 and 400 status codes like Prometheus does.
2020-11-01 22:43:51 +00:00
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
2021-05-13 13:16:16 +00:00
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
2020-11-01 22:43:51 +00:00
_ = resp . Body . Close ( )
c . packetsDropped . Inc ( )
2021-02-17 19:23:38 +00:00
return true
2020-11-01 22:43:51 +00:00
}
2020-07-28 17:52:00 +00:00
// Unexpected status code returned
2020-08-30 18:39:45 +00:00
retriesCount ++
2020-07-28 17:52:00 +00:00
retryDuration *= 2
if retryDuration > time . Minute {
retryDuration = time . Minute
}
body , err := ioutil . ReadAll ( resp . Body )
_ = resp . Body . Close ( )
if err != nil {
2020-09-16 19:34:01 +00:00
logger . Errorf ( "cannot read response body from %q during retry #%d: %s" , c . sanitizedURL , retriesCount , err )
2020-07-28 17:52:00 +00:00
} else {
2020-08-30 18:39:45 +00:00
logger . Errorf ( "unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; " +
2020-09-16 19:34:01 +00:00
"re-sending the block in %.3f seconds" , len ( block ) , c . sanitizedURL , retriesCount , statusCode , body , retryDuration . Seconds ( ) )
2020-07-28 17:52:00 +00:00
}
2021-01-26 22:23:10 +00:00
t := timerpool . Get ( retryDuration )
2020-07-28 17:52:00 +00:00
select {
case <- c . stopCh :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2021-02-17 19:23:38 +00:00
return false
2020-07-28 17:52:00 +00:00
case <- t . C :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2020-02-23 11:35:47 +00:00
}
2020-07-28 17:52:00 +00:00
c . retriesCount . Inc ( )
goto again
2020-04-17 12:51:29 +00:00
}
2021-01-26 22:19:35 +00:00
2022-06-27 09:31:16 +00:00
var remoteWriteRejectedLogger = logger . WithThrottler ( "remoteWriteRejected" , 5 * time . Second )
2021-01-26 22:19:35 +00:00
type rateLimiter struct {
perSecondLimit int64
2021-02-28 22:28:20 +00:00
// mu protects budget and deadline from concurrent access.
mu sync . Mutex
2021-01-26 22:19:35 +00:00
// The current budget. It is increased by perSecondLimit every second.
budget int64
// The next deadline for increasing the budget by perSecondLimit
deadline time . Time
limitReached * metrics . Counter
}
func ( rl * rateLimiter ) register ( dataLen int , stopCh <- chan struct { } ) {
limit := rl . perSecondLimit
if limit <= 0 {
return
}
2021-02-28 22:28:20 +00:00
rl . mu . Lock ( )
defer rl . mu . Unlock ( )
2021-01-26 22:19:35 +00:00
for rl . budget <= 0 {
2021-02-28 22:58:31 +00:00
if d := time . Until ( rl . deadline ) ; d > 0 {
2021-01-26 22:19:35 +00:00
rl . limitReached . Inc ( )
2021-01-26 22:23:10 +00:00
t := timerpool . Get ( d )
2021-01-26 22:19:35 +00:00
select {
case <- stopCh :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2021-01-26 22:19:35 +00:00
return
case <- t . C :
2021-01-26 22:23:10 +00:00
timerpool . Put ( t )
2021-01-26 22:19:35 +00:00
}
}
rl . budget += limit
2021-02-28 22:58:31 +00:00
rl . deadline = time . Now ( ) . Add ( time . Second )
2021-01-26 22:19:35 +00:00
}
rl . budget -= int64 ( dataLen )
}