2020-04-29 14:27:08 +00:00
package discoveryutils
import (
2023-01-06 03:34:47 +00:00
"context"
2020-05-04 17:48:02 +00:00
"crypto/tls"
2020-05-19 14:35:47 +00:00
"flag"
2020-05-04 17:48:02 +00:00
"fmt"
2023-01-06 03:34:47 +00:00
"io"
2020-05-04 17:48:02 +00:00
"net"
2020-04-29 14:27:08 +00:00
"net/http"
2023-01-06 03:34:47 +00:00
"net/url"
2020-05-04 17:48:02 +00:00
"strings"
2020-05-19 14:35:47 +00:00
"sync"
2020-04-29 14:27:08 +00:00
"time"
2020-05-04 17:48:02 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
2020-12-24 08:56:10 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
2020-05-19 14:35:47 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
2021-02-01 18:02:51 +00:00
"github.com/VictoriaMetrics/metrics"
2020-04-29 14:27:08 +00:00
)
2020-05-19 14:35:47 +00:00
var (
2020-06-20 14:52:49 +00:00
maxConcurrency = flag . Int ( "promscrape.discovery.concurrency" , 100 , "The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.)" )
2020-05-19 14:35:47 +00:00
maxWaitTime = flag . Duration ( "promscrape.discovery.concurrentWaitTime" , time . Minute , "The maximum duration for waiting to perform API requests " +
"if more than -promscrape.discovery.concurrency requests are simultaneously performed" )
)
2020-04-29 14:27:08 +00:00
var defaultClient = & http . Client {
Timeout : 30 * time . Second ,
}
2023-01-06 03:34:47 +00:00
var (
concurrencyLimitCh chan struct { }
concurrencyLimitChOnce sync . Once
)
const (
// BlockingClientReadTimeout is the maximum duration for waiting the response from GetBlockingAPI*
BlockingClientReadTimeout = 10 * time . Minute
// DefaultClientReadTimeout is the maximum duration for waiting the response from GetAPI*
DefaultClientReadTimeout = time . Minute
)
func concurrencyLimitChInit ( ) {
concurrencyLimitCh = make ( chan struct { } , * maxConcurrency )
}
2020-04-29 14:27:08 +00:00
// GetHTTPClient returns default client for http API requests.
func GetHTTPClient ( ) * http . Client {
return defaultClient
}
2020-05-04 17:48:02 +00:00
2023-01-06 05:13:02 +00:00
// Client is http client, which talks to the given apiServer passed to NewClient().
2020-05-04 17:48:02 +00:00
type Client struct {
2023-01-06 03:34:47 +00:00
// client is used for short requests.
client * HTTPClient
2020-12-03 17:50:50 +00:00
// blockingClient is used for long-polling requests.
2023-01-06 03:34:47 +00:00
blockingClient * HTTPClient
2020-12-03 17:50:50 +00:00
apiServer string
2021-04-03 21:40:08 +00:00
2023-01-06 03:34:47 +00:00
setHTTPHeaders func ( req * http . Request )
setHTTPProxyHeaders func ( req * http . Request )
clientCtx context . Context
clientCancel context . CancelFunc
}
// HTTPClient is a wrapper around http.Client with timeouts.
type HTTPClient struct {
2023-01-07 09:26:31 +00:00
client * http . Client
ReadTimeout time . Duration
2020-05-04 17:48:02 +00:00
}
2023-07-27 21:47:53 +00:00
func ( hc * HTTPClient ) stop ( ) {
// Close idle connections to server in order to free up resources.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724
hc . client . CloseIdleConnections ( )
}
2023-01-06 05:13:02 +00:00
var defaultDialer = & net . Dialer { }
2022-07-06 23:25:31 +00:00
2021-04-03 21:40:08 +00:00
// NewClient returns new Client for the given args.
2021-10-26 18:21:08 +00:00
func NewClient ( apiServer string , ac * promauth . Config , proxyURL * proxy . URL , proxyAC * promauth . Config ) ( * Client , error ) {
2023-01-06 03:34:47 +00:00
u , err := url . Parse ( apiServer )
if err != nil {
2023-01-06 05:13:02 +00:00
return nil , fmt . Errorf ( "cannot parse apiServer=%q: %w" , apiServer , err )
2023-01-06 03:34:47 +00:00
}
2020-10-12 10:38:21 +00:00
2023-01-06 05:13:02 +00:00
dialFunc := defaultDialer . DialContext
if u . Scheme == "unix" {
// special case for unix socket connection
2023-01-06 03:34:47 +00:00
dialAddr := u . Path
2023-01-06 05:13:02 +00:00
apiServer = "http://unix"
dialFunc = func ( ctx context . Context , _ , _ string ) ( net . Conn , error ) {
return defaultDialer . DialContext ( ctx , "unix" , dialAddr )
2020-10-12 10:38:21 +00:00
}
}
2020-12-24 08:52:37 +00:00
2023-01-06 05:13:02 +00:00
isTLS := u . Scheme == "https"
2021-04-03 21:40:08 +00:00
var tlsCfg * tls . Config
2021-03-09 16:54:09 +00:00
if isTLS {
2020-05-04 17:48:02 +00:00
tlsCfg = ac . NewTLSConfig ( )
}
2023-01-06 03:34:47 +00:00
2023-01-06 05:13:02 +00:00
var proxyURLFunc func ( * http . Request ) ( * url . URL , error )
if pu := proxyURL . GetURL ( ) ; pu != nil {
proxyURLFunc = http . ProxyURL ( pu )
2020-12-24 08:56:10 +00:00
}
2023-01-06 03:34:47 +00:00
2023-01-06 05:13:02 +00:00
client := & http . Client {
Timeout : DefaultClientReadTimeout ,
Transport : & http . Transport {
TLSClientConfig : tlsCfg ,
Proxy : proxyURLFunc ,
TLSHandshakeTimeout : 10 * time . Second ,
MaxIdleConnsPerHost : * maxConcurrency ,
ResponseHeaderTimeout : DefaultClientReadTimeout ,
DialContext : dialFunc ,
2023-01-06 03:34:47 +00:00
} ,
}
blockingClient := & http . Client {
2023-01-06 05:13:02 +00:00
Timeout : BlockingClientReadTimeout ,
Transport : & http . Transport {
TLSClientConfig : tlsCfg ,
Proxy : proxyURLFunc ,
TLSHandshakeTimeout : 10 * time . Second ,
MaxIdleConnsPerHost : 1000 ,
ResponseHeaderTimeout : BlockingClientReadTimeout ,
DialContext : dialFunc ,
} ,
2021-04-03 21:40:08 +00:00
}
2020-05-04 17:48:02 +00:00
2023-01-06 03:34:47 +00:00
setHTTPHeaders := func ( req * http . Request ) { }
if ac != nil {
2023-01-06 05:13:02 +00:00
setHTTPHeaders = func ( req * http . Request ) {
ac . SetHeaders ( req , true )
}
}
setHTTPProxyHeaders := func ( req * http . Request ) { }
if proxyAC != nil {
setHTTPProxyHeaders = func ( req * http . Request ) {
proxyURL . SetHeaders ( proxyAC , req )
}
2023-01-06 03:34:47 +00:00
}
ctx , cancel := context . WithCancel ( context . Background ( ) )
2020-05-19 14:35:47 +00:00
2023-01-06 05:13:02 +00:00
c := & Client {
client : & HTTPClient {
2023-01-07 09:26:31 +00:00
client : client ,
ReadTimeout : DefaultClientReadTimeout ,
2023-01-06 05:13:02 +00:00
} ,
blockingClient : & HTTPClient {
2023-01-07 09:26:31 +00:00
client : blockingClient ,
ReadTimeout : BlockingClientReadTimeout ,
2023-01-06 05:13:02 +00:00
} ,
2023-01-06 03:34:47 +00:00
apiServer : apiServer ,
setHTTPHeaders : setHTTPHeaders ,
setHTTPProxyHeaders : setHTTPProxyHeaders ,
clientCtx : ctx ,
clientCancel : cancel ,
2023-01-06 05:13:02 +00:00
}
return c , nil
2020-12-03 17:47:40 +00:00
}
2023-01-18 05:47:11 +00:00
// Context returns context for the client requests.
func ( c * Client ) Context ( ) context . Context {
return c . clientCtx
}
2021-06-22 10:33:37 +00:00
// GetAPIResponseWithReqParams returns response for given absolute path with optional callback for request.
2023-01-06 05:13:02 +00:00
func ( c * Client ) GetAPIResponseWithReqParams ( path string , modifyRequest func ( request * http . Request ) ) ( [ ] byte , error ) {
return c . getAPIResponse ( path , modifyRequest )
2021-06-22 10:33:37 +00:00
}
2020-05-04 17:48:02 +00:00
// GetAPIResponse returns response for the given absolute path.
func ( c * Client ) GetAPIResponse ( path string ) ( [ ] byte , error ) {
2021-06-22 10:33:37 +00:00
return c . getAPIResponse ( path , nil )
}
// GetAPIResponse returns response for the given absolute path with optional callback for request.
2023-01-06 03:34:47 +00:00
func ( c * Client ) getAPIResponse ( path string , modifyRequest func ( request * http . Request ) ) ( [ ] byte , error ) {
2020-05-19 14:35:47 +00:00
// Limit the number of concurrent API requests.
concurrencyLimitChOnce . Do ( concurrencyLimitChInit )
t := timerpool . Get ( * maxWaitTime )
select {
case concurrencyLimitCh <- struct { } { } :
timerpool . Put ( t )
case <- t . C :
timerpool . Put ( t )
return nil , fmt . Errorf ( "too many outstanding requests to %q; try increasing -promscrape.discovery.concurrentWaitTime=%s or -promscrape.discovery.concurrency=%d" ,
c . apiServer , * maxWaitTime , * maxConcurrency )
}
2023-01-06 05:13:02 +00:00
defer func ( ) {
<- concurrencyLimitCh
} ( )
2023-01-18 05:47:11 +00:00
return c . getAPIResponseWithParamsAndClientCtx ( c . clientCtx , c . client , path , modifyRequest , nil )
2020-12-03 17:47:40 +00:00
}
2020-05-19 14:35:47 +00:00
2020-12-03 17:47:40 +00:00
// GetBlockingAPIResponse returns response for given absolute path with blocking client and optional callback for api response,
2023-01-06 03:34:47 +00:00
func ( c * Client ) GetBlockingAPIResponse ( path string , inspectResponse func ( resp * http . Response ) ) ( [ ] byte , error ) {
2023-01-18 05:47:11 +00:00
return c . getAPIResponseWithParamsAndClientCtx ( c . clientCtx , c . blockingClient , path , nil , inspectResponse )
}
// GetBlockingAPIResponseCtx returns response for given absolute path with blocking client and optional callback for api response,
func ( c * Client ) GetBlockingAPIResponseCtx ( ctx context . Context , path string , inspectResponse func ( resp * http . Response ) ) ( [ ] byte , error ) {
return c . getAPIResponseWithParamsAndClientCtx ( ctx , c . blockingClient , path , nil , inspectResponse )
2020-12-03 17:47:40 +00:00
}
2021-06-22 10:33:37 +00:00
// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for request and for response.
2023-01-18 05:47:11 +00:00
func ( c * Client ) getAPIResponseWithParamsAndClientCtx ( ctx context . Context , client * HTTPClient , path string , modifyRequest func ( req * http . Request ) , inspectResponse func ( resp * http . Response ) ) ( [ ] byte , error ) {
2020-05-04 17:48:02 +00:00
requestURL := c . apiServer + path
2023-01-06 03:34:47 +00:00
u , err := url . Parse ( requestURL )
if err != nil {
return nil , fmt . Errorf ( "cannot parse %q: %w" , requestURL , err )
}
2023-01-07 09:13:03 +00:00
deadline := time . Now ( ) . Add ( client . ReadTimeout )
2023-01-18 05:47:11 +00:00
ctx , cancel := context . WithDeadline ( ctx , deadline )
2023-01-06 03:34:47 +00:00
defer cancel ( )
req , err := http . NewRequestWithContext ( ctx , "GET" , u . String ( ) , nil )
if err != nil {
return nil , fmt . Errorf ( "cannot create request for %q: %w" , requestURL , err )
2021-04-03 21:40:08 +00:00
}
2023-01-06 03:34:47 +00:00
c . setHTTPHeaders ( req )
c . setHTTPProxyHeaders ( req )
2021-06-22 10:33:37 +00:00
if modifyRequest != nil {
2023-01-06 03:34:47 +00:00
modifyRequest ( req )
2021-06-22 10:33:37 +00:00
}
2020-12-03 17:47:40 +00:00
2023-02-24 20:11:44 +00:00
resp , err := doRequestWithPossibleRetry ( client , req )
2023-01-06 03:34:47 +00:00
if err != nil {
2020-06-30 19:58:18 +00:00
return nil , fmt . Errorf ( "cannot fetch %q: %w" , requestURL , err )
2020-05-04 17:48:02 +00:00
}
2023-01-06 05:13:02 +00:00
data , err := io . ReadAll ( resp . Body )
_ = resp . Body . Close ( )
2023-01-06 03:34:47 +00:00
if err != nil {
2023-01-06 05:13:02 +00:00
return nil , fmt . Errorf ( "cannot read response from %q: %w" , requestURL , err )
2023-01-06 03:34:47 +00:00
}
2020-12-03 17:47:40 +00:00
if inspectResponse != nil {
2023-01-06 03:34:47 +00:00
inspectResponse ( resp )
2020-12-03 17:47:40 +00:00
}
2023-01-06 03:34:47 +00:00
statusCode := resp . StatusCode
if statusCode != http . StatusOK {
2020-05-04 17:48:02 +00:00
return nil , fmt . Errorf ( "unexpected status code returned from %q: %d; expecting %d; response body: %q" ,
2023-01-06 03:34:47 +00:00
requestURL , statusCode , http . StatusOK , data )
2020-05-04 17:48:02 +00:00
}
return data , nil
}
2020-08-13 19:31:42 +00:00
2022-12-09 02:29:10 +00:00
// APIServer returns the API server address
func ( c * Client ) APIServer ( ) string {
return c . apiServer
}
2023-01-06 03:34:47 +00:00
// Stop cancels all in-flight requests
func ( c * Client ) Stop ( ) {
c . clientCancel ( )
2023-07-27 21:47:53 +00:00
c . client . stop ( )
c . blockingClient . stop ( )
2023-01-06 03:34:47 +00:00
}
2023-02-24 20:11:44 +00:00
func doRequestWithPossibleRetry ( hc * HTTPClient , req * http . Request ) ( * http . Response , error ) {
2023-01-06 05:13:02 +00:00
discoveryRequests . Inc ( )
2023-01-06 03:34:47 +00:00
2023-02-24 19:39:56 +00:00
var (
reqErr error
resp * http . Response
)
// Return true if the request execution is completed and retry is not required
attempt := func ( ) bool {
resp , reqErr = hc . client . Do ( req )
if reqErr == nil {
2023-01-06 03:34:47 +00:00
statusCode := resp . StatusCode
2023-02-24 20:11:44 +00:00
if statusCode != http . StatusTooManyRequests {
2023-02-24 19:39:56 +00:00
return true
2022-08-16 11:52:38 +00:00
}
2023-02-24 20:11:44 +00:00
} else if reqErr != net . ErrClosed && ! strings . Contains ( reqErr . Error ( ) , "broken pipe" ) {
2023-02-24 19:39:56 +00:00
return true
2020-08-13 19:31:42 +00:00
}
2023-02-24 19:39:56 +00:00
return false
}
if attempt ( ) {
return resp , reqErr
}
2023-02-24 20:11:44 +00:00
// The first attempt was unsuccessful. Use exponential backoff for further attempts.
// Perform the second attempt immediately after the first attempt - this should help
// in cases when the remote side closes the keep-alive connection before the first attempt.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3293
2023-02-24 19:39:56 +00:00
sleepTime := time . Second
2023-02-24 20:11:44 +00:00
// It is expected that the deadline is already set to req.Context(), so the loop below
// should eventually finish if all the attempt() calls are unsuccessful.
ctx := req . Context ( )
2023-02-24 19:39:56 +00:00
for {
discoveryRetries . Inc ( )
if attempt ( ) {
return resp , reqErr
}
2021-05-13 07:38:43 +00:00
sleepTime += sleepTime
2023-02-24 20:11:44 +00:00
if ! SleepCtx ( ctx , sleepTime ) {
return resp , reqErr
2021-05-13 07:38:43 +00:00
}
2020-08-13 19:31:42 +00:00
}
}
2021-02-01 18:02:51 +00:00
var (
discoveryRequests = metrics . NewCounter ( ` vm_promscrape_discovery_requests_total ` )
2022-08-16 11:52:38 +00:00
discoveryRetries = metrics . NewCounter ( ` vm_promscrape_discovery_retries_total ` )
2021-02-01 18:02:51 +00:00
)
2023-02-24 20:11:44 +00:00
// SleepCtx sleeps for sleepDuration.
//
// It immediately returns false on ctx cancel or deadline, without waiting for sleepDuration.
func SleepCtx ( ctx context . Context , sleepDuration time . Duration ) bool {
t := timerpool . Get ( sleepDuration )
defer timerpool . Put ( t )
select {
case <- ctx . Done ( ) :
return false
case <- t . C :
return true
}
}