github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promscrape: properly retry requests on the server closed connection before returning the first response byte error during service discover API calls and target scrapes

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-08-13 22:31:42 +03:00
parent c402265e88
commit eead3ee8ec
2 changed files with 34 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
lib/promscrape/client.go
View file

@ -152,21 +152,19 @@ var (
)
func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response, deadline time.Time) error {
attempts := 0
again:
// Use DoDeadline instead of Do even if hc.ReadTimeout is already set in order to guarantee the given deadline
// across multiple retries.
err := hc.DoDeadline(req, resp, deadline)
if err == nil {
return nil
for {
// Use DoDeadline instead of Do even if hc.ReadTimeout is already set in order to guarantee the given deadline
// across multiple retries.
err := hc.DoDeadline(req, resp, deadline)
if err == nil {
return nil
}
if err != fasthttp.ErrConnectionClosed {
return err
}
// Retry request if the server closes the keep-alive connection unless deadline exceeds.
if time.Since(deadline) >= 0 {
return fmt.Errorf("the server closes all the connection attempts: %w", err)
}
}
if err != fasthttp.ErrConnectionClosed {
return err
}
// Retry request if the server closes the keep-alive connection unless deadline exceeds.
attempts++
if attempts > 3 {
return fmt.Errorf("the server closed 3 subsequent connections: %w", err)
}
goto again
}

22
lib/promscrape/discoveryutils/client.go
View file

@ -110,8 +110,8 @@ func (c *Client) GetAPIResponse(path string) ([]byte, error) {
req.Header.Set("Authorization", c.ac.Authorization)
}
var resp fasthttp.Response
// There is no need in calling DoTimeout, since the timeout is already set in c.hc.ReadTimeout above.
if err := c.hc.Do(&req, &resp); err != nil {
deadline := time.Now().Add(c.hc.ReadTimeout)
if err := doRequestWithPossibleRetry(c.hc, &req, &resp, deadline); err != nil {
return nil, fmt.Errorf("cannot fetch %q: %w", requestURL, err)
}
var data []byte
@ -131,3 +131,21 @@ func (c *Client) GetAPIResponse(path string) ([]byte, error) {
}
return data, nil
}
func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response, deadline time.Time) error {
for {
// Use DoDeadline instead of Do even if hc.ReadTimeout is already set in order to guarantee the given deadline
// across multiple retries.
err := hc.DoDeadline(req, resp, deadline)
if err == nil {
return nil
}
if err != fasthttp.ErrConnectionClosed {
return err
}
// Retry request if the server closes the keep-alive connection unless deadline exceeds.
if time.Since(deadline) >= 0 {
return fmt.Errorf("the server closes all the connection attempts: %w", err)
}
}
}