From 1ff35996cd71530783cb923a053baf37d8a6a832 Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <roman@victoriametrics.com>
Date: Thu, 24 Aug 2023 00:08:04 +0200
Subject: [PATCH] vmagent: retry failed write request on the closed connection
(#4857)
* vmagent: retry failed write request on the closed connection
Retry failed write request on the closed connection immediately,
without waiting for backoff. This should improve data delivery speed
and reduce amount of error logs emitted by vmagent when using idle connections.
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
Signed-off-by: hagen1778 <roman@victoriametrics.com>
* vmagent: retry failed write request on the closed connection
Re-instantinate request before retry as body could have been already spoiled.
Signed-off-by: hagen1778 <roman@victoriametrics.com>
---------
Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Nikolay <nik@victoriametrics.com>
---
app/vmagent/remotewrite/client.go | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go
index 05162e22d3..fa0720efba 100644
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -2,6 +2,7 @@ package remotewrite
import (
"bytes"
+ "errors"
"fmt"
"io"
"net/http"
@@ -322,6 +323,19 @@ func (c *client) runWorker() {
}
func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+ req := c.newRequest(url, body)
+ resp, err := c.hc.Do(req)
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ // it is likely connection become stale. So we do one more attempt,
+ // so we do a one more attempt in hope request will succeed.
+ // If not, the error should be handled by the caller as usual.
+ req = c.newRequest(url, body)
+ resp, err = c.hc.Do(req)
+ }
+ return resp, err
+}
+
+func (c *client) newRequest(url string, body []byte) *http.Request {
reqBody := bytes.NewBuffer(body)
req, err := http.NewRequest(http.MethodPost, url, reqBody)
if err != nil {
@@ -345,7 +359,7 @@ func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
}
}
- return c.hc.Do(req)
+ return req
}
// sendBlockHTTP sends the given block to c.remoteWriteURL.