From 992a1c0a3a2e4bb6ac9ad9be7c9621a3ac850a26 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 24 Aug 2023 00:08:04 +0200 Subject: [PATCH] vmagent: retry failed write request on the closed connection (#4857) * vmagent: retry failed write request on the closed connection Retry failed write request on the closed connection immediately, without waiting for backoff. This should improve data delivery speed and reduce amount of error logs emitted by vmagent when using idle connections. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139 Signed-off-by: hagen1778 * vmagent: retry failed write request on the closed connection Re-instantinate request before retry as body could have been already spoiled. Signed-off-by: hagen1778 --------- Signed-off-by: hagen1778 Co-authored-by: Nikolay --- app/vmagent/remotewrite/client.go | 16 +++++++++++++++- docs/CHANGELOG.md | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index 05162e22d3..fa0720efba 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -2,6 +2,7 @@ package remotewrite import ( "bytes" + "errors" "fmt" "io" "net/http" @@ -322,6 +323,19 @@ func (c *client) runWorker() { } func (c *client) doRequest(url string, body []byte) (*http.Response, error) { + req := c.newRequest(url, body) + resp, err := c.hc.Do(req) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + // it is likely connection become stale. So we do one more attempt, + // so we do a one more attempt in hope request will succeed. + // If not, the error should be handled by the caller as usual. + req = c.newRequest(url, body) + resp, err = c.hc.Do(req) + } + return resp, err +} + +func (c *client) newRequest(url string, body []byte) *http.Request { reqBody := bytes.NewBuffer(body) req, err := http.NewRequest(http.MethodPost, url, reqBody) if err != nil { @@ -345,7 +359,7 @@ func (c *client) doRequest(url string, body []byte) (*http.Response, error) { logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err) } } - return c.hc.Do(req) + return req } // sendBlockHTTP sends the given block to c.remoteWriteURL. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 22b04dadb2..4a19830508 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,6 +26,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): add support for server-side copy of existing backups. See [these docs](https://docs.victoriametrics.com/vmbackup.html#server-side-copy-of-the-existing-backup) for details. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly handle `unexpected EOF` error when parsing metrics in Prometheus exposition format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4817). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): retry failed write request on the closed connection immediately, without waiting for backoff. This should improve data delivery speed and reduce amount of error logs emitted by vmagent when using idle connections. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): make the warning message more noticeable for text fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4848). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add button for auto-formatting PromQL/MetricsQL queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4681). Thanks to @aramattamara for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4694).