From e963d6c789c2d6583651e1b47175f82b117be56b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 13 Feb 2024 02:36:35 +0200 Subject: [PATCH] app/vmagent/remotewrite: add -remoteWrite.tlsHandshakeTimeout command-line flag for tuning tls handshake timeout to -remoteWrite.url Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1699 --- app/vmagent/remotewrite/client.go | 3 ++- docs/CHANGELOG.md | 1 + docs/vmagent.md | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index d5d4baefb6..2181828434 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -35,6 +35,7 @@ var ( proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+ "Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234") + tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url") tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url") tlsCertFile = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+ "to the corresponding -remoteWrite.url") @@ -122,7 +123,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste tr := &http.Transport{ DialContext: statDial, TLSClientConfig: tlsCfg, - TLSHandshakeTimeout: 10 * time.Second, + TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx), MaxConnsPerHost: 2 * concurrency, MaxIdleConnsPerHost: 2 * concurrency, IdleConnTimeout: time.Minute, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 970167ca9d..c6535c5ac8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -37,6 +37,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: all VictoriaMetrics components: do not close connections to `-httpListenAddr` every 2 minutes. This behavior didn't help spreading load among multiple backend servers behind load-balancing TCP proxy. Instead, it could lead to hard-to-debug issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1304#issuecomment-1636997037). If you still need periodically closing client connections because of some reason, then pass the desired timeout to `-http.connTimeout` command-line flag. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add support for [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication)-based request routing to different backends depending on the subject of the TLS certificate provided by the client. See [these docs](https://docs.victoriametrics.com/vmauth.html#mtls-based-request-routing). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1547). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html) and [single-node VictoriaMetrics](https://docs.victoriametrics.com): add support for data ingestion via [DataDog lambda extension](https://docs.datadoghq.com/serverless/libraries_integrations/extension/) aka `/api/beta/sketches` endpoint. See [these docs](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3091). Thanks to @AndrewChubatiuk for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5584). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.tlsHandshakeTimeout` command-line flag for tuning the timeout needed for establishing TLS connections to `-remoteWrite.url`. Setting bigger tls handshake timeouts should reduce the probability of `http: TLS handshake error from ...: EOF` errors at the remote storage side under high load. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1699). * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add `-disableReroutingOnUnavailable` command-line flag to `vminsert`, which can be used for reducing resource usage spikes at `vmstorage` nodes during rolling restart. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#improving-re-routing-performance-during-restart). Thanks to @Muxa1L for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5713). * FEATURE: add `-search.resetRollupResultCacheOnStartup` command-line flag for resetting [query cache](https://docs.victoriametrics.com/#rollup-result-cache) on startup. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/834). * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Targets scraped/s` stat panel showing the number of targets scraped by the vmagent per-second. diff --git a/docs/vmagent.md b/docs/vmagent.md index d855f66436..52e0db1955 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -2119,6 +2119,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Optional path to client-side TLS certificate file to use when connecting to the corresponding -remoteWrite.url Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -remoteWrite.tlsHandshakeTimeout array + The timeout for estabilishing tls connections to the corresponding -remoteWrite.url (default 20s) + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to default value. -remoteWrite.tlsInsecureSkipVerify array Whether to skip tls verification when connecting to the corresponding -remoteWrite.url Supports array of values separated by comma or specified via multiple flags.