From dfb05c884b9015da924c9fc7ee83a11587962f60 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 8 Jun 2023 17:31:05 +0200 Subject: [PATCH] lib/vmselectapi: suppress "broken pipe" error logs on vmstorage side (#4418) The "broken pipe" error is emitted when the connection has been interrupted abruptly. It could happen due to unexpected network glitch or because connection was interrupted by remote client. In both cases, remote client will notice connection breach and handle it on its own. No need in logging this error on both: server and client side. This change should reduce the amount of log noise on vmstorage side. In the same time, it is not expected to lose any information, since important logs should be still emitted by the vmselect. To conduct an experiment for testing this change see the following instructions: 1. Setup vmcluster with at least 2 storage nodes, 1 vminsert and 1 vmselect 2. Run vmselect with complexity limit checked on the client side: `-search.maxSamplesPerQuery=1` 3. Ingest some data and query it back: `count({__name__!=""})` 4. Observe the logs on vmselect and vmstorage side Before the change, vmselect will log message about complexity limits exceeded. When this happens, vmselect closes network connections to vmstorage nodes signalizing that it doesn't expect any data back. Both vmstorage processes will try to push data to the connection and will fail with "broken pipe" error, means that vmselect closed the connection. After the change, vmstorages should remain silent. And vmselect will continue emittin the error message about complexity limits exceeded. Signed-off-by: hagen1778 --- lib/vmselectapi/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/vmselectapi/server.go b/lib/vmselectapi/server.go index b0c393bd59..d06b93f6b1 100644 --- a/lib/vmselectapi/server.go +++ b/lib/vmselectapi/server.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net" + "strings" "sync" "sync/atomic" "time" @@ -256,6 +257,13 @@ func (s *Server) processConn(bc *handshake.BufferedConn) error { // Remote client gracefully closed the connection. return nil } + if err == net.ErrClosed || strings.Contains(err.Error(), "broken pipe") { + // The connection has been interrupted abruptly. + // It could happen due to unexpected network glitch or because connection was + // interrupted by remote client. In both cases, remote client will notice + // connection breach and handle it on its own. No need in mirroring the error here. + return nil + } if errors.Is(err, storage.ErrDeadlineExceeded) { return fmt.Errorf("cannot process vmselect request in %d seconds: %w", ctx.timeout, err) }