From 0397b3f0f7b334ab3a201c6960c30ee00fba8d1b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 18 May 2023 10:37:56 -0700 Subject: [PATCH] lib/handshake: do not pollute logs with `cannot read hello` messages on TCP health checks Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1762 --- app/vminsert/clusternative/request_handler.go | 4 ++++ app/vmstorage/servers/vminsert.go | 6 ++++-- docs/CHANGELOG.md | 3 ++- lib/handshake/handshake.go | 11 +++++++++++ lib/vmselectapi/server.go | 4 +++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app/vminsert/clusternative/request_handler.go b/app/vminsert/clusternative/request_handler.go index 6e2c13c9e..a3d0e3864 100644 --- a/app/vminsert/clusternative/request_handler.go +++ b/app/vminsert/clusternative/request_handler.go @@ -1,6 +1,7 @@ package clusternative import ( + "errors" "fmt" "net" @@ -24,6 +25,9 @@ var ( func InsertHandler(c net.Conn) error { bc, err := handshake.VMInsertServer(c, 0) if err != nil { + if errors.Is(err, handshake.ErrIgnoreHealthcheck) { + return nil + } return fmt.Errorf("cannot perform vminsert handshake with client %q: %w", c.RemoteAddr(), err) } return stream.Parse(bc, func(rows []storage.MetricRow) error { diff --git a/app/vmstorage/servers/vminsert.go b/app/vmstorage/servers/vminsert.go index 7ed838e8e..c1b083b6c 100644 --- a/app/vmstorage/servers/vminsert.go +++ b/app/vmstorage/servers/vminsert.go @@ -1,6 +1,7 @@ package servers import ( + "errors" "flag" "fmt" "net" @@ -73,7 +74,6 @@ func (s *VMInsertServer) run() { } logger.Panicf("FATAL: cannot process vminsert conns at %s: %s", s.ln.Addr(), err) } - logger.Infof("accepted vminsert conn from %s", c.RemoteAddr()) if !s.connsMap.Add(c) { // The server is closed. @@ -98,7 +98,9 @@ func (s *VMInsertServer) run() { // c is stopped inside VMInsertServer.MustStop return } - logger.Errorf("cannot perform vminsert handshake with client %q: %s", c.RemoteAddr(), err) + if !errors.Is(err, handshake.ErrIgnoreHealthcheck) { + logger.Errorf("cannot perform vminsert handshake with client %q: %s", c.RemoteAddr(), err) + } _ = c.Close() return } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 54326cccf..199d9220f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,8 +32,9 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: do not execute the incoming request if it has been canceled by the client before the execution start. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4223). * FEATURE: support time formats with timezones. For example, `2024-01-02+02:00` means `January 2, 2024` at `+02:00` time zone. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). * FEATURE: expose `process_*` metrics at `/metrics` page of all the VictoriaMetrics components under Windows OS. See [this pull request](https://github.com/VictoriaMetrics/metrics/pull/47). -* FEATURE: reduce the amounts of unimprotant `INFO` logging during VictoriaMetrics startup / shutdown. This should improve visibility for potentially important logs. +* FEATURE: reduce the amounts of unimportant `INFO` logging during VictoriaMetrics startup / shutdown. This should improve visibility for potentially important logs. * FEATURE: upgrade base docker image (alpine) from 3.17.3 to 3.18.0. See [alpine 3.18.0 release notes](https://www.alpinelinux.org/posts/Alpine-3.18.0-released.html). +* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): do not pullute logs with `cannot read hello: cannot read message with size 11: EOF` messages at `vmstorage` during TCP health checks performed by [Consul](https://developer.hashicorp.com/consul/docs/services/usage/checks) or [other services](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-health-check/). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1762). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): support the ability to filter [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs) targets in more optimal way via new `filter` option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4183). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [consulagent_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consulagent_sd_configs). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3953). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): emit a warning if too small value is passed to `-remoteWrite.maxDiskUsagePerURL` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195). diff --git a/lib/handshake/handshake.go b/lib/handshake/handshake.go index d4e40d32e..b17f40c08 100644 --- a/lib/handshake/handshake.go +++ b/lib/handshake/handshake.go @@ -1,6 +1,7 @@ package handshake import ( + "errors" "fmt" "io" "net" @@ -55,8 +56,18 @@ func VMSelectServer(c net.Conn, compressionLevel int) (*BufferedConn, error) { return genericServer(c, vmselectHello, compressionLevel) } +// ErrIgnoreHealthcheck means the TCP healthckeck, which must be ignored. +// +// The TCP healthcheck is performed by opening and then immediately closing the connection. +var ErrIgnoreHealthcheck = fmt.Errorf("TCP healthcheck - ignore it") + func genericServer(c net.Conn, msg string, compressionLevel int) (*BufferedConn, error) { if err := readMessage(c, msg); err != nil { + if errors.Is(err, io.EOF) { + // This is TCP healthcheck, which must be ignored in order to prevent from logs pollution. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1762 + return nil, ErrIgnoreHealthcheck + } return nil, fmt.Errorf("cannot read hello: %w", err) } if err := writeMessage(c, successResponse); err != nil { diff --git a/lib/vmselectapi/server.go b/lib/vmselectapi/server.go index 907e33841..b0c393bd5 100644 --- a/lib/vmselectapi/server.go +++ b/lib/vmselectapi/server.go @@ -198,7 +198,9 @@ func (s *Server) run() { // c is closed inside Server.MustStop return } - logger.Errorf("cannot perform vmselect handshake with client %q: %s", c.RemoteAddr(), err) + if !errors.Is(err, handshake.ErrIgnoreHealthcheck) { + logger.Errorf("cannot perform vmselect handshake with client %q: %s", c.RemoteAddr(), err) + } _ = c.Close() return }