From d387da142e4005569162a709cf966a07a279d427 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 3 Sep 2020 22:22:26 +0300 Subject: [PATCH] lib/httpserver: add `-http.connTimeout` command-line flag for limiting the lifetime for incoming http connections This can be useful for balancing incoming connections among multiple services. --- lib/httpserver/httpserver.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index 2424a6ff6e..5ef7057e9f 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil" "github.com/VictoriaMetrics/metrics" @@ -43,6 +44,8 @@ var ( shutdownDelay = flag.Duration("http.shutdownDelay", 0, "Optional delay before http server shutdown. During this dealy the servier returns non-OK responses "+ "from /health page, so load balancers can route new requests to other servers") idleConnTimeout = flag.Duration("http.idleConnTimeout", time.Minute, "Timeout for incoming idle http connections") + connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, "Incoming http connections are closed after the configured timeout. This may help spreading incoming load "+ + "among a cluster of services behind load balancer") ) var ( @@ -111,6 +114,11 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler) { // since these timeouts must be controlled by request handlers. ErrorLog: logger.StdErrorLogger(), + + ConnContext: func(ctx context.Context, c net.Conn) context.Context { + startTime := fasttime.UnixTimestamp() + return context.WithValue(ctx, connStartTimeKey, &startTime) + }, } serversLock.Lock() servers[addr] = &s @@ -124,6 +132,15 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler) { } } +func whetherToCloseConn(r *http.Request) bool { + ctx := r.Context() + v := ctx.Value(connStartTimeKey) + st, ok := v.(*uint64) + return ok && fasttime.UnixTimestamp()-*st > uint64(*connTimeout/time.Second) +} + +var connStartTimeKey = interface{}("startTime") + // Stop stops the http server on the given addr, which has been started // via Serve func. func Stop(addr string) error { @@ -168,9 +185,14 @@ func gzipHandler(s *server, rh RequestHandler) http.HandlerFunc { } var metricsHandlerDuration = metrics.NewHistogram(`vm_http_request_duration_seconds{path="/metrics"}`) +var connTimeoutClosedConns = metrics.NewCounter(`vm_http_conn_timeout_closed_conns_total`) func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh RequestHandler) { requestsTotal.Inc() + if whetherToCloseConn(r) { + connTimeoutClosedConns.Inc() + w.Header().Set("Connection", "close") + } path, err := getCanonicalPath(r.URL.Path) if err != nil { Errorf(w, r, "cannot get canonical path: %s", err)