From 77f76371d040d9cac936e0cd8496a88e51ee70c8 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 24 Apr 2023 14:52:06 +0200 Subject: [PATCH] http server: limit max concurrent requests (#4185) * lib/httpserver: introduce `-http.maxConcurrentRequests` command-line flag Introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied. Signed-off-by: hagen1778 * lib/httpserver: mention http.maxConcurrentRequests in docs Signed-off-by: hagen1778 --------- Signed-off-by: hagen1778 --- README.md | 4 ++++ app/vmagent/README.md | 2 ++ app/vmalert/README.md | 2 ++ app/vmauth/README.md | 2 ++ app/vmbackup/README.md | 2 ++ app/vmbackupmanager/README.md | 2 ++ app/vmgateway/README.md | 2 ++ app/vmrestore/README.md | 2 ++ docs/CHANGELOG.md | 1 + docs/Cluster-VictoriaMetrics.md | 6 ++++++ docs/README.md | 2 ++ docs/Single-server-VictoriaMetrics.md | 2 ++ docs/vmagent.md | 2 ++ docs/vmalert.md | 2 ++ docs/vmauth.md | 2 ++ docs/vmbackup.md | 2 ++ docs/vmbackupmanager.md | 2 ++ docs/vmgateway.md | 2 ++ docs/vmrestore.md | 2 ++ lib/httpserver/httpserver.go | 26 +++++++++++++++++++++++--- 20 files changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a2738ccbc..2775a2f75 100644 --- a/README.md +++ b/README.md @@ -2231,6 +2231,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 5a65edec7..9ed20561b 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -1211,6 +1211,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 16b9450d0..205bb0b72 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -932,6 +932,8 @@ The shortlist of configuration flags is the following: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 75dd56c73..3f14986e8 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -285,6 +285,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmbackup/README.md b/app/vmbackup/README.md index bd73b0410..2e72a399f 100644 --- a/app/vmbackup/README.md +++ b/app/vmbackup/README.md @@ -207,6 +207,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time- Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmbackupmanager/README.md b/app/vmbackupmanager/README.md index 56af3501e..619fa5ff1 100644 --- a/app/vmbackupmanager/README.md +++ b/app/vmbackupmanager/README.md @@ -394,6 +394,8 @@ command-line flags: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmgateway/README.md b/app/vmgateway/README.md index 2eeae6f75..e6dbb2529 100644 --- a/app/vmgateway/README.md +++ b/app/vmgateway/README.md @@ -349,6 +349,8 @@ The shortlist of configuration flags include the following: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/app/vmrestore/README.md b/app/vmrestore/README.md index 28295537f..109656dc9 100644 --- a/app/vmrestore/README.md +++ b/app/vmrestore/README.md @@ -111,6 +111,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 04f511c88..087ee6a1e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -22,6 +22,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): store backup creation and completion time in `backup_complete.ignore` file of backup contents. This is useful to determine point in time when backup was created and completed. * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details. * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case. +* FEATURE: introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied. * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores. * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index bf722d8fd..1712c55d9 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -855,6 +855,8 @@ Below is the output for `/path/to/vminsert -help`: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string @@ -1064,6 +1066,8 @@ Below is the output for `/path/to/vmselect -help`: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string @@ -1284,6 +1288,8 @@ Below is the output for `/path/to/vmstorage -help`: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/README.md b/docs/README.md index e83b65020..cb152c7cb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2232,6 +2232,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 6c062c948..ac1f7d162 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2235,6 +2235,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmagent.md b/docs/vmagent.md index 4b640355e..f497c47d0 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1215,6 +1215,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmalert.md b/docs/vmalert.md index 86e3f2d9c..e2589e218 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -936,6 +936,8 @@ The shortlist of configuration flags is the following: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmauth.md b/docs/vmauth.md index a0c930f56..1ec339b69 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -289,6 +289,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmbackup.md b/docs/vmbackup.md index ab3438045..de1091eea 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -211,6 +211,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time- Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index 4a31b9f64..2dd8594e2 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -398,6 +398,8 @@ command-line flags: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmgateway.md b/docs/vmgateway.md index 8d11694ff..02c1f44af 100644 --- a/docs/vmgateway.md +++ b/docs/vmgateway.md @@ -353,6 +353,8 @@ The shortlist of configuration flags include the following: Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/docs/vmrestore.md b/docs/vmrestore.md index 4bbe3278c..fd3e55214 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -115,6 +115,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) + -http.maxConcurrentRequests int + The maximum number of concurrent HTTP requests. Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem.Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served.If set to zero - no limits are applied. -http.maxGracefulShutdownDuration duration The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s) -http.pathPrefix string diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index a8335c3b8..becd8e8ae 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -51,6 +51,10 @@ var ( shutdownDelay = flag.Duration("http.shutdownDelay", 0, `Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers`) idleConnTimeout = flag.Duration("http.idleConnTimeout", time.Minute, "Timeout for incoming idle http connections") connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, `Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem`) + maxConcurrentRequests = flag.Int("http.maxConcurrentRequests", 0, "The maximum number of concurrent HTTP requests. "+ + "Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem."+ + "Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served."+ + "If set to zero - no limits are applied.") ) var ( @@ -60,6 +64,7 @@ var ( type server struct { shutdownDelayDeadline int64 + concurrencyLimiter chan struct{} s *http.Server } @@ -71,9 +76,9 @@ type server struct { // In such cases the caller must serve the request. type RequestHandler func(w http.ResponseWriter, r *http.Request) bool -// Serve starts an http server on the given addr with the given optional rh. +// Serve starts a http server on the given addr with the given optional rh. // -// By default all the responses are transparently compressed, since egress traffic is usually expensive. +// By default, all the responses are transparently compressed, since egress traffic is usually expensive. // // The compression is also disabled if -http.disableResponseCompression flag is set. // @@ -136,6 +141,7 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler) { return context.WithValue(ctx, connDeadlineTimeKey, &deadline) }, } + s.concurrencyLimiter = make(chan struct{}, *maxConcurrentRequests) serversLock.Lock() servers[addr] = &s serversLock.Unlock() @@ -341,6 +347,19 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques if !CheckBasicAuth(w, r) { return } + + if *maxConcurrentRequests > 0 { + select { + case s.concurrencyLimiter <- struct{}{}: + default: + Errorf(w, r, "couldn't start processing the request at path %q, "+ + "since -http.maxConcurrentRequests=%d concurrent requests are executed.", r.URL.Path, *maxConcurrentRequests) + limitReachedRequestErrors.Inc() + return + } + defer func() { <-s.concurrencyLimiter }() + } + if rh(w, r) { return } @@ -429,7 +448,8 @@ var ( pprofDefaultRequests = metrics.NewCounter(`vm_http_requests_total{path="/debug/pprof/default"}`) faviconRequests = metrics.NewCounter(`vm_http_requests_total{path="/favicon.ico"}`) - unsupportedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="unsupported"}`) + unsupportedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="unsupported"}`) + limitReachedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="concurrency limit"}`) requestsTotal = metrics.NewCounter(`vm_http_requests_all_total`) )