github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/{vminsert,vmselect}: improve error messages when VictoriaMetrics cannot handle too high number of concurrent inserts / selects

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-01-17 13:24:37 +02:00
parent ffe352ad31
commit a9f683423c
2 changed files with 13 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
app/vminsert/concurrencylimiter/concurrencylimiter.go
View file

@ -12,17 +12,14 @@ import (
"github.com/VictoriaMetrics/metrics" "github.com/VictoriaMetrics/metrics"
) )
var maxConcurrentInserts = flag.Int("maxConcurrentInserts", runtime.GOMAXPROCS(-1)*4, "The maximum number of concurrent inserts")
var ( var (
// ch is the channel for limiting concurrent calls to Do. maxConcurrentInserts = flag.Int("maxConcurrentInserts", runtime.GOMAXPROCS(-1)*4, "The maximum number of concurrent inserts; see also `-insert.maxQueueDuration`")
ch chan struct{} maxQueueDuration = flag.Duration("insert.maxQueueDuration", 30*time.Second, "The maximum duration for waiting in the queue for insert requests due to `-maxConcurrentInserts`")
// waitDuration is the amount of time to wait until at least a single
// concurrent Do call out of cap(ch) inserts is complete.
waitDuration = time.Second * 30
) )
// ch is the channel for limiting concurrent calls to Do.
var ch chan struct{}
// Init initializes concurrencylimiter. // Init initializes concurrencylimiter.
// //
// Init must be called after flag.Parse call. // Init must be called after flag.Parse call.
@ -43,9 +40,9 @@ func Do(f func() error) error {
} }
// All the workers are busy. // All the workers are busy.
// Sleep for up to waitDuration. // Sleep for up to *maxQueueDuration.
concurrencyLimitReached.Inc() concurrencyLimitReached.Inc()
t := timerpool.Get(waitDuration) t := timerpool.Get(*maxQueueDuration)
select { select {
case ch <- struct{}{}: case ch <- struct{}{}:
timerpool.Put(t) timerpool.Put(t)
@ -56,7 +53,9 @@ func Do(f func() error) error {
timerpool.Put(t) timerpool.Put(t)
concurrencyLimitTimeout.Inc() concurrencyLimitTimeout.Inc()
return &httpserver.ErrorWithStatusCode{ return &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch)), Err: fmt.Errorf("cannot handle more than %d concurrent inserts during %s; possible solutions: "+
"increase `-insert.maxQueueDuration`, increase `-maxConcurrentInserts`, "+
"decrease `-search.maxConcurrentRequests`, increase server capacity", *maxConcurrentInserts, *maxQueueDuration),
StatusCode: http.StatusServiceUnavailable, StatusCode: http.StatusServiceUnavailable,
} }
} }

4
app/vmselect/main.go
View file

@ -112,7 +112,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
timerpool.Put(t) timerpool.Put(t)
concurrencyLimitTimeout.Inc() concurrencyLimitTimeout.Inc()
err := &httpserver.ErrorWithStatusCode{ err := &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("cannot handle more than %d concurrent requests", cap(concurrencyCh)), Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
*maxConcurrentRequests, *maxQueueDuration),
StatusCode: http.StatusServiceUnavailable, StatusCode: http.StatusServiceUnavailable,
} }
httpserver.Errorf(w, "%s", err) httpserver.Errorf(w, "%s", err)