From 307281e9221a8e19aa530eed5a4ed57f0990a27e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 7 Aug 2020 08:47:32 +0300 Subject: [PATCH] lib/storage: slow down concurrent searches when the number of concurrent inserts reaches the limit This should improve data ingestion performance when heavy searches are executed See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/648 See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/618 --- lib/storage/storage.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 1791ebce5a..acac917bcb 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1120,11 +1120,17 @@ func (s *Storage) AddRows(mrs []MetricRow, precisionBits uint8) error { // Sleep for a while until giving up atomic.AddUint64(&s.addRowsConcurrencyLimitReached, 1) t := timerpool.Get(addRowsTimeout) + + // Prioritize data ingestion over concurrent searches. + storagepacelimiter.Search.Inc() + select { case addRowsConcurrencyCh <- struct{}{}: timerpool.Put(t) + storagepacelimiter.Search.Dec() case <-t.C: timerpool.Put(t) + storagepacelimiter.Search.Dec() atomic.AddUint64(&s.addRowsConcurrencyLimitTimeout, 1) atomic.AddUint64(&s.addRowsConcurrencyDroppedRows, uint64(len(mrs))) return fmt.Errorf("cannot add %d rows to storage in %s, since it is overloaded with %d concurrent writers; add more CPUs or reduce load",