From 7335743d57a53a79384248d1b9f08f5daa8815b4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 8 Jul 2020 17:29:57 +0300 Subject: [PATCH] lib/storage: limit the maximum concurrency for data ingestion to GOMAXPROCS Previously the concurrency has been limited to GOMAXPROCS*2. This had little sense, since every call to Storage.AddRows is bound to CPU, so the maximum ingestion bandwidth is achieved when the number of concurrent calls to Storage.AddRows is limited to the number of CPUs, i.e. to GOMAXPROCS. --- lib/storage/storage.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 74f23c318..9664a81ea 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1122,7 +1122,10 @@ func (s *Storage) AddRows(mrs []MetricRow, precisionBits uint8) error { } var ( - addRowsConcurrencyCh = make(chan struct{}, runtime.GOMAXPROCS(-1)*2) + // Limit the concurrency for data ingestion to GOMAXPROCS, since this operation + // is CPU bound, so there is no sense in running more than GOMAXPROCS concurrent + // goroutines on data ingestion path. + addRowsConcurrencyCh = make(chan struct{}, runtime.GOMAXPROCS(-1)) addRowsTimeout = 30 * time.Second ) @@ -1183,6 +1186,9 @@ func (s *Storage) add(rows []rawRow, mrs []MetricRow, precisionBits uint8) ([]ra } if s.getTSIDFromCache(&r.TSID, mr.MetricNameRaw) { // Fast path - the TSID for the given MetricName has been found in cache and isn't deleted. + // There is no need in checking whether r.TSID.MetricID is deleted, since tsidCache doesn't + // contain MetricName->TSID entries for deleted time series. + // See Storage.DeleteMetrics code for details. prevTSID = r.TSID prevMetricNameRaw = mr.MetricNameRaw continue @@ -1229,6 +1235,9 @@ func (s *Storage) add(rows []rawRow, mrs []MetricRow, precisionBits uint8) ([]ra } if s.getTSIDFromCache(&r.TSID, mr.MetricNameRaw) { // Fast path - the TSID for the given MetricName has been found in cache and isn't deleted. + // There is no need in checking whether r.TSID.MetricID is deleted, since tsidCache doesn't + // contain MetricName->TSID entries for deleted time series. + // See Storage.DeleteMetrics code for details. prevTSID = r.TSID prevMetricNameRaw = mr.MetricNameRaw continue