From 9307de1b927eca26860f1c6bb55453235017b321 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 24 Sep 2019 00:49:21 +0300
Subject: [PATCH] lib/storage: limit the number of metricIDs in tag->metricIDs
 row

This reduces the overhead on index and metaindex in lib/mergeset
---
 lib/storage/index_db.go | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 942b63fc87..d2a59edb8b 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -2336,12 +2336,23 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) {
 		if err := mp.Init(item); err != nil {
 			logger.Panicf("FATAL: cannot parse tag->metricIDs row during merge: %s", err)
 		}
+		if mp.MetricIDsLen() >= maxMetricIDsPerRow {
+			if len(tmm.pendingMetricIDs) > 0 {
+				dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
+			}
+			dstData = append(dstData, item...)
+			dstItems = append(dstItems, dstData[len(dstData)-len(item):])
+			continue
+		}
 		if len(tmm.pendingMetricIDs) > 0 && !mp.EqualPrefix(mpPrev) {
 			dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
 		}
 		mp.ParseMetricIDs()
 		tmm.pendingMetricIDs = append(tmm.pendingMetricIDs, mp.MetricIDs...)
 		mpPrev, mp = mp, mpPrev
+		if len(tmm.pendingMetricIDs) >= maxMetricIDsPerRow {
+			dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
+		}
 	}
 	if len(tmm.pendingMetricIDs) > 0 {
 		dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
@@ -2350,6 +2361,11 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) {
 	return data, dstItems
 }
 
+// maxMetricIDsPerRow limits the number of metricIDs in tag->metricIDs row.
+//
+// This reduces overhead on index and metaindex in lib/mergeset.
+const maxMetricIDsPerRow = 64
+
 type uint64Sorter []uint64
 
 func (s uint64Sorter) Len() int { return len(s) }