From a6086cde7868954b4e956e1104b975db8658b4cc Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 24 Sep 2019 00:49:21 +0300
Subject: [PATCH] lib/storage: limit the number of metricIDs in tag->metricIDs
row
This reduces the overhead on index and metaindex in lib/mergeset
---
lib/storage/index_db.go | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 6751a7f7f5..8c92f01a4c 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -2265,12 +2265,23 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) {
if err := mp.Init(item); err != nil {
logger.Panicf("FATAL: cannot parse tag->metricIDs row during merge: %s", err)
}
+ if mp.MetricIDsLen() >= maxMetricIDsPerRow {
+ if len(tmm.pendingMetricIDs) > 0 {
+ dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
+ }
+ dstData = append(dstData, item...)
+ dstItems = append(dstItems, dstData[len(dstData)-len(item):])
+ continue
+ }
if len(tmm.pendingMetricIDs) > 0 && !mp.EqualPrefix(mpPrev) {
dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
}
mp.ParseMetricIDs()
tmm.pendingMetricIDs = append(tmm.pendingMetricIDs, mp.MetricIDs...)
mpPrev, mp = mp, mpPrev
+ if len(tmm.pendingMetricIDs) >= maxMetricIDsPerRow {
+ dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
+ }
}
if len(tmm.pendingMetricIDs) > 0 {
dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev)
@@ -2279,6 +2290,11 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) {
return data, dstItems
}
+// maxMetricIDsPerRow limits the number of metricIDs in tag->metricIDs row.
+//
+// This reduces overhead on index and metaindex in lib/mergeset.
+const maxMetricIDsPerRow = 64
+
type uint64Sorter []uint64
func (s uint64Sorter) Len() int { return len(s) }