From 9307de1b927eca26860f1c6bb55453235017b321 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin <valyala@gmail.com> Date: Tue, 24 Sep 2019 00:49:21 +0300 Subject: [PATCH] lib/storage: limit the number of metricIDs in tag->metricIDs row This reduces the overhead on index and metaindex in lib/mergeset --- lib/storage/index_db.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 942b63fc87..d2a59edb8b 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -2336,12 +2336,23 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) { if err := mp.Init(item); err != nil { logger.Panicf("FATAL: cannot parse tag->metricIDs row during merge: %s", err) } + if mp.MetricIDsLen() >= maxMetricIDsPerRow { + if len(tmm.pendingMetricIDs) > 0 { + dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev) + } + dstData = append(dstData, item...) + dstItems = append(dstItems, dstData[len(dstData)-len(item):]) + continue + } if len(tmm.pendingMetricIDs) > 0 && !mp.EqualPrefix(mpPrev) { dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev) } mp.ParseMetricIDs() tmm.pendingMetricIDs = append(tmm.pendingMetricIDs, mp.MetricIDs...) mpPrev, mp = mp, mpPrev + if len(tmm.pendingMetricIDs) >= maxMetricIDsPerRow { + dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev) + } } if len(tmm.pendingMetricIDs) > 0 { dstData, dstItems = tmm.flushPendingMetricIDs(dstData, dstItems, mpPrev) @@ -2350,6 +2361,11 @@ func mergeTagToMetricIDsRows(data []byte, items [][]byte) ([]byte, [][]byte) { return data, dstItems } +// maxMetricIDsPerRow limits the number of metricIDs in tag->metricIDs row. +// +// This reduces overhead on index and metaindex in lib/mergeset. +const maxMetricIDsPerRow = 64 + type uint64Sorter []uint64 func (s uint64Sorter) Len() int { return len(s) }