From 950b126a09a2813170e49fc3d9022f2fef395c1a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 8 Feb 2024 13:51:06 +0200 Subject: [PATCH] lib/{storage,mergeset}: do not create index blocks with sizes exceeding 64Kb in common case This should reduce memory fragmentation and memory usage for indexdb/indexBlocks and storage/indexBlocks caches --- lib/mergeset/block_stream_writer.go | 9 ++++++--- lib/storage/block_stream_writer.go | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/mergeset/block_stream_writer.go b/lib/mergeset/block_stream_writer.go index 8341b83183..a184d7a4b7 100644 --- a/lib/mergeset/block_stream_writer.go +++ b/lib/mergeset/block_stream_writer.go @@ -148,12 +148,15 @@ func (bsw *blockStreamWriter) WriteBlock(ib *inmemoryBlock) { bsw.lensBlockOffset += uint64(bsw.bh.lensBlockSize) // Write blockHeader + unpackedIndexBlockBufLen := len(bsw.unpackedIndexBlockBuf) bsw.unpackedIndexBlockBuf = bsw.bh.Marshal(bsw.unpackedIndexBlockBuf) + if len(bsw.unpackedIndexBlockBuf) > maxIndexBlockSize { + bsw.unpackedIndexBlockBuf = bsw.unpackedIndexBlockBuf[:unpackedIndexBlockBufLen] + bsw.flushIndexData() + bsw.unpackedIndexBlockBuf = bsw.bh.Marshal(bsw.unpackedIndexBlockBuf) + } bsw.bh.Reset() bsw.mr.blockHeadersCount++ - if len(bsw.unpackedIndexBlockBuf) >= maxIndexBlockSize { - bsw.flushIndexData() - } } // The maximum size of index block with multiple blockHeaders. diff --git a/lib/storage/block_stream_writer.go b/lib/storage/block_stream_writer.go index 0029534e7f..9d90c77412 100644 --- a/lib/storage/block_stream_writer.go +++ b/lib/storage/block_stream_writer.go @@ -143,11 +143,14 @@ func (bsw *blockStreamWriter) WriteExternalBlock(b *Block, ph *partHeader, rowsM atomic.AddUint64(×tampsBlocksMerged, 1) atomic.AddUint64(×tampsBytesSaved, uint64(len(timestampsData))) } + indexDataLen := len(bsw.indexData) bsw.indexData = append(bsw.indexData, headerData...) - bsw.mr.RegisterBlockHeader(&b.bh) - if len(bsw.indexData) >= maxBlockSize { + if len(bsw.indexData) > maxBlockSize { + bsw.indexData = bsw.indexData[:indexDataLen] bsw.flushIndexData() + bsw.indexData = append(bsw.indexData, headerData...) } + bsw.mr.RegisterBlockHeader(&b.bh) if !usePrevTimestamps { bsw.prevTimestampsData = append(bsw.prevTimestampsData[:0], timestampsData...) bsw.prevTimestampsBlockOffset = bsw.timestampsBlockOffset