diff --git a/lib/mergeset/block_stream_writer.go b/lib/mergeset/block_stream_writer.go index 8341b8318..a184d7a4b 100644 --- a/lib/mergeset/block_stream_writer.go +++ b/lib/mergeset/block_stream_writer.go @@ -148,12 +148,15 @@ func (bsw *blockStreamWriter) WriteBlock(ib *inmemoryBlock) { bsw.lensBlockOffset += uint64(bsw.bh.lensBlockSize) // Write blockHeader + unpackedIndexBlockBufLen := len(bsw.unpackedIndexBlockBuf) bsw.unpackedIndexBlockBuf = bsw.bh.Marshal(bsw.unpackedIndexBlockBuf) + if len(bsw.unpackedIndexBlockBuf) > maxIndexBlockSize { + bsw.unpackedIndexBlockBuf = bsw.unpackedIndexBlockBuf[:unpackedIndexBlockBufLen] + bsw.flushIndexData() + bsw.unpackedIndexBlockBuf = bsw.bh.Marshal(bsw.unpackedIndexBlockBuf) + } bsw.bh.Reset() bsw.mr.blockHeadersCount++ - if len(bsw.unpackedIndexBlockBuf) >= maxIndexBlockSize { - bsw.flushIndexData() - } } // The maximum size of index block with multiple blockHeaders. diff --git a/lib/storage/block_stream_writer.go b/lib/storage/block_stream_writer.go index 0029534e7..9d90c7741 100644 --- a/lib/storage/block_stream_writer.go +++ b/lib/storage/block_stream_writer.go @@ -143,11 +143,14 @@ func (bsw *blockStreamWriter) WriteExternalBlock(b *Block, ph *partHeader, rowsM atomic.AddUint64(×tampsBlocksMerged, 1) atomic.AddUint64(×tampsBytesSaved, uint64(len(timestampsData))) } + indexDataLen := len(bsw.indexData) bsw.indexData = append(bsw.indexData, headerData...) - bsw.mr.RegisterBlockHeader(&b.bh) - if len(bsw.indexData) >= maxBlockSize { + if len(bsw.indexData) > maxBlockSize { + bsw.indexData = bsw.indexData[:indexDataLen] bsw.flushIndexData() + bsw.indexData = append(bsw.indexData, headerData...) } + bsw.mr.RegisterBlockHeader(&b.bh) if !usePrevTimestamps { bsw.prevTimestampsData = append(bsw.prevTimestampsData[:0], timestampsData...) bsw.prevTimestampsBlockOffset = bsw.timestampsBlockOffset