github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/{mergeset,storage}: tune compression levels for small blocks

Browse source 
This should reduce CPU usage spent on compression
This commit is contained in:
Aliaksandr Valialkin 2022-02-25 15:32:27 +02:00
parent e757ebc58b
commit 59877d9f32
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
2 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
lib/mergeset/inmemory_part.go
View file

@ -50,7 +50,8 @@ func (mp *inmemoryPart) Init(ib *inmemoryBlock) {
// Use the minimum possible compressLevel for compressing inmemoryPart,
// since it will be merged into file part soon.
compressLevel := 0
// See https://github.com/facebook/zstd/releases/tag/v1.3.4 for details about negative compression level
compressLevel := -5
mp.bh.firstItem, mp.bh.commonPrefix, mp.bh.itemsCount, mp.bh.marshalType = ib.MarshalUnsortedData(&mp.sb, mp.bh.firstItem[:0], mp.bh.commonPrefix[:0], compressLevel)
mp.ph.itemsCount = uint64(len(ib.items))

7
lib/storage/partition.go
View file

@ -1255,6 +1255,13 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}) erro
func getCompressLevelForRowsCount(rowsCount, blocksCount uint64) int {
avgRowsPerBlock := rowsCount / blocksCount
// See https://github.com/facebook/zstd/releases/tag/v1.3.4 about negative compression levels.
if avgRowsPerBlock <= 10 {
return -5
}
if avgRowsPerBlock <= 50 {
return -2
}
if avgRowsPerBlock <= 200 {
return -1
}