github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 15:16:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/{mergeset,storage}: allow merging smaller number of small parts

Browse source 
While this may increase CPU and disk IO usage needed for background merge,
this also recudes CPU usage during queries in production. This is because
such queries tend to read recently added data and it is better to have lower number
of parts for such data in order to reduce CPU usage.

This partially reverts ebf8da3730
This commit is contained in:
Aliaksandr Valialkin 2021-02-21 21:25:58 +02:00
parent 34195218e1
commit b8a5ee2e93
3 changed files with 4 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/mergeset/table.go
View file

@ -1333,12 +1333,6 @@ func appendPartsToMerge(dst, src []*partWrapper, maxPartsToMerge int, maxItems u
for _, pw := range a {
itemsSum += pw.p.ph.itemsCount
}
if itemsSum < 1e6 && len(a) < maxPartsToMerge {
// Do not merge parts with too small number of items if the number of source parts
// isn't equal to maxPartsToMerge. This should reduce CPU usage and disk IO usage
// for small parts merge.
continue
}
if itemsSum > maxItems {
// There is no sense in checking the remaining bigger parts.
break

6
lib/storage/partition.go
View file

@ -1469,12 +1469,6 @@ func appendPartsToMerge(dst, src []*partWrapper, maxPartsToMerge int, maxRows ui
continue
}
rowsCount := getRowsCount(a)
if rowsCount < 1e6 && len(a) < maxPartsToMerge {
// Do not merge parts with too small number of rows if the number of source parts
// isn't equal to maxPartsToMerge. This should reduce CPU usage and disk IO usage
// for small parts merge.
continue
}
if rowsCount > maxRows {
// There is no need in verifying remaining parts with higher number of rows
needFreeSpace = true

6
lib/storage/partition_test.go
View file

@ -26,9 +26,11 @@ func TestAppendPartsToMerge(t *testing.T) {
testAppendPartsToMerge(t, 2, []uint64{4, 2, 4}, []uint64{4, 4})
testAppendPartsToMerge(t, 2, []uint64{1, 3, 7, 2}, nil)
testAppendPartsToMerge(t, 3, []uint64{1, 3, 7, 2}, []uint64{1, 2, 3})
testAppendPartsToMerge(t, 4, []uint64{1, 3, 7, 2}, nil)
testAppendPartsToMerge(t, 4, []uint64{1, 3, 7, 2}, []uint64{1, 2, 3})
testAppendPartsToMerge(t, 5, []uint64{1, 3, 7, 2}, nil)
testAppendPartsToMerge(t, 4, []uint64{1e6, 3e6, 7e6, 2e6}, []uint64{1e6, 2e6, 3e6})
testAppendPartsToMerge(t, 4, []uint64{2, 3, 7, 2}, []uint64{2, 2, 3, 7})
testAppendPartsToMerge(t, 4, []uint64{2, 3, 7, 2}, []uint64{2, 2, 3})
testAppendPartsToMerge(t, 5, []uint64{2, 3, 7, 2}, nil)
testAppendPartsToMerge(t, 3, []uint64{11, 1, 10, 100, 10}, []uint64{10, 10, 11})
}