github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-11 15:34:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmstorage: disable final merge by default, since it may result in high disk IO and CPU usage without measurable benefits such as increased query performance and reduced disk space usage

Browse source
This commit is contained in:
Aliaksandr Valialkin 2021-01-08 00:09:00 +02:00
parent 0477991b4d
commit 9dcb18e03d
3 changed files with 11 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/vmstorage/main.go
View file

@ -30,9 +30,9 @@ var (
// DataPath is a path to storage data.
DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
"disk IO usage and CPU usage")
finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
"Zero value disables final merge")
bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")

2
docs/CHANGELOG.md
View file

@ -2,6 +2,8 @@
# tip
* FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration.
# [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0)

11
lib/storage/partition.go
View file

@ -976,7 +976,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
if !errors.Is(err, errNothingToMerge) {
return err
}
if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
// We have free time for merging into bigger parts.
// This should improve select performance.
lastMergeTime = fasttime.UnixTimestamp()
@ -998,17 +998,18 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
}
}
var finalMergeDelaySeconds = uint64(30)
// Disable final merge by default, since it may lead to high disk IO and CPU usage
// at the beginning of every month when merging data for the previous month.
var finalMergeDelaySeconds = uint64(0)
// SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data.
//
// This function may be called only before Storage initialization.
func SetFinalMergeDelay(delay time.Duration) {
delaySeconds := int(delay.Seconds() + 0.5)
if delaySeconds <= 0 {
if delay <= 0 {
return
}
finalMergeDelaySeconds = uint64(delaySeconds)
finalMergeDelaySeconds = uint64(delay.Seconds() + 1)
}
func maxRowsByPath(path string) uint64 {