github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmstorage: add -finalMergeDelay command-line flag for configuring the delay before final merge for per-month partitions after no new data is ingested to it

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-10-07 17:35:42 +03:00
parent e9d99021b0
commit af90b3121c
2 changed files with 18 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/vmstorage/main.go
View file

@ -28,6 +28,9 @@ var (
// DataPath is a path to storage data.
DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
"disk IO usage and CPU usage")
bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
@ -65,6 +68,7 @@ func InitWithoutMetrics() {
logger.Fatalf("invalid `-precisionBits`: %s", err)
}
storage.SetFinalMergeDelay(*finalMergeDelay)
storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)

15
lib/storage/partition.go
View file

@ -958,7 +958,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
if !errors.Is(err, errNothingToMerge) {
return err
}
if fasttime.UnixTimestamp()-lastMergeTime > 30 {
if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
// We have free time for merging into bigger parts.
// This should improve select performance.
lastMergeTime = fasttime.UnixTimestamp()
@ -980,6 +980,19 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
}
}
var finalMergeDelaySeconds = uint64(30)
// SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data.
//
// This function may be called only before Storage initialization.
func SetFinalMergeDelay(delay time.Duration) {
delaySeconds := int(delay.Seconds() + 0.5)
if delaySeconds <= 0 {
return
}
finalMergeDelaySeconds = uint64(delaySeconds)
}
func maxRowsByPath(path string) uint64 {
freeSpace := fs.MustGetFreeSpace(path)