github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/fs: optimize MustGetFreeSpace performance by caching the results for up to 2 seconds

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-06-04 13:13:00 +03:00
parent fa103875a0
commit 3d0a0b3785
3 changed files with 33 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
lib/fs/fs.go
View file

@ -6,8 +6,10 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"sync"
"sync/atomic" "sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
@ -296,6 +298,35 @@ func CreateFlockFile(dir string) (*os.File, error) {
// MustGetFreeSpace returns free space for the given directory path. // MustGetFreeSpace returns free space for the given directory path.
func MustGetFreeSpace(path string) uint64 { func MustGetFreeSpace(path string) uint64 {
// Try obtaining cached value at first.
freeSpaceMapLock.Lock()
defer freeSpaceMapLock.Unlock()
e, ok := freeSpaceMap[path]
if ok && fasttime.UnixTimestamp()-e.updateTime < 2 {
// Fast path - the entry is fresh.
return e.freeSpace
}
// Slow path.
// Determine the amount of free space at path.
e.freeSpace = mustGetFreeSpace(path)
e.updateTime = fasttime.UnixTimestamp()
freeSpaceMap[path] = e
return e.freeSpace
}
var (
freeSpaceMap = make(map[string]freeSpaceEntry)
freeSpaceMapLock sync.Mutex
)
type freeSpaceEntry struct {
updateTime uint64
freeSpace uint64
}
func mustGetFreeSpace(path string) uint64 {
d, err := os.Open(path) d, err := os.Open(path)
if err != nil { if err != nil {
logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err) logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)

31
lib/storage/partition.go
View file

@ -927,7 +927,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
} }
func maxRowsByPath(path string) uint64 { func maxRowsByPath(path string) uint64 {
freeSpace := mustGetFreeDiskSpace(path) freeSpace := fs.MustGetFreeSpace(path)
// Calculate the maximum number of rows in the output merge part // Calculate the maximum number of rows in the output merge part
// by dividing the freeSpace by the number of concurrent // by dividing the freeSpace by the number of concurrent
@ -942,35 +942,6 @@ func maxRowsByPath(path string) uint64 {
return maxRows return maxRows
} }
func mustGetFreeDiskSpace(path string) uint64 {
// Try obtaining the cache value at first.
freeSpaceMapLock.Lock()
defer freeSpaceMapLock.Unlock()
e, ok := freeSpaceMap[path]
if ok && fasttime.UnixTimestamp()-e.updateTime < 2 {
// Fast path - the entry is fresh.
return e.freeSpace
}
// Slow path.
// Determine the amount of free space on bigPartsPath.
e.freeSpace = fs.MustGetFreeSpace(path)
e.updateTime = fasttime.UnixTimestamp()
freeSpaceMap[path] = e
return e.freeSpace
}
var (
freeSpaceMap = make(map[string]freeSpaceEntry)
freeSpaceMapLock sync.Mutex
)
type freeSpaceEntry struct {
updateTime uint64
freeSpace uint64
}
func (pt *partition) mergeBigParts(isFinal bool) error { func (pt *partition) mergeBigParts(isFinal bool) error {
bigMergeConcurrencyLimitCh <- struct{}{} bigMergeConcurrencyLimitCh <- struct{}{}
defer func() { defer func() {