diff --git a/Makefile b/Makefile index 2c9a5a819..4f7095fc2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \ - git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}'))) + git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17))) PKG_TAG ?= $(shell git tag -l --points-at HEAD) ifeq ($(PKG_TAG),) diff --git a/app/vmselect/netstorage/fadvise_freebsd.go b/app/vmselect/netstorage/fadvise_freebsd.go new file mode 100644 index 000000000..4bbbf132a --- /dev/null +++ b/app/vmselect/netstorage/fadvise_freebsd.go @@ -0,0 +1,15 @@ +package netstorage + +import ( + "os" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "golang.org/x/sys/unix" +) + +func mustFadviseRandomRead(f *os.File) { + fd := int(f.Fd()) + if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_RANDOM|unix.FADV_WILLNEED); err != nil { + logger.Panicf("FATAL: error returned from unix.Fadvise(RANDOM|WILLNEED): %s", err) + } +} diff --git a/app/vmselect/promql/aggr_incremental_test.go b/app/vmselect/promql/aggr_incremental_test.go index f9a78b136..94a4e49b9 100644 --- a/app/vmselect/promql/aggr_incremental_test.go +++ b/app/vmselect/promql/aggr_incremental_test.go @@ -179,7 +179,8 @@ func compareValues(vs1, vs2 []float64) error { } continue } - if v1 != v2 { + eps := math.Abs(v1 - v2) + if eps > 1e-14 { return fmt.Errorf("unexpected value; got %v; want %v", v1, v2) } } diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 275210ffe..eba0447b3 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -182,7 +182,8 @@ func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpecte t.Fatalf("unexpected value; got %v; want %v", v, vExpected) } } else { - if v != vExpected { + eps := math.Abs(v - vExpected) + if eps > 1e-14 { t.Fatalf("unexpected value; got %v; want %v", v, vExpected) } } diff --git a/go.mod b/go.mod index 4057eb0ae..b4312afb2 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/klauspost/compress v1.7.6 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/valyala/fastjson v1.4.1 - github.com/valyala/gozstd v1.6.0 + github.com/valyala/gozstd v1.6.1 github.com/valyala/histogram v1.0.1 github.com/valyala/quicktemplate v1.2.0 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a diff --git a/go.sum b/go.sum index ae5abbea0..00f5c52fc 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/y github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/valyala/gozstd v1.6.0 h1:34qKK75C6Dx9zof2JqUiunfJQ87Up6vTHXABWDyCH+g= -github.com/valyala/gozstd v1.6.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.6.1 h1:oFN2mNW0kOr1fEKJuLpDwakNb6Y9fElVEBZmPEsFTUw= +github.com/valyala/gozstd v1.6.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg= github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto= github.com/valyala/quicktemplate v1.2.0 h1:BaO1nHTkspYzmAjPXj0QiDJxai96tlcZyKcI9dyEGvM= diff --git a/lib/filestream/filestream_freebsd.go b/lib/filestream/filestream_freebsd.go new file mode 100644 index 000000000..332d7dda5 --- /dev/null +++ b/lib/filestream/filestream_freebsd.go @@ -0,0 +1,64 @@ +package filestream + +import ( + "fmt" + "syscall" + + "golang.org/x/sys/unix" +) + +func (st *streamTracker) adviseDontNeed(n int, fdatasync bool) error { + st.length += uint64(n) + if st.fd == 0 { + return nil + } + if st.length < dontNeedBlockSize { + return nil + } + blockSize := st.length - (st.length % dontNeedBlockSize) + if fdatasync { + if err := unixFdatasync(int(st.fd)); err != nil { + return fmt.Errorf("unix.Fdatasync error: %s", err) + } + } + if err := unix.Fadvise(int(st.fd), int64(st.offset), int64(blockSize), unix.FADV_DONTNEED); err != nil { + return fmt.Errorf("unix.Fadvise(FADV_DONTNEEDED, %d, %d) error: %s", st.offset, blockSize, err) + } + st.offset += blockSize + st.length -= blockSize + return nil +} + +func (st *streamTracker) close() error { + if st.fd == 0 { + return nil + } + // Advise the whole file as it shouldn't be cached. + if err := unix.Fadvise(int(st.fd), 0, 0, unix.FADV_DONTNEED); err != nil { + return fmt.Errorf("unix.Fadvise(FADV_DONTNEEDED, 0, 0) error: %s", err) + } + return nil +} + +// unix.Fdatasync is missing, so put it here +func unixFdatasync(fd int) (err error) { + _, _, e1 := unix.Syscall(unix.SYS_FDATASYNC, uintptr(fd), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case unix.EAGAIN: + return syscall.EAGAIN + case unix.EINVAL: + return syscall.EINVAL + case unix.ENOENT: + return syscall.ENOENT + } + return e +} diff --git a/lib/fs/fs.go b/lib/fs/fs.go index 617c835a6..060c09c45 100644 --- a/lib/fs/fs.go +++ b/lib/fs/fs.go @@ -400,3 +400,20 @@ func CreateFlockFile(dir string) (*os.File, error) { } return flockF, nil } + +// MustGetFreeSpace returns free space for the given directory path. +func MustGetFreeSpace(path string) uint64 { + d, err := os.Open(path) + if err != nil { + logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err) + } + defer MustClose(d) + + fd := d.Fd() + var stat unix.Statfs_t + if err := unix.Fstatfs(int(fd), &stat); err != nil { + logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err) + } + freeSpace := uint64(stat.Bavail) * uint64(stat.Bsize) + return freeSpace +} diff --git a/lib/memory/memory_bsd.go b/lib/memory/memory_bsd.go new file mode 100644 index 000000000..238caec76 --- /dev/null +++ b/lib/memory/memory_bsd.go @@ -0,0 +1,17 @@ +// +build freebsd openbsd dragonfly netbsd + +package memory + +import ( + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" +) + +// This code has been adopted from https://github.com/pbnjay/memory + +func sysTotalMemory() int { + s, err := sysctlUint64("hw.physmem") + if err != nil { + logger.Panicf("FATAL: cannot determine system memory: %s", err) + } + return int(s) +} diff --git a/lib/memory/memory_darwin.go b/lib/memory/memory_darwin.go index 4d05da995..85dffbdf4 100644 --- a/lib/memory/memory_darwin.go +++ b/lib/memory/memory_darwin.go @@ -1,9 +1,6 @@ package memory import ( - "syscall" - "unsafe" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) @@ -15,16 +12,3 @@ func sysTotalMemory() int { } return int(s) } - -func sysctlUint64(name string) (uint64, error) { - s, err := syscall.Sysctl(name) - if err != nil { - return 0, err - } - // hack because the string conversion above drops a \0 - b := []byte(s) - if len(b) < 8 { - b = append(b, 0) - } - return *(*uint64)(unsafe.Pointer(&b[0])), nil -} diff --git a/lib/memory/sysctl.go b/lib/memory/sysctl.go new file mode 100644 index 000000000..950a22cba --- /dev/null +++ b/lib/memory/sysctl.go @@ -0,0 +1,22 @@ +// +build darwin freebsd openbsd dragonfly netbsd + +package memory + +import ( + "syscall" + "unsafe" +) + +// This has been adapted from github.com/pbnjay/memory. +func sysctlUint64(name string) (uint64, error) { + s, err := syscall.Sysctl(name) + if err != nil { + return 0, err + } + // hack because the string conversion above drops a \0 + b := []byte(s) + if len(b) < 8 { + b = append(b, 0) + } + return *(*uint64)(unsafe.Pointer(&b[0])), nil +} diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index 5daa9de2b..e646b9c87 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -16,7 +16,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg" - "golang.org/x/sys/unix" ) // maxParts is the maximum number of parts in the table. @@ -804,19 +803,7 @@ func (tb *Table) maxOutPartItems() uint64 { } func (tb *Table) maxOutPartItemsSlow() uint64 { - // Determine the amount of free space on tb.path. - d, err := os.Open(tb.path) - if err != nil { - logger.Panicf("FATAL: cannot determine free disk space on %q: %s", tb.path, err) - } - defer fs.MustClose(d) - - fd := d.Fd() - var stat unix.Statfs_t - if err := unix.Fstatfs(int(fd), &stat); err != nil { - logger.Panicf("FATAL: cannot determine free disk space on %q: %s", tb.path, err) - } - freeSpace := stat.Bavail * uint64(stat.Bsize) + freeSpace := fs.MustGetFreeSpace(tb.path) // Calculate the maximum number of items in the output merge part // by dividing the freeSpace by 4 and by the number of concurrent diff --git a/lib/storage/partition.go b/lib/storage/partition.go index c511c681a..6f53fc2f4 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -19,7 +19,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" - "golang.org/x/sys/unix" ) func maxRowsPerSmallPart() uint64 { @@ -838,18 +837,7 @@ func mustGetFreeDiskSpace(path string) uint64 { // Slow path. // Determine the amount of free space on bigPartsPath. - d, err := os.Open(path) - if err != nil { - logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err) - } - defer fs.MustClose(d) - - fd := d.Fd() - var stat unix.Statfs_t - if err := unix.Fstatfs(int(fd), &stat); err != nil { - logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err) - } - e.freeSpace = stat.Bavail * uint64(stat.Bsize) + e.freeSpace = fs.MustGetFreeSpace(path) e.updateTime = time.Now() freeSpaceMap[path] = e return e.freeSpace diff --git a/vendor/github.com/valyala/gozstd/Makefile b/vendor/github.com/valyala/gozstd/Makefile index b48c0f994..1f151b7b6 100644 --- a/vendor/github.com/valyala/gozstd/Makefile +++ b/vendor/github.com/valyala/gozstd/Makefile @@ -26,7 +26,7 @@ endif clean: rm -f $(LIBZSTD_NAME) - cd zstd && make clean + cd zstd && $(MAKE) clean update-zstd: rm -rf zstd-tmp diff --git a/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.a b/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.a new file mode 100644 index 000000000..394a31a31 Binary files /dev/null and b/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.a differ diff --git a/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.go b/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.go new file mode 100644 index 000000000..eb5bca0f8 --- /dev/null +++ b/vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.go @@ -0,0 +1,6 @@ +package gozstd + +/* +#cgo LDFLAGS: ${SRCDIR}/libzstd_freebsd_amd64.a +*/ +import "C" diff --git a/vendor/modules.txt b/vendor/modules.txt index 026ea95d0..68a36a53a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/valyala/fastjson github.com/valyala/fastjson/fastfloat # github.com/valyala/fastrand v1.0.0 github.com/valyala/fastrand -# github.com/valyala/gozstd v1.6.0 +# github.com/valyala/gozstd v1.6.1 github.com/valyala/gozstd # github.com/valyala/histogram v1.0.1 github.com/valyala/histogram