all: port to FreeBSD on GOARCH=amd64

Makefile

@@ -1,7 +1,7 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}')))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)

app/vmselect/netstorage/fadvise_freebsd.go

@@ -0,0 +1,15 @@
+package netstorage
+
+import (
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"golang.org/x/sys/unix"
+)
+
+func mustFadviseRandomRead(f *os.File) {
+	fd := int(f.Fd())
+	if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_RANDOM|unix.FADV_WILLNEED); err != nil {
+		logger.Panicf("FATAL: error returned from unix.Fadvise(RANDOM|WILLNEED): %s", err)
+	}
+}

app/vmselect/promql/aggr_incremental_test.go

@@ -179,7 +179,8 @@ func compareValues(vs1, vs2 []float64) error {
 			}
 			continue
 		}
-		if v1 != v2 {
+		eps := math.Abs(v1 - v2)
+		if eps > 1e-14 {
 			return fmt.Errorf("unexpected value; got %v; want %v", v1, v2)
 		}
 	}

app/vmselect/promql/rollup_test.go

@@ -182,7 +182,8 @@ func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpecte
 				t.Fatalf("unexpected value; got %v; want %v", v, vExpected)
 			}
 		} else {
-			if v != vExpected {
+			eps := math.Abs(v - vExpected)
+			if eps > 1e-14 {
 				t.Fatalf("unexpected value; got %v; want %v", v, vExpected)
 			}
 		}

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/klauspost/compress v1.7.6
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/valyala/fastjson v1.4.1
-	github.com/valyala/gozstd v1.6.0
+	github.com/valyala/gozstd v1.6.1
 	github.com/valyala/histogram v1.0.1
 	github.com/valyala/quicktemplate v1.2.0
 	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a

go.sum

@@ -41,8 +41,8 @@ github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/y
 github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
-github.com/valyala/gozstd v1.6.0 h1:34qKK75C6Dx9zof2JqUiunfJQ87Up6vTHXABWDyCH+g=
+github.com/valyala/gozstd v1.6.1 h1:oFN2mNW0kOr1fEKJuLpDwakNb6Y9fElVEBZmPEsFTUw=
-github.com/valyala/gozstd v1.6.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/valyala/gozstd v1.6.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg=
 github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto=
 github.com/valyala/quicktemplate v1.2.0 h1:BaO1nHTkspYzmAjPXj0QiDJxai96tlcZyKcI9dyEGvM=

lib/filestream/filestream_freebsd.go

@@ -0,0 +1,64 @@
+package filestream
+
+import (
+	"fmt"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+func (st *streamTracker) adviseDontNeed(n int, fdatasync bool) error {
+	st.length += uint64(n)
+	if st.fd == 0 {
+		return nil
+	}
+	if st.length < dontNeedBlockSize {
+		return nil
+	}
+	blockSize := st.length - (st.length % dontNeedBlockSize)
+	if fdatasync {
+		if err := unixFdatasync(int(st.fd)); err != nil {
+			return fmt.Errorf("unix.Fdatasync error: %s", err)
+		}
+	}
+	if err := unix.Fadvise(int(st.fd), int64(st.offset), int64(blockSize), unix.FADV_DONTNEED); err != nil {
+		return fmt.Errorf("unix.Fadvise(FADV_DONTNEEDED, %d, %d) error: %s", st.offset, blockSize, err)
+	}
+	st.offset += blockSize
+	st.length -= blockSize
+	return nil
+}
+
+func (st *streamTracker) close() error {
+	if st.fd == 0 {
+		return nil
+	}
+	// Advise the whole file as it shouldn't be cached.
+	if err := unix.Fadvise(int(st.fd), 0, 0, unix.FADV_DONTNEED); err != nil {
+		return fmt.Errorf("unix.Fadvise(FADV_DONTNEEDED, 0, 0) error: %s", err)
+	}
+	return nil
+}
+
+// unix.Fdatasync is missing, so put it here
+func unixFdatasync(fd int) (err error) {
+	_, _, e1 := unix.Syscall(unix.SYS_FDATASYNC, uintptr(fd), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return nil
+	case unix.EAGAIN:
+		return syscall.EAGAIN
+	case unix.EINVAL:
+		return syscall.EINVAL
+	case unix.ENOENT:
+		return syscall.ENOENT
+	}
+	return e
+}

lib/fs/fs.go

@@ -400,3 +400,20 @@ func CreateFlockFile(dir string) (*os.File, error) {
 	}
 	return flockF, nil
 }
+
+// MustGetFreeSpace returns free space for the given directory path.
+func MustGetFreeSpace(path string) uint64 {
+	d, err := os.Open(path)
+	if err != nil {
+		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
+	}
+	defer MustClose(d)
+
+	fd := d.Fd()
+	var stat unix.Statfs_t
+	if err := unix.Fstatfs(int(fd), &stat); err != nil {
+		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
+	}
+	freeSpace := uint64(stat.Bavail) * uint64(stat.Bsize)
+	return freeSpace
+}

lib/memory/memory_bsd.go

@@ -0,0 +1,17 @@
+// +build freebsd openbsd dragonfly netbsd
+
+package memory
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// This code has been adopted from https://github.com/pbnjay/memory
+
+func sysTotalMemory() int {
+	s, err := sysctlUint64("hw.physmem")
+	if err != nil {
+		logger.Panicf("FATAL: cannot determine system memory: %s", err)
+	}
+	return int(s)
+}

lib/memory/memory_darwin.go

@@ -1,9 +1,6 @@
 package memory
 
 import (
-	"syscall"
-	"unsafe"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -15,16 +12,3 @@ func sysTotalMemory() int {
 	}
 	return int(s)
 }
-
-func sysctlUint64(name string) (uint64, error) {
-	s, err := syscall.Sysctl(name)
-	if err != nil {
-		return 0, err
-	}
-	// hack because the string conversion above drops a \0
-	b := []byte(s)
-	if len(b) < 8 {
-		b = append(b, 0)
-	}
-	return *(*uint64)(unsafe.Pointer(&b[0])), nil
-}

lib/memory/sysctl.go

@@ -0,0 +1,22 @@
+// +build darwin freebsd openbsd dragonfly netbsd
+
+package memory
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// This has been adapted from github.com/pbnjay/memory.
+func sysctlUint64(name string) (uint64, error) {
+	s, err := syscall.Sysctl(name)
+	if err != nil {
+		return 0, err
+	}
+	// hack because the string conversion above drops a \0
+	b := []byte(s)
+	if len(b) < 8 {
+		b = append(b, 0)
+	}
+	return *(*uint64)(unsafe.Pointer(&b[0])), nil
+}

lib/mergeset/table.go

@@ -16,7 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
-	"golang.org/x/sys/unix"
 )
 
 // maxParts is the maximum number of parts in the table.
@@ -804,19 +803,7 @@ func (tb *Table) maxOutPartItems() uint64 {
 }
 
 func (tb *Table) maxOutPartItemsSlow() uint64 {
-	// Determine the amount of free space on tb.path.
+	freeSpace := fs.MustGetFreeSpace(tb.path)
-	d, err := os.Open(tb.path)
-	if err != nil {
-		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", tb.path, err)
-	}
-	defer fs.MustClose(d)
-
-	fd := d.Fd()
-	var stat unix.Statfs_t
-	if err := unix.Fstatfs(int(fd), &stat); err != nil {
-		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", tb.path, err)
-	}
-	freeSpace := stat.Bavail * uint64(stat.Bsize)
 
 	// Calculate the maximum number of items in the output merge part
 	// by dividing the freeSpace by 4 and by the number of concurrent

lib/storage/partition.go

@@ -19,7 +19,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"golang.org/x/sys/unix"
 )
 
 func maxRowsPerSmallPart() uint64 {
@@ -838,18 +837,7 @@ func mustGetFreeDiskSpace(path string) uint64 {
 
 	// Slow path.
 	// Determine the amount of free space on bigPartsPath.
-	d, err := os.Open(path)
+	e.freeSpace = fs.MustGetFreeSpace(path)
-	if err != nil {
-		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
-	}
-	defer fs.MustClose(d)
-
-	fd := d.Fd()
-	var stat unix.Statfs_t
-	if err := unix.Fstatfs(int(fd), &stat); err != nil {
-		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
-	}
-	e.freeSpace = stat.Bavail * uint64(stat.Bsize)
 	e.updateTime = time.Now()
 	freeSpaceMap[path] = e
 	return e.freeSpace

vendor/github.com/valyala/gozstd/Makefile

@@ -26,7 +26,7 @@ endif
 
 clean:
 	rm -f $(LIBZSTD_NAME)
-	cd zstd && make clean
+	cd zstd && $(MAKE) clean
 
 update-zstd:
 	rm -rf zstd-tmp

vendor/github.com/valyala/gozstd/libzstd_freebsd_amd64.go

@@ -0,0 +1,6 @@
+package gozstd
+
+/*
+#cgo LDFLAGS: ${SRCDIR}/libzstd_freebsd_amd64.a
+*/
+import "C"

vendor/modules.txt

@@ -19,7 +19,7 @@ github.com/valyala/fastjson
 github.com/valyala/fastjson/fastfloat
 # github.com/valyala/fastrand v1.0.0
 github.com/valyala/fastrand
-# github.com/valyala/gozstd v1.6.0
+# github.com/valyala/gozstd v1.6.1
 github.com/valyala/gozstd
 # github.com/valyala/histogram v1.0.1
 github.com/valyala/histogram