diff --git a/.gitignore b/.gitignore index 172e57f11..40404d5d1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ _site /docs/.jekyll-metadata coverage.txt cspell.json +*~ diff --git a/Makefile b/Makefile index 14b6792db..3c9ba1d55 100644 --- a/Makefile +++ b/Makefile @@ -184,19 +184,19 @@ check-all: fmt vet golangci-lint govulncheck clean-checkers: remove-golangci-lint remove-govulncheck test: - go test ./lib/... ./app/... + DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/... test-race: - go test -race ./lib/... ./app/... + DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/... test-pure: - CGO_ENABLED=0 go test ./lib/... ./app/... + DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/... test-full: - go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... + DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... test-full-386: - GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... + DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... benchmark: go test -bench=. ./lib/... diff --git a/lib/envutil/envutil.go b/lib/envutil/envutil.go new file mode 100644 index 000000000..ebba7618a --- /dev/null +++ b/lib/envutil/envutil.go @@ -0,0 +1,19 @@ +package envutil + +import ( + "os" + "strconv" +) + +// GetenvBool retrieves the value of the environment variable named by the key, +// attempts to convert the value to bool type and returns the result. In order +// for conversion to succeed, the value must be any value supported by +// strconv.ParseBool() function, otherwise the function will return false. +func GetenvBool(key string) bool { + s := os.Getenv(key) + b, err := strconv.ParseBool(s) + if err != nil { + return false + } + return b +} diff --git a/lib/envutil/envutil_test.go b/lib/envutil/envutil_test.go new file mode 100644 index 000000000..04eb3b106 --- /dev/null +++ b/lib/envutil/envutil_test.go @@ -0,0 +1,37 @@ +package envutil + +import ( + "os" + "testing" +) + +func TestGetenvBool(t *testing.T) { + f := func(value string, want bool) { + t.Helper() + + key := "VM_LIB_ENVUTIL_TEST_GETENV_BOOL" + os.Setenv(key, value) + defer os.Unsetenv(key) + + if got := GetenvBool(key); got != want { + t.Errorf("GetenvBool(%s=%s) unexpected return value: got %t, want %t", key, value, got, want) + } + } + + f("TRUE", true) + f("True", true) + f("true", true) + f("T", true) + f("t", true) + f("1", true) + f("FALSE", false) + f("False", false) + f("false", false) + f("F", false) + f("f", false) + f("0", false) + + f("", false) + f("unsupported", false) + f("tRuE", false) +} diff --git a/lib/filestream/filestream.go b/lib/filestream/filestream.go index a04583256..38b27f4eb 100644 --- a/lib/filestream/filestream.go +++ b/lib/filestream/filestream.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" "github.com/VictoriaMetrics/metrics" @@ -18,6 +19,8 @@ var disableFadvise = flag.Bool("filestream.disableFadvise", false, "Whether to d "The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. "+ "In some rare cases it is better to disable the syscall if it uses too much CPU") +var disableFSyncForTesting = envutil.GetenvBool("DISABLE_FSYNC_FOR_TESTING") + const dontNeedBlockSize = 16 * 1024 * 1024 // ReadCloser is a standard interface for filestream Reader. @@ -241,8 +244,10 @@ func (w *Writer) MustClose() { putBufioWriter(w.bw) w.bw = nil - if err := w.f.Sync(); err != nil { - logger.Panicf("FATAL: cannot sync file %q: %d", w.f.Name(), err) + if !disableFSyncForTesting { + if err := w.f.Sync(); err != nil { + logger.Panicf("FATAL: cannot sync file %q: %d", w.f.Name(), err) + } } if err := w.st.close(); err != nil { logger.Panicf("FATAL: cannot close streamTracker for file %q: %s", w.f.Name(), err) diff --git a/lib/fs/fs.go b/lib/fs/fs.go index 9a451fb24..0a5bb6715 100644 --- a/lib/fs/fs.go +++ b/lib/fs/fs.go @@ -11,11 +11,14 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) +var disableFSyncForTesting = envutil.GetenvBool("DISABLE_FSYNC_FOR_TESTING") + var tmpFileNum atomic.Uint64 // MustSyncPath syncs contents of the given path. diff --git a/lib/fs/fs_solaris.go b/lib/fs/fs_solaris.go index b3d880446..fd036198c 100644 --- a/lib/fs/fs_solaris.go +++ b/lib/fs/fs_solaris.go @@ -30,9 +30,11 @@ func mustSyncPath(path string) { if err != nil { logger.Panicf("FATAL: cannot open file for fsync: %s", err) } - if err := d.Sync(); err != nil { - _ = d.Close() - logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err) + if !disableFSyncForTesting { + if err := d.Sync(); err != nil { + _ = d.Close() + logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err) + } } if err := d.Close(); err != nil { logger.Panicf("FATAL: cannot close %q: %s", path, err) diff --git a/lib/fs/fs_unix.go b/lib/fs/fs_unix.go index c7308fa37..907082a34 100644 --- a/lib/fs/fs_unix.go +++ b/lib/fs/fs_unix.go @@ -23,9 +23,11 @@ func mustSyncPath(path string) { if err != nil { logger.Panicf("FATAL: cannot open file for fsync: %s", err) } - if err := d.Sync(); err != nil { - _ = d.Close() - logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err) + if !disableFSyncForTesting { + if err := d.Sync(); err != nil { + _ = d.Close() + logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err) + } } if err := d.Close(); err != nil { logger.Panicf("FATAL: cannot close %q: %s", path, err)