2019-05-22 21:16:55 +00:00
|
|
|
package fs
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io"
|
2021-12-02 21:32:03 +00:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2019-05-22 21:16:55 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2019-08-12 11:44:24 +00:00
|
|
|
"regexp"
|
2022-09-13 10:10:33 +00:00
|
|
|
"strings"
|
2020-06-04 10:13:00 +00:00
|
|
|
"sync"
|
2019-08-12 11:44:24 +00:00
|
|
|
"sync/atomic"
|
2022-09-13 10:10:33 +00:00
|
|
|
"time"
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2020-06-04 10:13:00 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
2019-05-22 21:16:55 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
|
|
)
|
|
|
|
|
2021-02-26 22:37:07 +00:00
|
|
|
var tmpFileNum uint64
|
|
|
|
|
2019-06-11 20:13:04 +00:00
|
|
|
// MustSyncPath syncs contents of the given path.
|
|
|
|
func MustSyncPath(path string) {
|
2021-02-26 22:37:07 +00:00
|
|
|
mustSyncPath(path)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2023-04-14 04:42:11 +00:00
|
|
|
// MustWriteSync writes data to the file at path and then calls fsync on the created file.
|
2022-12-04 06:00:20 +00:00
|
|
|
//
|
|
|
|
// The fsync guarantees that the written data survives hardware reset after successful call.
|
|
|
|
//
|
|
|
|
// This function may leave the file at the path in inconsistent state on app crash
|
|
|
|
// in the middle of the write.
|
2023-04-14 05:41:12 +00:00
|
|
|
// Use MustWriteAtomic if the file at the path must be either written in full
|
2022-12-04 06:00:20 +00:00
|
|
|
// or not written at all on app crash in the middle of the write.
|
2023-04-14 04:42:11 +00:00
|
|
|
func MustWriteSync(path string, data []byte) {
|
2023-04-14 22:12:45 +00:00
|
|
|
f := filestream.MustCreate(path, false)
|
2022-12-04 06:00:20 +00:00
|
|
|
if _, err := f.Write(data); err != nil {
|
|
|
|
f.MustClose()
|
2023-02-13 12:27:13 +00:00
|
|
|
// Do not call MustRemoveAll(path), so the user could inspect
|
2022-12-04 06:00:20 +00:00
|
|
|
// the file contents during investigation of the issue.
|
2023-04-14 04:33:15 +00:00
|
|
|
logger.Panicf("FATAL: cannot write %d bytes to %q: %s", len(data), path, err)
|
2022-12-04 06:00:20 +00:00
|
|
|
}
|
|
|
|
// Sync and close the file.
|
|
|
|
f.MustClose()
|
|
|
|
}
|
|
|
|
|
2023-04-14 05:41:12 +00:00
|
|
|
// MustWriteAtomic atomically writes data to the given file path.
|
2019-05-22 21:16:55 +00:00
|
|
|
//
|
2022-12-04 06:00:20 +00:00
|
|
|
// This function returns only after the file is fully written and synced
|
2019-05-22 21:16:55 +00:00
|
|
|
// to the underlying storage.
|
2022-10-25 22:06:20 +00:00
|
|
|
//
|
2022-12-04 06:00:20 +00:00
|
|
|
// This function guarantees that the file at path either fully written or not written at all on app crash
|
|
|
|
// in the middle of the write.
|
|
|
|
//
|
2022-10-25 22:06:20 +00:00
|
|
|
// If the file at path already exists, then the file is overwritten atomically if canOverwrite is true.
|
|
|
|
// Otherwise error is returned.
|
2023-04-14 05:41:12 +00:00
|
|
|
func MustWriteAtomic(path string, data []byte, canOverwrite bool) {
|
2019-08-12 11:44:24 +00:00
|
|
|
// Check for the existing file. It is expected that
|
2023-04-14 05:41:12 +00:00
|
|
|
// the MustWriteAtomic function cannot be called concurrently
|
2019-08-12 11:44:24 +00:00
|
|
|
// with the same `path`.
|
2022-10-25 22:06:20 +00:00
|
|
|
if IsPathExist(path) && !canOverwrite {
|
2023-04-14 05:41:12 +00:00
|
|
|
logger.Panicf("FATAL: cannot create file %q, since it already exists", path)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2019-08-12 11:44:24 +00:00
|
|
|
|
2022-12-04 06:00:20 +00:00
|
|
|
// Write data to a temporary file.
|
2019-08-12 11:44:24 +00:00
|
|
|
n := atomic.AddUint64(&tmpFileNum, 1)
|
|
|
|
tmpPath := fmt.Sprintf("%s.tmp.%d", path, n)
|
2023-04-14 04:42:11 +00:00
|
|
|
MustWriteSync(tmpPath, data)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2022-12-04 06:00:20 +00:00
|
|
|
// Atomically move the temporary file from tmpPath to path.
|
2019-08-12 11:44:24 +00:00
|
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
|
|
// do not call MustRemoveAll(tmpPath) here, so the user could inspect
|
2022-12-04 06:00:20 +00:00
|
|
|
// the file contents during investigation of the issue.
|
2023-04-14 05:41:12 +00:00
|
|
|
logger.Panicf("FATAL: cannot move temporary file %q to %q: %s", tmpPath, path, err)
|
2019-08-12 11:44:24 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 21:16:55 +00:00
|
|
|
// Sync the containing directory, so the file is guaranteed to appear in the directory.
|
|
|
|
// See https://www.quora.com/When-should-you-fsync-the-containing-directory-in-addition-to-the-file-itself
|
|
|
|
absPath, err := filepath.Abs(path)
|
|
|
|
if err != nil {
|
2023-04-14 05:41:12 +00:00
|
|
|
logger.Panicf("FATAL: cannot obtain absolute path to %q: %s", path, err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2019-06-11 19:54:07 +00:00
|
|
|
parentDirPath := filepath.Dir(absPath)
|
2019-06-11 20:13:04 +00:00
|
|
|
MustSyncPath(parentDirPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2019-08-12 11:44:24 +00:00
|
|
|
// IsTemporaryFileName returns true if fn matches temporary file name pattern
|
2023-04-14 05:41:12 +00:00
|
|
|
// from MustWriteAtomic.
|
2019-08-12 11:44:24 +00:00
|
|
|
func IsTemporaryFileName(fn string) bool {
|
|
|
|
return tmpFileNameRe.MatchString(fn)
|
|
|
|
}
|
|
|
|
|
2023-04-14 05:41:12 +00:00
|
|
|
// tmpFileNameRe is regexp for temporary file name - see MustWriteAtomic for details.
|
2019-08-12 11:44:24 +00:00
|
|
|
var tmpFileNameRe = regexp.MustCompile(`\.tmp\.\d+$`)
|
|
|
|
|
2023-04-14 05:11:56 +00:00
|
|
|
// MustMkdirIfNotExist creates the given path dir if it isn't exist.
|
|
|
|
func MustMkdirIfNotExist(path string) {
|
2019-05-22 21:16:55 +00:00
|
|
|
if IsPathExist(path) {
|
2023-04-14 05:11:56 +00:00
|
|
|
return
|
|
|
|
}
|
2023-04-14 05:41:12 +00:00
|
|
|
mustMkdirSync(path)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2023-04-14 05:11:56 +00:00
|
|
|
// MustMkdirFailIfExist creates the given path dir if it isn't exist.
|
2019-05-22 21:16:55 +00:00
|
|
|
//
|
2023-04-14 05:11:56 +00:00
|
|
|
// If the directory at the given path already exists, then the function logs the error and exits.
|
|
|
|
func MustMkdirFailIfExist(path string) {
|
2019-05-22 21:16:55 +00:00
|
|
|
if IsPathExist(path) {
|
2023-04-14 05:11:56 +00:00
|
|
|
logger.Panicf("FATAL: the %q already exists", path)
|
|
|
|
}
|
2023-04-14 05:41:12 +00:00
|
|
|
mustMkdirSync(path)
|
2019-06-11 20:03:14 +00:00
|
|
|
}
|
|
|
|
|
2023-04-14 05:41:12 +00:00
|
|
|
func mustMkdirSync(path string) {
|
2019-06-11 20:03:14 +00:00
|
|
|
if err := os.MkdirAll(path, 0755); err != nil {
|
2023-04-14 05:41:12 +00:00
|
|
|
logger.Panicf("FATAL: cannot create directory: %s", err)
|
2019-06-11 20:03:14 +00:00
|
|
|
}
|
|
|
|
// Sync the parent directory, so the created directory becomes visible
|
|
|
|
// in the fs after power loss.
|
|
|
|
parentDirPath := filepath.Dir(path)
|
2019-06-11 20:13:04 +00:00
|
|
|
MustSyncPath(parentDirPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2019-12-02 19:34:35 +00:00
|
|
|
// RemoveDirContents removes all the contents of the given dir if it exists.
|
2019-05-22 21:16:55 +00:00
|
|
|
//
|
|
|
|
// It doesn't remove the dir itself, so the dir may be mounted
|
|
|
|
// to a separate partition.
|
|
|
|
func RemoveDirContents(dir string) {
|
|
|
|
if !IsPathExist(dir) {
|
|
|
|
// The path doesn't exist, so nothing to remove.
|
|
|
|
return
|
|
|
|
}
|
|
|
|
d, err := os.Open(dir)
|
|
|
|
if err != nil {
|
2022-12-04 06:00:20 +00:00
|
|
|
logger.Panicf("FATAL: cannot open dir: %s", err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
defer MustClose(d)
|
|
|
|
names, err := d.Readdirnames(-1)
|
|
|
|
if err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot read contents of the dir %q: %s", dir, err)
|
|
|
|
}
|
|
|
|
for _, name := range names {
|
|
|
|
if name == "." || name == ".." || name == "lost+found" {
|
|
|
|
// Skip special dirs.
|
|
|
|
continue
|
|
|
|
}
|
2023-04-14 03:12:24 +00:00
|
|
|
fullPath := filepath.Join(dir, name)
|
2019-06-11 22:53:43 +00:00
|
|
|
MustRemoveAll(fullPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2019-06-11 20:13:04 +00:00
|
|
|
MustSyncPath(dir)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// MustClose must close the given file f.
|
|
|
|
func MustClose(f *os.File) {
|
|
|
|
fname := f.Name()
|
|
|
|
if err := f.Close(); err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot close %q: %s", fname, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-04 16:09:40 +00:00
|
|
|
// MustFileSize returns file size for the given path.
|
|
|
|
func MustFileSize(path string) uint64 {
|
|
|
|
fi, err := os.Stat(path)
|
|
|
|
if err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot stat %q: %s", path, err)
|
|
|
|
}
|
|
|
|
if fi.IsDir() {
|
|
|
|
logger.Panicf("FATAL: %q must be a file, not a directory", path)
|
|
|
|
}
|
|
|
|
return uint64(fi.Size())
|
|
|
|
}
|
|
|
|
|
2019-05-22 21:16:55 +00:00
|
|
|
// IsPathExist returns whether the given path exists.
|
|
|
|
func IsPathExist(path string) bool {
|
|
|
|
if _, err := os.Stat(path); err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
logger.Panicf("FATAL: cannot stat %q: %s", path, err)
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2019-06-11 23:14:38 +00:00
|
|
|
func mustSyncParentDirIfExists(path string) {
|
|
|
|
parentDirPath := filepath.Dir(path)
|
|
|
|
if !IsPathExist(parentDirPath) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
MustSyncPath(parentDirPath)
|
|
|
|
}
|
|
|
|
|
2021-04-22 09:58:53 +00:00
|
|
|
// IsEmptyDir returns true if path points to empty directory.
|
|
|
|
func IsEmptyDir(path string) bool {
|
|
|
|
// See https://stackoverflow.com/a/30708914/274937
|
|
|
|
f, err := os.Open(path)
|
|
|
|
if err != nil {
|
2022-12-04 06:00:20 +00:00
|
|
|
logger.Panicf("FATAL: cannot open dir: %s", err)
|
2021-04-22 09:58:53 +00:00
|
|
|
}
|
|
|
|
_, err = f.Readdirnames(1)
|
|
|
|
MustClose(f)
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
logger.Panicf("FATAL: unexpected error when reading directory %q: %s", path, err)
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2022-09-13 10:10:33 +00:00
|
|
|
// MustRemoveDirAtomic removes the given dir atomically.
|
|
|
|
//
|
|
|
|
// It uses the following algorithm:
|
|
|
|
//
|
|
|
|
// 1. Atomically rename the "<dir>" to "<dir>.must-remove.<XYZ>",
|
|
|
|
// where <XYZ> is an unique number.
|
|
|
|
// 2. Remove the "<dir>.must-remove.XYZ" in background.
|
|
|
|
//
|
|
|
|
// If the process crashes after the step 1, then the directory must be removed
|
2022-09-13 12:48:20 +00:00
|
|
|
// on the next process start by calling MustRemoveTemporaryDirs on the parent directory.
|
2022-09-13 10:10:33 +00:00
|
|
|
func MustRemoveDirAtomic(dir string) {
|
2022-09-13 12:48:20 +00:00
|
|
|
if !IsPathExist(dir) {
|
|
|
|
return
|
|
|
|
}
|
2022-09-13 10:10:33 +00:00
|
|
|
n := atomic.AddUint64(&atomicDirRemoveCounter, 1)
|
|
|
|
tmpDir := fmt.Sprintf("%s.must-remove.%d", dir, n)
|
|
|
|
if err := os.Rename(dir, tmpDir); err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot move %s to %s: %s", dir, tmpDir, err)
|
|
|
|
}
|
|
|
|
MustRemoveAll(tmpDir)
|
2022-09-13 12:48:20 +00:00
|
|
|
parentDir := filepath.Dir(dir)
|
|
|
|
MustSyncPath(parentDir)
|
2022-09-13 10:10:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var atomicDirRemoveCounter = uint64(time.Now().UnixNano())
|
|
|
|
|
2023-04-15 05:08:43 +00:00
|
|
|
// MustReadDir reads directory entries at the given dir.
|
|
|
|
func MustReadDir(dir string) []os.DirEntry {
|
|
|
|
des, err := os.ReadDir(dir)
|
|
|
|
if err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot read directory contents: %s", err)
|
|
|
|
}
|
|
|
|
return des
|
|
|
|
}
|
|
|
|
|
2022-09-13 10:10:33 +00:00
|
|
|
// MustRemoveTemporaryDirs removes all the subdirectories with ".must-remove.<XYZ>" suffix.
|
|
|
|
//
|
2022-09-13 12:48:20 +00:00
|
|
|
// Such directories may be left on unclean shutdown during MustRemoveDirAtomic call.
|
2022-09-13 10:10:33 +00:00
|
|
|
func MustRemoveTemporaryDirs(dir string) {
|
2023-04-15 05:08:43 +00:00
|
|
|
des := MustReadDir(dir)
|
2023-03-18 04:03:34 +00:00
|
|
|
for _, de := range des {
|
|
|
|
if !IsDirOrSymlink(de) {
|
2022-09-13 10:10:33 +00:00
|
|
|
// Skip non-directories
|
|
|
|
continue
|
|
|
|
}
|
2023-03-18 04:03:34 +00:00
|
|
|
dirName := de.Name()
|
2023-02-24 20:38:42 +00:00
|
|
|
if IsScheduledForRemoval(dirName) {
|
2023-04-14 03:12:24 +00:00
|
|
|
fullPath := filepath.Join(dir, dirName)
|
2022-09-13 10:10:33 +00:00
|
|
|
MustRemoveAll(fullPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
MustSyncPath(dir)
|
|
|
|
}
|
|
|
|
|
2023-04-14 05:48:05 +00:00
|
|
|
// MustHardLinkFiles makes hard links for all the files from srcDir in dstDir.
|
|
|
|
func MustHardLinkFiles(srcDir, dstDir string) {
|
2023-04-14 05:41:12 +00:00
|
|
|
mustMkdirSync(dstDir)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-04-15 05:08:43 +00:00
|
|
|
des := MustReadDir(srcDir)
|
2023-03-18 04:03:34 +00:00
|
|
|
for _, de := range des {
|
|
|
|
if IsDirOrSymlink(de) {
|
2019-05-22 21:16:55 +00:00
|
|
|
// Skip directories.
|
|
|
|
continue
|
|
|
|
}
|
2023-03-18 04:03:34 +00:00
|
|
|
fn := de.Name()
|
2023-04-14 03:12:24 +00:00
|
|
|
srcPath := filepath.Join(srcDir, fn)
|
|
|
|
dstPath := filepath.Join(dstDir, fn)
|
2019-05-22 21:16:55 +00:00
|
|
|
if err := os.Link(srcPath, dstPath); err != nil {
|
2023-04-14 05:48:05 +00:00
|
|
|
logger.Panicf("FATAL: cannot link files: %s", err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-11 20:13:04 +00:00
|
|
|
MustSyncPath(dstDir)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2023-03-18 04:03:34 +00:00
|
|
|
// IsDirOrSymlink returns true if de is directory or symlink.
|
|
|
|
func IsDirOrSymlink(de os.DirEntry) bool {
|
|
|
|
return de.IsDir() || (de.Type()&os.ModeSymlink == os.ModeSymlink)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2023-04-14 05:52:45 +00:00
|
|
|
// MustSymlinkRelative creates relative symlink for srcPath in dstPath.
|
|
|
|
func MustSymlinkRelative(srcPath, dstPath string) {
|
2019-05-22 21:16:55 +00:00
|
|
|
baseDir := filepath.Dir(dstPath)
|
|
|
|
srcPathRel, err := filepath.Rel(baseDir, srcPath)
|
|
|
|
if err != nil {
|
2023-04-14 05:52:45 +00:00
|
|
|
logger.Panicf("FATAL: cannot make relative path for srcPath=%q: %s", srcPath, err)
|
|
|
|
}
|
|
|
|
if err := os.Symlink(srcPathRel, dstPath); err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot make a symlink: %s", err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-14 06:02:55 +00:00
|
|
|
// MustCopyDirectory copies all the files in srcPath to dstPath.
|
|
|
|
func MustCopyDirectory(srcPath, dstPath string) {
|
2023-04-15 05:08:43 +00:00
|
|
|
des := MustReadDir(srcPath)
|
2023-04-14 05:11:56 +00:00
|
|
|
MustMkdirIfNotExist(dstPath)
|
2022-08-21 21:02:19 +00:00
|
|
|
for _, de := range des {
|
|
|
|
if !de.Type().IsRegular() {
|
2021-02-10 12:37:14 +00:00
|
|
|
// Skip non-files
|
|
|
|
continue
|
|
|
|
}
|
2022-08-21 21:02:19 +00:00
|
|
|
src := filepath.Join(srcPath, de.Name())
|
|
|
|
dst := filepath.Join(dstPath, de.Name())
|
2023-04-14 06:02:55 +00:00
|
|
|
MustCopyFile(src, dst)
|
2021-02-10 12:37:14 +00:00
|
|
|
}
|
|
|
|
MustSyncPath(dstPath)
|
|
|
|
}
|
|
|
|
|
2023-04-14 06:02:55 +00:00
|
|
|
// MustCopyFile copies the file from srcPath to dstPath.
|
|
|
|
func MustCopyFile(srcPath, dstPath string) {
|
2021-02-10 12:37:14 +00:00
|
|
|
src, err := os.Open(srcPath)
|
|
|
|
if err != nil {
|
2023-04-14 06:02:55 +00:00
|
|
|
logger.Panicf("FATAL: cannot open srcPath: %s", err)
|
2021-02-10 12:37:14 +00:00
|
|
|
}
|
|
|
|
defer MustClose(src)
|
|
|
|
dst, err := os.Create(dstPath)
|
|
|
|
if err != nil {
|
2023-04-14 06:02:55 +00:00
|
|
|
logger.Panicf("FATAL: cannot create dstPath: %s", err)
|
2021-02-10 12:37:14 +00:00
|
|
|
}
|
|
|
|
defer MustClose(dst)
|
|
|
|
if _, err := io.Copy(dst, src); err != nil {
|
2023-04-14 06:02:55 +00:00
|
|
|
logger.Panicf("FATAL: cannot copy %q to %q: %s", srcPath, dstPath, err)
|
2021-02-10 12:37:14 +00:00
|
|
|
}
|
|
|
|
MustSyncPath(dstPath)
|
|
|
|
}
|
|
|
|
|
2023-04-14 21:39:26 +00:00
|
|
|
// MustReadData reads len(data) bytes from r.
|
|
|
|
func MustReadData(r filestream.ReadCloser, data []byte) {
|
2019-05-22 21:16:55 +00:00
|
|
|
n, err := io.ReadFull(r, data)
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
2023-04-14 21:39:26 +00:00
|
|
|
return
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2023-04-14 21:39:26 +00:00
|
|
|
logger.Panicf("FATAL: cannot read %d bytes from %s; read only %d bytes; error: %s", len(data), r.Path(), n, err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
if n != len(data) {
|
2023-04-14 21:39:26 +00:00
|
|
|
logger.Panicf("BUG: io.ReadFull read only %d bytes from %s; must read %d bytes", n, r.Path(), len(data))
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// MustWriteData writes data to w.
|
2023-04-14 21:32:43 +00:00
|
|
|
func MustWriteData(w filestream.WriteCloser, data []byte) {
|
2019-05-22 21:16:55 +00:00
|
|
|
if len(data) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
n, err := w.Write(data)
|
|
|
|
if err != nil {
|
2023-04-14 21:32:43 +00:00
|
|
|
logger.Panicf("FATAL: cannot write %d bytes to %s: %s", len(data), w.Path(), err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
if n != len(data) {
|
2023-04-14 21:32:43 +00:00
|
|
|
logger.Panicf("BUG: writer wrote %d bytes instead of %d bytes to %s", n, len(data), w.Path())
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
}
|
2019-08-12 22:45:22 +00:00
|
|
|
|
2023-04-15 02:49:54 +00:00
|
|
|
// MustCreateFlockFile creates FlockFilename file in the directory dir
|
2019-08-12 22:45:22 +00:00
|
|
|
// and returns the handler to the file.
|
2023-04-15 02:49:54 +00:00
|
|
|
func MustCreateFlockFile(dir string) *os.File {
|
|
|
|
flockFilepath := filepath.Join(dir, FlockFilename)
|
|
|
|
f, err := createFlockFile(flockFilepath)
|
|
|
|
if err != nil {
|
|
|
|
logger.Panicf("FATAL: cannot create lock file: %s; make sure a single process has exclusive access to %q", err, dir)
|
|
|
|
}
|
|
|
|
return f
|
2019-08-12 22:45:22 +00:00
|
|
|
}
|
2019-08-27 22:04:44 +00:00
|
|
|
|
2023-04-15 02:49:54 +00:00
|
|
|
// FlockFilename is the filename for the file created by MustCreateFlockFile().
|
2023-04-14 03:12:24 +00:00
|
|
|
const FlockFilename = "flock.lock"
|
|
|
|
|
2019-08-27 22:04:44 +00:00
|
|
|
// MustGetFreeSpace returns free space for the given directory path.
|
|
|
|
func MustGetFreeSpace(path string) uint64 {
|
2020-06-04 10:13:00 +00:00
|
|
|
// Try obtaining cached value at first.
|
|
|
|
freeSpaceMapLock.Lock()
|
|
|
|
defer freeSpaceMapLock.Unlock()
|
|
|
|
|
|
|
|
e, ok := freeSpaceMap[path]
|
|
|
|
if ok && fasttime.UnixTimestamp()-e.updateTime < 2 {
|
|
|
|
// Fast path - the entry is fresh.
|
|
|
|
return e.freeSpace
|
|
|
|
}
|
|
|
|
|
|
|
|
// Slow path.
|
|
|
|
// Determine the amount of free space at path.
|
|
|
|
e.freeSpace = mustGetFreeSpace(path)
|
|
|
|
e.updateTime = fasttime.UnixTimestamp()
|
|
|
|
freeSpaceMap[path] = e
|
|
|
|
return e.freeSpace
|
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
freeSpaceMap = make(map[string]freeSpaceEntry)
|
|
|
|
freeSpaceMapLock sync.Mutex
|
|
|
|
)
|
|
|
|
|
|
|
|
type freeSpaceEntry struct {
|
|
|
|
updateTime uint64
|
|
|
|
freeSpace uint64
|
|
|
|
}
|
2021-12-02 21:32:03 +00:00
|
|
|
|
|
|
|
// ReadFileOrHTTP reads path either from local filesystem or from http if path starts with http or https.
|
|
|
|
func ReadFileOrHTTP(path string) ([]byte, error) {
|
|
|
|
if isHTTPURL(path) {
|
|
|
|
// reads remote file via http or https, if url is given
|
|
|
|
resp, err := http.Get(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot fetch %q: %w", path, err)
|
|
|
|
}
|
2022-08-21 21:13:44 +00:00
|
|
|
data, err := io.ReadAll(resp.Body)
|
2021-12-02 21:32:03 +00:00
|
|
|
_ = resp.Body.Close()
|
2023-04-01 05:41:02 +00:00
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
if len(data) > 4*1024 {
|
|
|
|
data = data[:4*1024]
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("unexpected status code when fetching %q: %d, expecting %d; response: %q", path, resp.StatusCode, http.StatusOK, data)
|
|
|
|
}
|
2021-12-02 21:32:03 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot read %q: %s", path, err)
|
|
|
|
}
|
|
|
|
return data, nil
|
|
|
|
}
|
2022-08-21 20:51:13 +00:00
|
|
|
data, err := os.ReadFile(path)
|
2021-12-02 21:32:03 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot read %q: %w", path, err)
|
|
|
|
}
|
|
|
|
return data, nil
|
|
|
|
}
|
|
|
|
|
2021-12-02 22:08:42 +00:00
|
|
|
// GetFilepath returns full path to file for the given baseDir and path.
|
|
|
|
func GetFilepath(baseDir, path string) string {
|
|
|
|
if filepath.IsAbs(path) || isHTTPURL(path) {
|
|
|
|
return path
|
|
|
|
}
|
|
|
|
return filepath.Join(baseDir, path)
|
|
|
|
}
|
|
|
|
|
2021-12-02 21:32:03 +00:00
|
|
|
// isHTTPURL checks if a given targetURL is valid and contains a valid http scheme
|
|
|
|
func isHTTPURL(targetURL string) bool {
|
|
|
|
parsed, err := url.Parse(targetURL)
|
|
|
|
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != ""
|
|
|
|
|
|
|
|
}
|
2023-02-24 20:38:42 +00:00
|
|
|
|
2023-02-26 20:18:59 +00:00
|
|
|
// IsScheduledForRemoval returns true if the filename contains .must-remove. substring
|
|
|
|
func IsScheduledForRemoval(filename string) bool {
|
|
|
|
return strings.Contains(filename, ".must-remove.")
|
2023-02-24 20:38:42 +00:00
|
|
|
}
|