2019-11-07 19:05:39 +00:00
|
|
|
package fscommon
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
|
|
)
|
|
|
|
|
|
|
|
// FsyncFile fsyncs path contents and the parent directory contents.
|
|
|
|
func FsyncFile(path string) error {
|
|
|
|
if err := fsync(path); err != nil {
|
|
|
|
_ = os.RemoveAll(path)
|
2020-06-30 19:58:18 +00:00
|
|
|
return fmt.Errorf("cannot fsync file %q: %w", path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
dir := filepath.Dir(path)
|
|
|
|
if err := fsync(dir); err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return fmt.Errorf("cannot fsync dir %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// FsyncDir fsyncs dir contents.
|
|
|
|
func FsyncDir(dir string) error {
|
|
|
|
return fsync(dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
func fsync(path string) error {
|
|
|
|
f, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := f.Sync(); err != nil {
|
|
|
|
_ = f.Close()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return f.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
// AppendFiles appends all the files from dir to dst.
|
|
|
|
//
|
|
|
|
// All the appended files will have dir prefix.
|
|
|
|
func AppendFiles(dst []string, dir string) ([]string, error) {
|
|
|
|
d, err := os.Open(dir)
|
|
|
|
if err != nil {
|
2022-12-04 05:55:06 +00:00
|
|
|
return nil, fmt.Errorf("cannot open directory: %w", err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
dst, err = appendFilesInternal(dst, d)
|
|
|
|
if err1 := d.Close(); err1 != nil {
|
|
|
|
err = err1
|
|
|
|
}
|
|
|
|
return dst, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
|
|
|
|
dir := d.Name()
|
|
|
|
dfi, err := d.Stat()
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot stat %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if !dfi.IsDir() {
|
|
|
|
return nil, fmt.Errorf("%q isn't a directory", dir)
|
|
|
|
}
|
|
|
|
fis, err := d.Readdir(-1)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot read directory contents in %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
for _, fi := range fis {
|
|
|
|
name := fi.Name()
|
|
|
|
if name == "." || name == ".." {
|
|
|
|
continue
|
|
|
|
}
|
2021-12-22 11:10:15 +00:00
|
|
|
if isSpecialFile(name) {
|
|
|
|
// Do not take into account special files.
|
2019-11-19 16:35:31 +00:00
|
|
|
continue
|
|
|
|
}
|
2019-11-07 19:05:39 +00:00
|
|
|
path := dir + "/" + name
|
|
|
|
if fi.IsDir() {
|
|
|
|
// Process directory
|
|
|
|
dst, err = AppendFiles(dst, path)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot list %q: %w", path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
|
|
|
// Process file
|
|
|
|
dst = append(dst, path)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
pathOrig := path
|
|
|
|
again:
|
|
|
|
// Process symlink
|
|
|
|
pathReal, err := filepath.EvalSymlinks(pathOrig)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
|
|
|
|
// Skip symlink that points to nowhere.
|
|
|
|
continue
|
|
|
|
}
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot resolve symlink %q: %w", pathOrig, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
sfi, err := os.Stat(pathReal)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot stat %q from symlink %q: %w", pathReal, path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if sfi.IsDir() {
|
|
|
|
// Symlink points to directory
|
|
|
|
dstNew, err := AppendFiles(dst, pathReal)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot list files at %q from symlink %q: %w", pathReal, path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
pathReal += "/"
|
|
|
|
for i := len(dst); i < len(dstNew); i++ {
|
|
|
|
x := dstNew[i]
|
|
|
|
if !strings.HasPrefix(x, pathReal) {
|
|
|
|
return nil, fmt.Errorf("unexpected prefix for path %q; want %q", x, pathReal)
|
|
|
|
}
|
|
|
|
dstNew[i] = path + "/" + x[len(pathReal):]
|
|
|
|
}
|
|
|
|
dst = dstNew
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
|
|
|
// Symlink points to file
|
|
|
|
dst = append(dst, path)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Symlink points to symlink. Process it again.
|
|
|
|
pathOrig = pathReal
|
|
|
|
goto again
|
|
|
|
}
|
|
|
|
return dst, nil
|
|
|
|
}
|
|
|
|
|
2021-12-22 11:10:15 +00:00
|
|
|
func isSpecialFile(name string) bool {
|
|
|
|
return name == "flock.lock" || name == "restore-in-progress"
|
|
|
|
}
|
|
|
|
|
2019-11-07 19:05:39 +00:00
|
|
|
// RemoveEmptyDirs recursively removes empty directories under the given dir.
|
|
|
|
func RemoveEmptyDirs(dir string) error {
|
|
|
|
_, err := removeEmptyDirs(dir)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func removeEmptyDirs(dir string) (bool, error) {
|
|
|
|
d, err := os.Open(dir)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
ok, err := removeEmptyDirsInternal(d)
|
|
|
|
if err1 := d.Close(); err1 != nil {
|
|
|
|
err = err1
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
return ok, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func removeEmptyDirsInternal(d *os.File) (bool, error) {
|
|
|
|
dir := d.Name()
|
|
|
|
dfi, err := d.Stat()
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot stat %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if !dfi.IsDir() {
|
|
|
|
return false, fmt.Errorf("%q isn't a directory", dir)
|
|
|
|
}
|
|
|
|
fis, err := d.Readdir(-1)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot read directory contents in %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
dirEntries := 0
|
|
|
|
for _, fi := range fis {
|
|
|
|
name := fi.Name()
|
|
|
|
if name == "." || name == ".." {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
path := dir + "/" + name
|
|
|
|
if fi.IsDir() {
|
|
|
|
// Process directory
|
|
|
|
ok, err := removeEmptyDirs(path)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot list %q: %w", path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
dirEntries++
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
2021-12-22 11:10:15 +00:00
|
|
|
if isSpecialFile(name) {
|
|
|
|
// Do not take into account special files
|
2019-11-28 11:36:02 +00:00
|
|
|
continue
|
|
|
|
}
|
2019-11-07 19:05:39 +00:00
|
|
|
dirEntries++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
pathOrig := path
|
|
|
|
again:
|
|
|
|
// Process symlink
|
|
|
|
pathReal, err := filepath.EvalSymlinks(pathOrig)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
|
|
|
|
// Remove symlink that points to nowere.
|
|
|
|
logger.Infof("removing broken symlink %q", pathOrig)
|
|
|
|
if err := os.Remove(pathOrig); err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot remove %q: %w", pathOrig, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot resolve symlink %q: %w", pathOrig, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
sfi, err := os.Stat(pathReal)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot stat %q from symlink %q: %w", pathReal, path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if sfi.IsDir() {
|
|
|
|
// Symlink points to directory
|
|
|
|
ok, err := removeEmptyDirs(pathReal)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot list files at %q from symlink %q: %w", pathReal, path, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
dirEntries++
|
|
|
|
} else {
|
|
|
|
// Remove the symlink
|
|
|
|
logger.Infof("removing symlink that points to empty dir %q", pathOrig)
|
|
|
|
if err := os.Remove(pathOrig); err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot remove %q: %w", pathOrig, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
|
|
|
// Symlink points to file. Skip it.
|
|
|
|
dirEntries++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Symlink points to symlink. Process it again.
|
|
|
|
pathOrig = pathReal
|
|
|
|
goto again
|
|
|
|
}
|
|
|
|
if dirEntries > 0 {
|
|
|
|
return false, nil
|
|
|
|
}
|
2021-12-22 11:10:15 +00:00
|
|
|
// Use os.RemoveAll() instead of os.Remove(), since the dir may contain special files such as flock.lock and restore-in-progress,
|
2023-02-13 12:27:13 +00:00
|
|
|
// which must be ignored.
|
2021-12-22 11:10:15 +00:00
|
|
|
if err := os.RemoveAll(dir); err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return false, fmt.Errorf("cannot remove %q: %w", dir, err)
|
2019-11-07 19:05:39 +00:00
|
|
|
}
|
|
|
|
return true, nil
|
|
|
|
}
|
2020-01-09 13:24:26 +00:00
|
|
|
|
|
|
|
// IgnorePath returns true if the given path must be ignored.
|
|
|
|
func IgnorePath(path string) bool {
|
|
|
|
return strings.HasSuffix(path, ".ignore")
|
|
|
|
}
|
|
|
|
|
|
|
|
// BackupCompleteFilename is a filename, which is created in the destination fs when backup is complete.
|
|
|
|
const BackupCompleteFilename = "backup_complete.ignore"
|