package fscommon import ( "fmt" "os" "path/filepath" "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) // FsyncFile fsyncs path contents and the parent directory contents. func FsyncFile(path string) error { if err := fsync(path); err != nil { _ = os.RemoveAll(path) return fmt.Errorf("cannot fsync file %q: %w", path, err) } dir := filepath.Dir(path) if err := fsync(dir); err != nil { return fmt.Errorf("cannot fsync dir %q: %w", dir, err) } return nil } // FsyncDir fsyncs dir contents. func FsyncDir(dir string) error { return fsync(dir) } func fsync(path string) error { f, err := os.Open(path) if err != nil { return err } if err := f.Sync(); err != nil { _ = f.Close() return err } return f.Close() } // AppendFiles appends all the files from dir to dst. // // All the appended files will have dir prefix. func AppendFiles(dst []string, dir string) ([]string, error) { d, err := os.Open(dir) if err != nil { return nil, fmt.Errorf("cannot open directory: %w", err) } dst, err = appendFilesInternal(dst, d) if err1 := d.Close(); err1 != nil { err = err1 } return dst, err } func appendFilesInternal(dst []string, d *os.File) ([]string, error) { dir := d.Name() dfi, err := d.Stat() if err != nil { return nil, fmt.Errorf("cannot stat %q: %w", dir, err) } if !dfi.IsDir() { return nil, fmt.Errorf("%q isn't a directory", dir) } fis, err := d.Readdir(-1) if err != nil { return nil, fmt.Errorf("cannot read directory contents in %q: %w", dir, err) } for _, fi := range fis { name := fi.Name() if name == "." || name == ".." { continue } if isSpecialFile(name) { // Do not take into account special files. continue } path := dir + "/" + name if fi.IsDir() { // Process directory dst, err = AppendFiles(dst, path) if err != nil { return nil, fmt.Errorf("cannot list %q: %w", path, err) } continue } if fi.Mode()&os.ModeSymlink != os.ModeSymlink { // Process file dst = append(dst, path) continue } pathOrig := path again: // Process symlink pathReal, err := filepath.EvalSymlinks(pathOrig) if err != nil { if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") { // Skip symlink that points to nowhere. continue } return nil, fmt.Errorf("cannot resolve symlink %q: %w", pathOrig, err) } sfi, err := os.Stat(pathReal) if err != nil { return nil, fmt.Errorf("cannot stat %q from symlink %q: %w", pathReal, path, err) } if sfi.IsDir() { // Symlink points to directory dstNew, err := AppendFiles(dst, pathReal) if err != nil { return nil, fmt.Errorf("cannot list files at %q from symlink %q: %w", pathReal, path, err) } pathReal += "/" for i := len(dst); i < len(dstNew); i++ { x := dstNew[i] if !strings.HasPrefix(x, pathReal) { return nil, fmt.Errorf("unexpected prefix for path %q; want %q", x, pathReal) } dstNew[i] = path + "/" + x[len(pathReal):] } dst = dstNew continue } if sfi.Mode()&os.ModeSymlink != os.ModeSymlink { // Symlink points to file dst = append(dst, path) continue } // Symlink points to symlink. Process it again. pathOrig = pathReal goto again } return dst, nil } func isSpecialFile(name string) bool { return name == "flock.lock" || name == "restore-in-progress" } // RemoveEmptyDirs recursively removes empty directories under the given dir. func RemoveEmptyDirs(dir string) error { _, err := removeEmptyDirs(dir) return err } func removeEmptyDirs(dir string) (bool, error) { d, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { return true, nil } return false, err } ok, err := removeEmptyDirsInternal(d) if err1 := d.Close(); err1 != nil { err = err1 } if err != nil { return false, err } return ok, nil } func removeEmptyDirsInternal(d *os.File) (bool, error) { dir := d.Name() dfi, err := d.Stat() if err != nil { return false, fmt.Errorf("cannot stat %q: %w", dir, err) } if !dfi.IsDir() { return false, fmt.Errorf("%q isn't a directory", dir) } fis, err := d.Readdir(-1) if err != nil { return false, fmt.Errorf("cannot read directory contents in %q: %w", dir, err) } dirEntries := 0 for _, fi := range fis { name := fi.Name() if name == "." || name == ".." { continue } path := dir + "/" + name if fi.IsDir() { // Process directory ok, err := removeEmptyDirs(path) if err != nil { return false, fmt.Errorf("cannot list %q: %w", path, err) } if !ok { dirEntries++ } continue } if fi.Mode()&os.ModeSymlink != os.ModeSymlink { if isSpecialFile(name) { // Do not take into account special files continue } dirEntries++ continue } pathOrig := path again: // Process symlink pathReal, err := filepath.EvalSymlinks(pathOrig) if err != nil { if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") { // Remove symlink that points to nowere. logger.Infof("removing broken symlink %q", pathOrig) if err := os.Remove(pathOrig); err != nil { return false, fmt.Errorf("cannot remove %q: %w", pathOrig, err) } continue } return false, fmt.Errorf("cannot resolve symlink %q: %w", pathOrig, err) } sfi, err := os.Stat(pathReal) if err != nil { return false, fmt.Errorf("cannot stat %q from symlink %q: %w", pathReal, path, err) } if sfi.IsDir() { // Symlink points to directory ok, err := removeEmptyDirs(pathReal) if err != nil { return false, fmt.Errorf("cannot list files at %q from symlink %q: %w", pathReal, path, err) } if !ok { dirEntries++ } else { // Remove the symlink logger.Infof("removing symlink that points to empty dir %q", pathOrig) if err := os.Remove(pathOrig); err != nil { return false, fmt.Errorf("cannot remove %q: %w", pathOrig, err) } } continue } if sfi.Mode()&os.ModeSymlink != os.ModeSymlink { // Symlink points to file. Skip it. dirEntries++ continue } // Symlink points to symlink. Process it again. pathOrig = pathReal goto again } if dirEntries > 0 { return false, nil } // Use os.RemoveAll() instead of os.Remove(), since the dir may contain special files such as flock.lock and restore-in-progress, // which must be ignored. if err := os.RemoveAll(dir); err != nil { return false, fmt.Errorf("cannot remove %q: %w", dir, err) } return true, nil } // IgnorePath returns true if the given path must be ignored. func IgnorePath(path string) bool { return strings.HasSuffix(path, ".ignore") } // BackupCompleteFilename is a filename, which is created in the destination fs when backup is complete. const BackupCompleteFilename = "backup_complete.ignore"