github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/fs: attempt #2 to work around NFS issue with directory removal

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61
This commit is contained in:
Aliaksandr Valialkin 2019-06-12 01:07:02 +03:00
parent 89b928ff24
commit 2322c9a45a
1 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/fs/fs.go
View file

@ -221,10 +221,11 @@ func RemoveAllHard(path string) error {
if err == nil { if err == nil {
return nil return nil
} }
if !strings.Contains(err.Error(), "directory not empty") { if !isTemporaryNFSError(err) {
return err return err
} }
// This may be NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 . // NFS prevents from removing directories with open files.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
// Schedule for later directory removal. // Schedule for later directory removal.
select { select {
case removeDirCh <- path: case removeDirCh <- path:
@ -244,22 +245,29 @@ func dirRemover() {
if err == nil { if err == nil {
break break
} }
if !strings.Contains(err.Error(), "directory not empty") { if !isTemporaryNFSError(err) {
logger.Errorf("cannot remove %q: %s", path, err) logger.Errorf("cannot remove %q: %s", path, err)
break break
} }
// NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 . // NFS prevents from removing directories with open files.
// Sleep for a while and try again. // Sleep for a while and try again in the hope open files will be closed.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
attempts++ attempts++
if attempts > 50 { if attempts > 10 {
logger.Errorf("cannot remove %q in %d attempts: %s", path, attempts, err) logger.Errorf("cannot remove %q in %d attempts: %s", path, attempts, err)
break break
} }
time.Sleep(10 * time.Millisecond) time.Sleep(100 * time.Millisecond)
} }
} }
} }
func isTemporaryNFSError(err error) bool {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 for details.
errStr := err.Error()
return strings.Contains(errStr, "directory not empty") || strings.Contains(errStr, "device or resource busy")
}
func init() { func init() {
go dirRemover() go dirRemover()
} }