github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/fs: attempt #2 to work around NFS issue with directory removal

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61
This commit is contained in:
Aliaksandr Valialkin 2019-06-12 01:07:02 +03:00
parent f165500225
commit d897bc3f08
1 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/fs/fs.go
View file

@ -221,10 +221,11 @@ func RemoveAllHard(path string) error {
if err == nil {
return nil
}
if !strings.Contains(err.Error(), "directory not empty") {
if !isTemporaryNFSError(err) {
return err
}
// This may be NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
// NFS prevents from removing directories with open files.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
// Schedule for later directory removal.
select {
case removeDirCh <- path:
@ -244,22 +245,29 @@ func dirRemover() {
if err == nil {
break
}
if !strings.Contains(err.Error(), "directory not empty") {
if !isTemporaryNFSError(err) {
logger.Errorf("cannot remove %q: %s", path, err)
break
}
// NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
// Sleep for a while and try again.
// NFS prevents from removing directories with open files.
// Sleep for a while and try again in the hope open files will be closed.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
attempts++
if attempts > 50 {
if attempts > 10 {
logger.Errorf("cannot remove %q in %d attempts: %s", path, attempts, err)
break
}
time.Sleep(10 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
}
}
}
func isTemporaryNFSError(err error) bool {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 for details.
errStr := err.Error()
return strings.Contains(errStr, "directory not empty") || strings.Contains(errStr, "device or resource busy")
}
func init() {
go dirRemover()
}