From 2322c9a45a283400f13290e8e928bc24a550289b Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 12 Jun 2019 01:07:02 +0300
Subject: [PATCH] lib/fs: attempt #2 to work around NFS issue with directory
 removal

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61
---
 lib/fs/fs.go | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/lib/fs/fs.go b/lib/fs/fs.go
index 72573ebf8..46c6a0fb1 100644
--- a/lib/fs/fs.go
+++ b/lib/fs/fs.go
@@ -221,10 +221,11 @@ func RemoveAllHard(path string) error {
 	if err == nil {
 		return nil
 	}
-	if !strings.Contains(err.Error(), "directory not empty") {
+	if !isTemporaryNFSError(err) {
 		return err
 	}
-	// This may be NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
+	// NFS prevents from removing directories with open files.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
 	// Schedule for later directory removal.
 	select {
 	case removeDirCh <- path:
@@ -244,22 +245,29 @@ func dirRemover() {
 			if err == nil {
 				break
 			}
-			if !strings.Contains(err.Error(), "directory not empty") {
+			if !isTemporaryNFSError(err) {
 				logger.Errorf("cannot remove %q: %s", path, err)
 				break
 			}
-			// NFS-related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
-			// Sleep for a while and try again.
+			// NFS prevents from removing directories with open files.
+			// Sleep for a while and try again in the hope open files will be closed.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
 			attempts++
-			if attempts > 50 {
+			if attempts > 10 {
 				logger.Errorf("cannot remove %q in %d attempts: %s", path, attempts, err)
 				break
 			}
-			time.Sleep(10 * time.Millisecond)
+			time.Sleep(100 * time.Millisecond)
 		}
 	}
 }
 
+func isTemporaryNFSError(err error) bool {
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 for details.
+	errStr := err.Error()
+	return strings.Contains(errStr, "directory not empty") || strings.Contains(errStr, "device or resource busy")
+}
+
 func init() {
 	go dirRemover()
 }