From 340ada871d5d8e8ffdf5e60775722e2a32963ed3 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Tue, 13 Sep 2022 13:37:34 +0300
Subject: [PATCH] lib/storage: atomically remove partitions, which went outside
 the configured retention

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038
---
 lib/fs/fs.go             | 1 +
 lib/storage/partition.go | 4 ++--
 lib/storage/table.go     | 2 ++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/fs/fs.go b/lib/fs/fs.go
index b5c2928e8..245bc0814 100644
--- a/lib/fs/fs.go
+++ b/lib/fs/fs.go
@@ -211,6 +211,7 @@ func MustRemoveDirAtomic(dir string) {
 	if err := os.Rename(dir, tmpDir); err != nil {
 		logger.Panicf("FATAL: cannot move %s to %s: %s", dir, tmpDir, err)
 	}
+	MustSyncPath(dir)
 	MustRemoveAll(tmpDir)
 }
 
diff --git a/lib/storage/partition.go b/lib/storage/partition.go
index 082bc4f15..5ee2cc1e6 100644
--- a/lib/storage/partition.go
+++ b/lib/storage/partition.go
@@ -236,8 +236,8 @@ func (pt *partition) Drop() {
 	// Wait until all the pending transaction deletions are finished before removing partition directories.
 	pendingTxnDeletionsWG.Wait()
 
-	fs.MustRemoveAll(pt.smallPartsPath)
-	fs.MustRemoveAll(pt.bigPartsPath)
+	fs.MustRemoveDirAtomic(pt.smallPartsPath)
+	fs.MustRemoveDirAtomic(pt.bigPartsPath)
 	logger.Infof("partition %q has been dropped", pt.name)
 }
 
diff --git a/lib/storage/table.go b/lib/storage/table.go
index 25ea2edb9..cf005814b 100644
--- a/lib/storage/table.go
+++ b/lib/storage/table.go
@@ -103,6 +103,7 @@ func openTable(path string, getDeletedMetricIDs func() *uint64set.Set, retention
 	if err := fs.MkdirAllIfNotExist(smallPartitionsPath); err != nil {
 		return nil, fmt.Errorf("cannot create directory for small partitions %q: %w", smallPartitionsPath, err)
 	}
+	fs.MustRemoveTemporaryDirs(smallPartitionsPath)
 	smallSnapshotsPath := smallPartitionsPath + "/snapshots"
 	if err := fs.MkdirAllIfNotExist(smallSnapshotsPath); err != nil {
 		return nil, fmt.Errorf("cannot create %q: %w", smallSnapshotsPath, err)
@@ -113,6 +114,7 @@ func openTable(path string, getDeletedMetricIDs func() *uint64set.Set, retention
 	if err := fs.MkdirAllIfNotExist(bigPartitionsPath); err != nil {
 		return nil, fmt.Errorf("cannot create directory for big partitions %q: %w", bigPartitionsPath, err)
 	}
+	fs.MustRemoveTemporaryDirs(bigPartitionsPath)
 	bigSnapshotsPath := bigPartitionsPath + "/snapshots"
 	if err := fs.MkdirAllIfNotExist(bigSnapshotsPath); err != nil {
 		return nil, fmt.Errorf("cannot create %q: %w", bigSnapshotsPath, err)