From 0ed92585454080608fde827a9516d3100a127842 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 9 Dec 2019 15:42:57 +0200 Subject: [PATCH] lib/{mergeset,storage}: log info message when both source and destination part paths from txn are missing during startup This is expected condition after unclean shutdown (OOM, hard reset, `kill -9`) on NFS disk. --- lib/mergeset/table.go | 11 ++++++----- lib/storage/partition.go | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index edbefa7645..f966d2cac0 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -1152,11 +1152,12 @@ func runTransaction(txnLock *sync.RWMutex, pathPrefix, txnPath string) error { if err := os.Rename(srcPath, dstPath); err != nil { return fmt.Errorf("cannot rename %q to %q: %s", srcPath, dstPath, err) } - } else { - // Verify dstPath exists. - if !fs.IsPathExist(dstPath) { - return fmt.Errorf("cannot find both source and destination paths: %q -> %q", srcPath, dstPath) - } + } else if !fs.IsPathExist(dstPath) { + // Emit info message for the expected condition after unclean shutdown on NFS disk. + // The dstPath part may be missing because it could be already merged into bigger part + // while old source parts for the current txn weren't still deleted due to NFS locks. + logger.Infof("cannot find both source and destination paths: %q -> %q; this may be the case after unclean shutdown (OOM, `kill -9`, hard reset) on NFS disk", + srcPath, dstPath) } // Flush pathPrefix directory metadata to the underying storage. diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 39ab6af08e..9cc00d9913 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -1486,11 +1486,12 @@ func runTransaction(txnLock *sync.RWMutex, pathPrefix1, pathPrefix2, txnPath str if err := os.Rename(srcPath, dstPath); err != nil { return fmt.Errorf("cannot rename %q to %q: %s", srcPath, dstPath, err) } - } else { - // Verify dstPath exists. - if !fs.IsPathExist(dstPath) { - return fmt.Errorf("cannot find both source and destination paths: %q -> %q", srcPath, dstPath) - } + } else if !fs.IsPathExist(dstPath) { + // Emit info message for the expected condition after unclean shutdown on NFS disk. + // The dstPath part may be missing because it could be already merged into bigger part + // while old source parts for the current txn weren't still deleted due to NFS locks. + logger.Infof("cannot find both source and destination paths: %q -> %q; this may be the case after unclean shutdown (OOM, `kill -9`, hard reset) on NFS disk", + srcPath, dstPath) } } else { // Just remove srcPath.