From e72518e8c6ad5bc53f9830cc894c1dc6b32df72c Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 15 May 2020 17:03:04 +0300
Subject: [PATCH] lib/backup: donload only the remaining parts for partially
downloaded files after `vmrestore` restart
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/487
---
lib/backup/actions/restore.go | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/lib/backup/actions/restore.go b/lib/backup/actions/restore.go
index 29905e364f..2903f86910 100644
--- a/lib/backup/actions/restore.go
+++ b/lib/backup/actions/restore.go
@@ -113,10 +113,16 @@ func (r *Restore) Run() error {
partsToDelete := common.PartsDifference(dstParts, srcParts)
deleteSize := uint64(0)
if len(partsToDelete) > 0 {
- // Fully remove local file if certain parts from the remote part are missing.
+ // Remove only files with the missing part at offset 0.
+ // Assume other files are partially downloaded during the previous Restore.Run call,
+ // so only the last part in them may be incomplete.
+ // The last part for partially downloaded files will be re-downloaded later.
+ // This addresses https://github.com/VictoriaMetrics/VictoriaMetrics/issues/487 .
pathsToDelete := make(map[string]bool)
for _, p := range partsToDelete {
- pathsToDelete[p.Path] = true
+ if p.Offset == 0 {
+ pathsToDelete[p.Path] = true
+ }
}
logger.Infof("deleting %d files from %s", len(pathsToDelete), dst)
for path := range pathsToDelete {
@@ -153,7 +159,8 @@ func (r *Restore) Run() error {
logger.Infof("downloading %d parts from %s to %s", len(partsToCopy), src, dst)
bytesDownloaded := uint64(0)
err = runParallelPerPath(concurrency, perPath, func(parts []common.Part) error {
- // Sort partsToCopy in order to properly grow file size during downloading.
+ // Sort partsToCopy in order to properly grow file size during downloading
+ // and to properly resume downloading of incomplete files on the next Restore.Run call.
common.SortParts(parts)
for _, p := range parts {
logger.Infof("downloading %s from %s to %s", &p, src, dst)
@@ -169,7 +176,7 @@ func (r *Restore) Run() error {
return fmt.Errorf("cannot download %s to %s: %s", &p, dst, err)
}
if err := wc.Close(); err != nil {
- return fmt.Errorf("cannot close reader fro %s from %s: %s", &p, src, err)
+ return fmt.Errorf("cannot close reader from %s from %s: %s", &p, src, err)
}
}
return nil