From a50120a212084fb625d700771e90a470ca01d9a7 Mon Sep 17 00:00:00 2001
From: Zakhar Bessarab <z.bessarab@victoriametrics.com>
Date: Tue, 13 Dec 2022 21:32:57 +0400
Subject: [PATCH] lib/backup/azremote: fix copying for parts larger than 256M
 by using async copy (#3479)

* lib/backup/azremote: fix copying for parts larger than 256M by using async copy

* lib/backup/azremote: add description of an error for log message
---
 lib/backup/azremote/azblob.go | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/lib/backup/azremote/azblob.go b/lib/backup/azremote/azblob.go
index b17f2acd8..c248d122d 100644
--- a/lib/backup/azremote/azblob.go
+++ b/lib/backup/azremote/azblob.go
@@ -178,12 +178,35 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
 		return fmt.Errorf("failed to generate SAS token of src %q: %w", p.Path, err)
 	}
 
-	// Hotfix for SDK issue: https://github.com/Azure/azure-sdk-for-go/issues/19245
-	t = strings.Replace(t, "/?", "?", -1)
 	ctx := context.Background()
-	_, err = dbc.CopyFromURL(ctx, t, &blob.CopyFromURLOptions{})
+
+	// In order to support copy of files larger than 256MB, we need to use the async copy
+	// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url
+	_, err = dbc.StartCopyFromURL(ctx, t, &blob.StartCopyFromURLOptions{})
 	if err != nil {
-		return fmt.Errorf("cannot copy %q from %s to %s: %w", p.Path, src, fs, err)
+		return fmt.Errorf("cannot start async copy %q from %s to %s: %w", p.Path, src, fs, err)
+	}
+
+	var copyStatus *blob.CopyStatusType
+	var copyStatusDescription *string
+	for {
+		r, err := dbc.GetProperties(ctx, nil)
+		if err != nil {
+			return fmt.Errorf("failed to check copy status, cannot get properties of %q at %s: %w", p.Path, fs, err)
+		}
+
+		// After the copy will be finished status will be changed to success/failed/aborted
+		// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob-properties#response-headers - x-ms-copy-status
+		if *r.CopyStatus != blob.CopyStatusTypePending {
+			copyStatus = r.CopyStatus
+			copyStatusDescription = r.CopyStatusDescription
+			break
+		}
+		time.Sleep(5 * time.Second)
+	}
+
+	if *copyStatus != blob.CopyStatusTypeSuccess {
+		return fmt.Errorf("copy of %q from %s to %s failed: expected status %q, received %q (description: %q)", p.Path, src, fs, blob.CopyStatusTypeSuccess, *copyStatus, *copyStatusDescription)
 	}
 
 	return nil
@@ -290,12 +313,12 @@ func (fs *FS) HasFile(filePath string) (bool, error) {
 
 	ctx := context.Background()
 	_, err := bc.GetProperties(ctx, nil)
-	logger.Errorf("GetProperties(%q) returned %s", bc.URL(), err)
 	var azerr *azcore.ResponseError
 	if errors.As(err, &azerr) {
 		if azerr.ErrorCode == storageErrorCodeBlobNotFound {
 			return false, nil
 		}
+		logger.Errorf("GetProperties(%q) returned %s", bc.URL(), err)
 		return false, fmt.Errorf("unexpected error when obtaining properties for %q at %s (remote path %q): %w", filePath, fs, bc.URL(), err)
 	}