github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/backup/azremote: fix copying for parts larger than 256M by using async copy (#3479)

Browse source 
* lib/backup/azremote: fix copying for parts larger than 256M by using async copy

* lib/backup/azremote: add description of an error for log message
This commit is contained in:
Zakhar Bessarab 2022-12-13 21:32:57 +04:00 committed by GitHub
parent d3418bafc0
commit a50120a212
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 28 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
lib/backup/azremote/azblob.go
View file

@ -178,12 +178,35 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
return fmt.Errorf("failed to generate SAS token of src %q: %w", p.Path, err)
}
// Hotfix for SDK issue: https://github.com/Azure/azure-sdk-for-go/issues/19245
t = strings.Replace(t, "/?", "?", -1)
ctx := context.Background()
_, err = dbc.CopyFromURL(ctx, t, &blob.CopyFromURLOptions{})
// In order to support copy of files larger than 256MB, we need to use the async copy
// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url
_, err = dbc.StartCopyFromURL(ctx, t, &blob.StartCopyFromURLOptions{})
if err != nil {
return fmt.Errorf("cannot copy %q from %s to %s: %w", p.Path, src, fs, err)
return fmt.Errorf("cannot start async copy %q from %s to %s: %w", p.Path, src, fs, err)
}
var copyStatus *blob.CopyStatusType
var copyStatusDescription *string
for {
r, err := dbc.GetProperties(ctx, nil)
if err != nil {
return fmt.Errorf("failed to check copy status, cannot get properties of %q at %s: %w", p.Path, fs, err)
}
// After the copy will be finished status will be changed to success/failed/aborted
// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob-properties#response-headers - x-ms-copy-status
if *r.CopyStatus != blob.CopyStatusTypePending {
copyStatus = r.CopyStatus
copyStatusDescription = r.CopyStatusDescription
break
}
time.Sleep(5 * time.Second)
}
if *copyStatus != blob.CopyStatusTypeSuccess {
return fmt.Errorf("copy of %q from %s to %s failed: expected status %q, received %q (description: %q)", p.Path, src, fs, blob.CopyStatusTypeSuccess, *copyStatus, *copyStatusDescription)
}
return nil
@ -290,12 +313,12 @@ func (fs *FS) HasFile(filePath string) (bool, error) {
ctx := context.Background()
_, err := bc.GetProperties(ctx, nil)
logger.Errorf("GetProperties(%q) returned %s", bc.URL(), err)
var azerr *azcore.ResponseError
if errors.As(err, &azerr) {
if azerr.ErrorCode == storageErrorCodeBlobNotFound {
return false, nil
}
logger.Errorf("GetProperties(%q) returned %s", bc.URL(), err)
return false, fmt.Errorf("unexpected error when obtaining properties for %q at %s (remote path %q): %w", filePath, fs, bc.URL(), err)
}