2019-11-07 19:05:39 +00:00
|
|
|
// Copyright 2014 Google LLC
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2022-05-20 11:48:16 +00:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
2021-09-01 09:51:42 +00:00
|
|
|
"io"
|
2022-01-27 11:16:16 +00:00
|
|
|
"net"
|
2022-05-20 11:48:16 +00:00
|
|
|
"net/http"
|
2021-09-01 09:51:42 +00:00
|
|
|
"net/url"
|
|
|
|
"strings"
|
2019-11-07 19:05:39 +00:00
|
|
|
|
|
|
|
"cloud.google.com/go/internal"
|
2022-05-20 11:48:16 +00:00
|
|
|
"cloud.google.com/go/internal/version"
|
|
|
|
sinternal "cloud.google.com/go/storage/internal"
|
|
|
|
"github.com/google/uuid"
|
2019-11-07 19:05:39 +00:00
|
|
|
gax "github.com/googleapis/gax-go/v2"
|
2021-09-01 09:51:42 +00:00
|
|
|
"google.golang.org/api/googleapi"
|
|
|
|
"google.golang.org/grpc/codes"
|
|
|
|
"google.golang.org/grpc/status"
|
2019-11-07 19:05:39 +00:00
|
|
|
)
|
|
|
|
|
2022-01-27 11:16:16 +00:00
|
|
|
var defaultRetry *retryConfig = &retryConfig{}
|
2022-05-20 11:48:16 +00:00
|
|
|
var xGoogDefaultHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), sinternal.Version)
|
2022-01-27 11:16:16 +00:00
|
|
|
|
|
|
|
// run determines whether a retry is necessary based on the config and
|
|
|
|
// idempotency information. It then calls the function with or without retries
|
|
|
|
// as appropriate, using the configured settings.
|
2022-05-20 11:48:16 +00:00
|
|
|
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool, setHeader func(string, int)) error {
|
|
|
|
attempts := 1
|
|
|
|
invocationID := uuid.New().String()
|
|
|
|
|
2022-01-27 11:16:16 +00:00
|
|
|
if retry == nil {
|
|
|
|
retry = defaultRetry
|
|
|
|
}
|
|
|
|
if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever {
|
2022-05-20 11:48:16 +00:00
|
|
|
setHeader(invocationID, attempts)
|
2022-01-27 11:16:16 +00:00
|
|
|
return call()
|
|
|
|
}
|
|
|
|
bo := gax.Backoff{}
|
|
|
|
if retry.backoff != nil {
|
|
|
|
bo.Multiplier = retry.backoff.Multiplier
|
|
|
|
bo.Initial = retry.backoff.Initial
|
|
|
|
bo.Max = retry.backoff.Max
|
|
|
|
}
|
2022-11-10 11:55:47 +00:00
|
|
|
var errorFunc func(err error) bool = ShouldRetry
|
2022-01-27 11:16:16 +00:00
|
|
|
if retry.shouldRetry != nil {
|
|
|
|
errorFunc = retry.shouldRetry
|
|
|
|
}
|
2022-05-20 11:48:16 +00:00
|
|
|
|
2022-01-27 11:16:16 +00:00
|
|
|
return internal.Retry(ctx, bo, func() (stop bool, err error) {
|
2022-05-20 11:48:16 +00:00
|
|
|
setHeader(invocationID, attempts)
|
2019-11-07 19:05:39 +00:00
|
|
|
err = call()
|
2022-05-20 11:48:16 +00:00
|
|
|
attempts++
|
2022-01-27 11:16:16 +00:00
|
|
|
return !errorFunc(err), err
|
2019-11-07 19:05:39 +00:00
|
|
|
})
|
|
|
|
}
|
2021-09-01 09:51:42 +00:00
|
|
|
|
2022-05-20 11:48:16 +00:00
|
|
|
func setRetryHeaderHTTP(req interface{ Header() http.Header }) func(string, int) {
|
|
|
|
return func(invocationID string, attempts int) {
|
|
|
|
if req == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
header := req.Header()
|
|
|
|
invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts)
|
|
|
|
xGoogHeader := strings.Join([]string{invocationHeader, xGoogDefaultHeader}, " ")
|
|
|
|
header.Set("x-goog-api-client", xGoogHeader)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: Implement method setting header via context for gRPC
|
|
|
|
func setRetryHeaderGRPC(_ context.Context) func(string, int) {
|
|
|
|
return func(_ string, _ int) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-11-10 11:55:47 +00:00
|
|
|
// ShouldRetry returns true if an error is retryable, based on best practice
|
|
|
|
// guidance from GCS. See
|
|
|
|
// https://cloud.google.com/storage/docs/retry-strategy#go for more information
|
|
|
|
// on what errors are considered retryable.
|
|
|
|
//
|
|
|
|
// If you would like to customize retryable errors, use the WithErrorFunc to
|
|
|
|
// supply a RetryOption to your library calls. For example, to retry additional
|
|
|
|
// errors, you can write a custom func that wraps ShouldRetry and also specifies
|
|
|
|
// additional errors that should return true.
|
|
|
|
func ShouldRetry(err error) bool {
|
2022-01-27 11:16:16 +00:00
|
|
|
if err == nil {
|
|
|
|
return false
|
|
|
|
}
|
2022-05-20 11:48:16 +00:00
|
|
|
if errors.Is(err, io.ErrUnexpectedEOF) {
|
2021-09-01 09:51:42 +00:00
|
|
|
return true
|
|
|
|
}
|
2022-01-27 11:16:16 +00:00
|
|
|
|
2021-09-01 09:51:42 +00:00
|
|
|
switch e := err.(type) {
|
2022-01-27 11:16:16 +00:00
|
|
|
case *net.OpError:
|
|
|
|
if strings.Contains(e.Error(), "use of closed network connection") {
|
|
|
|
// TODO: check against net.ErrClosed (go 1.16+) instead of string
|
|
|
|
return true
|
|
|
|
}
|
2021-09-01 09:51:42 +00:00
|
|
|
case *googleapi.Error:
|
2022-01-27 11:16:16 +00:00
|
|
|
// Retry on 408, 429, and 5xx, according to
|
2021-09-01 09:51:42 +00:00
|
|
|
// https://cloud.google.com/storage/docs/exponential-backoff.
|
2022-01-27 11:16:16 +00:00
|
|
|
return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600)
|
2021-09-01 09:51:42 +00:00
|
|
|
case *url.Error:
|
2022-01-27 11:16:16 +00:00
|
|
|
// Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
|
2021-09-01 09:51:42 +00:00
|
|
|
// Unfortunately the error type is unexported, so we resort to string
|
|
|
|
// matching.
|
|
|
|
retriable := []string{"connection refused", "connection reset"}
|
|
|
|
for _, s := range retriable {
|
|
|
|
if strings.Contains(e.Error(), s) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case interface{ Temporary() bool }:
|
|
|
|
if e.Temporary() {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// HTTP 429, 502, 503, and 504 all map to gRPC UNAVAILABLE per
|
|
|
|
// https://grpc.github.io/grpc/core/md_doc_http-grpc-status-mapping.html.
|
|
|
|
//
|
|
|
|
// This is only necessary for the experimental gRPC-based media operations.
|
|
|
|
if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
// Unwrap is only supported in go1.13.x+
|
|
|
|
if e, ok := err.(interface{ Unwrap() error }); ok {
|
2022-11-10 11:55:47 +00:00
|
|
|
return ShouldRetry(e.Unwrap())
|
2021-09-01 09:51:42 +00:00
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|