VictoriaMetrics/vendor/cloud.google.com/go/storage/bucket.go

2190 lines
72 KiB
Go
Raw Normal View History

// Copyright 2014 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
2021-10-11 18:51:32 +00:00
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"time"
2021-10-11 18:51:32 +00:00
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
2022-04-12 09:51:54 +00:00
"github.com/googleapis/go-type-adapters/adapters"
"google.golang.org/api/googleapi"
2021-10-11 18:51:32 +00:00
"google.golang.org/api/iamcredentials/v1"
"google.golang.org/api/iterator"
2021-10-11 18:51:32 +00:00
"google.golang.org/api/option"
raw "google.golang.org/api/storage/v1"
2022-04-12 09:51:54 +00:00
"google.golang.org/genproto/googleapis/storage/v2"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
"google.golang.org/protobuf/proto"
)
// BucketHandle provides operations on a Google Cloud Storage bucket.
// Use Client.Bucket to get a handle.
type BucketHandle struct {
c *Client
name string
acl ACLHandle
defaultObjectACL ACLHandle
conds *BucketConditions
userProject string // project for Requester Pays buckets
2022-01-27 11:16:16 +00:00
retry *retryConfig
}
// Bucket returns a BucketHandle, which provides operations on the named bucket.
// This call does not perform any network operations.
//
// The supplied name must contain only lowercase letters, numbers, dashes,
// underscores, and dots. The full specification for valid bucket names can be
// found at:
// https://cloud.google.com/storage/docs/bucket-naming
func (c *Client) Bucket(name string) *BucketHandle {
2022-01-27 11:16:16 +00:00
retry := c.retry.clone()
return &BucketHandle{
c: c,
name: name,
acl: ACLHandle{
c: c,
bucket: name,
2022-01-27 11:16:16 +00:00
retry: retry,
},
defaultObjectACL: ACLHandle{
c: c,
bucket: name,
isDefault: true,
2022-01-27 11:16:16 +00:00
retry: retry,
},
2022-01-27 11:16:16 +00:00
retry: retry,
}
}
// Create creates the Bucket in the project.
// If attrs is nil the API defaults will be used.
func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) (err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create")
defer func() { trace.EndSpan(ctx, err) }()
var bkt *raw.Bucket
if attrs != nil {
bkt = attrs.toRawBucket()
} else {
bkt = &raw.Bucket{}
}
bkt.Name = b.name
// If there is lifecycle information but no location, explicitly set
// the location. This is a GCS quirk/bug.
if bkt.Location == "" && bkt.Lifecycle != nil {
bkt.Location = "US"
}
req := b.c.raw.Buckets.Insert(projectID, bkt)
setClientHeader(req.Header())
if attrs != nil && attrs.PredefinedACL != "" {
req.PredefinedAcl(attrs.PredefinedACL)
}
if attrs != nil && attrs.PredefinedDefaultObjectACL != "" {
req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL)
}
2022-05-20 11:48:16 +00:00
return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true, setRetryHeaderHTTP(req))
}
// Delete deletes the Bucket.
func (b *BucketHandle) Delete(ctx context.Context) (err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Delete")
defer func() { trace.EndSpan(ctx, err) }()
req, err := b.newDeleteCall()
if err != nil {
return err
}
2022-01-27 11:16:16 +00:00
2022-05-20 11:48:16 +00:00
return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true, setRetryHeaderHTTP(req))
}
func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
req := b.c.raw.Buckets.Delete(b.name)
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// ACL returns an ACLHandle, which provides access to the bucket's access control list.
// This controls who can list, create or overwrite the objects in a bucket.
// This call does not perform any network operations.
func (b *BucketHandle) ACL() *ACLHandle {
return &b.acl
}
// DefaultObjectACL returns an ACLHandle, which provides access to the bucket's default object ACLs.
// These ACLs are applied to newly created objects in this bucket that do not have a defined ACL.
// This call does not perform any network operations.
func (b *BucketHandle) DefaultObjectACL() *ACLHandle {
return &b.defaultObjectACL
}
// Object returns an ObjectHandle, which provides operations on the named object.
// This call does not perform any network operations such as fetching the object or verifying its existence.
// Use methods on ObjectHandle to perform network operations.
//
// name must consist entirely of valid UTF-8-encoded runes. The full specification
// for valid object names can be found at:
2021-02-08 23:12:07 +00:00
// https://cloud.google.com/storage/docs/naming-objects
func (b *BucketHandle) Object(name string) *ObjectHandle {
2022-01-27 11:16:16 +00:00
retry := b.retry.clone()
return &ObjectHandle{
c: b.c,
bucket: b.name,
object: name,
acl: ACLHandle{
c: b.c,
bucket: b.name,
object: name,
userProject: b.userProject,
2022-01-27 11:16:16 +00:00
retry: retry,
},
gen: -1,
userProject: b.userProject,
2022-01-27 11:16:16 +00:00
retry: retry,
}
}
// Attrs returns the metadata for the bucket.
func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Attrs")
defer func() { trace.EndSpan(ctx, err) }()
req, err := b.newGetCall()
if err != nil {
return nil, err
}
var resp *raw.Bucket
2022-01-27 11:16:16 +00:00
err = run(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
2022-05-20 11:48:16 +00:00
}, b.retry, true, setRetryHeaderHTTP(req))
2021-10-11 18:51:32 +00:00
var e *googleapi.Error
2022-05-20 11:48:16 +00:00
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
return nil, ErrBucketNotExist
}
if err != nil {
return nil, err
}
return newBucket(resp)
}
func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) {
req := b.c.raw.Buckets.Get(b.name).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// Update updates a bucket's attributes.
func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (attrs *BucketAttrs, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create")
defer func() { trace.EndSpan(ctx, err) }()
req, err := b.newPatchCall(&uattrs)
if err != nil {
return nil, err
}
if uattrs.PredefinedACL != "" {
req.PredefinedAcl(uattrs.PredefinedACL)
}
if uattrs.PredefinedDefaultObjectACL != "" {
req.PredefinedDefaultObjectAcl(uattrs.PredefinedDefaultObjectACL)
}
2022-01-27 11:16:16 +00:00
isIdempotent := b.conds != nil && b.conds.MetagenerationMatch != 0
var rawBucket *raw.Bucket
call := func() error {
rb, err := req.Context(ctx).Do()
rawBucket = rb
return err
}
2022-05-20 11:48:16 +00:00
if err := run(ctx, call, b.retry, isIdempotent, setRetryHeaderHTTP(req)); err != nil {
return nil, err
}
2022-01-27 11:16:16 +00:00
return newBucket(rawBucket)
}
func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
rb := uattrs.toRawBucket()
req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
2021-10-11 18:51:32 +00:00
// SignedURL returns a URL for the specified object. Signed URLs allow anyone
// access to a restricted resource for a limited time without needing a
// Google account or signing in. For more information about signed URLs, see
// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
//
// This method only requires the Method and Expires fields in the specified
// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the
// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
2021-10-18 12:23:46 +00:00
// If you are authenticating with a custom HTTP client, Service Account based
// auto-detection will be hindered.
//
2021-10-11 18:51:32 +00:00
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
// This requires the IAM Service Account Credentials API to be enabled
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
// If you do not want these fields set for you, you may pass them in through opts or use
// SignedURL(bucket, name string, opts *SignedURLOptions) instead.
func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return SignedURL(b.name, object, opts)
}
// Make a copy of opts so we don't modify the pointer parameter.
newopts := opts.clone()
if newopts.GoogleAccessID == "" {
id, err := b.detectDefaultGoogleAccessID()
if err != nil {
return "", err
}
newopts.GoogleAccessID = id
}
if newopts.SignBytes == nil && len(newopts.PrivateKey) == 0 {
2021-10-18 12:23:46 +00:00
if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
2021-10-11 18:51:32 +00:00
var sa struct {
PrivateKey string `json:"private_key"`
}
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.PrivateKey != "" {
newopts.PrivateKey = []byte(sa.PrivateKey)
}
}
// Don't error out if we can't unmarshal the private key from the client,
// fallback to the default sign function for the service account.
if len(newopts.PrivateKey) == 0 {
newopts.SignBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID)
}
}
return SignedURL(b.name, object, newopts)
}
2022-01-27 11:16:16 +00:00
// GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts.
// The generated URL and fields will then allow an unauthenticated client to perform multipart uploads.
//
// This method only requires the Expires field in the specified PostPolicyV4Options
// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey
// from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
// If you are authenticating with a custom HTTP client, Service Account based
// auto-detection will be hindered.
//
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
// This requires the IAM Service Account Credentials API to be enabled
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
// If you do not want these fields set for you, you may pass them in through opts or use
// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead.
func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) {
if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return GenerateSignedPostPolicyV4(b.name, object, opts)
}
// Make a copy of opts so we don't modify the pointer parameter.
newopts := opts.clone()
if newopts.GoogleAccessID == "" {
id, err := b.detectDefaultGoogleAccessID()
if err != nil {
return nil, err
}
newopts.GoogleAccessID = id
}
if newopts.SignBytes == nil && newopts.SignRawBytes == nil && len(newopts.PrivateKey) == 0 {
if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
var sa struct {
PrivateKey string `json:"private_key"`
}
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.PrivateKey != "" {
newopts.PrivateKey = []byte(sa.PrivateKey)
}
}
// Don't error out if we can't unmarshal the private key from the client,
// fallback to the default sign function for the service account.
if len(newopts.PrivateKey) == 0 {
newopts.SignRawBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID)
}
}
return GenerateSignedPostPolicyV4(b.name, object, newopts)
}
2021-10-11 18:51:32 +00:00
func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)")
2021-10-18 12:23:46 +00:00
if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
2021-10-11 18:51:32 +00:00
var sa struct {
ClientEmail string `json:"client_email"`
}
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.ClientEmail != "" {
return sa.ClientEmail, nil
} else if err != nil {
returnErr = err
} else {
returnErr = errors.New("storage: empty client email in credentials")
}
}
// Don't error out if we can't unmarshal, fallback to GCE check.
if metadata.OnGCE() {
email, err := metadata.Email("default")
if err == nil && email != "" {
return email, nil
} else if err != nil {
returnErr = err
} else {
returnErr = errors.New("got empty email from GCE metadata service")
}
}
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr)
}
func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {
return func(in []byte) ([]byte, error) {
ctx := context.Background()
// It's ok to recreate this service per call since we pass in the http client,
// circumventing the cost of recreating the auth/transport layer
svc, err := iamcredentials.NewService(ctx, option.WithHTTPClient(b.c.hc))
if err != nil {
return nil, fmt.Errorf("unable to create iamcredentials client: %v", err)
}
resp, err := svc.Projects.ServiceAccounts.SignBlob(fmt.Sprintf("projects/-/serviceAccounts/%s", email), &iamcredentials.SignBlobRequest{
Payload: base64.StdEncoding.EncodeToString(in),
}).Do()
if err != nil {
return nil, fmt.Errorf("unable to sign bytes: %v", err)
}
out, err := base64.StdEncoding.DecodeString(resp.SignedBlob)
if err != nil {
return nil, fmt.Errorf("unable to base64 decode response: %v", err)
}
return out, nil
}
}
// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
// Read-only fields are ignored by BucketHandle.Create.
type BucketAttrs struct {
// Name is the name of the bucket.
// This field is read-only.
Name string
// ACL is the list of access control rules on the bucket.
ACL []ACLRule
// BucketPolicyOnly is an alias for UniformBucketLevelAccess. Use of
// UniformBucketLevelAccess is recommended above the use of this field.
// Setting BucketPolicyOnly.Enabled OR UniformBucketLevelAccess.Enabled to
// true, will enable UniformBucketLevelAccess.
BucketPolicyOnly BucketPolicyOnly
// UniformBucketLevelAccess configures access checks to use only bucket-level IAM
// policies and ignore any ACL rules for the bucket.
// See https://cloud.google.com/storage/docs/uniform-bucket-level-access
// for more information.
UniformBucketLevelAccess UniformBucketLevelAccess
2021-07-07 13:10:33 +00:00
// PublicAccessPrevention is the setting for the bucket's
// PublicAccessPrevention policy, which can be used to prevent public access
// of data in the bucket. See
// https://cloud.google.com/storage/docs/public-access-prevention for more
// information.
PublicAccessPrevention PublicAccessPrevention
// DefaultObjectACL is the list of access controls to
// apply to new objects when no object ACL is provided.
DefaultObjectACL []ACLRule
// DefaultEventBasedHold is the default value for event-based hold on
// newly created objects in this bucket. It defaults to false.
DefaultEventBasedHold bool
// If not empty, applies a predefined set of access controls. It should be set
// only when creating a bucket.
// It is always empty for BucketAttrs returned from the service.
// See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert
// for valid values.
PredefinedACL string
// If not empty, applies a predefined set of default object access controls.
// It should be set only when creating a bucket.
// It is always empty for BucketAttrs returned from the service.
// See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert
// for valid values.
PredefinedDefaultObjectACL string
// Location is the location of the bucket. It defaults to "US".
Location string
// MetaGeneration is the metadata generation of the bucket.
// This field is read-only.
MetaGeneration int64
// StorageClass is the default storage class of the bucket. This defines
// how objects in the bucket are stored and determines the SLA
2020-01-16 12:17:34 +00:00
// and the cost of storage. Typical values are "STANDARD", "NEARLINE",
// "COLDLINE" and "ARCHIVE". Defaults to "STANDARD".
// See https://cloud.google.com/storage/docs/storage-classes for all
// valid values.
StorageClass string
// Created is the creation time of the bucket.
// This field is read-only.
Created time.Time
// VersioningEnabled reports whether this bucket has versioning enabled.
VersioningEnabled bool
// Labels are the bucket's labels.
Labels map[string]string
// RequesterPays reports whether the bucket is a Requester Pays bucket.
// Clients performing operations on Requester Pays buckets must provide
// a user project (see BucketHandle.UserProject), which will be billed
// for the operations.
RequesterPays bool
// Lifecycle is the lifecycle configuration for objects in the bucket.
Lifecycle Lifecycle
// Retention policy enforces a minimum retention time for all objects
// contained in the bucket. A RetentionPolicy of nil implies the bucket
// has no minimum data retention.
//
// This feature is in private alpha release. It is not currently available to
// most customers. It might be changed in backwards-incompatible ways and is not
// subject to any SLA or deprecation policy.
RetentionPolicy *RetentionPolicy
// The bucket's Cross-Origin Resource Sharing (CORS) configuration.
CORS []CORS
// The encryption configuration used by default for newly inserted objects.
Encryption *BucketEncryption
// The logging configuration.
Logging *BucketLogging
// The website configuration.
Website *BucketWebsite
// Etag is the HTTP/1.1 Entity tag for the bucket.
// This field is read-only.
Etag string
// LocationType describes how data is stored and replicated.
// Typical values are "multi-region", "region" and "dual-region".
// This field is read-only.
LocationType string
2021-09-30 14:54:37 +00:00
// The project number of the project the bucket belongs to.
// This field is read-only.
ProjectNumber uint64
2022-01-27 11:16:16 +00:00
// RPO configures the Recovery Point Objective (RPO) policy of the bucket.
// Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket.
// See https://cloud.google.com/storage/docs/managing-turbo-replication for
// more information.
RPO RPO
}
// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
// Use of UniformBucketLevelAccess is preferred above BucketPolicyOnly.
type BucketPolicyOnly struct {
// Enabled specifies whether access checks use only bucket-level IAM
// policies. Enabled may be disabled until the locked time.
Enabled bool
// LockedTime specifies the deadline for changing Enabled from true to
// false.
LockedTime time.Time
}
// UniformBucketLevelAccess configures access checks to use only bucket-level IAM
// policies.
type UniformBucketLevelAccess struct {
// Enabled specifies whether access checks use only bucket-level IAM
// policies. Enabled may be disabled until the locked time.
Enabled bool
// LockedTime specifies the deadline for changing Enabled from true to
// false.
LockedTime time.Time
}
2021-07-07 13:10:33 +00:00
// PublicAccessPrevention configures the Public Access Prevention feature, which
// can be used to disallow public access to any data in a bucket. See
// https://cloud.google.com/storage/docs/public-access-prevention for more
// information.
type PublicAccessPrevention int
const (
// PublicAccessPreventionUnknown is a zero value, used only if this field is
// not set in a call to GCS.
PublicAccessPreventionUnknown PublicAccessPrevention = iota
2021-10-11 18:51:32 +00:00
// PublicAccessPreventionUnspecified corresponds to a value of "unspecified".
// Deprecated: use PublicAccessPreventionInherited
2021-07-07 13:10:33 +00:00
PublicAccessPreventionUnspecified
// PublicAccessPreventionEnforced corresponds to a value of "enforced". This
// enforces Public Access Prevention on the bucket.
PublicAccessPreventionEnforced
2021-10-11 18:51:32 +00:00
// PublicAccessPreventionInherited corresponds to a value of "inherited"
// and is the default for buckets.
PublicAccessPreventionInherited
publicAccessPreventionUnknown string = ""
// TODO: remove unspecified when change is fully completed
publicAccessPreventionUnspecified = "unspecified"
publicAccessPreventionEnforced = "enforced"
publicAccessPreventionInherited = "inherited"
2021-07-07 13:10:33 +00:00
)
func (p PublicAccessPrevention) String() string {
switch p {
2021-10-11 18:51:32 +00:00
case PublicAccessPreventionInherited, PublicAccessPreventionUnspecified:
return publicAccessPreventionInherited
2021-07-07 13:10:33 +00:00
case PublicAccessPreventionEnforced:
return publicAccessPreventionEnforced
default:
return publicAccessPreventionUnknown
}
}
// Lifecycle is the lifecycle configuration for objects in the bucket.
type Lifecycle struct {
Rules []LifecycleRule
}
// RetentionPolicy enforces a minimum retention time for all objects
// contained in the bucket.
//
// Any attempt to overwrite or delete objects younger than the retention
// period will result in an error. An unlocked retention policy can be
// modified or removed from the bucket via the Update method. A
// locked retention policy cannot be removed or shortened in duration
// for the lifetime of the bucket.
//
// This feature is in private alpha release. It is not currently available to
// most customers. It might be changed in backwards-incompatible ways and is not
// subject to any SLA or deprecation policy.
type RetentionPolicy struct {
// RetentionPeriod specifies the duration that objects need to be
// retained. Retention duration must be greater than zero and less than
// 100 years. Note that enforcement of retention periods less than a day
// is not guaranteed. Such periods should only be used for testing
// purposes.
RetentionPeriod time.Duration
// EffectiveTime is the time from which the policy was enforced and
// effective. This field is read-only.
EffectiveTime time.Time
// IsLocked describes whether the bucket is locked. Once locked, an
// object retention policy cannot be modified.
// This field is read-only.
IsLocked bool
}
const (
2020-09-01 14:43:21 +00:00
// RFC3339 timestamp with only the date segment, used for CreatedBefore,
// CustomTimeBefore, and NoncurrentTimeBefore in LifecycleRule.
rfc3339Date = "2006-01-02"
// DeleteAction is a lifecycle action that deletes a live and/or archived
// objects. Takes precedence over SetStorageClass actions.
DeleteAction = "Delete"
// SetStorageClassAction changes the storage class of live and/or archived
// objects.
SetStorageClassAction = "SetStorageClass"
2022-06-28 11:55:33 +00:00
// AbortIncompleteMPUAction is a lifecycle action that aborts an incomplete
// multipart upload when the multipart upload meets the conditions specified
// in the lifecycle rule. The AgeInDays condition is the only allowed
// condition for this action. AgeInDays is measured from the time the
// multipart upload was created.
AbortIncompleteMPUAction = "AbortIncompleteMultipartUpload"
)
// LifecycleRule is a lifecycle configuration rule.
//
// When all the configured conditions are met by an object in the bucket, the
// configured action will automatically be taken on that object.
type LifecycleRule struct {
// Action is the action to take when all of the associated conditions are
// met.
Action LifecycleAction
// Condition is the set of conditions that must be met for the associated
// action to be taken.
Condition LifecycleCondition
}
// LifecycleAction is a lifecycle configuration action.
type LifecycleAction struct {
// Type is the type of action to take on matching objects.
//
2022-06-28 11:55:33 +00:00
// Acceptable values are storage.DeleteAction, storage.SetStorageClassAction,
// and storage.AbortIncompleteMPUAction.
Type string
// StorageClass is the storage class to set on matching objects if the Action
// is "SetStorageClass".
StorageClass string
}
// Liveness specifies whether the object is live or not.
type Liveness int
const (
// LiveAndArchived includes both live and archived objects.
LiveAndArchived Liveness = iota
// Live specifies that the object is still live.
Live
// Archived specifies that the object is archived.
Archived
)
// LifecycleCondition is a set of conditions used to match objects and take an
// action automatically.
//
// All configured conditions must be met for the associated action to be taken.
type LifecycleCondition struct {
// AgeInDays is the age of the object in days.
AgeInDays int64
// CreatedBefore is the time the object was created.
//
// This condition is satisfied when an object is created before midnight of
// the specified date in UTC.
CreatedBefore time.Time
2020-09-01 14:43:21 +00:00
// CustomTimeBefore is the CustomTime metadata field of the object. This
// condition is satisfied when an object's CustomTime timestamp is before
// midnight of the specified date in UTC.
//
// This condition can only be satisfied if CustomTime has been set.
CustomTimeBefore time.Time
// DaysSinceCustomTime is the days elapsed since the CustomTime date of the
// object. This condition can only be satisfied if CustomTime has been set.
DaysSinceCustomTime int64
// DaysSinceNoncurrentTime is the days elapsed since the noncurrent timestamp
// of the object. This condition is relevant only for versioned objects.
DaysSinceNoncurrentTime int64
// Liveness specifies the object's liveness. Relevant only for versioned objects
Liveness Liveness
2022-06-28 11:55:33 +00:00
// MatchesPrefix is the condition matching an object if any of the
// matches_prefix strings are an exact prefix of the object's name.
MatchesPrefix []string
// MatchesStorageClasses is the condition matching the object's storage
// class.
//
2020-01-16 12:17:34 +00:00
// Values include "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE".
MatchesStorageClasses []string
2022-06-28 11:55:33 +00:00
// MatchesSuffix is the condition matching an object if any of the
// matches_suffix strings are an exact suffix of the object's name.
MatchesSuffix []string
2020-09-01 14:43:21 +00:00
// NoncurrentTimeBefore is the noncurrent timestamp of the object. This
// condition is satisfied when an object's noncurrent timestamp is before
// midnight of the specified date in UTC.
//
// This condition is relevant only for versioned objects.
NoncurrentTimeBefore time.Time
// NumNewerVersions is the condition matching objects with a number of newer versions.
//
// If the value is N, this condition is satisfied when there are at least N
// versions (including the live version) newer than this version of the
// object.
NumNewerVersions int64
}
// BucketLogging holds the bucket's logging configuration, which defines the
// destination bucket and optional name prefix for the current bucket's
// logs.
type BucketLogging struct {
// The destination bucket where the current bucket's logs
// should be placed.
LogBucket string
// A prefix for log object names.
LogObjectPrefix string
}
// BucketWebsite holds the bucket's website configuration, controlling how the
// service behaves when accessing bucket contents as a web site. See
// https://cloud.google.com/storage/docs/static-website for more information.
type BucketWebsite struct {
// If the requested object path is missing, the service will ensure the path has
// a trailing '/', append this suffix, and attempt to retrieve the resulting
// object. This allows the creation of index.html objects to represent directory
// pages.
MainPageSuffix string
// If the requested object path is missing, and any mainPageSuffix object is
// missing, if applicable, the service will return the named object from this
// bucket as the content for a 404 Not Found result.
NotFoundPage string
}
func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
if b == nil {
return nil, nil
}
rp, err := toRetentionPolicy(b.RetentionPolicy)
if err != nil {
return nil, err
}
return &BucketAttrs{
Name: b.Name,
Location: b.Location,
MetaGeneration: b.Metageneration,
DefaultEventBasedHold: b.DefaultEventBasedHold,
StorageClass: b.StorageClass,
Created: convertTime(b.TimeCreated),
VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled,
ACL: toBucketACLRules(b.Acl),
DefaultObjectACL: toObjectACLRules(b.DefaultObjectAcl),
Labels: b.Labels,
RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
Lifecycle: toLifecycle(b.Lifecycle),
RetentionPolicy: rp,
CORS: toCORS(b.Cors),
Encryption: toBucketEncryption(b.Encryption),
Logging: toBucketLogging(b.Logging),
Website: toBucketWebsite(b.Website),
BucketPolicyOnly: toBucketPolicyOnly(b.IamConfiguration),
UniformBucketLevelAccess: toUniformBucketLevelAccess(b.IamConfiguration),
2021-07-07 13:10:33 +00:00
PublicAccessPrevention: toPublicAccessPrevention(b.IamConfiguration),
Etag: b.Etag,
LocationType: b.LocationType,
2021-09-30 14:54:37 +00:00
ProjectNumber: b.ProjectNumber,
2022-01-27 11:16:16 +00:00
RPO: toRPO(b),
}, nil
}
2022-04-12 09:51:54 +00:00
func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
if b == nil {
return nil
}
return &BucketAttrs{
Name: parseBucketName(b.GetName()),
Location: b.GetLocation(),
MetaGeneration: b.GetMetageneration(),
DefaultEventBasedHold: b.GetDefaultEventBasedHold(),
StorageClass: b.GetStorageClass(),
Created: b.GetCreateTime().AsTime(),
VersioningEnabled: b.GetVersioning().GetEnabled(),
ACL: toBucketACLRulesFromProto(b.GetAcl()),
DefaultObjectACL: toObjectACLRulesFromProto(b.GetDefaultObjectAcl()),
Labels: b.GetLabels(),
RequesterPays: b.GetBilling().GetRequesterPays(),
Lifecycle: toLifecycleFromProto(b.GetLifecycle()),
RetentionPolicy: toRetentionPolicyFromProto(b.GetRetentionPolicy()),
CORS: toCORSFromProto(b.GetCors()),
Encryption: toBucketEncryptionFromProto(b.GetEncryption()),
Logging: toBucketLoggingFromProto(b.GetLogging()),
Website: toBucketWebsiteFromProto(b.GetWebsite()),
BucketPolicyOnly: toBucketPolicyOnlyFromProto(b.GetIamConfig()),
UniformBucketLevelAccess: toUniformBucketLevelAccessFromProto(b.GetIamConfig()),
PublicAccessPrevention: toPublicAccessPreventionFromProto(b.GetIamConfig()),
LocationType: b.GetLocationType(),
RPO: toRPOFromProto(b),
}
}
// toRawBucket copies the editable attribute from b to the raw library's Bucket type.
func (b *BucketAttrs) toRawBucket() *raw.Bucket {
// Copy label map.
var labels map[string]string
if len(b.Labels) > 0 {
labels = make(map[string]string, len(b.Labels))
for k, v := range b.Labels {
labels[k] = v
}
}
// Ignore VersioningEnabled if it is false. This is OK because
// we only call this method when creating a bucket, and by default
// new buckets have versioning off.
var v *raw.BucketVersioning
if b.VersioningEnabled {
v = &raw.BucketVersioning{Enabled: true}
}
var bb *raw.BucketBilling
if b.RequesterPays {
bb = &raw.BucketBilling{RequesterPays: true}
}
var bktIAM *raw.BucketIamConfiguration
2021-07-07 13:10:33 +00:00
if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled || b.PublicAccessPrevention != PublicAccessPreventionUnknown {
bktIAM = &raw.BucketIamConfiguration{}
if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled {
bktIAM.UniformBucketLevelAccess = &raw.BucketIamConfigurationUniformBucketLevelAccess{
Enabled: true,
2021-07-07 13:10:33 +00:00
}
}
if b.PublicAccessPrevention != PublicAccessPreventionUnknown {
bktIAM.PublicAccessPrevention = b.PublicAccessPrevention.String()
}
}
return &raw.Bucket{
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toRawBucketACL(b.ACL),
DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toRawLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(),
Cors: toRawCORS(b.CORS),
Encryption: b.Encryption.toRawBucketEncryption(),
Logging: b.Logging.toRawBucketLogging(),
Website: b.Website.toRawBucketWebsite(),
IamConfiguration: bktIAM,
2022-01-27 11:16:16 +00:00
Rpo: b.RPO.String(),
}
}
2022-04-12 09:51:54 +00:00
func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
if b == nil {
return &storagepb.Bucket{}
}
// Copy label map.
var labels map[string]string
if len(b.Labels) > 0 {
labels = make(map[string]string, len(b.Labels))
for k, v := range b.Labels {
labels[k] = v
}
}
// Ignore VersioningEnabled if it is false. This is OK because
// we only call this method when creating a bucket, and by default
// new buckets have versioning off.
var v *storagepb.Bucket_Versioning
if b.VersioningEnabled {
v = &storagepb.Bucket_Versioning{Enabled: true}
}
var bb *storagepb.Bucket_Billing
if b.RequesterPays {
bb = &storage.Bucket_Billing{RequesterPays: true}
}
var bktIAM *storagepb.Bucket_IamConfig
if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled || b.PublicAccessPrevention != PublicAccessPreventionUnknown {
bktIAM = &storagepb.Bucket_IamConfig{}
if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled {
bktIAM.UniformBucketLevelAccess = &storagepb.Bucket_IamConfig_UniformBucketLevelAccess{
Enabled: true,
}
}
2022-05-20 11:48:16 +00:00
if b.PublicAccessPrevention != PublicAccessPreventionUnknown {
bktIAM.PublicAccessPrevention = b.PublicAccessPrevention.String()
}
2022-04-12 09:51:54 +00:00
}
return &storagepb.Bucket{
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toProtoBucketACL(b.ACL),
DefaultObjectAcl: toProtoObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toProtoLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toProtoRetentionPolicy(),
Cors: toProtoCORS(b.CORS),
Encryption: b.Encryption.toProtoBucketEncryption(),
Logging: b.Logging.toProtoBucketLogging(),
Website: b.Website.toProtoBucketWebsite(),
IamConfig: bktIAM,
Rpo: b.RPO.String(),
}
}
2022-05-20 11:48:16 +00:00
func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
if ua == nil {
return &storagepb.Bucket{}
}
// TODO(cathyo): Handle labels. Pending b/230510191.
var v *storagepb.Bucket_Versioning
if ua.VersioningEnabled != nil {
v = &storagepb.Bucket_Versioning{Enabled: optional.ToBool(ua.VersioningEnabled)}
}
var bb *storagepb.Bucket_Billing
if ua.RequesterPays != nil {
bb = &storage.Bucket_Billing{RequesterPays: optional.ToBool(ua.RequesterPays)}
}
var bktIAM *storagepb.Bucket_IamConfig
var ublaEnabled bool
var bktPolicyOnlyEnabled bool
if ua.UniformBucketLevelAccess != nil {
ublaEnabled = optional.ToBool(ua.UniformBucketLevelAccess.Enabled)
}
if ua.BucketPolicyOnly != nil {
bktPolicyOnlyEnabled = optional.ToBool(ua.BucketPolicyOnly.Enabled)
}
if ublaEnabled || bktPolicyOnlyEnabled {
bktIAM.UniformBucketLevelAccess = &storagepb.Bucket_IamConfig_UniformBucketLevelAccess{
Enabled: true,
}
}
if ua.PublicAccessPrevention != PublicAccessPreventionUnknown {
bktIAM.PublicAccessPrevention = ua.PublicAccessPrevention.String()
}
var defaultHold bool
if ua.DefaultEventBasedHold != nil {
defaultHold = optional.ToBool(ua.DefaultEventBasedHold)
}
var lifecycle Lifecycle
if ua.Lifecycle != nil {
lifecycle = *ua.Lifecycle
}
var bktACL []*storagepb.BucketAccessControl
if ua.acl != nil {
bktACL = toProtoBucketACL(ua.acl)
}
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
bktACL = nil
}
var bktDefaultObjectACL []*storagepb.ObjectAccessControl
if ua.defaultObjectACL != nil {
bktDefaultObjectACL = toProtoObjectACL(ua.defaultObjectACL)
}
if ua.PredefinedDefaultObjectACL != "" {
// Clear ACLs or the call will fail.
bktDefaultObjectACL = nil
}
return &storagepb.Bucket{
StorageClass: ua.StorageClass,
Acl: bktACL,
DefaultObjectAcl: bktDefaultObjectACL,
DefaultEventBasedHold: defaultHold,
Versioning: v,
Billing: bb,
Lifecycle: toProtoLifecycle(lifecycle),
RetentionPolicy: ua.RetentionPolicy.toProtoRetentionPolicy(),
Cors: toProtoCORS(ua.CORS),
Encryption: ua.Encryption.toProtoBucketEncryption(),
Logging: ua.Logging.toProtoBucketLogging(),
Website: ua.Website.toProtoBucketWebsite(),
IamConfig: bktIAM,
Rpo: ua.RPO.String(),
}
}
// CORS is the bucket's Cross-Origin Resource Sharing (CORS) configuration.
type CORS struct {
// MaxAge is the value to return in the Access-Control-Max-Age
// header used in preflight responses.
MaxAge time.Duration
// Methods is the list of HTTP methods on which to include CORS response
// headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list
// of methods, and means "any method".
Methods []string
// Origins is the list of Origins eligible to receive CORS response
// headers. Note: "*" is permitted in the list of origins, and means
// "any Origin".
Origins []string
// ResponseHeaders is the list of HTTP headers other than the simple
// response headers to give permission for the user-agent to share
// across domains.
ResponseHeaders []string
}
// BucketEncryption is a bucket's encryption configuration.
type BucketEncryption struct {
// A Cloud KMS key name, in the form
// projects/P/locations/L/keyRings/R/cryptoKeys/K, that will be used to encrypt
// objects inserted into this bucket, if no encryption method is specified.
// The key's location must be the same as the bucket's.
DefaultKMSKeyName string
}
// BucketAttrsToUpdate define the attributes to update during an Update call.
type BucketAttrsToUpdate struct {
// If set, updates whether the bucket uses versioning.
VersioningEnabled optional.Bool
// If set, updates whether the bucket is a Requester Pays bucket.
RequesterPays optional.Bool
// DefaultEventBasedHold is the default value for event-based hold on
// newly created objects in this bucket.
DefaultEventBasedHold optional.Bool
// BucketPolicyOnly is an alias for UniformBucketLevelAccess. Use of
// UniformBucketLevelAccess is recommended above the use of this field.
// Setting BucketPolicyOnly.Enabled OR UniformBucketLevelAccess.Enabled to
// true, will enable UniformBucketLevelAccess. If both BucketPolicyOnly and
// UniformBucketLevelAccess are set, the value of UniformBucketLevelAccess
// will take precedence.
BucketPolicyOnly *BucketPolicyOnly
// UniformBucketLevelAccess configures access checks to use only bucket-level IAM
// policies and ignore any ACL rules for the bucket.
// See https://cloud.google.com/storage/docs/uniform-bucket-level-access
// for more information.
UniformBucketLevelAccess *UniformBucketLevelAccess
2021-07-07 13:10:33 +00:00
// PublicAccessPrevention is the setting for the bucket's
// PublicAccessPrevention policy, which can be used to prevent public access
// of data in the bucket. See
// https://cloud.google.com/storage/docs/public-access-prevention for more
// information.
PublicAccessPrevention PublicAccessPrevention
2021-02-08 23:12:07 +00:00
// StorageClass is the default storage class of the bucket. This defines
// how objects in the bucket are stored and determines the SLA
// and the cost of storage. Typical values are "STANDARD", "NEARLINE",
// "COLDLINE" and "ARCHIVE". Defaults to "STANDARD".
// See https://cloud.google.com/storage/docs/storage-classes for all
// valid values.
StorageClass string
// If set, updates the retention policy of the bucket. Using
// RetentionPolicy.RetentionPeriod = 0 will delete the existing policy.
//
// This feature is in private alpha release. It is not currently available to
// most customers. It might be changed in backwards-incompatible ways and is not
// subject to any SLA or deprecation policy.
RetentionPolicy *RetentionPolicy
// If set, replaces the CORS configuration with a new configuration.
// An empty (rather than nil) slice causes all CORS policies to be removed.
CORS []CORS
// If set, replaces the encryption configuration of the bucket. Using
// BucketEncryption.DefaultKMSKeyName = "" will delete the existing
// configuration.
Encryption *BucketEncryption
// If set, replaces the lifecycle configuration of the bucket.
Lifecycle *Lifecycle
// If set, replaces the logging configuration of the bucket.
Logging *BucketLogging
// If set, replaces the website configuration of the bucket.
Website *BucketWebsite
// If not empty, applies a predefined set of access controls.
// See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch.
PredefinedACL string
// If not empty, applies a predefined set of default object access controls.
// See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch.
PredefinedDefaultObjectACL string
2022-01-27 11:16:16 +00:00
// RPO configures the Recovery Point Objective (RPO) policy of the bucket.
// Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket.
// See https://cloud.google.com/storage/docs/managing-turbo-replication for
// more information.
RPO RPO
2022-05-20 11:48:16 +00:00
// acl is the list of access control rules on the bucket.
// It is unexported and only used internally by the gRPC client.
// Library users should use ACLHandle methods directly.
acl []ACLRule
// defaultObjectACL is the list of access controls to
// apply to new objects when no object ACL is provided.
// It is unexported and only used internally by the gRPC client.
// Library users should use ACLHandle methods directly.
defaultObjectACL []ACLRule
setLabels map[string]string
deleteLabels map[string]bool
}
// SetLabel causes a label to be added or modified when ua is used
// in a call to Bucket.Update.
func (ua *BucketAttrsToUpdate) SetLabel(name, value string) {
if ua.setLabels == nil {
ua.setLabels = map[string]string{}
}
ua.setLabels[name] = value
}
// DeleteLabel causes a label to be deleted when ua is used in a
// call to Bucket.Update.
func (ua *BucketAttrsToUpdate) DeleteLabel(name string) {
if ua.deleteLabels == nil {
ua.deleteLabels = map[string]bool{}
}
ua.deleteLabels[name] = true
}
func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
rb := &raw.Bucket{}
if ua.CORS != nil {
rb.Cors = toRawCORS(ua.CORS)
rb.ForceSendFields = append(rb.ForceSendFields, "Cors")
}
if ua.DefaultEventBasedHold != nil {
rb.DefaultEventBasedHold = optional.ToBool(ua.DefaultEventBasedHold)
rb.ForceSendFields = append(rb.ForceSendFields, "DefaultEventBasedHold")
}
if ua.RetentionPolicy != nil {
if ua.RetentionPolicy.RetentionPeriod == 0 {
rb.NullFields = append(rb.NullFields, "RetentionPolicy")
rb.RetentionPolicy = nil
} else {
rb.RetentionPolicy = ua.RetentionPolicy.toRawRetentionPolicy()
}
}
if ua.VersioningEnabled != nil {
rb.Versioning = &raw.BucketVersioning{
Enabled: optional.ToBool(ua.VersioningEnabled),
ForceSendFields: []string{"Enabled"},
}
}
if ua.RequesterPays != nil {
rb.Billing = &raw.BucketBilling{
RequesterPays: optional.ToBool(ua.RequesterPays),
ForceSendFields: []string{"RequesterPays"},
}
}
if ua.BucketPolicyOnly != nil {
rb.IamConfiguration = &raw.BucketIamConfiguration{
UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{
Enabled: ua.BucketPolicyOnly.Enabled,
ForceSendFields: []string{"Enabled"},
},
}
}
if ua.UniformBucketLevelAccess != nil {
rb.IamConfiguration = &raw.BucketIamConfiguration{
UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{
Enabled: ua.UniformBucketLevelAccess.Enabled,
ForceSendFields: []string{"Enabled"},
},
}
}
2021-07-07 13:10:33 +00:00
if ua.PublicAccessPrevention != PublicAccessPreventionUnknown {
if rb.IamConfiguration == nil {
rb.IamConfiguration = &raw.BucketIamConfiguration{}
}
rb.IamConfiguration.PublicAccessPrevention = ua.PublicAccessPrevention.String()
}
if ua.Encryption != nil {
if ua.Encryption.DefaultKMSKeyName == "" {
rb.NullFields = append(rb.NullFields, "Encryption")
rb.Encryption = nil
} else {
rb.Encryption = ua.Encryption.toRawBucketEncryption()
}
}
if ua.Lifecycle != nil {
rb.Lifecycle = toRawLifecycle(*ua.Lifecycle)
2020-06-18 23:40:30 +00:00
rb.ForceSendFields = append(rb.ForceSendFields, "Lifecycle")
}
if ua.Logging != nil {
if *ua.Logging == (BucketLogging{}) {
rb.NullFields = append(rb.NullFields, "Logging")
rb.Logging = nil
} else {
rb.Logging = ua.Logging.toRawBucketLogging()
}
}
if ua.Website != nil {
if *ua.Website == (BucketWebsite{}) {
rb.NullFields = append(rb.NullFields, "Website")
rb.Website = nil
} else {
rb.Website = ua.Website.toRawBucketWebsite()
}
}
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
rb.Acl = nil
rb.ForceSendFields = append(rb.ForceSendFields, "Acl")
}
if ua.PredefinedDefaultObjectACL != "" {
// Clear ACLs or the call will fail.
rb.DefaultObjectAcl = nil
rb.ForceSendFields = append(rb.ForceSendFields, "DefaultObjectAcl")
}
2022-01-27 11:16:16 +00:00
2021-02-08 23:12:07 +00:00
rb.StorageClass = ua.StorageClass
2022-01-27 11:16:16 +00:00
rb.Rpo = ua.RPO.String()
if ua.setLabels != nil || ua.deleteLabels != nil {
rb.Labels = map[string]string{}
for k, v := range ua.setLabels {
rb.Labels[k] = v
}
if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 {
rb.ForceSendFields = append(rb.ForceSendFields, "Labels")
}
for l := range ua.deleteLabels {
rb.NullFields = append(rb.NullFields, "Labels."+l)
}
}
return rb
}
// If returns a new BucketHandle that applies a set of preconditions.
// Preconditions already set on the BucketHandle are ignored.
// Operations on the new handle will return an error if the preconditions are not
// satisfied. The only valid preconditions for buckets are MetagenerationMatch
// and MetagenerationNotMatch.
func (b *BucketHandle) If(conds BucketConditions) *BucketHandle {
b2 := *b
b2.conds = &conds
return &b2
}
// BucketConditions constrain bucket methods to act on specific metagenerations.
//
// The zero value is an empty set of constraints.
type BucketConditions struct {
// MetagenerationMatch specifies that the bucket must have the given
// metageneration for the operation to occur.
// If MetagenerationMatch is zero, it has no effect.
MetagenerationMatch int64
// MetagenerationNotMatch specifies that the bucket must not have the given
// metageneration for the operation to occur.
// If MetagenerationNotMatch is zero, it has no effect.
MetagenerationNotMatch int64
}
func (c *BucketConditions) validate(method string) error {
if *c == (BucketConditions{}) {
return fmt.Errorf("storage: %s: empty conditions", method)
}
if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 {
return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
}
return nil
}
// UserProject returns a new BucketHandle that passes the project ID as the user
// project for all subsequent calls. Calls with a user project will be billed to that
// project rather than to the bucket's owning project.
//
// A user project is required for all operations on Requester Pays buckets.
func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
b2 := *b
b2.userProject = projectID
b2.acl.userProject = projectID
b2.defaultObjectACL.userProject = projectID
return &b2
}
// LockRetentionPolicy locks a bucket's retention policy until a previously-configured
// RetentionPeriod past the EffectiveTime. Note that if RetentionPeriod is set to less
// than a day, the retention policy is treated as a development configuration and locking
// will have no effect. The BucketHandle must have a metageneration condition that
// matches the bucket's metageneration. See BucketHandle.If.
//
// This feature is in private alpha release. It is not currently available to
// most customers. It might be changed in backwards-incompatible ways and is not
// subject to any SLA or deprecation policy.
func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error {
var metageneration int64
if b.conds != nil {
metageneration = b.conds.MetagenerationMatch
}
req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration)
2022-01-27 11:16:16 +00:00
return run(ctx, func() error {
_, err := req.Context(ctx).Do()
return err
2022-05-20 11:48:16 +00:00
}, b.retry, true, setRetryHeaderHTTP(req))
}
// applyBucketConds modifies the provided call using the conditions in conds.
// call is something that quacks like a *raw.WhateverCall.
func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
if conds == nil {
return nil
}
if err := conds.validate(method); err != nil {
return err
}
cval := reflect.ValueOf(call)
switch {
case conds.MetagenerationMatch != 0:
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
return nil
}
2022-04-12 09:51:54 +00:00
// applyBucketConds modifies the provided request message using the conditions
// in conds. msg is a protobuf Message that has fields if_metageneration_match
// and if_metageneration_not_match.
func applyBucketCondsProto(method string, conds *BucketConditions, msg proto.Message) error {
rmsg := msg.ProtoReflect()
if conds == nil {
return nil
}
if err := conds.validate(method); err != nil {
return err
}
switch {
case conds.MetagenerationMatch != 0:
if !setConditionProtoField(rmsg, "if_metageneration_match", conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionProtoField(rmsg, "if_metageneration_not_match", conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
return nil
}
func (rp *RetentionPolicy) toRawRetentionPolicy() *raw.BucketRetentionPolicy {
if rp == nil {
return nil
}
return &raw.BucketRetentionPolicy{
RetentionPeriod: int64(rp.RetentionPeriod / time.Second),
}
}
2022-04-12 09:51:54 +00:00
func (rp *RetentionPolicy) toProtoRetentionPolicy() *storagepb.Bucket_RetentionPolicy {
if rp == nil {
return nil
}
return &storagepb.Bucket_RetentionPolicy{
RetentionPeriod: int64(rp.RetentionPeriod / time.Second),
}
}
func toRetentionPolicy(rp *raw.BucketRetentionPolicy) (*RetentionPolicy, error) {
2022-05-20 11:48:16 +00:00
if rp == nil || rp.EffectiveTime == "" {
return nil, nil
}
t, err := time.Parse(time.RFC3339, rp.EffectiveTime)
if err != nil {
return nil, err
}
return &RetentionPolicy{
RetentionPeriod: time.Duration(rp.RetentionPeriod) * time.Second,
EffectiveTime: t,
IsLocked: rp.IsLocked,
}, nil
}
2022-04-12 09:51:54 +00:00
func toRetentionPolicyFromProto(rp *storagepb.Bucket_RetentionPolicy) *RetentionPolicy {
if rp == nil {
return nil
}
return &RetentionPolicy{
RetentionPeriod: time.Duration(rp.GetRetentionPeriod()) * time.Second,
EffectiveTime: rp.GetEffectiveTime().AsTime(),
IsLocked: rp.GetIsLocked(),
}
}
func toRawCORS(c []CORS) []*raw.BucketCors {
var out []*raw.BucketCors
for _, v := range c {
out = append(out, &raw.BucketCors{
MaxAgeSeconds: int64(v.MaxAge / time.Second),
Method: v.Methods,
Origin: v.Origins,
ResponseHeader: v.ResponseHeaders,
})
}
return out
}
2022-04-12 09:51:54 +00:00
func toProtoCORS(c []CORS) []*storagepb.Bucket_Cors {
var out []*storagepb.Bucket_Cors
for _, v := range c {
out = append(out, &storagepb.Bucket_Cors{
MaxAgeSeconds: int32(v.MaxAge / time.Second),
Method: v.Methods,
Origin: v.Origins,
ResponseHeader: v.ResponseHeaders,
})
}
return out
}
func toCORS(rc []*raw.BucketCors) []CORS {
var out []CORS
for _, v := range rc {
out = append(out, CORS{
MaxAge: time.Duration(v.MaxAgeSeconds) * time.Second,
Methods: v.Method,
Origins: v.Origin,
ResponseHeaders: v.ResponseHeader,
})
}
return out
}
2022-04-12 09:51:54 +00:00
func toCORSFromProto(rc []*storagepb.Bucket_Cors) []CORS {
var out []CORS
for _, v := range rc {
out = append(out, CORS{
MaxAge: time.Duration(v.GetMaxAgeSeconds()) * time.Second,
Methods: v.GetMethod(),
Origins: v.GetOrigin(),
ResponseHeaders: v.GetResponseHeader(),
})
}
return out
}
2022-06-28 11:55:33 +00:00
// Used to handle breaking change in Autogen Storage client OLM Age field
// from int64 to *int64 gracefully in the manual client
// TODO(#6240): Method should be removed once breaking change is made and introduced to this client
func setAgeCondition(age int64, ageField interface{}) {
c := reflect.ValueOf(ageField).Elem()
switch c.Kind() {
case reflect.Int64:
c.SetInt(age)
case reflect.Ptr:
c.Set(reflect.ValueOf(&age))
}
}
func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
var rl raw.BucketLifecycle
if len(l.Rules) == 0 {
2020-06-18 23:40:30 +00:00
rl.ForceSendFields = []string{"Rule"}
}
for _, r := range l.Rules {
rr := &raw.BucketLifecycleRule{
Action: &raw.BucketLifecycleRuleAction{
Type: r.Action.Type,
StorageClass: r.Action.StorageClass,
},
Condition: &raw.BucketLifecycleRuleCondition{
2020-09-01 14:43:21 +00:00
DaysSinceCustomTime: r.Condition.DaysSinceCustomTime,
DaysSinceNoncurrentTime: r.Condition.DaysSinceNoncurrentTime,
2022-06-28 11:55:33 +00:00
MatchesPrefix: r.Condition.MatchesPrefix,
2020-09-01 14:43:21 +00:00
MatchesStorageClass: r.Condition.MatchesStorageClasses,
2022-06-28 11:55:33 +00:00
MatchesSuffix: r.Condition.MatchesSuffix,
2020-09-01 14:43:21 +00:00
NumNewerVersions: r.Condition.NumNewerVersions,
},
}
2022-06-28 11:55:33 +00:00
setAgeCondition(r.Condition.AgeInDays, &rr.Condition.Age)
switch r.Condition.Liveness {
case LiveAndArchived:
rr.Condition.IsLive = nil
case Live:
rr.Condition.IsLive = googleapi.Bool(true)
case Archived:
rr.Condition.IsLive = googleapi.Bool(false)
}
if !r.Condition.CreatedBefore.IsZero() {
rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date)
}
2020-09-01 14:43:21 +00:00
if !r.Condition.CustomTimeBefore.IsZero() {
rr.Condition.CustomTimeBefore = r.Condition.CustomTimeBefore.Format(rfc3339Date)
}
if !r.Condition.NoncurrentTimeBefore.IsZero() {
rr.Condition.NoncurrentTimeBefore = r.Condition.NoncurrentTimeBefore.Format(rfc3339Date)
}
rl.Rule = append(rl.Rule, rr)
}
return &rl
}
2022-04-12 09:51:54 +00:00
func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle {
var rl storagepb.Bucket_Lifecycle
for _, r := range l.Rules {
rr := &storagepb.Bucket_Lifecycle_Rule{
Action: &storagepb.Bucket_Lifecycle_Rule_Action{
Type: r.Action.Type,
StorageClass: r.Action.StorageClass,
},
Condition: &storagepb.Bucket_Lifecycle_Rule_Condition{
// Note: The Apiary types use int64 (even though the Discovery
// doc states "format: int32"), so the client types used int64,
// but the proto uses int32 so we have a potentially lossy
// conversion.
AgeDays: proto.Int32(int32(r.Condition.AgeInDays)),
DaysSinceCustomTime: proto.Int32(int32(r.Condition.DaysSinceCustomTime)),
DaysSinceNoncurrentTime: proto.Int32(int32(r.Condition.DaysSinceNoncurrentTime)),
2022-06-28 11:55:33 +00:00
MatchesPrefix: r.Condition.MatchesPrefix,
2022-04-12 09:51:54 +00:00
MatchesStorageClass: r.Condition.MatchesStorageClasses,
2022-06-28 11:55:33 +00:00
MatchesSuffix: r.Condition.MatchesSuffix,
2022-04-12 09:51:54 +00:00
NumNewerVersions: proto.Int32(int32(r.Condition.NumNewerVersions)),
},
}
switch r.Condition.Liveness {
case LiveAndArchived:
rr.Condition.IsLive = nil
case Live:
rr.Condition.IsLive = proto.Bool(true)
case Archived:
rr.Condition.IsLive = proto.Bool(false)
}
if !r.Condition.CreatedBefore.IsZero() {
rr.Condition.CreatedBefore = adapters.TimeToProtoDate(r.Condition.CreatedBefore)
}
if !r.Condition.CustomTimeBefore.IsZero() {
rr.Condition.CustomTimeBefore = adapters.TimeToProtoDate(r.Condition.CustomTimeBefore)
}
if !r.Condition.NoncurrentTimeBefore.IsZero() {
rr.Condition.NoncurrentTimeBefore = adapters.TimeToProtoDate(r.Condition.NoncurrentTimeBefore)
}
rl.Rule = append(rl.Rule, rr)
}
return &rl
}
2022-06-28 11:55:33 +00:00
// Used to handle breaking change in Autogen Storage client OLM Age field
// from int64 to *int64 gracefully in the manual client
// TODO(#6240): Method should be removed once breaking change is made and introduced to this client
func getAgeCondition(ageField interface{}) int64 {
v := reflect.ValueOf(ageField)
if v.Kind() == reflect.Int64 {
return v.Interface().(int64)
} else if v.Kind() == reflect.Ptr {
if val, ok := v.Interface().(*int64); ok {
return *val
}
}
return 0
}
func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
var l Lifecycle
if rl == nil {
return l
}
for _, rr := range rl.Rule {
r := LifecycleRule{
Action: LifecycleAction{
Type: rr.Action.Type,
StorageClass: rr.Action.StorageClass,
},
Condition: LifecycleCondition{
2020-09-01 14:43:21 +00:00
DaysSinceCustomTime: rr.Condition.DaysSinceCustomTime,
DaysSinceNoncurrentTime: rr.Condition.DaysSinceNoncurrentTime,
2022-06-28 11:55:33 +00:00
MatchesPrefix: rr.Condition.MatchesPrefix,
2020-09-01 14:43:21 +00:00
MatchesStorageClasses: rr.Condition.MatchesStorageClass,
2022-06-28 11:55:33 +00:00
MatchesSuffix: rr.Condition.MatchesSuffix,
2020-09-01 14:43:21 +00:00
NumNewerVersions: rr.Condition.NumNewerVersions,
},
}
2022-06-28 11:55:33 +00:00
r.Condition.AgeInDays = getAgeCondition(rr.Condition.Age)
2020-02-26 18:46:17 +00:00
if rr.Condition.IsLive == nil {
r.Condition.Liveness = LiveAndArchived
2020-02-26 18:46:17 +00:00
} else if *rr.Condition.IsLive {
r.Condition.Liveness = Live
2020-02-26 18:46:17 +00:00
} else {
r.Condition.Liveness = Archived
}
if rr.Condition.CreatedBefore != "" {
r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
}
2020-09-01 14:43:21 +00:00
if rr.Condition.CustomTimeBefore != "" {
r.Condition.CustomTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.CustomTimeBefore)
}
if rr.Condition.NoncurrentTimeBefore != "" {
r.Condition.NoncurrentTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.NoncurrentTimeBefore)
}
l.Rules = append(l.Rules, r)
}
return l
}
2022-04-12 09:51:54 +00:00
func toLifecycleFromProto(rl *storagepb.Bucket_Lifecycle) Lifecycle {
var l Lifecycle
if rl == nil {
return l
}
for _, rr := range rl.GetRule() {
r := LifecycleRule{
Action: LifecycleAction{
Type: rr.GetAction().GetType(),
StorageClass: rr.GetAction().GetStorageClass(),
},
Condition: LifecycleCondition{
AgeInDays: int64(rr.GetCondition().GetAgeDays()),
DaysSinceCustomTime: int64(rr.GetCondition().GetDaysSinceCustomTime()),
DaysSinceNoncurrentTime: int64(rr.GetCondition().GetDaysSinceNoncurrentTime()),
2022-06-28 11:55:33 +00:00
MatchesPrefix: rr.GetCondition().GetMatchesPrefix(),
2022-04-12 09:51:54 +00:00
MatchesStorageClasses: rr.GetCondition().GetMatchesStorageClass(),
2022-06-28 11:55:33 +00:00
MatchesSuffix: rr.GetCondition().GetMatchesSuffix(),
2022-04-12 09:51:54 +00:00
NumNewerVersions: int64(rr.GetCondition().GetNumNewerVersions()),
},
}
if rr.GetCondition().IsLive == nil {
r.Condition.Liveness = LiveAndArchived
} else if rr.GetCondition().GetIsLive() {
r.Condition.Liveness = Live
} else {
r.Condition.Liveness = Archived
}
if rr.GetCondition().GetCreatedBefore() != nil {
r.Condition.CreatedBefore = adapters.ProtoDateToUTCTime(rr.GetCondition().GetCreatedBefore())
}
if rr.GetCondition().GetCustomTimeBefore() != nil {
r.Condition.CustomTimeBefore = adapters.ProtoDateToUTCTime(rr.GetCondition().GetCustomTimeBefore())
}
if rr.GetCondition().GetNoncurrentTimeBefore() != nil {
r.Condition.NoncurrentTimeBefore = adapters.ProtoDateToUTCTime(rr.GetCondition().GetNoncurrentTimeBefore())
}
l.Rules = append(l.Rules, r)
}
return l
}
func (e *BucketEncryption) toRawBucketEncryption() *raw.BucketEncryption {
if e == nil {
return nil
}
return &raw.BucketEncryption{
DefaultKmsKeyName: e.DefaultKMSKeyName,
}
}
2022-04-12 09:51:54 +00:00
func (e *BucketEncryption) toProtoBucketEncryption() *storagepb.Bucket_Encryption {
if e == nil {
return nil
}
return &storagepb.Bucket_Encryption{
DefaultKmsKey: e.DefaultKMSKeyName,
}
}
func toBucketEncryption(e *raw.BucketEncryption) *BucketEncryption {
if e == nil {
return nil
}
return &BucketEncryption{DefaultKMSKeyName: e.DefaultKmsKeyName}
}
2022-04-12 09:51:54 +00:00
func toBucketEncryptionFromProto(e *storagepb.Bucket_Encryption) *BucketEncryption {
if e == nil {
return nil
}
return &BucketEncryption{DefaultKMSKeyName: e.GetDefaultKmsKey()}
}
func (b *BucketLogging) toRawBucketLogging() *raw.BucketLogging {
if b == nil {
return nil
}
return &raw.BucketLogging{
LogBucket: b.LogBucket,
LogObjectPrefix: b.LogObjectPrefix,
}
}
2022-04-12 09:51:54 +00:00
func (b *BucketLogging) toProtoBucketLogging() *storagepb.Bucket_Logging {
if b == nil {
return nil
}
return &storagepb.Bucket_Logging{
2022-06-28 11:55:33 +00:00
LogBucket: bucketResourceName(globalProjectAlias, b.LogBucket),
2022-04-12 09:51:54 +00:00
LogObjectPrefix: b.LogObjectPrefix,
}
}
func toBucketLogging(b *raw.BucketLogging) *BucketLogging {
if b == nil {
return nil
}
return &BucketLogging{
LogBucket: b.LogBucket,
LogObjectPrefix: b.LogObjectPrefix,
}
}
2022-04-12 09:51:54 +00:00
func toBucketLoggingFromProto(b *storagepb.Bucket_Logging) *BucketLogging {
if b == nil {
return nil
}
2022-06-28 11:55:33 +00:00
lb := parseBucketName(b.GetLogBucket())
2022-04-12 09:51:54 +00:00
return &BucketLogging{
2022-06-28 11:55:33 +00:00
LogBucket: lb,
2022-04-12 09:51:54 +00:00
LogObjectPrefix: b.GetLogObjectPrefix(),
}
}
func (w *BucketWebsite) toRawBucketWebsite() *raw.BucketWebsite {
if w == nil {
return nil
}
return &raw.BucketWebsite{
MainPageSuffix: w.MainPageSuffix,
NotFoundPage: w.NotFoundPage,
}
}
2022-04-12 09:51:54 +00:00
func (w *BucketWebsite) toProtoBucketWebsite() *storagepb.Bucket_Website {
if w == nil {
return nil
}
return &storagepb.Bucket_Website{
MainPageSuffix: w.MainPageSuffix,
NotFoundPage: w.NotFoundPage,
}
}
func toBucketWebsite(w *raw.BucketWebsite) *BucketWebsite {
if w == nil {
return nil
}
return &BucketWebsite{
MainPageSuffix: w.MainPageSuffix,
NotFoundPage: w.NotFoundPage,
}
}
2022-04-12 09:51:54 +00:00
func toBucketWebsiteFromProto(w *storagepb.Bucket_Website) *BucketWebsite {
if w == nil {
return nil
}
return &BucketWebsite{
MainPageSuffix: w.GetMainPageSuffix(),
NotFoundPage: w.GetNotFoundPage(),
}
}
func toBucketPolicyOnly(b *raw.BucketIamConfiguration) BucketPolicyOnly {
if b == nil || b.BucketPolicyOnly == nil || !b.BucketPolicyOnly.Enabled {
return BucketPolicyOnly{}
}
lt, err := time.Parse(time.RFC3339, b.BucketPolicyOnly.LockedTime)
if err != nil {
return BucketPolicyOnly{
Enabled: true,
}
}
return BucketPolicyOnly{
Enabled: true,
LockedTime: lt,
}
}
2022-04-12 09:51:54 +00:00
func toBucketPolicyOnlyFromProto(b *storagepb.Bucket_IamConfig) BucketPolicyOnly {
if b == nil || !b.GetUniformBucketLevelAccess().GetEnabled() {
return BucketPolicyOnly{}
}
return BucketPolicyOnly{
Enabled: true,
LockedTime: b.GetUniformBucketLevelAccess().GetLockTime().AsTime(),
}
}
func toUniformBucketLevelAccess(b *raw.BucketIamConfiguration) UniformBucketLevelAccess {
if b == nil || b.UniformBucketLevelAccess == nil || !b.UniformBucketLevelAccess.Enabled {
return UniformBucketLevelAccess{}
}
lt, err := time.Parse(time.RFC3339, b.UniformBucketLevelAccess.LockedTime)
if err != nil {
return UniformBucketLevelAccess{
Enabled: true,
}
}
return UniformBucketLevelAccess{
Enabled: true,
LockedTime: lt,
}
}
2022-04-12 09:51:54 +00:00
func toUniformBucketLevelAccessFromProto(b *storagepb.Bucket_IamConfig) UniformBucketLevelAccess {
if b == nil || !b.GetUniformBucketLevelAccess().GetEnabled() {
return UniformBucketLevelAccess{}
}
return UniformBucketLevelAccess{
Enabled: true,
LockedTime: b.GetUniformBucketLevelAccess().GetLockTime().AsTime(),
}
}
2021-07-07 13:10:33 +00:00
func toPublicAccessPrevention(b *raw.BucketIamConfiguration) PublicAccessPrevention {
if b == nil {
return PublicAccessPreventionUnknown
}
switch b.PublicAccessPrevention {
2021-10-11 18:51:32 +00:00
case publicAccessPreventionInherited, publicAccessPreventionUnspecified:
return PublicAccessPreventionInherited
2021-07-07 13:10:33 +00:00
case publicAccessPreventionEnforced:
return PublicAccessPreventionEnforced
default:
return PublicAccessPreventionUnknown
}
}
2022-04-12 09:51:54 +00:00
func toPublicAccessPreventionFromProto(b *storagepb.Bucket_IamConfig) PublicAccessPrevention {
if b == nil {
return PublicAccessPreventionUnknown
}
switch b.GetPublicAccessPrevention() {
case publicAccessPreventionInherited, publicAccessPreventionUnspecified:
return PublicAccessPreventionInherited
case publicAccessPreventionEnforced:
return PublicAccessPreventionEnforced
default:
return PublicAccessPreventionUnknown
}
}
2022-01-27 11:16:16 +00:00
func toRPO(b *raw.Bucket) RPO {
if b == nil {
return RPOUnknown
}
switch b.Rpo {
case rpoDefault:
return RPODefault
case rpoAsyncTurbo:
return RPOAsyncTurbo
default:
return RPOUnknown
}
}
2022-04-12 09:51:54 +00:00
func toRPOFromProto(b *storagepb.Bucket) RPO {
if b == nil {
return RPOUnknown
}
switch b.GetRpo() {
case rpoDefault:
return RPODefault
case rpoAsyncTurbo:
return RPOAsyncTurbo
default:
return RPOUnknown
}
}
2020-09-23 11:23:39 +00:00
// Objects returns an iterator over the objects in the bucket that match the
// Query q. If q is nil, no filtering is done. Objects will be iterated over
// lexicographically by name.
//
// Note: The returned iterator is not safe for concurrent operations without explicit synchronization.
func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
it := &ObjectIterator{
ctx: ctx,
bucket: b,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
if q != nil {
it.query = *q
}
return it
}
2022-01-27 11:16:16 +00:00
// Retryer returns a bucket handle that is configured with custom retry
// behavior as specified by the options that are passed to it. All operations
// on the new handle will use the customized retry configuration.
// Retry options set on a object handle will take precedence over options set on
// the bucket handle.
// These retry options will merge with the client's retry configuration (if set)
// for the returned handle. Options passed into this method will take precedence
// over retry options on the client. Note that you must explicitly pass in each
// option you want to override.
func (b *BucketHandle) Retryer(opts ...RetryOption) *BucketHandle {
b2 := *b
var retry *retryConfig
if b.retry != nil {
// merge the options with the existing retry
retry = b.retry
} else {
retry = &retryConfig{}
}
for _, opt := range opts {
opt.apply(retry)
}
b2.retry = retry
b2.acl.retry = retry
b2.defaultObjectACL.retry = retry
return &b2
}
// An ObjectIterator is an iterator over ObjectAttrs.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
type ObjectIterator struct {
ctx context.Context
bucket *BucketHandle
query Query
pageInfo *iterator.PageInfo
nextFunc func() error
items []*ObjectAttrs
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
//
// Note: This method is not safe for concurrent operations without explicit synchronization.
func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
//
2020-09-23 11:23:39 +00:00
// In addition, if Next returns an error other than iterator.Done, all
// subsequent calls will return the same error. To continue iteration, a new
// `ObjectIterator` must be created. Since objects are ordered lexicographically
// by name, `Query.StartOffset` can be used to create a new iterator which will
// start at the desired place. See
// https://pkg.go.dev/cloud.google.com/go/storage?tab=doc#hdr-Listing_objects.
//
// If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will
// have a non-empty Prefix field, and a zero value for all other fields. These
// represent prefixes.
//
// Note: This method is not safe for concurrent operations without explicit synchronization.
func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.bucket.c.raw.Objects.List(it.bucket.name)
setClientHeader(req.Header())
2021-02-08 23:12:07 +00:00
projection := it.query.Projection
if projection == ProjectionDefault {
projection = ProjectionFull
}
req.Projection(projection.String())
req.Delimiter(it.query.Delimiter)
req.Prefix(it.query.Prefix)
2020-09-01 14:43:21 +00:00
req.StartOffset(it.query.StartOffset)
req.EndOffset(it.query.EndOffset)
req.Versions(it.query.Versions)
2022-04-12 09:51:54 +00:00
req.IncludeTrailingDelimiter(it.query.IncludeTrailingDelimiter)
2019-11-19 19:29:35 +00:00
if len(it.query.fieldSelection) > 0 {
req.Fields("nextPageToken", googleapi.Field(it.query.fieldSelection))
}
req.PageToken(pageToken)
if it.bucket.userProject != "" {
req.UserProject(it.bucket.userProject)
}
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Objects
var err error
2022-01-27 11:16:16 +00:00
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
2022-05-20 11:48:16 +00:00
}, it.bucket.retry, true, setRetryHeaderHTTP(req))
if err != nil {
2021-10-11 18:51:32 +00:00
var e *googleapi.Error
2022-05-20 11:48:16 +00:00
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
err = ErrBucketNotExist
}
return "", err
}
for _, item := range resp.Items {
it.items = append(it.items, newObject(item))
}
for _, prefix := range resp.Prefixes {
it.items = append(it.items, &ObjectAttrs{Prefix: prefix})
}
return resp.NextPageToken, nil
}
// Buckets returns an iterator over the buckets in the project. You may
// optionally set the iterator's Prefix field to restrict the list to buckets
// whose names begin with the prefix. By default, all buckets in the project
// are returned.
//
// Note: The returned iterator is not safe for concurrent operations without explicit synchronization.
func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator {
it := &BucketIterator{
ctx: ctx,
client: c,
projectID: projectID,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.buckets) },
func() interface{} { b := it.buckets; it.buckets = nil; return b })
return it
}
// A BucketIterator is an iterator over BucketAttrs.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
type BucketIterator struct {
// Prefix restricts the iterator to buckets whose names begin with it.
Prefix string
ctx context.Context
client *Client
projectID string
buckets []*BucketAttrs
pageInfo *iterator.PageInfo
nextFunc func() error
}
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
//
// Note: This method is not safe for concurrent operations without explicit synchronization.
func (it *BucketIterator) Next() (*BucketAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
b := it.buckets[0]
it.buckets = it.buckets[1:]
return b, nil
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
//
// Note: This method is not safe for concurrent operations without explicit synchronization.
func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
2022-05-20 11:48:16 +00:00
// TODO: When the transport-agnostic client interface is integrated into the Veneer,
// this method should be removed, and the iterator should be initialized by the
// transport-specific client implementations.
func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, err error) {
req := it.client.raw.Buckets.List(it.projectID)
setClientHeader(req.Header())
req.Projection("full")
req.Prefix(it.Prefix)
req.PageToken(pageToken)
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Buckets
2022-01-27 11:16:16 +00:00
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
2022-05-20 11:48:16 +00:00
}, it.client.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return "", err
}
for _, item := range resp.Items {
b, err := newBucket(item)
if err != nil {
return "", err
}
it.buckets = append(it.buckets, b)
}
return resp.NextPageToken, nil
}
2022-01-27 11:16:16 +00:00
// RPO (Recovery Point Objective) configures the turbo replication feature. See
// https://cloud.google.com/storage/docs/managing-turbo-replication for more information.
type RPO int
const (
// RPOUnknown is a zero value. It may be returned from bucket.Attrs() if RPO
// is not present in the bucket metadata, that is, the bucket is not dual-region.
// This value is also used if the RPO field is not set in a call to GCS.
RPOUnknown RPO = iota
// RPODefault represents default replication. It is used to reset RPO on an
// existing bucket that has this field set to RPOAsyncTurbo. Otherwise it
// is equivalent to RPOUnknown, and is always ignored. This value is valid
// for dual- or multi-region buckets.
RPODefault
// RPOAsyncTurbo represents turbo replication and is used to enable Turbo
// Replication on a bucket. This value is only valid for dual-region buckets.
RPOAsyncTurbo
rpoUnknown string = ""
rpoDefault = "DEFAULT"
rpoAsyncTurbo = "ASYNC_TURBO"
)
func (rpo RPO) String() string {
switch rpo {
case RPODefault:
return rpoDefault
case RPOAsyncTurbo:
return rpoAsyncTurbo
default:
return rpoUnknown
}
}