diff --git a/go.mod b/go.mod index a575c7bf6..79dabb836 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/VictoriaMetrics/VictoriaMetrics go 1.17 require ( - cloud.google.com/go/storage v1.22.1 + cloud.google.com/go/storage v1.23.0 github.com/VictoriaMetrics/fastcache v1.10.0 // Do not use the original github.com/valyala/fasthttp because of issues @@ -11,7 +11,7 @@ require ( github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.1 github.com/VictoriaMetrics/metricsql v0.44.0 - github.com/aws/aws-sdk-go v1.44.37 + github.com/aws/aws-sdk-go v1.44.43 github.com/cespare/xxhash/v2 v2.1.2 // TODO: switch back to https://github.com/cheggaaa/pb/v3 when v3-pooling branch @@ -22,16 +22,16 @@ require ( github.com/influxdata/influxdb v1.9.7 github.com/klauspost/compress v1.15.6 github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9 - github.com/urfave/cli/v2 v2.10.1 + github.com/urfave/cli/v2 v2.10.3 github.com/valyala/fastjson v1.6.3 github.com/valyala/fastrand v1.1.0 github.com/valyala/fasttemplate v1.2.1 github.com/valyala/gozstd v1.17.0 github.com/valyala/quicktemplate v1.7.0 - golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 - golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb - golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c - google.golang.org/api v0.84.0 + golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e + golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 + golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b + google.golang.org/api v0.85.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -76,7 +76,7 @@ require ( golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect + google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.0 // indirect ) diff --git a/go.sum b/go.sum index 2432b16ee..7076687a4 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,9 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0 h1:wWRIaDURQA8xxHguFCshYepGlrWIrbBnAmc7wfg07qY= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v48.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -144,8 +145,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= -github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.43 h1:gILXnQAOkfAV9dhdXOUlnVTGM3AiOQFqwQmJJ9R7rUE= +github.com/aws/aws-sdk-go v1.44.43/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -820,8 +821,8 @@ github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.10.1 h1:34qJSQxqF/4fqJ7oiAV5WoXaTFlGG9QNM+qxpY3W3gs= -github.com/urfave/cli/v2 v2.10.1/go.mod h1:MaQ2eKodtz1fFzu2U0jL+tVjoWmG134POMRjyXJK6+8= +github.com/urfave/cli/v2 v2.10.3 h1:oi571Fxz5aHugfBAJd5nkwSk3fzATXtMlpxdLylSCMo= +github.com/urfave/cli/v2 v2.10.3/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= @@ -998,8 +999,9 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1019,8 +1021,9 @@ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 h1:+jnHzr9VPj32ykQVai5DNahi9+NSp7yYuCsl5eAQtL0= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1129,8 +1132,9 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1275,8 +1279,9 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0 h1:8rJoHuRxx+vCmZtAO/3k1dRLvYNVyTJtZ5oaFZvhgvc= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1367,8 +1372,9 @@ google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33 h1:3L4edWcjDHPWGcMl1N0YH1NSoasyvfEcZCe2rUbxHfs= +google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= diff --git a/vendor/cloud.google.com/go/storage/.release-please-manifest.json b/vendor/cloud.google.com/go/storage/.release-please-manifest.json index a068903e0..110758e0c 100644 --- a/vendor/cloud.google.com/go/storage/.release-please-manifest.json +++ b/vendor/cloud.google.com/go/storage/.release-please-manifest.json @@ -1,3 +1,3 @@ { - "storage": "1.22.1" + "storage": "1.23.0" } \ No newline at end of file diff --git a/vendor/cloud.google.com/go/storage/CHANGES.md b/vendor/cloud.google.com/go/storage/CHANGES.md index d88a1b2c8..4a80b8ff4 100644 --- a/vendor/cloud.google.com/go/storage/CHANGES.md +++ b/vendor/cloud.google.com/go/storage/CHANGES.md @@ -1,6 +1,19 @@ # Changes +## [1.23.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.22.1...storage/v1.23.0) (2022-06-23) + + +### Features + +* **storage:** add support for OLM Prefix/Suffix ([#5929](https://github.com/googleapis/google-cloud-go/issues/5929)) ([ec21d10](https://github.com/googleapis/google-cloud-go/commit/ec21d10d6d1b01aa97a52560319775041707690d)) +* **storage:** support AbortIncompleteMultipartUpload LifecycleAction ([#5812](https://github.com/googleapis/google-cloud-go/issues/5812)) ([fdec929](https://github.com/googleapis/google-cloud-go/commit/fdec929b9da6e01dda0ab3c72544d44d6bd82bd4)), refs [#5795](https://github.com/googleapis/google-cloud-go/issues/5795) + + +### Bug Fixes + +* **storage:** allow for Age *int64 type and int64 type ([#6230](https://github.com/googleapis/google-cloud-go/issues/6230)) ([cc7acb8](https://github.com/googleapis/google-cloud-go/commit/cc7acb8bffb31828e9e96d4834a65f9728494473)) + ### [1.22.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.22.0...storage/v1.22.1) (2022-05-19) diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index 0c2374008..13d79ee97 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -67,6 +67,8 @@ type ProjectTeam struct { } // ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object. +// ACLHandle on an object operates on the latest generation of that object by default. +// Selecting a specific generation of an object is not currently supported by the client. type ACLHandle struct { c *Client bucket string diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index 101f695b5..2148e6896 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -645,6 +645,13 @@ const ( // SetStorageClassAction changes the storage class of live and/or archived // objects. SetStorageClassAction = "SetStorageClass" + + // AbortIncompleteMPUAction is a lifecycle action that aborts an incomplete + // multipart upload when the multipart upload meets the conditions specified + // in the lifecycle rule. The AgeInDays condition is the only allowed + // condition for this action. AgeInDays is measured from the time the + // multipart upload was created. + AbortIncompleteMPUAction = "AbortIncompleteMultipartUpload" ) // LifecycleRule is a lifecycle configuration rule. @@ -665,9 +672,8 @@ type LifecycleRule struct { type LifecycleAction struct { // Type is the type of action to take on matching objects. // - // Acceptable values are "Delete" to delete matching objects and - // "SetStorageClass" to set the storage class defined in StorageClass on - // matching objects. + // Acceptable values are storage.DeleteAction, storage.SetStorageClassAction, + // and storage.AbortIncompleteMPUAction. Type string // StorageClass is the storage class to set on matching objects if the Action @@ -719,12 +725,20 @@ type LifecycleCondition struct { // Liveness specifies the object's liveness. Relevant only for versioned objects Liveness Liveness + // MatchesPrefix is the condition matching an object if any of the + // matches_prefix strings are an exact prefix of the object's name. + MatchesPrefix []string + // MatchesStorageClasses is the condition matching the object's storage // class. // // Values include "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE". MatchesStorageClasses []string + // MatchesSuffix is the condition matching an object if any of the + // matches_suffix strings are an exact suffix of the object's name. + MatchesSuffix []string + // NoncurrentTimeBefore is the noncurrent timestamp of the object. This // condition is satisfied when an object's noncurrent timestamp is before // midnight of the specified date in UTC. @@ -1489,6 +1503,19 @@ func toCORSFromProto(rc []*storagepb.Bucket_Cors) []CORS { return out } +// Used to handle breaking change in Autogen Storage client OLM Age field +// from int64 to *int64 gracefully in the manual client +// TODO(#6240): Method should be removed once breaking change is made and introduced to this client +func setAgeCondition(age int64, ageField interface{}) { + c := reflect.ValueOf(ageField).Elem() + switch c.Kind() { + case reflect.Int64: + c.SetInt(age) + case reflect.Ptr: + c.Set(reflect.ValueOf(&age)) + } +} + func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { var rl raw.BucketLifecycle if len(l.Rules) == 0 { @@ -1501,14 +1528,17 @@ func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { StorageClass: r.Action.StorageClass, }, Condition: &raw.BucketLifecycleRuleCondition{ - Age: r.Condition.AgeInDays, DaysSinceCustomTime: r.Condition.DaysSinceCustomTime, DaysSinceNoncurrentTime: r.Condition.DaysSinceNoncurrentTime, + MatchesPrefix: r.Condition.MatchesPrefix, MatchesStorageClass: r.Condition.MatchesStorageClasses, + MatchesSuffix: r.Condition.MatchesSuffix, NumNewerVersions: r.Condition.NumNewerVersions, }, } + setAgeCondition(r.Condition.AgeInDays, &rr.Condition.Age) + switch r.Condition.Liveness { case LiveAndArchived: rr.Condition.IsLive = nil @@ -1549,7 +1579,9 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { AgeDays: proto.Int32(int32(r.Condition.AgeInDays)), DaysSinceCustomTime: proto.Int32(int32(r.Condition.DaysSinceCustomTime)), DaysSinceNoncurrentTime: proto.Int32(int32(r.Condition.DaysSinceNoncurrentTime)), + MatchesPrefix: r.Condition.MatchesPrefix, MatchesStorageClass: r.Condition.MatchesStorageClasses, + MatchesSuffix: r.Condition.MatchesSuffix, NumNewerVersions: proto.Int32(int32(r.Condition.NumNewerVersions)), }, } @@ -1577,6 +1609,21 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { return &rl } +// Used to handle breaking change in Autogen Storage client OLM Age field +// from int64 to *int64 gracefully in the manual client +// TODO(#6240): Method should be removed once breaking change is made and introduced to this client +func getAgeCondition(ageField interface{}) int64 { + v := reflect.ValueOf(ageField) + if v.Kind() == reflect.Int64 { + return v.Interface().(int64) + } else if v.Kind() == reflect.Ptr { + if val, ok := v.Interface().(*int64); ok { + return *val + } + } + return 0 +} + func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { var l Lifecycle if rl == nil { @@ -1589,13 +1636,15 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { StorageClass: rr.Action.StorageClass, }, Condition: LifecycleCondition{ - AgeInDays: rr.Condition.Age, DaysSinceCustomTime: rr.Condition.DaysSinceCustomTime, DaysSinceNoncurrentTime: rr.Condition.DaysSinceNoncurrentTime, + MatchesPrefix: rr.Condition.MatchesPrefix, MatchesStorageClasses: rr.Condition.MatchesStorageClass, + MatchesSuffix: rr.Condition.MatchesSuffix, NumNewerVersions: rr.Condition.NumNewerVersions, }, } + r.Condition.AgeInDays = getAgeCondition(rr.Condition.Age) if rr.Condition.IsLive == nil { r.Condition.Liveness = LiveAndArchived @@ -1634,7 +1683,9 @@ func toLifecycleFromProto(rl *storagepb.Bucket_Lifecycle) Lifecycle { AgeInDays: int64(rr.GetCondition().GetAgeDays()), DaysSinceCustomTime: int64(rr.GetCondition().GetDaysSinceCustomTime()), DaysSinceNoncurrentTime: int64(rr.GetCondition().GetDaysSinceNoncurrentTime()), + MatchesPrefix: rr.GetCondition().GetMatchesPrefix(), MatchesStorageClasses: rr.GetCondition().GetMatchesStorageClass(), + MatchesSuffix: rr.GetCondition().GetMatchesSuffix(), NumNewerVersions: int64(rr.GetCondition().GetNumNewerVersions()), }, } @@ -1708,7 +1759,7 @@ func (b *BucketLogging) toProtoBucketLogging() *storagepb.Bucket_Logging { return nil } return &storagepb.Bucket_Logging{ - LogBucket: b.LogBucket, + LogBucket: bucketResourceName(globalProjectAlias, b.LogBucket), LogObjectPrefix: b.LogObjectPrefix, } } @@ -1727,8 +1778,9 @@ func toBucketLoggingFromProto(b *storagepb.Bucket_Logging) *BucketLogging { if b == nil { return nil } + lb := parseBucketName(b.GetLogBucket()) return &BucketLogging{ - LogBucket: b.GetLogBucket(), + LogBucket: lb, LogObjectPrefix: b.GetLogObjectPrefix(), } } diff --git a/vendor/cloud.google.com/go/storage/client.go b/vendor/cloud.google.com/go/storage/client.go index 87ee41927..f26a52a76 100644 --- a/vendor/cloud.google.com/go/storage/client.go +++ b/vendor/cloud.google.com/go/storage/client.go @@ -16,6 +16,8 @@ package storage import ( "context" + "io" + "time" gax "github.com/googleapis/gax-go/v2" "google.golang.org/api/option" @@ -56,9 +58,9 @@ type storageClient interface { // Object metadata methods. - DeleteObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) error - GetObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) - UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) + DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error + GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) + UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) // Default Object ACL methods. @@ -83,8 +85,8 @@ type storageClient interface { ComposeObject(ctx context.Context, req *composeObjectRequest, opts ...storageOption) (*ObjectAttrs, error) RewriteObject(ctx context.Context, req *rewriteObjectRequest, opts ...storageOption) (*rewriteObjectResponse, error) - OpenReader(ctx context.Context, r *Reader, opts ...storageOption) error - OpenWriter(ctx context.Context, w *Writer, opts ...storageOption) error + NewRangeReader(ctx context.Context, params *newRangeReaderParams, opts ...storageOption) (*Reader, error) + OpenWriter(params *openWriterParams, opts ...storageOption) (*io.PipeWriter, error) // IAM methods. @@ -99,6 +101,11 @@ type storageClient interface { UpdateHMACKey(ctx context.Context, desc *hmacKeyDesc, attrs *HMACKeyAttrsToUpdate, opts ...storageOption) (*HMACKey, error) CreateHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) (*HMACKey, error) DeleteHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) error + + // Notification methods. + ListNotifications(ctx context.Context, bucket string, opts ...storageOption) (map[string]*Notification, error) + CreateNotification(ctx context.Context, bucket string, n *Notification, opts ...storageOption) (*Notification, error) + DeleteNotification(ctx context.Context, bucket string, id string, opts ...storageOption) error } // settings contains transport-agnostic configuration for API calls made via @@ -211,6 +218,64 @@ type userProjectOption struct { func (o *userProjectOption) Apply(s *settings) { s.userProject = o.project } +type openWriterParams struct { + // Writer configuration + + // ctx is the context used by the writer routine to make all network calls + // and to manage the writer routine - see `Writer.ctx`. + // Required. + ctx context.Context + // chunkSize - see `Writer.ChunkSize`. + // Optional. + chunkSize int + // chunkRetryDeadline - see `Writer.ChunkRetryDeadline`. + // Optional. + chunkRetryDeadline time.Duration + + // Object/request properties + + // bucket - see `Writer.o.bucket`. + // Required. + bucket string + // attrs - see `Writer.ObjectAttrs`. + // Required. + attrs *ObjectAttrs + // conds - see `Writer.o.conds`. + // Optional. + conds *Conditions + // encryptionKey - see `Writer.o.encryptionKey` + // Optional. + encryptionKey []byte + // sendCRC32C - see `Writer.SendCRC32C`. + // Optional. + sendCRC32C bool + + // Writer callbacks + + // donec - see `Writer.donec`. + // Required. + donec chan struct{} + // setError callback for reporting errors - see `Writer.error`. + // Required. + setError func(error) + // progress callback for reporting upload progress - see `Writer.progress`. + // Required. + progress func(int64) + // setObj callback for reporting the resulting object - see `Writer.obj`. + // Required. + setObj func(*ObjectAttrs) +} + +type newRangeReaderParams struct { + bucket string + conds *Conditions + encryptionKey []byte + gen int64 + length int64 + object string + offset int64 +} + type composeObjectRequest struct { dstBucket string dstObject string diff --git a/vendor/cloud.google.com/go/storage/grpc_client.go b/vendor/cloud.google.com/go/storage/grpc_client.go index db914d61f..cc246cf72 100644 --- a/vendor/cloud.google.com/go/storage/grpc_client.go +++ b/vendor/cloud.google.com/go/storage/grpc_client.go @@ -16,8 +16,11 @@ package storage import ( "context" + "fmt" + "io" "os" + "cloud.google.com/go/internal/trace" gapic "cloud.google.com/go/storage/internal/apiv2" "google.golang.org/api/iterator" "google.golang.org/api/option" @@ -27,6 +30,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" fieldmaskpb "google.golang.org/protobuf/types/known/fieldmaskpb" ) @@ -36,7 +40,7 @@ const ( // connection pool may be necessary for jobs that require // high throughput and/or leverage many concurrent streams. // - // This is an experimental API and not intended for public use. + // This is only used for the gRPC client. defaultConnPoolSize = 4 // globalProjectAlias is the project ID alias used for global buckets. @@ -47,8 +51,6 @@ const ( // defaultGRPCOptions returns a set of the default client options // for gRPC client initialization. -// -// This is an experimental API and not intended for public use. func defaultGRPCOptions() []option.ClientOption { defaults := []option.ClientOption{ option.WithGRPCConnectionPool(defaultConnPoolSize), @@ -80,8 +82,6 @@ func defaultGRPCOptions() []option.ClientOption { // grpcStorageClient is the gRPC API implementation of the transport-agnostic // storageClient interface. -// -// This is an experimental API and not intended for public use. type grpcStorageClient struct { raw *gapic.Client settings *settings @@ -89,8 +89,6 @@ type grpcStorageClient struct { // newGRPCStorageClient initializes a new storageClient that uses the gRPC // Storage API. -// -// This is an experimental API and not intended for public use. func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (storageClient, error) { s := initSettings(opts...) s.clientOption = append(defaultGRPCOptions(), s.clientOption...) @@ -404,14 +402,119 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q // Object metadata methods. -func (c *grpcStorageClient) DeleteObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) error { - return errMethodNotSupported +func (c *grpcStorageClient) DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error { + s := callSettings(c.settings, opts...) + req := &storagepb.DeleteObjectRequest{ + Bucket: bucketResourceName(globalProjectAlias, bucket), + Object: object, + } + if err := applyCondsProto("grpcStorageClient.DeleteObject", gen, conds, req); err != nil { + return err + } + if s.userProject != "" { + ctx = setUserProjectMetadata(ctx, s.userProject) + } + err := run(ctx, func() error { + return c.raw.DeleteObject(ctx, req, s.gax...) + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { + return ErrObjectNotExist + } + return err } -func (c *grpcStorageClient) GetObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { - return nil, errMethodNotSupported + +func (c *grpcStorageClient) GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { + s := callSettings(c.settings, opts...) + req := &storagepb.GetObjectRequest{ + Bucket: bucketResourceName(globalProjectAlias, bucket), + Object: object, + } + if err := applyCondsProto("grpcStorageClient.GetObject", gen, conds, req); err != nil { + return nil, err + } + if s.userProject != "" { + ctx = setUserProjectMetadata(ctx, s.userProject) + } + if encryptionKey != nil { + req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(encryptionKey) + } + + var attrs *ObjectAttrs + err := run(ctx, func() error { + res, err := c.raw.GetObject(ctx, req, s.gax...) + attrs = newObjectFromProto(res) + + return err + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + + if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { + return nil, ErrObjectNotExist + } + + return attrs, err } -func (c *grpcStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { - return nil, errMethodNotSupported + +func (c *grpcStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { + s := callSettings(c.settings, opts...) + o := uattrs.toProtoObject(bucketResourceName(globalProjectAlias, bucket), object) + req := &storagepb.UpdateObjectRequest{ + Object: o, + } + if err := applyCondsProto("grpcStorageClient.UpdateObject", gen, conds, req); err != nil { + return nil, err + } + if s.userProject != "" { + ctx = setUserProjectMetadata(ctx, s.userProject) + } + if encryptionKey != nil { + req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(encryptionKey) + } + + var paths []string + fieldMask := &fieldmaskpb.FieldMask{ + Paths: paths, + } + if uattrs.EventBasedHold != nil { + fieldMask.Paths = append(fieldMask.Paths, "event_based_hold") + } + if uattrs.TemporaryHold != nil { + fieldMask.Paths = append(fieldMask.Paths, "temporary_hold") + } + if uattrs.ContentType != nil { + fieldMask.Paths = append(fieldMask.Paths, "content_type") + } + if uattrs.ContentLanguage != nil { + fieldMask.Paths = append(fieldMask.Paths, "content_language") + } + if uattrs.ContentEncoding != nil { + fieldMask.Paths = append(fieldMask.Paths, "content_encoding") + } + if uattrs.ContentDisposition != nil { + fieldMask.Paths = append(fieldMask.Paths, "content_disposition") + } + if uattrs.CacheControl != nil { + fieldMask.Paths = append(fieldMask.Paths, "cache_control") + } + if !uattrs.CustomTime.IsZero() { + fieldMask.Paths = append(fieldMask.Paths, "custom_time") + } + + // TODO(cathyo): Handle ACL and PredefinedACL. Pending b/233617896. + // TODO(cathyo): Handle metadata. Pending b/230510191. + + req.UpdateMask = fieldMask + + var attrs *ObjectAttrs + err := run(ctx, func() error { + res, err := c.raw.UpdateObject(ctx, req, s.gax...) + attrs = newObjectFromProto(res) + return err + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + if e, ok := status.FromError(err); ok && e.Code() == codes.NotFound { + return nil, ErrObjectNotExist + } + + return attrs, err } // Default Object ACL methods. @@ -503,9 +606,17 @@ func (c *grpcStorageClient) UpdateBucketACL(ctx context.Context, bucket string, func (c *grpcStorageClient) DeleteObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, opts ...storageOption) error { return errMethodNotSupported } + +// ListObjectACLs retrieves object ACL entries. By default, it operates on the latest generation of this object. +// Selecting a specific generation of this object is not currently supported by the client. func (c *grpcStorageClient) ListObjectACLs(ctx context.Context, bucket, object string, opts ...storageOption) ([]ACLRule, error) { - return nil, errMethodNotSupported + o, err := c.GetObject(ctx, bucket, object, defaultGen, nil, nil, opts...) + if err != nil { + return nil, err + } + return o.ACL, nil } + func (c *grpcStorageClient) UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) (*ACLRule, error) { return nil, errMethodNotSupported } @@ -519,11 +630,200 @@ func (c *grpcStorageClient) RewriteObject(ctx context.Context, req *rewriteObjec return nil, errMethodNotSupported } -func (c *grpcStorageClient) OpenReader(ctx context.Context, r *Reader, opts ...storageOption) error { - return errMethodNotSupported +func (c *grpcStorageClient) NewRangeReader(ctx context.Context, params *newRangeReaderParams, opts ...storageOption) (r *Reader, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.grpcStorageClient.NewRangeReader") + defer func() { trace.EndSpan(ctx, err) }() + + if params.conds != nil { + if err := params.conds.validate("grpcStorageClient.NewRangeReader"); err != nil { + return nil, err + } + } + + s := callSettings(c.settings, opts...) + + // A negative length means "read to the end of the object", but the + // read_limit field it corresponds to uses zero to mean the same thing. Thus + // we coerce the length to 0 to read to the end of the object. + if params.length < 0 { + params.length = 0 + } + + b := bucketResourceName(globalProjectAlias, params.bucket) + // TODO(noahdietz): Use encryptionKey to set relevant request fields. + req := &storagepb.ReadObjectRequest{ + Bucket: b, + Object: params.object, + } + // The default is a negative value, which means latest. + if params.gen >= 0 { + req.Generation = params.gen + } + + // Define a function that initiates a Read with offset and length, assuming + // we have already read seen bytes. + reopen := func(seen int64) (*readStreamResponse, context.CancelFunc, error) { + // If the context has already expired, return immediately without making + // we call. + if err := ctx.Err(); err != nil { + return nil, nil, err + } + + cc, cancel := context.WithCancel(ctx) + + start := params.offset + seen + // Only set a ReadLimit if length is greater than zero, because zero + // means read it all. + if params.length > 0 { + req.ReadLimit = params.length - seen + } + req.ReadOffset = start + + if err := applyCondsProto("gRPCReader.reopen", params.gen, params.conds, req); err != nil { + cancel() + return nil, nil, err + } + + var stream storagepb.Storage_ReadObjectClient + var msg *storagepb.ReadObjectResponse + var err error + + err = run(cc, func() error { + stream, err = c.raw.ReadObject(cc, req, s.gax...) + if err != nil { + return err + } + + msg, err = stream.Recv() + + return err + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + if err != nil { + // Close the stream context we just created to ensure we don't leak + // resources. + cancel() + return nil, nil, err + } + + return &readStreamResponse{stream, msg}, cancel, nil + } + + res, cancel, err := reopen(0) + if err != nil { + return nil, err + } + + // The first message was Recv'd on stream open, use it to populate the + // object metadata. + msg := res.response + obj := msg.GetMetadata() + // This is the size of the entire object, even if only a range was requested. + size := obj.GetSize() + + r = &Reader{ + Attrs: ReaderObjectAttrs{ + Size: size, + ContentType: obj.GetContentType(), + ContentEncoding: obj.GetContentEncoding(), + CacheControl: obj.GetCacheControl(), + LastModified: obj.GetUpdateTime().AsTime(), + Metageneration: obj.GetMetageneration(), + Generation: obj.GetGeneration(), + }, + reader: &gRPCReader{ + stream: res.stream, + reopen: reopen, + cancel: cancel, + size: size, + // Store the content from the first Recv in the + // client buffer for reading later. + leftovers: msg.GetChecksummedData().GetContent(), + }, + } + + cr := msg.GetContentRange() + if cr != nil { + r.Attrs.StartOffset = cr.GetStart() + r.remain = cr.GetEnd() - cr.GetStart() + 1 + } else { + r.remain = size + } + + // Only support checksums when reading an entire object, not a range. + if checksums := msg.GetObjectChecksums(); checksums != nil && checksums.Crc32C != nil && params.offset == 0 && params.length == 0 { + r.wantCRC = checksums.GetCrc32C() + r.checkCRC = true + } + + return r, nil } -func (c *grpcStorageClient) OpenWriter(ctx context.Context, w *Writer, opts ...storageOption) error { - return errMethodNotSupported + +func (c *grpcStorageClient) OpenWriter(params *openWriterParams, opts ...storageOption) (*io.PipeWriter, error) { + var offset int64 + errorf := params.setError + progress := params.progress + setObj := params.setObj + + pr, pw := io.Pipe() + gw := newGRPCWriter(c, params, pr) + + // This function reads the data sent to the pipe and sends sets of messages + // on the gRPC client-stream as the buffer is filled. + go func() { + defer close(params.donec) + + // Loop until there is an error or the Object has been finalized. + for { + // Note: This blocks until either the buffer is full or EOF is read. + recvd, doneReading, err := gw.read() + if err != nil { + err = checkCanceled(err) + errorf(err) + pr.CloseWithError(err) + return + } + + // TODO(noahdietz): Send encryption key via CommonObjectRequestParams. + + // The chunk buffer is full, but there is no end in sight. This + // means that a resumable upload will need to be used to send + // multiple chunks, until we are done reading data. Start a + // resumable upload if it has not already been started. + // Otherwise, all data will be sent over a single gRPC stream. + if !doneReading && gw.upid == "" { + err = gw.startResumableUpload() + if err != nil { + err = checkCanceled(err) + errorf(err) + pr.CloseWithError(err) + return + } + } + + o, off, finalized, err := gw.uploadBuffer(recvd, offset, doneReading) + if err != nil { + err = checkCanceled(err) + errorf(err) + pr.CloseWithError(err) + return + } + // At this point, the current buffer has been uploaded. Capture the + // committed offset here in case the upload was not finalized and + // another chunk is to be uploaded. + offset = off + progress(offset) + + // When we are done reading data and the chunk has been finalized, + // we are done. + if doneReading && finalized { + // Build Object from server's response. + setObj(newObjectFromProto(o)) + return + } + } + }() + + return pw, nil } // IAM methods. @@ -599,6 +899,75 @@ func (c *grpcStorageClient) DeleteHMACKey(ctx context.Context, desc *hmacKeyDesc return errMethodNotSupported } +// Notification methods. + +func (c *grpcStorageClient) ListNotifications(ctx context.Context, bucket string, opts ...storageOption) (n map[string]*Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.grpcStorageClient.ListNotifications") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + if s.userProject != "" { + ctx = setUserProjectMetadata(ctx, s.userProject) + } + req := &storagepb.ListNotificationsRequest{ + Parent: bucketResourceName(globalProjectAlias, bucket), + } + var notifications []*storagepb.Notification + err = run(ctx, func() error { + gitr := c.raw.ListNotifications(ctx, req, s.gax...) + for { + // PageSize is not set and fallbacks to the API default pageSize of 100. + items, nextPageToken, err := gitr.InternalFetch(int(req.GetPageSize()), req.GetPageToken()) + if err != nil { + return err + } + notifications = append(notifications, items...) + // If there are no more results, nextPageToken is empty and err is nil. + if nextPageToken == "" { + return err + } + req.PageToken = nextPageToken + } + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + if err != nil { + return nil, err + } + + return notificationsToMapFromProto(notifications), nil +} + +func (c *grpcStorageClient) CreateNotification(ctx context.Context, bucket string, n *Notification, opts ...storageOption) (ret *Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.grpcStorageClient.CreateNotification") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + req := &storagepb.CreateNotificationRequest{ + Parent: bucketResourceName(globalProjectAlias, bucket), + Notification: toProtoNotification(n), + } + var pbn *storagepb.Notification + err = run(ctx, func() error { + var err error + pbn, err = c.raw.CreateNotification(ctx, req, s.gax...) + return err + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) + if err != nil { + return nil, err + } + return toNotificationFromProto(pbn), err +} + +func (c *grpcStorageClient) DeleteNotification(ctx context.Context, bucket string, id string, opts ...storageOption) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.grpcStorageClient.DeleteNotification") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + req := &storagepb.DeleteNotificationRequest{Name: id} + return run(ctx, func() error { + return c.raw.DeleteNotification(ctx, req, s.gax...) + }, s.retry, s.idempotent, setRetryHeaderGRPC(ctx)) +} + // setUserProjectMetadata appends a project ID to the outgoing Context metadata // via the x-goog-user-project system parameter defined at // https://cloud.google.com/apis/docs/system-parameters. This is only for @@ -607,3 +976,396 @@ func (c *grpcStorageClient) DeleteHMACKey(ctx context.Context, desc *hmacKeyDesc func setUserProjectMetadata(ctx context.Context, project string) context.Context { return metadata.AppendToOutgoingContext(ctx, "x-goog-user-project", project) } + +type readStreamResponse struct { + stream storagepb.Storage_ReadObjectClient + response *storagepb.ReadObjectResponse +} + +type gRPCReader struct { + seen, size int64 + stream storagepb.Storage_ReadObjectClient + reopen func(seen int64) (*readStreamResponse, context.CancelFunc, error) + leftovers []byte + cancel context.CancelFunc +} + +// Read reads bytes into the user's buffer from an open gRPC stream. +func (r *gRPCReader) Read(p []byte) (int, error) { + // No stream to read from, either never initiliazed or Close was called. + // Note: There is a potential concurrency issue if multiple routines are + // using the same reader. One encounters an error and the stream is closed + // and then reopened while the other routine attempts to read from it. + if r.stream == nil { + return 0, fmt.Errorf("reader has been closed") + } + + // The entire object has been read by this reader, return EOF. + if r.size != 0 && r.size == r.seen { + return 0, io.EOF + } + + var n int + // Read leftovers and return what was available to conform to the Reader + // interface: https://pkg.go.dev/io#Reader. + if len(r.leftovers) > 0 { + n = copy(p, r.leftovers) + r.seen += int64(n) + r.leftovers = r.leftovers[n:] + return n, nil + } + + // Attempt to Recv the next message on the stream. + msg, err := r.recv() + if err != nil { + return 0, err + } + + // TODO: Determine if we need to capture incremental CRC32C for this + // chunk. The Object CRC32C checksum is captured when directed to read + // the entire Object. If directed to read a range, we may need to + // calculate the range's checksum for verification if the checksum is + // present in the response here. + // TODO: Figure out if we need to support decompressive transcoding + // https://cloud.google.com/storage/docs/transcoding. + content := msg.GetChecksummedData().GetContent() + n = copy(p[n:], content) + leftover := len(content) - n + if leftover > 0 { + // Wasn't able to copy all of the data in the message, store for + // future Read calls. + r.leftovers = content[n:] + } + r.seen += int64(n) + + return n, nil +} + +// Close cancels the read stream's context in order for it to be closed and +// collected. +func (r *gRPCReader) Close() error { + if r.cancel != nil { + r.cancel() + } + r.stream = nil + return nil +} + +// recv attempts to Recv the next message on the stream. In the event +// that a retryable error is encountered, the stream will be closed, reopened, +// and Recv again. This will attempt to Recv until one of the following is true: +// +// * Recv is successful +// * A non-retryable error is encountered +// * The Reader's context is canceled +// +// The last error received is the one that is returned, which could be from +// an attempt to reopen the stream. +func (r *gRPCReader) recv() (*storagepb.ReadObjectResponse, error) { + msg, err := r.stream.Recv() + if err != nil && shouldRetry(err) { + // This will "close" the existing stream and immediately attempt to + // reopen the stream, but will backoff if further attempts are necessary. + // Reopening the stream Recvs the first message, so if retrying is + // successful, the next logical chunk will be returned. + msg, err = r.reopenStream() + } + + return msg, err +} + +// reopenStream "closes" the existing stream and attempts to reopen a stream and +// sets the Reader's stream and cancelStream properties in the process. +func (r *gRPCReader) reopenStream() (*storagepb.ReadObjectResponse, error) { + // Close existing stream and initialize new stream with updated offset. + r.Close() + + res, cancel, err := r.reopen(r.seen) + if err != nil { + return nil, err + } + r.stream = res.stream + r.cancel = cancel + return res.response, nil +} + +func newGRPCWriter(c *grpcStorageClient, params *openWriterParams, r io.Reader) *gRPCWriter { + size := params.chunkSize + if params.chunkSize == 0 { + // TODO: Should we actually use the minimum of 256 KB here when the user + // indicates they want minimal memory usage? We cannot do a zero-copy, + // bufferless upload like HTTP/JSON can. + // TODO: We need to determine if we can avoid starting a + // resumable upload when the user *plans* to send more than bufSize but + // with a bufferless upload. + size = maxPerMessageWriteSize + } + + return &gRPCWriter{ + buf: make([]byte, size), + c: c, + ctx: params.ctx, + reader: r, + bucket: params.bucket, + attrs: params.attrs, + conds: params.conds, + encryptionKey: params.encryptionKey, + sendCRC32C: params.sendCRC32C, + } +} + +// gRPCWriter is a wrapper around the the gRPC client-stream API that manages +// sending chunks of data provided by the user over the stream. +type gRPCWriter struct { + c *grpcStorageClient + buf []byte + reader io.Reader + + ctx context.Context + + bucket string + attrs *ObjectAttrs + conds *Conditions + encryptionKey []byte + + sendCRC32C bool + + // The gRPC client-stream used for sending buffers. + stream storagepb.Storage_WriteObjectClient + + // The Resumable Upload ID started by a gRPC-based Writer. + upid string +} + +// startResumableUpload initializes a Resumable Upload with gRPC and sets the +// upload ID on the Writer. +func (w *gRPCWriter) startResumableUpload() error { + spec, err := w.writeObjectSpec() + if err != nil { + return err + } + upres, err := w.c.raw.StartResumableWrite(w.ctx, &storagepb.StartResumableWriteRequest{ + WriteObjectSpec: spec, + }) + + w.upid = upres.GetUploadId() + return err +} + +// queryProgress is a helper that queries the status of the resumable upload +// associated with the given upload ID. +func (w *gRPCWriter) queryProgress() (int64, error) { + q, err := w.c.raw.QueryWriteStatus(w.ctx, &storagepb.QueryWriteStatusRequest{UploadId: w.upid}) + + // q.GetCommittedSize() will return 0 if q is nil. + return q.GetPersistedSize(), err +} + +// uploadBuffer opens a Write stream and uploads the buffer at the given offset (if +// uploading a chunk for a resumable uploadBuffer), and will mark the write as +// finished if we are done receiving data from the user. The resulting write +// offset after uploading the buffer is returned, as well as a boolean +// indicating if the Object has been finalized. If it has been finalized, the +// final Object will be returned as well. Finalizing the upload is primarily +// important for Resumable Uploads. A simple or multi-part upload will always +// be finalized once the entire buffer has been written. +func (w *gRPCWriter) uploadBuffer(recvd int, start int64, doneReading bool) (*storagepb.Object, int64, bool, error) { + var err error + var finishWrite bool + var sent, limit int = 0, maxPerMessageWriteSize + offset := start + toWrite := w.buf[:recvd] + for { + first := sent == 0 + // This indicates that this is the last message and the remaining + // data fits in one message. + belowLimit := recvd-sent <= limit + if belowLimit { + limit = recvd - sent + } + if belowLimit && doneReading { + finishWrite = true + } + + // Prepare chunk section for upload. + data := toWrite[sent : sent+limit] + req := &storagepb.WriteObjectRequest{ + Data: &storagepb.WriteObjectRequest_ChecksummedData{ + ChecksummedData: &storagepb.ChecksummedData{ + Content: data, + }, + }, + WriteOffset: offset, + FinishWrite: finishWrite, + } + + // Open a new stream and set the first_message field on the request. + // The first message on the WriteObject stream must either be the + // Object or the Resumable Upload ID. + if first { + w.stream, err = w.c.raw.WriteObject(w.ctx) + if err != nil { + return nil, 0, false, err + } + + if w.upid != "" { + req.FirstMessage = &storagepb.WriteObjectRequest_UploadId{UploadId: w.upid} + } else { + spec, err := w.writeObjectSpec() + if err != nil { + return nil, 0, false, err + } + req.FirstMessage = &storagepb.WriteObjectRequest_WriteObjectSpec{ + WriteObjectSpec: spec, + } + } + + // TODO: Currently the checksums are only sent on the first message + // of the stream, but in the future, we must also support sending it + // on the *last* message of the stream (instead of the first). + if w.sendCRC32C { + req.ObjectChecksums = &storagepb.ObjectChecksums{ + Crc32C: proto.Uint32(w.attrs.CRC32C), + Md5Hash: w.attrs.MD5, + } + } + } + + err = w.stream.Send(req) + if err == io.EOF { + // err was io.EOF. The client-side of a stream only gets an EOF on Send + // when the backend closes the stream and wants to return an error + // status. Closing the stream receives the status as an error. + _, err = w.stream.CloseAndRecv() + + // Retriable errors mean we should start over and attempt to + // resend the entire buffer via a new stream. + // If not retriable, falling through will return the error received + // from closing the stream. + if shouldRetry(err) { + sent = 0 + finishWrite = false + // TODO: Add test case for failure modes of querying progress. + offset, err = w.determineOffset(start) + if err == nil { + continue + } + } + } + if err != nil { + return nil, 0, false, err + } + + // Update the immediate stream's sent total and the upload offset with + // the data sent. + sent += len(data) + offset += int64(len(data)) + + // Not done sending data, do not attempt to commit it yet, loop around + // and send more data. + if recvd-sent > 0 { + continue + } + + // Done sending data. Close the stream to "commit" the data sent. + resp, finalized, err := w.commit() + // Retriable errors mean we should start over and attempt to + // resend the entire buffer via a new stream. + // If not retriable, falling through will return the error received + // from closing the stream. + if shouldRetry(err) { + sent = 0 + finishWrite = false + offset, err = w.determineOffset(start) + if err == nil { + continue + } + } + if err != nil { + return nil, 0, false, err + } + + return resp.GetResource(), offset, finalized, nil + } +} + +// determineOffset either returns the offset given to it in the case of a simple +// upload, or queries the write status in the case a resumable upload is being +// used. +func (w *gRPCWriter) determineOffset(offset int64) (int64, error) { + // For a Resumable Upload, we must start from however much data + // was committed. + if w.upid != "" { + committed, err := w.queryProgress() + if err != nil { + return 0, err + } + offset = committed + } + return offset, nil +} + +// commit closes the stream to commit the data sent and potentially receive +// the finalized object if finished uploading. If the last request sent +// indicated that writing was finished, the Object will be finalized and +// returned. If not, then the Object will be nil, and the boolean returned will +// be false. +func (w *gRPCWriter) commit() (*storagepb.WriteObjectResponse, bool, error) { + finalized := true + resp, err := w.stream.CloseAndRecv() + if err == io.EOF { + // Closing a stream for a resumable upload finish_write = false results + // in an EOF which can be ignored, as we aren't done uploading yet. + finalized = false + err = nil + } + // Drop the stream reference as it has been closed. + w.stream = nil + + return resp, finalized, err +} + +// writeObjectSpec constructs a WriteObjectSpec proto using the Writer's +// ObjectAttrs and applies its Conditions. This is only used for gRPC. +func (w *gRPCWriter) writeObjectSpec() (*storagepb.WriteObjectSpec, error) { + // To avoid modifying the ObjectAttrs embeded in the calling writer, deref + // the ObjectAttrs pointer to make a copy, then assign the desired name to + // the attribute. + attrs := *w.attrs + + spec := &storagepb.WriteObjectSpec{ + Resource: attrs.toProtoObject(w.bucket), + } + // WriteObject doesn't support the generation condition, so use -1. + if err := applyCondsProto("WriteObject", -1, w.conds, spec); err != nil { + return nil, err + } + return spec, nil +} + +// read copies the data in the reader to the given buffer and reports how much +// data was read into the buffer and if there is no more data to read (EOF). +func (w *gRPCWriter) read() (int, bool, error) { + // Set n to -1 to start the Read loop. + var n, recvd int = -1, 0 + var err error + for err == nil && n != 0 { + // The routine blocks here until data is received. + n, err = w.reader.Read(w.buf[recvd:]) + recvd += n + } + var done bool + if err == io.EOF { + done = true + err = nil + } + return recvd, done, err +} + +func checkCanceled(err error) error { + if status.Code(err) == codes.Canceled { + return context.Canceled + } + + return err +} diff --git a/vendor/cloud.google.com/go/storage/http_client.go b/vendor/cloud.google.com/go/storage/http_client.go index 1ec55150f..45c175f89 100644 --- a/vendor/cloud.google.com/go/storage/http_client.go +++ b/vendor/cloud.google.com/go/storage/http_client.go @@ -16,14 +16,21 @@ package storage import ( "context" + "encoding/base64" "errors" "fmt" + "io" + "io/ioutil" "net/http" "net/url" "os" "reflect" + "strconv" "strings" + "time" + "cloud.google.com/go/internal/optional" + "cloud.google.com/go/internal/trace" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" @@ -378,14 +385,143 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q // Object metadata methods. -func (c *httpStorageClient) DeleteObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) error { - return errMethodNotSupported +func (c *httpStorageClient) DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error { + s := callSettings(c.settings, opts...) + req := c.raw.Objects.Delete(bucket, object).Context(ctx) + if err := applyConds("Delete", gen, conds, req); err != nil { + return err + } + if s.userProject != "" { + req.UserProject(s.userProject) + } + err := run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req)) + var e *googleapi.Error + if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound { + return ErrObjectNotExist + } + return err } -func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { - return nil, errMethodNotSupported + +func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { + s := callSettings(c.settings, opts...) + req := c.raw.Objects.Get(bucket, object).Projection("full").Context(ctx) + if err := applyConds("Attrs", gen, conds, req); err != nil { + return nil, err + } + if s.userProject != "" { + req.UserProject(s.userProject) + } + if err := setEncryptionHeaders(req.Header(), encryptionKey, false); err != nil { + return nil, err + } + var obj *raw.Object + var err error + err = run(ctx, func() error { + obj, err = req.Context(ctx).Do() + return err + }, s.retry, s.idempotent, setRetryHeaderHTTP(req)) + var e *googleapi.Error + if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound { + return nil, ErrObjectNotExist + } + if err != nil { + return nil, err + } + return newObject(obj), nil } -func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { - return nil, errMethodNotSupported + +func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { + s := callSettings(c.settings, opts...) + + var attrs ObjectAttrs + // Lists of fields to send, and set to null, in the JSON. + var forceSendFields, nullFields []string + if uattrs.ContentType != nil { + attrs.ContentType = optional.ToString(uattrs.ContentType) + // For ContentType, sending the empty string is a no-op. + // Instead we send a null. + if attrs.ContentType == "" { + nullFields = append(nullFields, "ContentType") + } else { + forceSendFields = append(forceSendFields, "ContentType") + } + } + if uattrs.ContentLanguage != nil { + attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) + // For ContentLanguage it's an error to send the empty string. + // Instead we send a null. + if attrs.ContentLanguage == "" { + nullFields = append(nullFields, "ContentLanguage") + } else { + forceSendFields = append(forceSendFields, "ContentLanguage") + } + } + if uattrs.ContentEncoding != nil { + attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) + forceSendFields = append(forceSendFields, "ContentEncoding") + } + if uattrs.ContentDisposition != nil { + attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) + forceSendFields = append(forceSendFields, "ContentDisposition") + } + if uattrs.CacheControl != nil { + attrs.CacheControl = optional.ToString(uattrs.CacheControl) + forceSendFields = append(forceSendFields, "CacheControl") + } + if uattrs.EventBasedHold != nil { + attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold) + forceSendFields = append(forceSendFields, "EventBasedHold") + } + if uattrs.TemporaryHold != nil { + attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) + forceSendFields = append(forceSendFields, "TemporaryHold") + } + if !uattrs.CustomTime.IsZero() { + attrs.CustomTime = uattrs.CustomTime + forceSendFields = append(forceSendFields, "CustomTime") + } + if uattrs.Metadata != nil { + attrs.Metadata = uattrs.Metadata + if len(attrs.Metadata) == 0 { + // Sending the empty map is a no-op. We send null instead. + nullFields = append(nullFields, "Metadata") + } else { + forceSendFields = append(forceSendFields, "Metadata") + } + } + if uattrs.ACL != nil { + attrs.ACL = uattrs.ACL + // It's an error to attempt to delete the ACL, so + // we don't append to nullFields here. + forceSendFields = append(forceSendFields, "Acl") + } + rawObj := attrs.toRawObject(bucket) + rawObj.ForceSendFields = forceSendFields + rawObj.NullFields = nullFields + call := c.raw.Objects.Patch(bucket, object, rawObj).Projection("full").Context(ctx) + if err := applyConds("Update", gen, conds, call); err != nil { + return nil, err + } + if s.userProject != "" { + call.UserProject(s.userProject) + } + if uattrs.PredefinedACL != "" { + call.PredefinedAcl(uattrs.PredefinedACL) + } + if err := setEncryptionHeaders(call.Header(), encryptionKey, false); err != nil { + return nil, err + } + var obj *raw.Object + var err error + err = run(ctx, func() error { obj, err = call.Do(); return err }, s.retry, s.idempotent, setRetryHeaderHTTP(call)) + var e *googleapi.Error + if errors.As(err, &e) && e.Code == http.StatusNotFound { + return nil, ErrObjectNotExist + } + if err != nil { + return nil, err + } + return newObject(obj), nil } // Default Object ACL methods. @@ -479,9 +615,25 @@ func configureACLCall(ctx context.Context, userProject string, call interface{ H func (c *httpStorageClient) DeleteObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, opts ...storageOption) error { return errMethodNotSupported } + +// ListObjectACLs retrieves object ACL entries. By default, it operates on the latest generation of this object. +// Selecting a specific generation of this object is not currently supported by the client. func (c *httpStorageClient) ListObjectACLs(ctx context.Context, bucket, object string, opts ...storageOption) ([]ACLRule, error) { - return nil, errMethodNotSupported + s := callSettings(c.settings, opts...) + var acls *raw.ObjectAccessControls + var err error + req := c.raw.ObjectAccessControls.List(bucket, object) + configureACLCall(ctx, s.userProject, req) + err = run(ctx, func() error { + acls, err = req.Do() + return err + }, s.retry, s.idempotent, setRetryHeaderHTTP(req)) + if err != nil { + return nil, err + } + return toObjectACLRules(acls.Items), nil } + func (c *httpStorageClient) UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) (*ACLRule, error) { return nil, errMethodNotSupported } @@ -495,11 +647,305 @@ func (c *httpStorageClient) RewriteObject(ctx context.Context, req *rewriteObjec return nil, errMethodNotSupported } -func (c *httpStorageClient) OpenReader(ctx context.Context, r *Reader, opts ...storageOption) error { - return errMethodNotSupported +func (c *httpStorageClient) NewRangeReader(ctx context.Context, params *newRangeReaderParams, opts ...storageOption) (r *Reader, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.httpStorageClient.NewRangeReader") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + + if params.offset < 0 && params.length >= 0 { + return nil, fmt.Errorf("storage: invalid offset %d < 0 requires negative length", params.offset) + } + if params.conds != nil { + if err := params.conds.validate("NewRangeReader"); err != nil { + return nil, err + } + } + u := &url.URL{ + Scheme: c.scheme, + Host: c.readHost, + Path: fmt.Sprintf("/%s/%s", params.bucket, params.object), + } + verb := "GET" + if params.length == 0 { + verb = "HEAD" + } + req, err := http.NewRequest(verb, u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if s.userProject != "" { + req.Header.Set("X-Goog-User-Project", s.userProject) + } + // TODO(noahdietz): add option for readCompressed. + // if o.readCompressed { + // req.Header.Set("Accept-Encoding", "gzip") + // } + if err := setEncryptionHeaders(req.Header, params.encryptionKey, false); err != nil { + return nil, err + } + + // Define a function that initiates a Read with offset and length, assuming we + // have already read seen bytes. + reopen := func(seen int64) (*http.Response, error) { + // If the context has already expired, return immediately without making a + // call. + if err := ctx.Err(); err != nil { + return nil, err + } + start := params.offset + seen + if params.length < 0 && start < 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d", start)) + } else if params.length < 0 && start > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start)) + } else if params.length > 0 { + // The end character isn't affected by how many bytes we've seen. + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, params.offset+params.length-1)) + } + // We wait to assign conditions here because the generation number can change in between reopen() runs. + if err := setConditionsHeaders(req.Header, params.conds); err != nil { + return nil, err + } + // If an object generation is specified, include generation as query string parameters. + if params.gen >= 0 { + req.URL.RawQuery = fmt.Sprintf("generation=%d", params.gen) + } + + var res *http.Response + err = run(ctx, func() error { + res, err = c.hc.Do(req) + if err != nil { + return err + } + if res.StatusCode == http.StatusNotFound { + res.Body.Close() + return ErrObjectNotExist + } + if res.StatusCode < 200 || res.StatusCode > 299 { + body, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + return &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + Body: string(body), + } + } + + partialContentNotSatisfied := + !decompressiveTranscoding(res) && + start > 0 && params.length != 0 && + res.StatusCode != http.StatusPartialContent + + if partialContentNotSatisfied { + res.Body.Close() + return errors.New("storage: partial request not satisfied") + } + + // With "Content-Encoding": "gzip" aka decompressive transcoding, GCS serves + // back the whole file regardless of the range count passed in as per: + // https://cloud.google.com/storage/docs/transcoding#range, + // thus we have to manually move the body forward by seen bytes. + if decompressiveTranscoding(res) && seen > 0 { + _, _ = io.CopyN(ioutil.Discard, res.Body, seen) + } + + // If a generation hasn't been specified, and this is the first response we get, let's record the + // generation. In future requests we'll use this generation as a precondition to avoid data races. + if params.gen < 0 && res.Header.Get("X-Goog-Generation") != "" { + gen64, err := strconv.ParseInt(res.Header.Get("X-Goog-Generation"), 10, 64) + if err != nil { + return err + } + params.gen = gen64 + } + return nil + }, s.retry, s.idempotent, setRetryHeaderHTTP(nil)) + if err != nil { + return nil, err + } + return res, nil + } + + res, err := reopen(0) + if err != nil { + return nil, err + } + var ( + size int64 // total size of object, even if a range was requested. + checkCRC bool + crc uint32 + startOffset int64 // non-zero if range request. + ) + if res.StatusCode == http.StatusPartialContent { + cr := strings.TrimSpace(res.Header.Get("Content-Range")) + if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + // Content range is formatted -/. We take + // the total size. + size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) + if err != nil { + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + + dashIndex := strings.Index(cr, "-") + if dashIndex >= 0 { + startOffset, err = strconv.ParseInt(cr[len("bytes="):dashIndex], 10, 64) + if err != nil { + return nil, fmt.Errorf("storage: invalid Content-Range %q: %v", cr, err) + } + } + } else { + size = res.ContentLength + // Check the CRC iff all of the following hold: + // - We asked for content (length != 0). + // - We got all the content (status != PartialContent). + // - The server sent a CRC header. + // - The Go http stack did not uncompress the file. + // - We were not served compressed data that was uncompressed on download. + // The problem with the last two cases is that the CRC will not match -- GCS + // computes it on the compressed contents, but we compute it on the + // uncompressed contents. + if params.length != 0 && !res.Uncompressed && !uncompressedByServer(res) { + crc, checkCRC = parseCRC32c(res) + } + } + + remain := res.ContentLength + body := res.Body + if params.length == 0 { + remain = 0 + body.Close() + body = emptyBody + } + var metaGen int64 + if res.Header.Get("X-Goog-Metageneration") != "" { + metaGen, err = strconv.ParseInt(res.Header.Get("X-Goog-Metageneration"), 10, 64) + if err != nil { + return nil, err + } + } + + var lm time.Time + if res.Header.Get("Last-Modified") != "" { + lm, err = http.ParseTime(res.Header.Get("Last-Modified")) + if err != nil { + return nil, err + } + } + + attrs := ReaderObjectAttrs{ + Size: size, + ContentType: res.Header.Get("Content-Type"), + ContentEncoding: res.Header.Get("Content-Encoding"), + CacheControl: res.Header.Get("Cache-Control"), + LastModified: lm, + StartOffset: startOffset, + Generation: params.gen, + Metageneration: metaGen, + } + return &Reader{ + Attrs: attrs, + size: size, + remain: remain, + wantCRC: crc, + checkCRC: checkCRC, + reader: &httpReader{ + reopen: reopen, + body: body, + }, + }, nil } -func (c *httpStorageClient) OpenWriter(ctx context.Context, w *Writer, opts ...storageOption) error { - return errMethodNotSupported + +func (c *httpStorageClient) OpenWriter(params *openWriterParams, opts ...storageOption) (*io.PipeWriter, error) { + s := callSettings(c.settings, opts...) + errorf := params.setError + setObj := params.setObj + progress := params.progress + attrs := params.attrs + + mediaOpts := []googleapi.MediaOption{ + googleapi.ChunkSize(params.chunkSize), + } + if c := attrs.ContentType; c != "" { + mediaOpts = append(mediaOpts, googleapi.ContentType(c)) + } + if params.chunkRetryDeadline != 0 { + mediaOpts = append(mediaOpts, googleapi.ChunkRetryDeadline(params.chunkRetryDeadline)) + } + + pr, pw := io.Pipe() + + go func() { + defer close(params.donec) + + rawObj := attrs.toRawObject(params.bucket) + if params.sendCRC32C { + rawObj.Crc32c = encodeUint32(attrs.CRC32C) + } + if attrs.MD5 != nil { + rawObj.Md5Hash = base64.StdEncoding.EncodeToString(attrs.MD5) + } + call := c.raw.Objects.Insert(params.bucket, rawObj). + Media(pr, mediaOpts...). + Projection("full"). + Context(params.ctx). + Name(params.attrs.Name) + call.ProgressUpdater(func(n, _ int64) { progress(n) }) + + if attrs.KMSKeyName != "" { + call.KmsKeyName(attrs.KMSKeyName) + } + if attrs.PredefinedACL != "" { + call.PredefinedAcl(attrs.PredefinedACL) + } + if err := setEncryptionHeaders(call.Header(), params.encryptionKey, false); err != nil { + errorf(err) + pr.CloseWithError(err) + return + } + var resp *raw.Object + err := applyConds("NewWriter", params.attrs.Generation, params.conds, call) + if err == nil { + if s.userProject != "" { + call.UserProject(s.userProject) + } + // TODO(tritone): Remove this code when Uploads begin to support + // retry attempt header injection with "client header" injection. + setClientHeader(call.Header()) + + // The internals that perform call.Do automatically retry both the initial + // call to set up the upload as well as calls to upload individual chunks + // for a resumable upload (as long as the chunk size is non-zero). Hence + // there is no need to add retries here. + + // Retry only when the operation is idempotent or the retry policy is RetryAlways. + isIdempotent := params.conds != nil && (params.conds.GenerationMatch >= 0 || params.conds.DoesNotExist == true) + var useRetry bool + if (s.retry == nil || s.retry.policy == RetryIdempotent) && isIdempotent { + useRetry = true + } else if s.retry != nil && s.retry.policy == RetryAlways { + useRetry = true + } + if useRetry { + if s.retry != nil { + call.WithRetry(s.retry.backoff, s.retry.shouldRetry) + } else { + call.WithRetry(nil, nil) + } + } + resp, err = call.Do() + } + if err != nil { + errorf(err) + pr.CloseWithError(err) + return + } + setObj(newObject(resp)) + }() + + return pw, nil } // IAM methods. @@ -575,3 +1021,96 @@ func (c *httpStorageClient) CreateHMACKey(ctx context.Context, desc *hmacKeyDesc func (c *httpStorageClient) DeleteHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) error { return errMethodNotSupported } + +// Notification methods. + +// ListNotifications returns all the Notifications configured for this bucket, as a map indexed by notification ID. +// +// Note: This API does not support pagination. However, entity limits cap the number of notifications on a single bucket, +// so all results will be returned in the first response. See https://cloud.google.com/storage/quotas#buckets. +func (c *httpStorageClient) ListNotifications(ctx context.Context, bucket string, opts ...storageOption) (n map[string]*Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.httpStorageClient.ListNotifications") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + call := c.raw.Notifications.List(bucket) + if s.userProject != "" { + call.UserProject(s.userProject) + } + var res *raw.Notifications + err = run(ctx, func() error { + res, err = call.Context(ctx).Do() + return err + }, s.retry, true, setRetryHeaderHTTP(call)) + if err != nil { + return nil, err + } + return notificationsToMap(res.Items), nil +} + +func (c *httpStorageClient) CreateNotification(ctx context.Context, bucket string, n *Notification, opts ...storageOption) (ret *Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.httpStorageClient.CreateNotification") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + call := c.raw.Notifications.Insert(bucket, toRawNotification(n)) + if s.userProject != "" { + call.UserProject(s.userProject) + } + var rn *raw.Notification + err = run(ctx, func() error { + rn, err = call.Context(ctx).Do() + return err + }, s.retry, s.idempotent, setRetryHeaderHTTP(call)) + if err != nil { + return nil, err + } + return toNotification(rn), nil +} + +func (c *httpStorageClient) DeleteNotification(ctx context.Context, bucket string, id string, opts ...storageOption) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.httpStorageClient.DeleteNotification") + defer func() { trace.EndSpan(ctx, err) }() + + s := callSettings(c.settings, opts...) + call := c.raw.Notifications.Delete(bucket, id) + if s.userProject != "" { + call.UserProject(s.userProject) + } + return run(ctx, func() error { + return call.Context(ctx).Do() + }, s.retry, s.idempotent, setRetryHeaderHTTP(call)) +} + +type httpReader struct { + body io.ReadCloser + seen int64 + reopen func(seen int64) (*http.Response, error) +} + +func (r *httpReader) Read(p []byte) (int, error) { + n := 0 + for len(p[n:]) > 0 { + m, err := r.body.Read(p[n:]) + n += m + r.seen += int64(m) + if err == nil || err == io.EOF { + return n, err + } + // Read failed (likely due to connection issues), but we will try to reopen + // the pipe and continue. Send a ranged read request that takes into account + // the number of bytes we've already seen. + res, err := r.reopen(r.seen) + if err != nil { + // reopen already retries + return n, err + } + r.body.Close() + r.body = res.Body + } + return n, nil +} + +func (r *httpReader) Close() error { + return r.body.Close() +} diff --git a/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go b/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go index 803f9d260..ced732ca6 100644 --- a/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go +++ b/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go @@ -113,7 +113,7 @@ func defaultCallOptions() *CallOptions { } } -// internalClient is an interface that defines the methods availaible from Cloud Storage API. +// internalClient is an interface that defines the methods available from Cloud Storage API. type internalClient interface { Close() error setGoogleClientInfo(...string) @@ -312,13 +312,40 @@ func (c *Client) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRe // true, or else it is an error. // // For a resumable write, the client should instead call -// StartResumableWrite() and provide that method an WriteObjectSpec. +// StartResumableWrite(), populating a WriteObjectSpec into that request. // They should then attach the returned upload_id to the first message of -// each following call to Create. If there is an error or the connection is -// broken during the resumable Create(), the client should check the status -// of the Create() by calling QueryWriteStatus() and continue writing from -// the returned persisted_size. This may be less than the amount of data the -// client previously sent. +// each following call to WriteObject. If the stream is closed before +// finishing the upload (either explicitly by the client or due to a network +// error or an error response from the server), the client should do as +// follows: +// +// Check the result Status of the stream, to determine if writing can be +// resumed on this stream or must be restarted from scratch (by calling +// StartResumableWrite()). The resumable errors are DEADLINE_EXCEEDED, +// INTERNAL, and UNAVAILABLE. For each case, the client should use binary +// exponential backoff before retrying. Additionally, writes can be +// resumed after RESOURCE_EXHAUSTED errors, but only after taking +// appropriate measures, which may include reducing aggregate send rate +// across clients and/or requesting a quota increase for your project. +// +// If the call to WriteObject returns ABORTED, that indicates +// concurrent attempts to update the resumable write, caused either by +// multiple racing clients or by a single client where the previous +// request was timed out on the client side but nonetheless reached the +// server. In this case the client should take steps to prevent further +// concurrent writes (e.g., increase the timeouts, stop using more than +// one process to perform the upload, etc.), and then should follow the +// steps below for resuming the upload. +// +// For resumable errors, the client should call QueryWriteStatus() and +// then continue writing from the returned persisted_size. This may be +// less than the amount of data the client previously sent. Note also that +// it is acceptable to send data starting at an offset earlier than the +// returned persisted_size; in this case, the service will skip data at +// offsets that were already persisted (without checking that it matches +// the previously written data), and write only the data starting from the +// persisted offset. This behavior can make client-side handling simpler +// in some cases. // // The service will not view the object as complete until the client has // sent a WriteObjectRequest with finish_write set to true. Sending any @@ -326,6 +353,10 @@ func (c *Client) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRe // true will cause an error. The client should check the response it // receives to determine how much data the service was able to commit and // whether the service views the object as complete. +// +// Attempting to resume an already finalized object will result in an OK +// status, with a WriteObjectResponse containing the finalized object’s +// metadata. func (c *Client) WriteObject(ctx context.Context, opts ...gax.CallOption) (storagepb.Storage_WriteObjectClient, error) { return c.internalClient.WriteObject(ctx, opts...) } diff --git a/vendor/cloud.google.com/go/storage/internal/version.go b/vendor/cloud.google.com/go/storage/internal/version.go index 08bddba74..639553700 100644 --- a/vendor/cloud.google.com/go/storage/internal/version.go +++ b/vendor/cloud.google.com/go/storage/internal/version.go @@ -15,4 +15,4 @@ package internal // Version is the current tagged release of the library. -const Version = "1.22.1" +const Version = "1.23.0" diff --git a/vendor/cloud.google.com/go/storage/notifications.go b/vendor/cloud.google.com/go/storage/notifications.go index dd43822b6..fe3e3ae06 100644 --- a/vendor/cloud.google.com/go/storage/notifications.go +++ b/vendor/cloud.google.com/go/storage/notifications.go @@ -22,6 +22,7 @@ import ( "cloud.google.com/go/internal/trace" raw "google.golang.org/api/storage/v1" + storagepb "google.golang.org/genproto/googleapis/storage/v2" ) // A Notification describes how to send Cloud PubSub messages when certain @@ -91,6 +92,30 @@ func toNotification(rn *raw.Notification) *Notification { return n } +func toNotificationFromProto(pbn *storagepb.Notification) *Notification { + n := &Notification{ + ID: pbn.GetName(), + EventTypes: pbn.GetEventTypes(), + ObjectNamePrefix: pbn.GetObjectNamePrefix(), + CustomAttributes: pbn.GetCustomAttributes(), + PayloadFormat: pbn.GetPayloadFormat(), + } + n.TopicProjectID, n.TopicID = parseNotificationTopic(pbn.Topic) + return n +} + +func toProtoNotification(n *Notification) *storagepb.Notification { + return &storagepb.Notification{ + Name: n.ID, + Topic: fmt.Sprintf("//pubsub.googleapis.com/projects/%s/topics/%s", + n.TopicProjectID, n.TopicID), + EventTypes: n.EventTypes, + ObjectNamePrefix: n.ObjectNamePrefix, + CustomAttributes: n.CustomAttributes, + PayloadFormat: n.PayloadFormat, + } +} + var topicRE = regexp.MustCompile("^//pubsub.googleapis.com/projects/([^/]+)/topics/([^/]+)") // parseNotificationTopic extracts the project and topic IDs from from the full @@ -179,6 +204,14 @@ func notificationsToMap(rns []*raw.Notification) map[string]*Notification { return m } +func notificationsToMapFromProto(ns []*storagepb.Notification) map[string]*Notification { + m := map[string]*Notification{} + for _, n := range ns { + m[n.Name] = toNotificationFromProto(n) + } + return m +} + // DeleteNotification deletes the notification with the given ID. func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err error) { ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.DeleteNotification") diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index ab108b039..7a28b6745 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -29,7 +29,6 @@ import ( "cloud.google.com/go/internal/trace" "google.golang.org/api/googleapi" - storagepb "google.golang.org/genproto/googleapis/storage/v2" ) var crc32cTable = crc32.MakeTable(crc32.Castagnoli) @@ -95,7 +94,7 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader") defer func() { trace.EndSpan(ctx, err) }() - if o.c.gc != nil { + if o.c.tc != nil { return o.newRangeReaderWithGRPC(ctx, offset, length) } @@ -383,16 +382,7 @@ type Reader struct { gotCRC uint32 // running crc reopen func(seen int64) (*http.Response, error) - // The following fields are only for use in the gRPC hybrid client. - stream storagepb.Storage_ReadObjectClient - reopenWithGRPC func(seen int64) (*readStreamResponse, context.CancelFunc, error) - leftovers []byte - cancelStream context.CancelFunc -} - -type readStreamResponse struct { - stream storagepb.Storage_ReadObjectClient - response *storagepb.ReadObjectResponse + reader io.ReadCloser } // Close closes the Reader. It must be called when done reading. @@ -401,14 +391,18 @@ func (r *Reader) Close() error { return r.body.Close() } - r.closeStream() + // TODO(noahdietz): Complete integration means returning this call's return + // value, which for gRPC will always be nil. + if r.reader != nil { + return r.reader.Close() + } return nil } func (r *Reader) Read(p []byte) (int, error) { read := r.readWithRetry - if r.reopenWithGRPC != nil { - read = r.readWithGRPC + if r.reader != nil { + read = r.reader.Read } n, err := read(p) @@ -438,129 +432,23 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.newRangeReaderWithGRPC") defer func() { trace.EndSpan(ctx, err) }() - if o.c.gc == nil { - err = fmt.Errorf("handle doesn't have a gRPC client initialized") - return - } if err = o.validate(); err != nil { return } - // A negative length means "read to the end of the object", but the - // read_limit field it corresponds to uses zero to mean the same thing. Thus - // we coerce the length to 0 to read to the end of the object. - if length < 0 { - length = 0 + params := &newRangeReaderParams{ + bucket: o.bucket, + object: o.object, + gen: o.gen, + offset: offset, + length: length, + encryptionKey: o.encryptionKey, + conds: o.conds, } - // For now, there are only globally unique buckets, and "_" is the alias - // project ID for such buckets. - b := bucketResourceName("_", o.bucket) - req := &storagepb.ReadObjectRequest{ - Bucket: b, - Object: o.object, - } - // The default is a negative value, which means latest. - if o.gen >= 0 { - req.Generation = o.gen - } + r, err = o.c.tc.NewRangeReader(ctx, params) - // Define a function that initiates a Read with offset and length, assuming - // we have already read seen bytes. - reopen := func(seen int64) (*readStreamResponse, context.CancelFunc, error) { - // If the context has already expired, return immediately without making - // we call. - if err := ctx.Err(); err != nil { - return nil, nil, err - } - - cc, cancel := context.WithCancel(ctx) - - start := offset + seen - // Only set a ReadLimit if length is greater than zero, because zero - // means read it all. - if length > 0 { - req.ReadLimit = length - seen - } - req.ReadOffset = start - - if err := applyCondsProto("reopenWithGRPC", o.gen, o.conds, req); err != nil { - cancel() - return nil, nil, err - } - - var stream storagepb.Storage_ReadObjectClient - var msg *storagepb.ReadObjectResponse - var err error - - err = run(cc, func() error { - stream, err = o.c.gc.ReadObject(cc, req) - if err != nil { - return err - } - - msg, err = stream.Recv() - - return err - }, o.retry, true, setRetryHeaderGRPC(ctx)) - if err != nil { - // Close the stream context we just created to ensure we don't leak - // resources. - cancel() - return nil, nil, err - } - - return &readStreamResponse{stream, msg}, cancel, nil - } - - res, cancel, err := reopen(0) - if err != nil { - return nil, err - } - - r = &Reader{ - stream: res.stream, - reopenWithGRPC: reopen, - cancelStream: cancel, - } - - // The first message was Recv'd on stream open, use it to populate the - // object metadata. - msg := res.response - obj := msg.GetMetadata() - // This is the size of the entire object, even if only a range was requested. - size := obj.GetSize() - - r.Attrs = ReaderObjectAttrs{ - Size: size, - ContentType: obj.GetContentType(), - ContentEncoding: obj.GetContentEncoding(), - CacheControl: obj.GetCacheControl(), - LastModified: obj.GetUpdateTime().AsTime(), - Metageneration: obj.GetMetageneration(), - Generation: obj.GetGeneration(), - } - - r.size = size - cr := msg.GetContentRange() - if cr != nil { - r.Attrs.StartOffset = cr.GetStart() - r.remain = cr.GetEnd() - cr.GetStart() + 1 - } else { - r.remain = size - } - - // Only support checksums when reading an entire object, not a range. - if checksums := msg.GetObjectChecksums(); checksums != nil && checksums.Crc32C != nil && offset == 0 && length == 0 { - r.wantCRC = checksums.GetCrc32C() - r.checkCRC = true - } - - // Store the content from the first Recv in the client buffer for reading - // later. - r.leftovers = msg.GetChecksummedData().GetContent() - - return r, nil + return r, err } func (r *Reader) readWithRetry(p []byte) (int, error) { @@ -586,112 +474,6 @@ func (r *Reader) readWithRetry(p []byte) (int, error) { return n, nil } -// closeStream cancels a stream's context in order for it to be closed and -// collected. -// -// This is an experimental API and not intended for public use. -func (r *Reader) closeStream() { - if r.cancelStream != nil { - r.cancelStream() - } - r.stream = nil -} - -// readWithGRPC reads bytes into the user's buffer from an open gRPC stream. -// -// This is an experimental API and not intended for public use. -func (r *Reader) readWithGRPC(p []byte) (int, error) { - // No stream to read from, either never initiliazed or Close was called. - // Note: There is a potential concurrency issue if multiple routines are - // using the same reader. One encounters an error and the stream is closed - // and then reopened while the other routine attempts to read from it. - if r.stream == nil { - return 0, fmt.Errorf("reader has been closed") - } - - // The entire object has been read by this reader, return EOF. - if r.size != 0 && r.size == r.seen { - return 0, io.EOF - } - - var n int - // Read leftovers and return what was available to conform to the Reader - // interface: https://pkg.go.dev/io#Reader. - if len(r.leftovers) > 0 { - n = copy(p, r.leftovers) - r.seen += int64(n) - r.leftovers = r.leftovers[n:] - return n, nil - } - - // Attempt to Recv the next message on the stream. - msg, err := r.recv() - if err != nil { - return 0, err - } - - // TODO: Determine if we need to capture incremental CRC32C for this - // chunk. The Object CRC32C checksum is captured when directed to read - // the entire Object. If directed to read a range, we may need to - // calculate the range's checksum for verification if the checksum is - // present in the response here. - // TODO: Figure out if we need to support decompressive transcoding - // https://cloud.google.com/storage/docs/transcoding. - content := msg.GetChecksummedData().GetContent() - n = copy(p[n:], content) - leftover := len(content) - n - if leftover > 0 { - // Wasn't able to copy all of the data in the message, store for - // future Read calls. - r.leftovers = content[n:] - } - r.seen += int64(n) - - return n, nil -} - -// recv attempts to Recv the next message on the stream. In the event -// that a retryable error is encountered, the stream will be closed, reopened, -// and Recv again. This will attempt to Recv until one of the following is true: -// -// * Recv is successful -// * A non-retryable error is encountered -// * The Reader's context is canceled -// -// The last error received is the one that is returned, which could be from -// an attempt to reopen the stream. -// -// This is an experimental API and not intended for public use. -func (r *Reader) recv() (*storagepb.ReadObjectResponse, error) { - msg, err := r.stream.Recv() - if err != nil && shouldRetry(err) { - // This will "close" the existing stream and immediately attempt to - // reopen the stream, but will backoff if further attempts are necessary. - // Reopening the stream Recvs the first message, so if retrying is - // successful, the next logical chunk will be returned. - msg, err = r.reopenStream(r.seen) - } - - return msg, err -} - -// reopenStream "closes" the existing stream and attempts to reopen a stream and -// sets the Reader's stream and cancelStream properties in the process. -// -// This is an experimental API and not intended for public use. -func (r *Reader) reopenStream(seen int64) (*storagepb.ReadObjectResponse, error) { - // Close existing stream and initialize new stream with updated offset. - r.closeStream() - - res, cancel, err := r.reopenWithGRPC(r.seen) - if err != nil { - return nil, err - } - r.stream = res.stream - r.cancelStream = cancel - return res.response, nil -} - // Size returns the size of the object in bytes. // The returned value is always the same and is not affected by // calls to Read or Close. diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 719f39723..e5f7d4734 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -40,7 +40,6 @@ import ( "cloud.google.com/go/internal/optional" "cloud.google.com/go/internal/trace" "cloud.google.com/go/storage/internal" - gapic "cloud.google.com/go/storage/internal/apiv2" "github.com/googleapis/gax-go/v2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" @@ -84,6 +83,14 @@ const ( // ScopeReadWrite grants permissions to manage your // data in Google Cloud Storage. ScopeReadWrite = raw.DevstorageReadWriteScope + + // aes256Algorithm is the AES256 encryption algorithm used with the + // Customer-Supplied Encryption Keys feature. + aes256Algorithm = "AES256" + + // defaultGen indicates the latest object generation by default, + // using a negative value. + defaultGen = int64(-1) ) // TODO: remove this once header with invocation ID is applied to all methods. @@ -106,10 +113,8 @@ type Client struct { creds *google.Credentials retry *retryConfig - // gc is an optional gRPC-based, GAPIC client. - // - // This is an experimental field and not intended for public use. - gc *gapic.Client + // tc is the transport-agnostic client implemented with either gRPC or HTTP. + tc storageClient } // NewClient creates a new Google Cloud Storage client. @@ -205,12 +210,12 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error // This is an experimental API and not intended for public use. func newGRPCClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { opts = append(defaultGRPCOptions(), opts...) - g, err := gapic.NewClient(ctx, opts...) + tc, err := newGRPCStorageClient(ctx, withClientOptions(opts...)) if err != nil { return nil, err } - return &Client{gc: g}, nil + return &Client{tc: tc}, nil } // Close closes the Client. @@ -221,8 +226,8 @@ func (c *Client) Close() error { c.hc = nil c.raw = nil c.creds = nil - if c.gc != nil { - return c.gc.Close() + if c.tc != nil { + return c.tc.Close() } return nil } @@ -1231,6 +1236,49 @@ func (o *ObjectAttrs) toProtoObject(b string) *storagepb.Object { } } +// toProtoObject copies the attributes to update from uattrs to the proto library's Object type. +func (uattrs *ObjectAttrsToUpdate) toProtoObject(bucket, object string) *storagepb.Object { + o := &storagepb.Object{ + Name: object, + Bucket: bucket, + } + if uattrs == nil { + return o + } + + if uattrs.EventBasedHold != nil { + o.EventBasedHold = proto.Bool(optional.ToBool(uattrs.EventBasedHold)) + } + if uattrs.TemporaryHold != nil { + o.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) + } + if uattrs.ContentType != nil { + o.ContentType = optional.ToString(uattrs.ContentType) + } + if uattrs.ContentLanguage != nil { + o.ContentLanguage = optional.ToString(uattrs.ContentLanguage) + } + if uattrs.ContentEncoding != nil { + o.ContentEncoding = optional.ToString(uattrs.ContentEncoding) + } + if uattrs.ContentDisposition != nil { + o.ContentDisposition = optional.ToString(uattrs.ContentDisposition) + } + if uattrs.CacheControl != nil { + o.CacheControl = optional.ToString(uattrs.CacheControl) + } + if !uattrs.CustomTime.IsZero() { + o.CustomTime = toProtoTimestamp(uattrs.CustomTime) + } + if uattrs.ACL != nil { + o.Acl = toProtoObjectACL(uattrs.ACL) + } + + // TODO(cathyo): Handle metadata. Pending b/230510191. + + return o +} + // ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. type ObjectAttrs struct { // Bucket is the name of the bucket containing this GCS object. @@ -1312,6 +1360,10 @@ type ObjectAttrs struct { // Metadata represents user-provided metadata, in key/value pairs. // It can be nil if no metadata is provided. + // + // For object downloads using Reader, metadata keys are sent as headers. + // Therefore, avoid setting metadata keys using characters that are not valid + // for headers. See https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6. Metadata map[string]string // Generation is the generation number of the object's content. @@ -1992,13 +2044,26 @@ func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) erro if copySource { cs = "copy-source-" } - headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") + headers.Set("x-goog-"+cs+"encryption-algorithm", aes256Algorithm) headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) keyHash := sha256.Sum256(key) headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) return nil } +// toProtoCommonObjectRequestParams sets customer-supplied encryption to the proto library's CommonObjectRequestParams. +func toProtoCommonObjectRequestParams(key []byte) *storagepb.CommonObjectRequestParams { + if key == nil { + return nil + } + keyHash := sha256.Sum256(key) + return &storagepb.CommonObjectRequestParams{ + EncryptionAlgorithm: aes256Algorithm, + EncryptionKeyBytes: key, + EncryptionKeySha256Bytes: keyHash[:], + } +} + // ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { r := c.raw.Projects.ServiceAccount.Get(projectID) diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index c616603e4..1908826e6 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -24,12 +24,9 @@ import ( "time" "unicode/utf8" - "github.com/golang/protobuf/proto" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" storagepb "google.golang.org/genproto/googleapis/storage/v2" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) const ( @@ -123,16 +120,6 @@ type Writer struct { mu sync.Mutex err error - - // The gRPC client-stream used for sending buffers. - // - // This is an experimental API and not intended for public use. - stream storagepb.Storage_WriteObjectClient - - // The Resumable Upload ID started by a gRPC-based Writer. - // - // This is an experimental API and not intended for public use. - upid string } func (w *Writer) open() error { @@ -249,8 +236,8 @@ func (w *Writer) Write(p []byte) (n int, err error) { } if !w.opened { // gRPC client has been initialized - use gRPC to upload. - if w.o.c.gc != nil { - if err := w.openGRPC(); err != nil { + if w.o.c.tc != nil { + if err := w.openWriter(); err != nil { return 0, err } } else if err := w.open(); err != nil { @@ -277,7 +264,11 @@ func (w *Writer) Write(p []byte) (n int, err error) { // can be retrieved by calling Attrs. func (w *Writer) Close() error { if !w.opened { - if err := w.open(); err != nil { + if w.o.c.tc != nil { + if err := w.openWriter(); err != nil { + return err + } + } else if err := w.open(); err != nil { return err } } @@ -293,6 +284,35 @@ func (w *Writer) Close() error { return w.err } +func (w *Writer) openWriter() (err error) { + if err := w.validateWriteAttrs(); err != nil { + return err + } + + go w.monitorCancel() + params := &openWriterParams{ + ctx: w.ctx, + chunkSize: w.ChunkSize, + chunkRetryDeadline: w.ChunkRetryDeadline, + bucket: w.o.bucket, + attrs: &w.ObjectAttrs, + conds: w.o.conds, + encryptionKey: w.o.encryptionKey, + sendCRC32C: w.SendCRC32C, + donec: w.donec, + setError: w.error, + progress: w.progress, + setObj: func(o *ObjectAttrs) { w.obj = o }, + } + w.pw, err = w.o.c.tc.OpenWriter(params) + if err != nil { + return err + } + w.opened = true + + return nil +} + // monitorCancel is intended to be used as a background goroutine. It monitors the // context, and when it observes that the context has been canceled, it manually // closes things that do not take a context. @@ -361,333 +381,3 @@ func (w *Writer) error(err error) { w.err = err w.mu.Unlock() } - -// openGRPC initializes a pipe for the user to write data to, and a routine to -// read from that pipe and upload the data to GCS via gRPC. -// -// This is an experimental API and not intended for public use. -func (w *Writer) openGRPC() error { - if err := w.validateWriteAttrs(); err != nil { - return err - } - - pr, pw := io.Pipe() - w.pw = pw - w.opened = true - - go w.monitorCancel() - - bufSize := w.ChunkSize - if w.ChunkSize == 0 { - // TODO: Should we actually use the minimum of 256 KB here when the user - // indicates they want minimal memory usage? We cannot do a zero-copy, - // bufferless upload like HTTP/JSON can. - // TODO: We need to determine if we can avoid starting a - // resumable upload when the user *plans* to send more than bufSize but - // with a bufferless upload. - bufSize = maxPerMessageWriteSize - } - buf := make([]byte, bufSize) - - var offset int64 - - // This function reads the data sent to the pipe and sends sets of messages - // on the gRPC client-stream as the buffer is filled. - go func() { - defer close(w.donec) - - // Loop until there is an error or the Object has been finalized. - for { - // Note: This blocks until either the buffer is full or EOF is read. - recvd, doneReading, err := read(pr, buf) - if err != nil { - err = checkCanceled(err) - w.error(err) - pr.CloseWithError(err) - return - } - toWrite := buf[:recvd] - - // TODO: Figure out how to set up encryption via CommonObjectRequestParams. - - // The chunk buffer is full, but there is no end in sight. This - // means that a resumable upload will need to be used to send - // multiple chunks, until we are done reading data. Start a - // resumable upload if it has not already been started. - // Otherwise, all data will be sent over a single gRPC stream. - if !doneReading && w.upid == "" { - err = w.startResumableUpload() - if err != nil { - err = checkCanceled(err) - w.error(err) - pr.CloseWithError(err) - return - } - } - - o, off, finalized, err := w.uploadBuffer(toWrite, recvd, offset, doneReading) - if err != nil { - err = checkCanceled(err) - w.error(err) - pr.CloseWithError(err) - return - } - // At this point, the current buffer has been uploaded. Capture the - // committed offset here in case the upload was not finalized and - // another chunk is to be uploaded. - offset = off - w.progress(offset) - - // When we are done reading data and the chunk has been finalized, - // we are done. - if doneReading && finalized { - // Build Object from server's response. - w.obj = newObjectFromProto(o) - return - } - } - }() - - return nil -} - -// startResumableUpload initializes a Resumable Upload with gRPC and sets the -// upload ID on the Writer. -// -// This is an experimental API and not intended for public use. -func (w *Writer) startResumableUpload() error { - spec, err := w.writeObjectSpec() - if err != nil { - return err - } - upres, err := w.o.c.gc.StartResumableWrite(w.ctx, &storagepb.StartResumableWriteRequest{ - WriteObjectSpec: spec, - }) - - w.upid = upres.GetUploadId() - return err -} - -// queryProgress is a helper that queries the status of the resumable upload -// associated with the given upload ID. -// -// This is an experimental API and not intended for public use. -func (w *Writer) queryProgress() (int64, error) { - q, err := w.o.c.gc.QueryWriteStatus(w.ctx, &storagepb.QueryWriteStatusRequest{UploadId: w.upid}) - - // q.GetCommittedSize() will return 0 if q is nil. - return q.GetPersistedSize(), err -} - -// uploadBuffer opens a Write stream and uploads the buffer at the given offset (if -// uploading a chunk for a resumable uploadBuffer), and will mark the write as -// finished if we are done receiving data from the user. The resulting write -// offset after uploading the buffer is returned, as well as a boolean -// indicating if the Object has been finalized. If it has been finalized, the -// final Object will be returned as well. Finalizing the upload is primarily -// important for Resumable Uploads. A simple or multi-part upload will always -// be finalized once the entire buffer has been written. -// -// This is an experimental API and not intended for public use. -func (w *Writer) uploadBuffer(buf []byte, recvd int, start int64, doneReading bool) (*storagepb.Object, int64, bool, error) { - var err error - var finishWrite bool - var sent, limit int = 0, maxPerMessageWriteSize - offset := start - for { - first := sent == 0 - // This indicates that this is the last message and the remaining - // data fits in one message. - belowLimit := recvd-sent <= limit - if belowLimit { - limit = recvd - sent - } - if belowLimit && doneReading { - finishWrite = true - } - - // Prepare chunk section for upload. - data := buf[sent : sent+limit] - req := &storagepb.WriteObjectRequest{ - Data: &storagepb.WriteObjectRequest_ChecksummedData{ - ChecksummedData: &storagepb.ChecksummedData{ - Content: data, - }, - }, - WriteOffset: offset, - FinishWrite: finishWrite, - } - - // Open a new stream and set the first_message field on the request. - // The first message on the WriteObject stream must either be the - // Object or the Resumable Upload ID. - if first { - w.stream, err = w.o.c.gc.WriteObject(w.ctx) - if err != nil { - return nil, 0, false, err - } - - if w.upid != "" { - req.FirstMessage = &storagepb.WriteObjectRequest_UploadId{UploadId: w.upid} - } else { - spec, err := w.writeObjectSpec() - if err != nil { - return nil, 0, false, err - } - req.FirstMessage = &storagepb.WriteObjectRequest_WriteObjectSpec{ - WriteObjectSpec: spec, - } - } - - // TODO: Currently the checksums are only sent on the first message - // of the stream, but in the future, we must also support sending it - // on the *last* message of the stream (instead of the first). - if w.SendCRC32C { - req.ObjectChecksums = &storagepb.ObjectChecksums{ - Crc32C: proto.Uint32(w.CRC32C), - Md5Hash: w.MD5, - } - } - } - - err = w.stream.Send(req) - if err == io.EOF { - // err was io.EOF. The client-side of a stream only gets an EOF on Send - // when the backend closes the stream and wants to return an error - // status. Closing the stream receives the status as an error. - _, err = w.stream.CloseAndRecv() - - // Retriable errors mean we should start over and attempt to - // resend the entire buffer via a new stream. - // If not retriable, falling through will return the error received - // from closing the stream. - if shouldRetry(err) { - sent = 0 - finishWrite = false - // TODO: Add test case for failure modes of querying progress. - offset, err = w.determineOffset(start) - if err == nil { - continue - } - } - } - if err != nil { - return nil, 0, false, err - } - - // Update the immediate stream's sent total and the upload offset with - // the data sent. - sent += len(data) - offset += int64(len(data)) - - // Not done sending data, do not attempt to commit it yet, loop around - // and send more data. - if recvd-sent > 0 { - continue - } - - // Done sending data. Close the stream to "commit" the data sent. - resp, finalized, err := w.commit() - // Retriable errors mean we should start over and attempt to - // resend the entire buffer via a new stream. - // If not retriable, falling through will return the error received - // from closing the stream. - if shouldRetry(err) { - sent = 0 - finishWrite = false - offset, err = w.determineOffset(start) - if err == nil { - continue - } - } - if err != nil { - return nil, 0, false, err - } - - return resp.GetResource(), offset, finalized, nil - } -} - -// determineOffset either returns the offset given to it in the case of a simple -// upload, or queries the write status in the case a resumable upload is being -// used. -// -// This is an experimental API and not intended for public use. -func (w *Writer) determineOffset(offset int64) (int64, error) { - // For a Resumable Upload, we must start from however much data - // was committed. - if w.upid != "" { - committed, err := w.queryProgress() - if err != nil { - return 0, err - } - offset = committed - } - return offset, nil -} - -// commit closes the stream to commit the data sent and potentially receive -// the finalized object if finished uploading. If the last request sent -// indicated that writing was finished, the Object will be finalized and -// returned. If not, then the Object will be nil, and the boolean returned will -// be false. -// -// This is an experimental API and not intended for public use. -func (w *Writer) commit() (*storagepb.WriteObjectResponse, bool, error) { - finalized := true - resp, err := w.stream.CloseAndRecv() - if err == io.EOF { - // Closing a stream for a resumable upload finish_write = false results - // in an EOF which can be ignored, as we aren't done uploading yet. - finalized = false - err = nil - } - // Drop the stream reference as it has been closed. - w.stream = nil - - return resp, finalized, err -} - -// writeObjectSpec constructs a WriteObjectSpec proto using the Writer's -// ObjectAttrs and applies its Conditions. This is only used for gRPC. -// -// This is an experimental API and not intended for public use. -func (w *Writer) writeObjectSpec() (*storagepb.WriteObjectSpec, error) { - spec := &storagepb.WriteObjectSpec{ - Resource: w.ObjectAttrs.toProtoObject(w.o.bucket), - } - // WriteObject doesn't support the generation condition, so use -1. - if err := applyCondsProto("WriteObject", -1, w.o.conds, spec); err != nil { - return nil, err - } - return spec, nil -} - -// read copies the data in the reader to the given buffer and reports how much -// data was read into the buffer and if there is no more data to read (EOF). -// -// This is an experimental API and not intended for public use. -func read(r io.Reader, buf []byte) (int, bool, error) { - // Set n to -1 to start the Read loop. - var n, recvd int = -1, 0 - var err error - for err == nil && n != 0 { - // The routine blocks here until data is received. - n, err = r.Read(buf[recvd:]) - recvd += n - } - var done bool - if err == io.EOF { - done = true - err = nil - } - return recvd, done, err -} - -func checkCanceled(err error) error { - if status.Code(err) == codes.Canceled { - return context.Canceled - } - - return err -} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 4a1a0f9ef..9deff7719 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -16792,43 +16792,6 @@ var awsPartition = partition{ }, }, }, - "redshift-serverless": service{ - Endpoints: serviceEndpoints{ - endpointKey{ - Region: "ap-northeast-1", - }: endpoint{}, - endpointKey{ - Region: "ap-northeast-2", - }: endpoint{}, - endpointKey{ - Region: "ap-southeast-1", - }: endpoint{}, - endpointKey{ - Region: "ap-southeast-2", - }: endpoint{}, - endpointKey{ - Region: "eu-central-1", - }: endpoint{}, - endpointKey{ - Region: "eu-north-1", - }: endpoint{}, - endpointKey{ - Region: "eu-west-1", - }: endpoint{}, - endpointKey{ - Region: "eu-west-2", - }: endpoint{}, - endpointKey{ - Region: "us-east-1", - }: endpoint{}, - endpointKey{ - Region: "us-east-2", - }: endpoint{}, - endpointKey{ - Region: "us-west-2", - }: endpoint{}, - }, - }, "rekognition": service{ Endpoints: serviceEndpoints{ endpointKey{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 3bf33f8de..e6612ead4 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.44.37" +const SDKVersion = "1.44.43" diff --git a/vendor/golang.org/x/oauth2/authhandler/authhandler.go b/vendor/golang.org/x/oauth2/authhandler/authhandler.go index 69967cf87..9bc6cd7bc 100644 --- a/vendor/golang.org/x/oauth2/authhandler/authhandler.go +++ b/vendor/golang.org/x/oauth2/authhandler/authhandler.go @@ -13,11 +13,36 @@ import ( "golang.org/x/oauth2" ) +const ( + // Parameter keys for AuthCodeURL method to support PKCE. + codeChallengeKey = "code_challenge" + codeChallengeMethodKey = "code_challenge_method" + + // Parameter key for Exchange method to support PKCE. + codeVerifierKey = "code_verifier" +) + +// PKCEParams holds parameters to support PKCE. +type PKCEParams struct { + Challenge string // The unpadded, base64-url-encoded string of the encrypted code verifier. + ChallengeMethod string // The encryption method (ex. S256). + Verifier string // The original, non-encrypted secret. +} + // AuthorizationHandler is a 3-legged-OAuth helper that prompts // the user for OAuth consent at the specified auth code URL // and returns an auth code and state upon approval. type AuthorizationHandler func(authCodeURL string) (code string, state string, err error) +// TokenSourceWithPKCE is an enhanced version of TokenSource with PKCE support. +// +// The pkce parameter supports PKCE flow, which uses code challenge and code verifier +// to prevent CSRF attacks. A unique code challenge and code verifier should be generated +// by the caller at runtime. See https://www.oauth.com/oauth2-servers/pkce/ for more info. +func TokenSourceWithPKCE(ctx context.Context, config *oauth2.Config, state string, authHandler AuthorizationHandler, pkce *PKCEParams) oauth2.TokenSource { + return oauth2.ReuseTokenSource(nil, authHandlerSource{config: config, ctx: ctx, authHandler: authHandler, state: state, pkce: pkce}) +} + // TokenSource returns an oauth2.TokenSource that fetches access tokens // using 3-legged-OAuth flow. // @@ -33,7 +58,7 @@ type AuthorizationHandler func(authCodeURL string) (code string, state string, e // and response before exchanging the auth code for OAuth token to prevent CSRF // attacks. func TokenSource(ctx context.Context, config *oauth2.Config, state string, authHandler AuthorizationHandler) oauth2.TokenSource { - return oauth2.ReuseTokenSource(nil, authHandlerSource{config: config, ctx: ctx, authHandler: authHandler, state: state}) + return TokenSourceWithPKCE(ctx, config, state, authHandler, nil) } type authHandlerSource struct { @@ -41,10 +66,17 @@ type authHandlerSource struct { config *oauth2.Config authHandler AuthorizationHandler state string + pkce *PKCEParams } func (source authHandlerSource) Token() (*oauth2.Token, error) { - url := source.config.AuthCodeURL(source.state) + // Step 1: Obtain auth code. + var authCodeUrlOptions []oauth2.AuthCodeOption + if source.pkce != nil && source.pkce.Challenge != "" && source.pkce.ChallengeMethod != "" { + authCodeUrlOptions = []oauth2.AuthCodeOption{oauth2.SetAuthURLParam(codeChallengeKey, source.pkce.Challenge), + oauth2.SetAuthURLParam(codeChallengeMethodKey, source.pkce.ChallengeMethod)} + } + url := source.config.AuthCodeURL(source.state, authCodeUrlOptions...) code, state, err := source.authHandler(url) if err != nil { return nil, err @@ -52,5 +84,11 @@ func (source authHandlerSource) Token() (*oauth2.Token, error) { if state != source.state { return nil, errors.New("state mismatch in 3-legged-OAuth flow") } - return source.config.Exchange(source.ctx, code) + + // Step 2: Exchange auth code for access token. + var exchangeOptions []oauth2.AuthCodeOption + if source.pkce != nil && source.pkce.Verifier != "" { + exchangeOptions = []oauth2.AuthCodeOption{oauth2.SetAuthURLParam(codeVerifierKey, source.pkce.Verifier)} + } + return source.config.Exchange(source.ctx, code, exchangeOptions...) } diff --git a/vendor/golang.org/x/oauth2/google/default.go b/vendor/golang.org/x/oauth2/google/default.go index 024a104b0..7ed02cd41 100644 --- a/vendor/golang.org/x/oauth2/google/default.go +++ b/vendor/golang.org/x/oauth2/google/default.go @@ -54,11 +54,14 @@ type CredentialsParams struct { // Optional. Subject string - // AuthHandler is the AuthorizationHandler used for 3-legged OAuth flow. Optional. + // AuthHandler is the AuthorizationHandler used for 3-legged OAuth flow. Required for 3LO flow. AuthHandler authhandler.AuthorizationHandler - // State is a unique string used with AuthHandler. Optional. + // State is a unique string used with AuthHandler. Required for 3LO flow. State string + + // PKCE is used to support PKCE flow. Optional for 3LO flow. + PKCE *authhandler.PKCEParams } func (params CredentialsParams) deepCopy() CredentialsParams { @@ -176,7 +179,7 @@ func CredentialsFromJSONWithParams(ctx context.Context, jsonData []byte, params if config != nil { return &Credentials{ ProjectID: "", - TokenSource: authhandler.TokenSource(ctx, config, params.State, params.AuthHandler), + TokenSource: authhandler.TokenSourceWithPKCE(ctx, config, params.State, params.AuthHandler, params.PKCE), JSON: jsonData, }, nil } diff --git a/vendor/golang.org/x/sys/unix/syscall_aix.go b/vendor/golang.org/x/sys/unix/syscall_aix.go index ad22c33db..ac579c60f 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix.go @@ -217,12 +217,12 @@ func Accept(fd int) (nfd int, sa Sockaddr, err error) { return } -func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { +func recvmsgRaw(fd int, iov []Iovec, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { // Recvmsg not implemented on AIX return -1, -1, -1, ENOSYS } -func sendmsgN(fd int, p, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { +func sendmsgN(fd int, iov []Iovec, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { // SendmsgN not implemented on AIX return -1, ENOSYS } diff --git a/vendor/golang.org/x/sys/unix/syscall_bsd.go b/vendor/golang.org/x/sys/unix/syscall_bsd.go index 9c87c5f07..c437fc5d7 100644 --- a/vendor/golang.org/x/sys/unix/syscall_bsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_bsd.go @@ -325,27 +325,26 @@ func GetsockoptString(fd, level, opt int) (string, error) { //sys sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen _Socklen) (err error) //sys recvmsg(s int, msg *Msghdr, flags int) (n int, err error) -func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { +func recvmsgRaw(fd int, iov []Iovec, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { var msg Msghdr msg.Name = (*byte)(unsafe.Pointer(rsa)) msg.Namelen = uint32(SizeofSockaddrAny) - var iov Iovec - if len(p) > 0 { - iov.Base = (*byte)(unsafe.Pointer(&p[0])) - iov.SetLen(len(p)) - } var dummy byte if len(oob) > 0 { // receive at least one normal byte - if len(p) == 0 { - iov.Base = &dummy - iov.SetLen(1) + if emptyIovecs(iov) { + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) + iov = iova[:] } msg.Control = (*byte)(unsafe.Pointer(&oob[0])) msg.SetControllen(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = recvmsg(fd, &msg, flags); err != nil { return } @@ -356,31 +355,32 @@ func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn //sys sendmsg(s int, msg *Msghdr, flags int) (n int, err error) -func sendmsgN(fd int, p, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { +func sendmsgN(fd int, iov []Iovec, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { var msg Msghdr msg.Name = (*byte)(unsafe.Pointer(ptr)) msg.Namelen = uint32(salen) - var iov Iovec - if len(p) > 0 { - iov.Base = (*byte)(unsafe.Pointer(&p[0])) - iov.SetLen(len(p)) - } var dummy byte + var empty bool if len(oob) > 0 { // send at least one normal byte - if len(p) == 0 { - iov.Base = &dummy - iov.SetLen(1) + empty := emptyIovecs(iov) + if empty { + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) + iov = iova[:] } msg.Control = (*byte)(unsafe.Pointer(&oob[0])) msg.SetControllen(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = sendmsg(fd, &msg, flags); err != nil { return 0, err } - if len(oob) > 0 && len(p) == 0 { + if len(oob) > 0 && empty { n = 0 } return n, nil diff --git a/vendor/golang.org/x/sys/unix/syscall_illumos.go b/vendor/golang.org/x/sys/unix/syscall_illumos.go index 8d5f294c4..e48244a9c 100644 --- a/vendor/golang.org/x/sys/unix/syscall_illumos.go +++ b/vendor/golang.org/x/sys/unix/syscall_illumos.go @@ -20,10 +20,9 @@ func bytes2iovec(bs [][]byte) []Iovec { for i, b := range bs { iovecs[i].SetLen(len(b)) if len(b) > 0 { - // somehow Iovec.Base on illumos is (*int8), not (*byte) - iovecs[i].Base = (*int8)(unsafe.Pointer(&b[0])) + iovecs[i].Base = &b[0] } else { - iovecs[i].Base = (*int8)(unsafe.Pointer(&_zero)) + iovecs[i].Base = (*byte)(unsafe.Pointer(&_zero)) } } return iovecs diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index c8d203212..5e4a94f73 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -1499,18 +1499,13 @@ func KeyctlRestrictKeyring(ringid int, keyType string, restriction string) error //sys keyctlRestrictKeyringByType(cmd int, arg2 int, keyType string, restriction string) (err error) = SYS_KEYCTL //sys keyctlRestrictKeyring(cmd int, arg2 int) (err error) = SYS_KEYCTL -func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { +func recvmsgRaw(fd int, iov []Iovec, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { var msg Msghdr msg.Name = (*byte)(unsafe.Pointer(rsa)) msg.Namelen = uint32(SizeofSockaddrAny) - var iov Iovec - if len(p) > 0 { - iov.Base = &p[0] - iov.SetLen(len(p)) - } var dummy byte if len(oob) > 0 { - if len(p) == 0 { + if emptyIovecs(iov) { var sockType int sockType, err = GetsockoptInt(fd, SOL_SOCKET, SO_TYPE) if err != nil { @@ -1518,15 +1513,19 @@ func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn } // receive at least one normal byte if sockType != SOCK_DGRAM { - iov.Base = &dummy - iov.SetLen(1) + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) + iov = iova[:] } } msg.Control = &oob[0] msg.SetControllen(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = recvmsg(fd, &msg, flags); err != nil { return } @@ -1535,18 +1534,15 @@ func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn return } -func sendmsgN(fd int, p, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { +func sendmsgN(fd int, iov []Iovec, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { var msg Msghdr msg.Name = (*byte)(ptr) msg.Namelen = uint32(salen) - var iov Iovec - if len(p) > 0 { - iov.Base = &p[0] - iov.SetLen(len(p)) - } var dummy byte + var empty bool if len(oob) > 0 { - if len(p) == 0 { + empty := emptyIovecs(iov) + if empty { var sockType int sockType, err = GetsockoptInt(fd, SOL_SOCKET, SO_TYPE) if err != nil { @@ -1554,19 +1550,22 @@ func sendmsgN(fd int, p, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags i } // send at least one normal byte if sockType != SOCK_DGRAM { - iov.Base = &dummy - iov.SetLen(1) + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) } } msg.Control = &oob[0] msg.SetControllen(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = sendmsg(fd, &msg, flags); err != nil { return 0, err } - if len(oob) > 0 && len(p) == 0 { + if len(oob) > 0 && empty { n = 0 } return n, nil diff --git a/vendor/golang.org/x/sys/unix/syscall_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/syscall_openbsd_mips64.go index 30f285343..1378489f8 100644 --- a/vendor/golang.org/x/sys/unix/syscall_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/syscall_openbsd_mips64.go @@ -26,6 +26,10 @@ func (msghdr *Msghdr) SetControllen(length int) { msghdr.Controllen = uint32(length) } +func (msghdr *Msghdr) SetIovlen(length int) { + msghdr.Iovlen = uint32(length) +} + func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } diff --git a/vendor/golang.org/x/sys/unix/syscall_solaris.go b/vendor/golang.org/x/sys/unix/syscall_solaris.go index 932996c75..b5ec457cd 100644 --- a/vendor/golang.org/x/sys/unix/syscall_solaris.go +++ b/vendor/golang.org/x/sys/unix/syscall_solaris.go @@ -451,26 +451,25 @@ func Accept(fd int) (nfd int, sa Sockaddr, err error) { //sys recvmsg(s int, msg *Msghdr, flags int) (n int, err error) = libsocket.__xnet_recvmsg -func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { +func recvmsgRaw(fd int, iov []Iovec, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) { var msg Msghdr msg.Name = (*byte)(unsafe.Pointer(rsa)) msg.Namelen = uint32(SizeofSockaddrAny) - var iov Iovec - if len(p) > 0 { - iov.Base = (*int8)(unsafe.Pointer(&p[0])) - iov.SetLen(len(p)) - } - var dummy int8 + var dummy byte if len(oob) > 0 { // receive at least one normal byte - if len(p) == 0 { - iov.Base = &dummy - iov.SetLen(1) + if emptyIovecs(iov) { + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) + iov = iova[:] } msg.Accrightslen = int32(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = recvmsg(fd, &msg, flags); n == -1 { return } @@ -480,30 +479,31 @@ func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn //sys sendmsg(s int, msg *Msghdr, flags int) (n int, err error) = libsocket.__xnet_sendmsg -func sendmsgN(fd int, p, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { +func sendmsgN(fd int, iov []Iovec, oob []byte, ptr unsafe.Pointer, salen _Socklen, flags int) (n int, err error) { var msg Msghdr msg.Name = (*byte)(unsafe.Pointer(ptr)) msg.Namelen = uint32(salen) - var iov Iovec - if len(p) > 0 { - iov.Base = (*int8)(unsafe.Pointer(&p[0])) - iov.SetLen(len(p)) - } - var dummy int8 + var dummy byte + var empty bool if len(oob) > 0 { // send at least one normal byte - if len(p) == 0 { - iov.Base = &dummy - iov.SetLen(1) + empty = emptyIovecs(iov) + if empty { + var iova [1]Iovec + iova[0].Base = &dummy + iova[0].SetLen(1) + iov = iova[:] } msg.Accrightslen = int32(len(oob)) } - msg.Iov = &iov - msg.Iovlen = 1 + if len(iov) > 0 { + msg.Iov = &iov[0] + msg.SetIovlen(len(iov)) + } if n, err = sendmsg(fd, &msg, flags); err != nil { return 0, err } - if len(oob) > 0 && len(p) == 0 { + if len(oob) > 0 && empty { n = 0 } return n, nil diff --git a/vendor/golang.org/x/sys/unix/syscall_unix.go b/vendor/golang.org/x/sys/unix/syscall_unix.go index 70508afc1..1ff5060b5 100644 --- a/vendor/golang.org/x/sys/unix/syscall_unix.go +++ b/vendor/golang.org/x/sys/unix/syscall_unix.go @@ -338,8 +338,13 @@ func Recvfrom(fd int, p []byte, flags int) (n int, from Sockaddr, err error) { } func Recvmsg(fd int, p, oob []byte, flags int) (n, oobn int, recvflags int, from Sockaddr, err error) { + var iov [1]Iovec + if len(p) > 0 { + iov[0].Base = &p[0] + iov[0].SetLen(len(p)) + } var rsa RawSockaddrAny - n, oobn, recvflags, err = recvmsgRaw(fd, p, oob, flags, &rsa) + n, oobn, recvflags, err = recvmsgRaw(fd, iov[:], oob, flags, &rsa) // source address is only specified if the socket is unconnected if rsa.Addr.Family != AF_UNSPEC { from, err = anyToSockaddr(fd, &rsa) @@ -347,12 +352,42 @@ func Recvmsg(fd int, p, oob []byte, flags int) (n, oobn int, recvflags int, from return } +// RecvmsgBuffers receives a message from a socket using the recvmsg +// system call. The flags are passed to recvmsg. Any non-control data +// read is scattered into the buffers slices. The results are: +// - n is the number of non-control data read into bufs +// - oobn is the number of control data read into oob; this may be interpreted using [ParseSocketControlMessage] +// - recvflags is flags returned by recvmsg +// - from is the address of the sender +func RecvmsgBuffers(fd int, buffers [][]byte, oob []byte, flags int) (n, oobn int, recvflags int, from Sockaddr, err error) { + iov := make([]Iovec, len(buffers)) + for i := range buffers { + if len(buffers[i]) > 0 { + iov[i].Base = &buffers[i][0] + iov[i].SetLen(len(buffers[i])) + } else { + iov[i].Base = (*byte)(unsafe.Pointer(&_zero)) + } + } + var rsa RawSockaddrAny + n, oobn, recvflags, err = recvmsgRaw(fd, iov, oob, flags, &rsa) + if err == nil && rsa.Addr.Family != AF_UNSPEC { + from, err = anyToSockaddr(fd, &rsa) + } + return +} + func Sendmsg(fd int, p, oob []byte, to Sockaddr, flags int) (err error) { _, err = SendmsgN(fd, p, oob, to, flags) return } func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) { + var iov [1]Iovec + if len(p) > 0 { + iov[0].Base = &p[0] + iov[0].SetLen(len(p)) + } var ptr unsafe.Pointer var salen _Socklen if to != nil { @@ -361,7 +396,32 @@ func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) return 0, err } } - return sendmsgN(fd, p, oob, ptr, salen, flags) + return sendmsgN(fd, iov[:], oob, ptr, salen, flags) +} + +// SendmsgBuffers sends a message on a socket to an address using the sendmsg +// system call. The flags are passed to sendmsg. Any non-control data written +// is gathered from buffers. The function returns the number of bytes written +// to the socket. +func SendmsgBuffers(fd int, buffers [][]byte, oob []byte, to Sockaddr, flags int) (n int, err error) { + iov := make([]Iovec, len(buffers)) + for i := range buffers { + if len(buffers[i]) > 0 { + iov[i].Base = &buffers[i][0] + iov[i].SetLen(len(buffers[i])) + } else { + iov[i].Base = (*byte)(unsafe.Pointer(&_zero)) + } + } + var ptr unsafe.Pointer + var salen _Socklen + if to != nil { + ptr, salen, err = to.sockaddr() + if err != nil { + return 0, err + } + } + return sendmsgN(fd, iov, oob, ptr, salen, flags) } func Send(s int, buf []byte, flags int) (err error) { @@ -484,3 +544,13 @@ func Lutimes(path string, tv []Timeval) error { } return UtimesNanoAt(AT_FDCWD, path, ts, AT_SYMLINK_NOFOLLOW) } + +// emptyIovec reports whether there are no bytes in the slice of Iovec. +func emptyIovecs(iov []Iovec) bool { + for i := range iov { + if iov[i].Len > 0 { + return false + } + } + return true +} diff --git a/vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go index ad4aad279..c1a9b83ad 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go @@ -178,7 +178,7 @@ type Linger struct { } type Iovec struct { - Base *int8 + Base *byte Len uint64 } diff --git a/vendor/google.golang.org/api/internal/gensupport/resumable.go b/vendor/google.golang.org/api/internal/gensupport/resumable.go index 0eae147fa..0c659188d 100644 --- a/vendor/google.golang.org/api/internal/gensupport/resumable.go +++ b/vendor/google.golang.org/api/internal/gensupport/resumable.go @@ -10,8 +10,12 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "time" + + "github.com/google/uuid" + "google.golang.org/api/internal" ) // ResumableUpload is used by the generated APIs to provide resumable uploads. @@ -38,6 +42,11 @@ type ResumableUpload struct { // ChunkRetryDeadline configures the per-chunk deadline after which no further // retries should happen. ChunkRetryDeadline time.Duration + + // Track current request invocation ID and attempt count for retry metric + // headers. + invocationID string + attempts int } // Progress returns the number of bytes uploaded at this point. @@ -72,6 +81,10 @@ func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader, req.Header.Set("Content-Type", rx.MediaType) req.Header.Set("User-Agent", rx.UserAgent) + baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version + invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", rx.invocationID, rx.attempts) + req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " ")) + // Google's upload endpoint uses status code 308 for a // different purpose than the "308 Permanent Redirect" // since-standardized in RFC 7238. Because of the conflict in @@ -178,9 +191,11 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err for { var pause time.Duration - // Each chunk gets its own initialized-at-zero backoff. + // Each chunk gets its own initialized-at-zero backoff and invocation ID. bo := rx.Retry.backoff() quitAfter := time.After(retryDeadline) + rx.attempts = 1 + rx.invocationID = uuid.New().String() // Retry loop for a single chunk. for { @@ -223,6 +238,7 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err break } + rx.attempts++ pause = bo.Pause() if resp != nil && resp.Body != nil { resp.Body.Close() diff --git a/vendor/google.golang.org/api/internal/gensupport/send.go b/vendor/google.golang.org/api/internal/gensupport/send.go index dd50cc20a..70a8e01c1 100644 --- a/vendor/google.golang.org/api/internal/gensupport/send.go +++ b/vendor/google.golang.org/api/internal/gensupport/send.go @@ -8,9 +8,12 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" + "strings" "time" + "github.com/google/uuid" "github.com/googleapis/gax-go/v2" ) @@ -71,6 +74,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r var resp *http.Response var err error + attempts := 1 + invocationID := uuid.New().String() + baseXGoogHeader := req.Header.Get("X-Goog-Api-Client") // Loop to retry the request, up to the context deadline. var pause time.Duration @@ -109,6 +115,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r } return resp, err } + invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", invocationID, attempts) + xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ") + req.Header.Set("X-Goog-Api-Client", xGoogHeader) resp, err = client.Do(req.WithContext(ctx)) @@ -123,6 +132,7 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r if req.GetBody == nil || !errorFunc(status, err) { break } + attempts++ var errBody error req.Body, errBody = req.GetBody() if errBody != nil { diff --git a/vendor/google.golang.org/api/internal/version.go b/vendor/google.golang.org/api/internal/version.go index be0f776af..40b776bd7 100644 --- a/vendor/google.golang.org/api/internal/version.go +++ b/vendor/google.golang.org/api/internal/version.go @@ -5,4 +5,4 @@ package internal // Version is the current tagged release of the library. -const Version = "0.84.0" +const Version = "0.85.0" diff --git a/vendor/modules.txt b/vendor/modules.txt index f76d20b22..a8814f5c6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,7 +11,7 @@ cloud.google.com/go/compute/metadata # cloud.google.com/go/iam v0.3.0 ## explicit; go 1.15 cloud.google.com/go/iam -# cloud.google.com/go/storage v1.22.1 +# cloud.google.com/go/storage v1.23.0 ## explicit; go 1.15 cloud.google.com/go/storage cloud.google.com/go/storage/internal @@ -34,7 +34,7 @@ github.com/VictoriaMetrics/metricsql/binaryop # github.com/VividCortex/ewma v1.2.0 ## explicit; go 1.12 github.com/VividCortex/ewma -# github.com/aws/aws-sdk-go v1.44.37 +# github.com/aws/aws-sdk-go v1.44.43 ## explicit; go 1.11 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn @@ -229,7 +229,7 @@ github.com/rivo/uniseg # github.com/russross/blackfriday/v2 v2.1.0 ## explicit github.com/russross/blackfriday/v2 -# github.com/urfave/cli/v2 v2.10.1 +# github.com/urfave/cli/v2 v2.10.3 ## explicit; go 1.18 github.com/urfave/cli/v2 # github.com/valyala/bytebufferpool v1.0.0 @@ -281,7 +281,7 @@ go.opencensus.io/trace/tracestate go.uber.org/atomic # go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 ## explicit; go 1.13 -# golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 +# golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e ## explicit; go 1.17 golang.org/x/net/context golang.org/x/net/context/ctxhttp @@ -293,7 +293,7 @@ golang.org/x/net/internal/socks golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace -# golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb +# golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 ## explicit; go 1.15 golang.org/x/oauth2 golang.org/x/oauth2/authhandler @@ -306,7 +306,7 @@ golang.org/x/oauth2/jwt # golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f ## explicit golang.org/x/sync/errgroup -# golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c +# golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b ## explicit; go 1.17 golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix @@ -321,7 +321,7 @@ golang.org/x/text/unicode/norm ## explicit; go 1.17 golang.org/x/xerrors golang.org/x/xerrors/internal -# google.golang.org/api v0.84.0 +# google.golang.org/api v0.85.0 ## explicit; go 1.15 google.golang.org/api/googleapi google.golang.org/api/googleapi/transport @@ -354,7 +354,7 @@ google.golang.org/appengine/internal/socket google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/socket google.golang.org/appengine/urlfetch -# google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad +# google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33 ## explicit; go 1.15 google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/iam/v1