vendor: make vendor-update

This commit is contained in:
Aliaksandr Valialkin 2023-02-18 23:05:15 -08:00
parent 35cf2fe3a3
commit f279ba3585
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
156 changed files with 5276 additions and 2968 deletions

38
go.mod
View file

@ -3,15 +3,15 @@ module github.com/VictoriaMetrics/VictoriaMetrics
go 1.19 go 1.19
require ( require (
cloud.google.com/go/storage v1.28.1 cloud.google.com/go/storage v1.29.0
github.com/VictoriaMetrics/fastcache v1.12.0 github.com/VictoriaMetrics/fastcache v1.12.0
// Do not use the original github.com/valyala/fasthttp because of issues // Do not use the original github.com/valyala/fasthttp because of issues
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.23.1 github.com/VictoriaMetrics/metrics v1.23.1
github.com/VictoriaMetrics/metricsql v0.51.2 github.com/VictoriaMetrics/metricsql v0.56.1
github.com/aws/aws-sdk-go v1.44.177 github.com/aws/aws-sdk-go v1.44.204
github.com/cespare/xxhash/v2 v2.2.0 github.com/cespare/xxhash/v2 v2.2.0
// TODO: switch back to https://github.com/cheggaaa/pb/v3 when v3-pooling branch // TODO: switch back to https://github.com/cheggaaa/pb/v3 when v3-pooling branch
@ -20,38 +20,38 @@ require (
github.com/dmitryk-dk/pb/v3 v3.0.9 github.com/dmitryk-dk/pb/v3 v3.0.9
github.com/golang/snappy v0.0.4 github.com/golang/snappy v0.0.4
github.com/influxdata/influxdb v1.11.0 github.com/influxdata/influxdb v1.11.0
github.com/klauspost/compress v1.15.14 github.com/klauspost/compress v1.15.15
github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9 github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9
github.com/urfave/cli/v2 v2.23.7 github.com/urfave/cli/v2 v2.24.4
github.com/valyala/fastjson v1.6.4 github.com/valyala/fastjson v1.6.4
github.com/valyala/fastrand v1.1.0 github.com/valyala/fastrand v1.1.0
github.com/valyala/fasttemplate v1.2.2 github.com/valyala/fasttemplate v1.2.2
github.com/valyala/gozstd v1.17.0 github.com/valyala/gozstd v1.18.0
github.com/valyala/quicktemplate v1.7.0 github.com/valyala/quicktemplate v1.7.0
golang.org/x/net v0.5.0 golang.org/x/net v0.7.0
golang.org/x/oauth2 v0.4.0 golang.org/x/oauth2 v0.5.0
golang.org/x/sys v0.4.0 golang.org/x/sys v0.5.0
google.golang.org/api v0.106.0 google.golang.org/api v0.110.0
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
) )
require ( require (
cloud.google.com/go v0.108.0 // indirect cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.10.0 // indirect cloud.google.com/go/iam v0.12.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/fatih/color v1.13.0 // indirect github.com/fatih/color v1.14.1 // indirect
github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/kit v0.12.0 // indirect
github.com/go-kit/log v0.2.1 // indirect github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
@ -73,10 +73,10 @@ require (
go.uber.org/atomic v1.10.0 // indirect go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 // indirect go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 // indirect
golang.org/x/sync v0.1.0 // indirect golang.org/x/sync v0.1.0 // indirect
golang.org/x/text v0.6.0 // indirect golang.org/x/text v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
google.golang.org/grpc v1.52.0 // indirect google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
) )

81
go.sum
View file

@ -15,8 +15,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@ -24,14 +24,14 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o=
cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY=
cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI= cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE=
cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
@ -42,8 +42,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go v48.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v48.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@ -91,8 +91,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0=
github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI= github.com/VictoriaMetrics/metricsql v0.56.1 h1:j+W4fA/gtozsZb4PlKDU0Ma2VOgl88xla4FEf29w94g=
github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VictoriaMetrics/metricsql v0.56.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@ -128,8 +128,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.177 h1:ckMJhU5Gj+4Rta+bJIUiUd7jvHom84aim3zkGPblq0s= github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ=
github.com/aws/aws-sdk-go v1.44.177/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -200,8 +200,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@ -227,8 +227,8 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
@ -399,7 +399,7 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@ -414,8 +414,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
@ -516,8 +516,8 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@ -552,7 +552,6 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -561,7 +560,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@ -769,8 +767,8 @@ github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.23.7 h1:YHDQ46s3VghFHFf1DdF+Sh7H4RqhcM+t0TmZRJx4oJY= github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU=
github.com/urfave/cli/v2 v2.23.7/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
@ -780,8 +778,8 @@ github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/gozstd v1.17.0 h1:M4Ds4MIrw+pD+s6vYtuFZ8D3iEw9htzfdytOV3C3iQU= github.com/valyala/gozstd v1.18.0 h1:f4BskcUZBnDrEJ2F+lVbNCMGOFBoGHEw71RBkCNR4IM=
github.com/valyala/gozstd v1.17.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/gozstd v1.18.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
github.com/valyala/quicktemplate v1.7.0 h1:LUPTJmlVcb46OOUY3IeD9DojFpAVbsG+5WFTcjMJzCM= github.com/valyala/quicktemplate v1.7.0 h1:LUPTJmlVcb46OOUY3IeD9DojFpAVbsG+5WFTcjMJzCM=
@ -934,8 +932,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -943,8 +941,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -1023,14 +1021,13 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -1043,8 +1040,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1146,8 +1143,8 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU=
google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1191,8 +1188,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 h1:EfLuoKW5WfkgVdDy7dTK8qSbH37AX5mj/MFh+bGPz14=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@ -1213,8 +1210,8 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

View file

@ -15,4 +15,4 @@
package internal package internal
// Version is the current tagged release of the library. // Version is the current tagged release of the library.
const Version = "1.15.1" const Version = "1.18.0"

View file

@ -1,5 +1,19 @@
# Changes # Changes
## [0.12.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.11.0...iam/v0.12.0) (2023-02-17)
### Features
* **iam:** Migrate to new stubs ([a61ddcd](https://github.com/googleapis/google-cloud-go/commit/a61ddcd3041c7af4a15109dc4431f9b327c497fb))
## [0.11.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.10.0...iam/v0.11.0) (2023-02-16)
### Features
* **iam:** Start generating proto stubs ([970d763](https://github.com/googleapis/google-cloud-go/commit/970d763531b54b2bc75d7ff26a20b6e05150cab8))
## [0.10.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.9.0...iam/v0.10.0) (2023-01-04) ## [0.10.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.9.0...iam/v0.10.0) (2023-01-04)

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.26.0 // protoc-gen-go v1.26.0
// protoc v3.21.9 // protoc v3.21.12
// source: google/iam/v1/iam_policy.proto // source: google/iam/v1/iam_policy.proto
package iampb package iampb

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.26.0 // protoc-gen-go v1.26.0
// protoc v3.21.9 // protoc v3.21.12
// source: google/iam/v1/options.proto // source: google/iam/v1/options.proto
package iampb package iampb

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.26.0 // protoc-gen-go v1.26.0
// protoc v3.21.9 // protoc v3.21.12
// source: google/iam/v1/policy.proto // source: google/iam/v1/policy.proto
package iampb package iampb

View file

@ -26,8 +26,8 @@ import (
"fmt" "fmt"
"time" "time"
pb "cloud.google.com/go/iam/apiv1/iampb"
gax "github.com/googleapis/gax-go/v2" gax "github.com/googleapis/gax-go/v2"
pb "google.golang.org/genproto/googleapis/iam/v1"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"

View file

@ -548,6 +548,15 @@
"release_level": "beta", "release_level": "beta",
"library_type": "GAPIC_AUTO" "library_type": "GAPIC_AUTO"
}, },
"cloud.google.com/go/datacatalog/lineage/apiv1": {
"distribution_name": "cloud.google.com/go/datacatalog/lineage/apiv1",
"description": "Data Lineage API",
"language": "Go",
"client_library_type": "generated",
"docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datacatalog/latest/lineage/apiv1",
"release_level": "beta",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/dataflow/apiv1beta3": { "cloud.google.com/go/dataflow/apiv1beta3": {
"distribution_name": "cloud.google.com/go/dataflow/apiv1beta3", "distribution_name": "cloud.google.com/go/dataflow/apiv1beta3",
"description": "Dataflow API", "description": "Dataflow API",
@ -710,6 +719,15 @@
"release_level": "beta", "release_level": "beta",
"library_type": "GAPIC_AUTO" "library_type": "GAPIC_AUTO"
}, },
"cloud.google.com/go/discoveryengine/apiv1beta": {
"distribution_name": "cloud.google.com/go/discoveryengine/apiv1beta",
"description": "Discovery Engine API",
"language": "Go",
"client_library_type": "generated",
"docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/discoveryengine/latest/apiv1beta",
"release_level": "beta",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/dlp/apiv2": { "cloud.google.com/go/dlp/apiv2": {
"distribution_name": "cloud.google.com/go/dlp/apiv2", "distribution_name": "cloud.google.com/go/dlp/apiv2",
"description": "Cloud Data Loss Prevention (DLP) API", "description": "Cloud Data Loss Prevention (DLP) API",
@ -1079,6 +1097,15 @@
"release_level": "beta", "release_level": "beta",
"library_type": "GAPIC_AUTO" "library_type": "GAPIC_AUTO"
}, },
"cloud.google.com/go/maps/mapsplatformdatasets/apiv1alpha": {
"distribution_name": "cloud.google.com/go/maps/mapsplatformdatasets/apiv1alpha",
"description": "Maps Platform Datasets API",
"language": "Go",
"client_library_type": "generated",
"docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/maps/latest/mapsplatformdatasets/apiv1alpha",
"release_level": "alpha",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/maps/routing/apiv2": { "cloud.google.com/go/maps/routing/apiv2": {
"distribution_name": "cloud.google.com/go/maps/routing/apiv2", "distribution_name": "cloud.google.com/go/maps/routing/apiv2",
"description": "Routes API", "description": "Routes API",

View file

@ -16,3 +16,28 @@ each package, which is the pattern followed by some other languages. External
tools would then talk to pkg.go.dev or some other service to get the overall tools would then talk to pkg.go.dev or some other service to get the overall
list of packages and use the `.repo-metadata.json` files to get the additional list of packages and use the `.repo-metadata.json` files to get the additional
metadata required. For now, `.repo-metadata-full.json` includes everything. metadata required. For now, `.repo-metadata-full.json` includes everything.
## cloudbuild.yaml
To kick off a build locally run from the repo root:
```bash
gcloud builds submit --project=cloud-devrel-kokoro-resources --config=internal/cloudbuild.yaml
```
### Updating OwlBot SHA
You may want to manually update the which version of the post processor will be
used -- to do this you need to update the SHA in the OwlBot lock file. Start by
running the following commands:
```bash
docker pull gcr.io/cloud-devrel-public-resources/owlbot-go:latest
docker inspect --format='{{index .RepoDigests 0}}' gcr.io/cloud-devrel-public-resources/owlbot-go:latest
```
This will give you a SHA. You can use this value to update the value in
`.github/.OwlBot.lock.yaml`.
*Note*: OwlBot will eventually open a pull request to update this value if it
discovers a new version of the container.

25
vendor/cloud.google.com/go/internal/cloudbuild.yaml generated vendored Normal file
View file

@ -0,0 +1,25 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# note: /workspace is a special directory in the docker image where all the files in this folder
# get placed on your behalf
timeout: 7200s # 2 hours
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'gcr.io/cloud-devrel-public-resources/owlbot-go', '-f', 'postprocessor/Dockerfile', '.']
dir: internal
images:
- gcr.io/cloud-devrel-public-resources/owlbot-go:latest

View file

@ -1,6 +1,19 @@
# Changes # Changes
## [1.29.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.28.1...storage/v1.29.0) (2023-01-19)
### Features
* **storage:** Add ComponentCount as part of ObjectAttrs ([#7230](https://github.com/googleapis/google-cloud-go/issues/7230)) ([a19bca6](https://github.com/googleapis/google-cloud-go/commit/a19bca60704b4fbb674cf51d828580aa653c8210))
* **storage:** Add REST client ([06a54a1](https://github.com/googleapis/google-cloud-go/commit/06a54a16a5866cce966547c51e203b9e09a25bc0))
### Documentation
* **storage/internal:** Corrected typos and spellings ([7357077](https://github.com/googleapis/google-cloud-go/commit/735707796d81d7f6f32fc3415800c512fe62297e))
## [1.28.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.28.0...storage/v1.28.1) (2022-12-02) ## [1.28.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.28.0...storage/v1.28.1) (2022-12-02)

View file

@ -35,6 +35,7 @@ import (
raw "google.golang.org/api/storage/v1" raw "google.golang.org/api/storage/v1"
dpb "google.golang.org/genproto/googleapis/type/date" dpb "google.golang.org/genproto/googleapis/type/date"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
) )
// BucketHandle provides operations on a Google Cloud Storage bucket. // BucketHandle provides operations on a Google Cloud Storage bucket.
@ -1389,12 +1390,12 @@ func (rp *RetentionPolicy) toProtoRetentionPolicy() *storagepb.Bucket_RetentionP
} }
// RetentionPeriod must be greater than 0, so if it is 0, the user left it // RetentionPeriod must be greater than 0, so if it is 0, the user left it
// unset, and so we should not send it in the request i.e. nil is sent. // unset, and so we should not send it in the request i.e. nil is sent.
var period *int64 var dur *durationpb.Duration
if rp.RetentionPeriod != 0 { if rp.RetentionPeriod != 0 {
period = proto.Int64(int64(rp.RetentionPeriod / time.Second)) dur = durationpb.New(rp.RetentionPeriod)
} }
return &storagepb.Bucket_RetentionPolicy{ return &storagepb.Bucket_RetentionPolicy{
RetentionPeriod: period, RetentionDuration: dur,
} }
} }
@ -1418,7 +1419,7 @@ func toRetentionPolicyFromProto(rp *storagepb.Bucket_RetentionPolicy) *Retention
return nil return nil
} }
return &RetentionPolicy{ return &RetentionPolicy{
RetentionPeriod: time.Duration(rp.GetRetentionPeriod()) * time.Second, RetentionPeriod: rp.GetRetentionDuration().AsDuration(),
EffectiveTime: rp.GetEffectiveTime().AsTime(), EffectiveTime: rp.GetEffectiveTime().AsTime(),
IsLocked: rp.GetIsLocked(), IsLocked: rp.GetIsLocked(),
} }

View file

@ -34,7 +34,6 @@ import (
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
fieldmaskpb "google.golang.org/protobuf/types/known/fieldmaskpb" fieldmaskpb "google.golang.org/protobuf/types/known/fieldmaskpb"
) )
@ -1496,11 +1495,16 @@ func (w *gRPCWriter) startResumableUpload() error {
if err != nil { if err != nil {
return err return err
} }
return run(w.ctx, func() error { req := &storagepb.StartResumableWriteRequest{
upres, err := w.c.raw.StartResumableWrite(w.ctx, &storagepb.StartResumableWriteRequest{
WriteObjectSpec: spec, WriteObjectSpec: spec,
CommonObjectRequestParams: toProtoCommonObjectRequestParams(w.encryptionKey), CommonObjectRequestParams: toProtoCommonObjectRequestParams(w.encryptionKey),
}) }
// TODO: Currently the checksums are only sent on the request to initialize
// the upload, but in the future, we must also support sending it
// on the *last* message of the stream.
req.ObjectChecksums = toProtoChecksums(w.sendCRC32C, w.attrs)
return run(w.ctx, func() error {
upres, err := w.c.raw.StartResumableWrite(w.ctx, req)
w.upid = upres.GetUploadId() w.upid = upres.GetUploadId()
return err return err
}, w.settings.retry, w.settings.idempotent, setRetryHeaderGRPC(w.ctx)) }, w.settings.retry, w.settings.idempotent, setRetryHeaderGRPC(w.ctx))
@ -1585,25 +1589,13 @@ func (w *gRPCWriter) uploadBuffer(recvd int, start int64, doneReading bool) (*st
WriteObjectSpec: spec, WriteObjectSpec: spec,
} }
req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(w.encryptionKey) req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(w.encryptionKey)
} // For a non-resumable upload, checksums must be sent in this message.
// TODO: Currently the checksums are only sent on the first message // TODO: Currently the checksums are only sent on the first message
// of the stream, but in the future, we must also support sending it // of the stream, but in the future, we must also support sending it
// on the *last* message of the stream (instead of the first). // on the *last* message of the stream (instead of the first).
if w.sendCRC32C { req.ObjectChecksums = toProtoChecksums(w.sendCRC32C, w.attrs)
req.ObjectChecksums = &storagepb.ObjectChecksums{
Crc32C: proto.Uint32(w.attrs.CRC32C),
}
}
if len(w.attrs.MD5) != 0 {
if cs := req.GetObjectChecksums(); cs == nil {
req.ObjectChecksums = &storagepb.ObjectChecksums{
Md5Hash: w.attrs.MD5,
}
} else {
cs.Md5Hash = w.attrs.MD5
}
} }
} }
err = w.stream.Send(req) err = w.stream.Send(req)

View file

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC // Copyright 2023 Google LLC
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -21,6 +21,11 @@
// //
// NOTE: This package is in alpha. It is not stable, and is likely to change. // NOTE: This package is in alpha. It is not stable, and is likely to change.
// //
// # General documentation
//
// For information about setting deadlines, reusing contexts, and more
// please visit https://pkg.go.dev/cloud.google.com/go.
//
// # Example usage // # Example usage
// //
// To get started with this package, create a client. // To get started with this package, create a client.
@ -73,9 +78,6 @@
// Individual methods on the client use the ctx given to them. // Individual methods on the client use the ctx given to them.
// //
// To close the open connection, use the Close() method. // To close the open connection, use the Close() method.
//
// For information about setting deadlines, reusing contexts, and more
// please visit https://pkg.go.dev/cloud.google.com/go.
package storage // import "cloud.google.com/go/storage/internal/apiv2" package storage // import "cloud.google.com/go/storage/internal/apiv2"
import ( import (

View file

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC // Copyright 2023 Google LLC
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -238,17 +238,26 @@ func (c *Client) LockBucketRetentionPolicy(ctx context.Context, req *storagepb.L
} }
// GetIamPolicy gets the IAM policy for a specified bucket or object. // GetIamPolicy gets the IAM policy for a specified bucket or object.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { func (c *Client) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
return c.internalClient.GetIamPolicy(ctx, req, opts...) return c.internalClient.GetIamPolicy(ctx, req, opts...)
} }
// SetIamPolicy updates an IAM policy for the specified bucket or object. // SetIamPolicy updates an IAM policy for the specified bucket or object.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { func (c *Client) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
return c.internalClient.SetIamPolicy(ctx, req, opts...) return c.internalClient.SetIamPolicy(ctx, req, opts...)
} }
// TestIamPermissions tests a set of permissions on the given bucket or object to see which, if // TestIamPermissions tests a set of permissions on the given bucket or object to see which, if
// any, are held by the caller. // any, are held by the caller.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) { func (c *Client) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) {
return c.internalClient.TestIamPermissions(ctx, req, opts...) return c.internalClient.TestIamPermissions(ctx, req, opts...)
} }
@ -1048,6 +1057,7 @@ func (c *gRPCClient) ReadObject(ctx context.Context, req *storagepb.ReadObjectRe
md := metadata.Pairs("x-goog-request-params", routingHeaders) md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md) ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ReadObject[0:len((*c.CallOptions).ReadObject):len((*c.CallOptions).ReadObject)], opts...)
var resp storagepb.Storage_ReadObjectClient var resp storagepb.Storage_ReadObjectClient
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error var err error

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC // Copyright 2023 Google LLC
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.

View file

@ -15,4 +15,4 @@
package internal package internal
// Version is the current tagged release of the library. // Version is the current tagged release of the library.
const Version = "1.28.1" const Version = "1.29.0"

View file

@ -1315,6 +1315,11 @@ type ObjectAttrs struct {
// later value but not to an earlier one. For more information see // later value but not to an earlier one. For more information see
// https://cloud.google.com/storage/docs/metadata#custom-time . // https://cloud.google.com/storage/docs/metadata#custom-time .
CustomTime time.Time CustomTime time.Time
// ComponentCount is the number of objects contained within a composite object.
// For non-composite objects, the value will be zero.
// This field is read-only.
ComponentCount int64
} }
// convertTime converts a time in RFC3339 format to time.Time. // convertTime converts a time in RFC3339 format to time.Time.
@ -1385,6 +1390,7 @@ func newObject(o *raw.Object) *ObjectAttrs {
Updated: convertTime(o.Updated), Updated: convertTime(o.Updated),
Etag: o.Etag, Etag: o.Etag,
CustomTime: convertTime(o.CustomTime), CustomTime: convertTime(o.CustomTime),
ComponentCount: o.ComponentCount,
} }
} }
@ -1419,6 +1425,7 @@ func newObjectFromProto(o *storagepb.Object) *ObjectAttrs {
Deleted: convertProtoTime(o.GetDeleteTime()), Deleted: convertProtoTime(o.GetDeleteTime()),
Updated: convertProtoTime(o.GetUpdateTime()), Updated: convertProtoTime(o.GetUpdateTime()),
CustomTime: convertProtoTime(o.GetCustomTime()), CustomTime: convertProtoTime(o.GetCustomTime()),
ComponentCount: int64(o.ComponentCount),
} }
} }
@ -1547,6 +1554,7 @@ var attrToFieldMap = map[string]string{
"Updated": "updated", "Updated": "updated",
"Etag": "etag", "Etag": "etag",
"CustomTime": "customTime", "CustomTime": "customTime",
"ComponentCount": "componentCount",
} }
// attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field // attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field
@ -1578,6 +1586,7 @@ var attrToProtoFieldMap = map[string]string{
"Owner": "owner", "Owner": "owner",
"CustomerKeySHA256": "customer_encryption", "CustomerKeySHA256": "customer_encryption",
"CustomTime": "custom_time", "CustomTime": "custom_time",
"ComponentCount": "component_count",
// MediaLink was explicitly excluded from the proto as it is an HTTP-ism. // MediaLink was explicitly excluded from the proto as it is an HTTP-ism.
// "MediaLink": "mediaLink", // "MediaLink": "mediaLink",
} }
@ -2079,6 +2088,25 @@ func toProtoCommonObjectRequestParams(key []byte) *storagepb.CommonObjectRequest
} }
} }
func toProtoChecksums(sendCRC32C bool, attrs *ObjectAttrs) *storagepb.ObjectChecksums {
var checksums *storagepb.ObjectChecksums
if sendCRC32C {
checksums = &storagepb.ObjectChecksums{
Crc32C: proto.Uint32(attrs.CRC32C),
}
}
if len(attrs.MD5) != 0 {
if checksums == nil {
checksums = &storagepb.ObjectChecksums{
Md5Hash: attrs.MD5,
}
} else {
checksums.Md5Hash = attrs.MD5
}
}
return checksums
}
// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. // ServiceAccount fetches the email address of the given project's Google Cloud Storage service account.
func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) {
o := makeStorageOpts(true, c.retry, "") o := makeStorageOpts(true, c.retry, "")

View file

@ -29,6 +29,7 @@ var aggrFuncs = map[string]bool{
"outliersk": true, "outliersk": true,
"quantile": true, "quantile": true,
"quantiles": true, "quantiles": true,
"share": true,
"stddev": true, "stddev": true,
"stdvar": true, "stdvar": true,
"sum": true, "sum": true,

View file

@ -392,7 +392,8 @@ func getTransformArgIdxForOptimization(funcName string, args []Expr) int {
return -1 return -1
case "limit_offset": case "limit_offset":
return 2 return 2
case "buckets_limit", "histogram_quantile", "histogram_share", "range_quantile", "range_trim_spikes": case "buckets_limit", "histogram_quantile", "histogram_share", "range_quantile",
"range_trim_outliers", "range_trim_spikes", "range_trim_zscore":
return 1 return 1
case "histogram_quantiles": case "histogram_quantiles":
return len(args) - 1 return len(args) - 1

View file

@ -42,6 +42,7 @@ var rollupFuncs = map[string]bool{
"lag": true, "lag": true,
"last_over_time": true, "last_over_time": true,
"lifetime": true, "lifetime": true,
"mad_over_time": true,
"max_over_time": true, "max_over_time": true,
"min_over_time": true, "min_over_time": true,
"mode_over_time": true, "mode_over_time": true,

View file

@ -74,6 +74,7 @@ var transformFuncs = map[string]bool{
"range_first": true, "range_first": true,
"range_last": true, "range_last": true,
"range_linear_regression": true, "range_linear_regression": true,
"range_mad": true,
"range_max": true, "range_max": true,
"range_min": true, "range_min": true,
"range_normalize": true, "range_normalize": true,
@ -81,7 +82,10 @@ var transformFuncs = map[string]bool{
"range_stddev": true, "range_stddev": true,
"range_stdvar": true, "range_stdvar": true,
"range_sum": true, "range_sum": true,
"range_trim_outliers": true,
"range_trim_spikes": true, "range_trim_spikes": true,
"range_trim_zscore": true,
"range_zscore": true,
"remove_resets": true, "remove_resets": true,
"round": true, "round": true,
"running_avg": true, "running_avg": true,

View file

@ -226,11 +226,23 @@ func NewCredentialsCommand(command *exec.Cmd, options ...func(*ProcessProvider))
return credentials.NewCredentials(p) return credentials.NewCredentials(p)
} }
type credentialProcessResponse struct { // A CredentialProcessResponse is the AWS credentials format that must be
// returned when executing an external credential_process.
type CredentialProcessResponse struct {
// As of this writing, the Version key must be set to 1. This might
// increment over time as the structure evolves.
Version int Version int
// The access key ID that identifies the temporary security credentials.
AccessKeyID string `json:"AccessKeyId"` AccessKeyID string `json:"AccessKeyId"`
// The secret access key that can be used to sign requests.
SecretAccessKey string SecretAccessKey string
// The token that users must pass to the service API to use the temporary credentials.
SessionToken string SessionToken string
// The date on which the current credentials expire.
Expiration *time.Time Expiration *time.Time
} }
@ -242,7 +254,7 @@ func (p *ProcessProvider) Retrieve() (credentials.Value, error) {
} }
// Serialize and validate response // Serialize and validate response
resp := &credentialProcessResponse{} resp := &CredentialProcessResponse{}
if err = json.Unmarshal(out, resp); err != nil { if err = json.Unmarshal(out, resp); err != nil {
return credentials.Value{ProviderName: ProviderName}, awserr.New( return credentials.Value{ProviderName: ProviderName}, awserr.New(
ErrCodeProcessProviderParse, ErrCodeProcessProviderParse,

File diff suppressed because it is too large Load diff

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go" const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK // SDKVersion is the version of this SDK
const SDKVersion = "1.44.177" const SDKVersion = "1.44.204"

View file

@ -35,6 +35,8 @@ func NewUnmarshalTypedError(exceptions map[string]func(protocol.ResponseMetadata
} }
} }
// NewUnmarshalTypedErrorWithOptions works similar to NewUnmarshalTypedError applying options to the UnmarshalTypedError
// before returning it
func NewUnmarshalTypedErrorWithOptions(exceptions map[string]func(protocol.ResponseMetadata) error, optFns ...func(*UnmarshalTypedError)) *UnmarshalTypedError { func NewUnmarshalTypedErrorWithOptions(exceptions map[string]func(protocol.ResponseMetadata) error, optFns ...func(*UnmarshalTypedError)) *UnmarshalTypedError {
unmarshaledError := NewUnmarshalTypedError(exceptions) unmarshaledError := NewUnmarshalTypedError(exceptions)
for _, fn := range optFns { for _, fn := range optFns {
@ -43,6 +45,11 @@ func NewUnmarshalTypedErrorWithOptions(exceptions map[string]func(protocol.Respo
return unmarshaledError return unmarshaledError
} }
// WithQueryCompatibility is a helper function to construct a functional option for use with NewUnmarshalTypedErrorWithOptions.
// The queryExceptions given act as an override for unmarshalling errors when query compatible error codes are found.
// See also [awsQueryCompatible trait]
//
// [awsQueryCompatible trait]: https://smithy.io/2.0/aws/protocols/aws-query-protocol.html#aws-protocols-awsquerycompatible-trait
func WithQueryCompatibility(queryExceptions map[string]func(protocol.ResponseMetadata, string) error) func(*UnmarshalTypedError) { func WithQueryCompatibility(queryExceptions map[string]func(protocol.ResponseMetadata, string) error) func(*UnmarshalTypedError) {
return func(typedError *UnmarshalTypedError) { return func(typedError *UnmarshalTypedError) {
typedError.queryExceptions = queryExceptions typedError.queryExceptions = queryExceptions

View file

@ -25,5 +25,5 @@ func add100Continue(r *request.Request) {
return return
} }
r.HTTPRequest.Header.Set("Expect", "100-Continue") r.HTTPRequest.Header.Set("Expect", "100-continue")
} }

View file

@ -56,12 +56,11 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
// AssumeRole API operation for AWS Security Token Service. // AssumeRole API operation for AWS Security Token Service.
// //
// Returns a set of temporary security credentials that you can use to access // Returns a set of temporary security credentials that you can use to access
// Amazon Web Services resources that you might not normally have access to. // Amazon Web Services resources. These temporary credentials consist of an
// These temporary credentials consist of an access key ID, a secret access // access key ID, a secret access key, and a security token. Typically, you
// key, and a security token. Typically, you use AssumeRole within your account // use AssumeRole within your account or for cross-account access. For a comparison
// or for cross-account access. For a comparison of AssumeRole with other API // of AssumeRole with other API operations that produce temporary credentials,
// operations that produce temporary credentials, see Requesting Temporary Security // see Requesting Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide. // in the IAM User Guide.
// //
@ -1103,13 +1102,15 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
// # Permissions // # Permissions
// //
// You can use the temporary credentials created by GetFederationToken in any // You can use the temporary credentials created by GetFederationToken in any
// Amazon Web Services service except the following: // Amazon Web Services service with the following exceptions:
// //
// - You cannot call any IAM operations using the CLI or the Amazon Web Services // - You cannot call any IAM operations using the CLI or the Amazon Web Services
// API. // API. This limitation does not apply to console sessions.
// //
// - You cannot call any STS operations except GetCallerIdentity. // - You cannot call any STS operations except GetCallerIdentity.
// //
// You can use temporary credentials for single sign-on (SSO) to the console.
//
// You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// to this operation. You can pass a single JSON policy document to use as an // to this operation. You can pass a single JSON policy document to use as an
// inline session policy. You can also specify up to 10 managed policy Amazon // inline session policy. You can also specify up to 10 managed policy Amazon

View file

@ -7,7 +7,6 @@ suits you.
![Color](https://user-images.githubusercontent.com/438920/96832689-03b3e000-13f4-11eb-9803-46f4c4de3406.jpg) ![Color](https://user-images.githubusercontent.com/438920/96832689-03b3e000-13f4-11eb-9803-46f4c4de3406.jpg)
## Install ## Install
```bash ```bash
@ -130,11 +129,11 @@ There might be a case where you want to explicitly disable/enable color output.
(for example if the output were piped directly to `less`). (for example if the output were piped directly to `less`).
The `color` package also disables color output if the [`NO_COLOR`](https://no-color.org) environment The `color` package also disables color output if the [`NO_COLOR`](https://no-color.org) environment
variable is set (regardless of its value). variable is set to a non-empty string.
`Color` has support to disable/enable colors programatically both globally and `Color` has support to disable/enable colors programmatically both globally and
for single color definitions. For example suppose you have a CLI app and a for single color definitions. For example suppose you have a CLI app and a
`--no-color` bool flag. You can easily disable the color output with: `-no-color` bool flag. You can easily disable the color output with:
```go ```go
var flagNoColor = flag.Bool("no-color", false, "Disable color output") var flagNoColor = flag.Bool("no-color", false, "Disable color output")
@ -167,11 +166,10 @@ To output color in GitHub Actions (or other CI systems that support ANSI colors)
* Save/Return previous values * Save/Return previous values
* Evaluate fmt.Formatter interface * Evaluate fmt.Formatter interface
## Credits ## Credits
* [Fatih Arslan](https://github.com/fatih) * [Fatih Arslan](https://github.com/fatih)
* Windows support via @mattn: [colorable](https://github.com/mattn/go-colorable) * Windows support via @mattn: [colorable](https://github.com/mattn/go-colorable)
## License ## License

View file

@ -19,10 +19,10 @@ var (
// set (regardless of its value). This is a global option and affects all // set (regardless of its value). This is a global option and affects all
// colors. For more control over each color block use the methods // colors. For more control over each color block use the methods
// DisableColor() individually. // DisableColor() individually.
NoColor = noColorExists() || os.Getenv("TERM") == "dumb" || NoColor = noColorIsSet() || os.Getenv("TERM") == "dumb" ||
(!isatty.IsTerminal(os.Stdout.Fd()) && !isatty.IsCygwinTerminal(os.Stdout.Fd())) (!isatty.IsTerminal(os.Stdout.Fd()) && !isatty.IsCygwinTerminal(os.Stdout.Fd()))
// Output defines the standard output of the print functions. By default // Output defines the standard output of the print functions. By default,
// os.Stdout is used. // os.Stdout is used.
Output = colorable.NewColorableStdout() Output = colorable.NewColorableStdout()
@ -35,10 +35,9 @@ var (
colorsCacheMu sync.Mutex // protects colorsCache colorsCacheMu sync.Mutex // protects colorsCache
) )
// noColorExists returns true if the environment variable NO_COLOR exists. // noColorIsSet returns true if the environment variable NO_COLOR is set to a non-empty string.
func noColorExists() bool { func noColorIsSet() bool {
_, exists := os.LookupEnv("NO_COLOR") return os.Getenv("NO_COLOR") != ""
return exists
} }
// Color defines a custom color object which is defined by SGR parameters. // Color defines a custom color object which is defined by SGR parameters.
@ -120,7 +119,7 @@ func New(value ...Attribute) *Color {
params: make([]Attribute, 0), params: make([]Attribute, 0),
} }
if noColorExists() { if noColorIsSet() {
c.noColor = boolPtr(true) c.noColor = boolPtr(true)
} }
@ -152,7 +151,7 @@ func (c *Color) Set() *Color {
return c return c
} }
fmt.Fprintf(Output, c.format()) fmt.Fprint(Output, c.format())
return c return c
} }
@ -164,16 +163,21 @@ func (c *Color) unset() {
Unset() Unset()
} }
func (c *Color) setWriter(w io.Writer) *Color { // SetWriter is used to set the SGR sequence with the given io.Writer. This is
// a low-level function, and users should use the higher-level functions, such
// as color.Fprint, color.Print, etc.
func (c *Color) SetWriter(w io.Writer) *Color {
if c.isNoColorSet() { if c.isNoColorSet() {
return c return c
} }
fmt.Fprintf(w, c.format()) fmt.Fprint(w, c.format())
return c return c
} }
func (c *Color) unsetWriter(w io.Writer) { // UnsetWriter resets all escape attributes and clears the output with the give
// io.Writer. Usually should be called after SetWriter().
func (c *Color) UnsetWriter(w io.Writer) {
if c.isNoColorSet() { if c.isNoColorSet() {
return return
} }
@ -192,20 +196,14 @@ func (c *Color) Add(value ...Attribute) *Color {
return c return c
} }
func (c *Color) prepend(value Attribute) {
c.params = append(c.params, 0)
copy(c.params[1:], c.params[0:])
c.params[0] = value
}
// Fprint formats using the default formats for its operands and writes to w. // Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string. // Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered. // It returns the number of bytes written and any write error encountered.
// On Windows, users should wrap w with colorable.NewColorable() if w is of // On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File. // type *os.File.
func (c *Color) Fprint(w io.Writer, a ...interface{}) (n int, err error) { func (c *Color) Fprint(w io.Writer, a ...interface{}) (n int, err error) {
c.setWriter(w) c.SetWriter(w)
defer c.unsetWriter(w) defer c.UnsetWriter(w)
return fmt.Fprint(w, a...) return fmt.Fprint(w, a...)
} }
@ -227,8 +225,8 @@ func (c *Color) Print(a ...interface{}) (n int, err error) {
// On Windows, users should wrap w with colorable.NewColorable() if w is of // On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File. // type *os.File.
func (c *Color) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { func (c *Color) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
c.setWriter(w) c.SetWriter(w)
defer c.unsetWriter(w) defer c.UnsetWriter(w)
return fmt.Fprintf(w, format, a...) return fmt.Fprintf(w, format, a...)
} }
@ -248,8 +246,8 @@ func (c *Color) Printf(format string, a ...interface{}) (n int, err error) {
// On Windows, users should wrap w with colorable.NewColorable() if w is of // On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File. // type *os.File.
func (c *Color) Fprintln(w io.Writer, a ...interface{}) (n int, err error) { func (c *Color) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
c.setWriter(w) c.SetWriter(w)
defer c.unsetWriter(w) defer c.UnsetWriter(w)
return fmt.Fprintln(w, a...) return fmt.Fprintln(w, a...)
} }
@ -396,7 +394,7 @@ func (c *Color) DisableColor() {
} }
// EnableColor enables the color output. Use it in conjunction with // EnableColor enables the color output. Use it in conjunction with
// DisableColor(). Otherwise this method has no side effects. // DisableColor(). Otherwise, this method has no side effects.
func (c *Color) EnableColor() { func (c *Color) EnableColor() {
c.noColor = boolPtr(false) c.noColor = boolPtr(false)
} }

View file

@ -20,7 +20,7 @@ Use simple and default helper functions with predefined foreground colors:
color.HiBlack("Bright black means gray..") color.HiBlack("Bright black means gray..")
color.HiWhite("Shiny white color!") color.HiWhite("Shiny white color!")
However there are times where custom color mixes are required. Below are some However, there are times when custom color mixes are required. Below are some
examples to create custom color objects and use the print functions of each examples to create custom color objects and use the print functions of each
separate color object. separate color object.
@ -68,7 +68,6 @@ You can also FprintXxx functions to pass your own io.Writer:
success := color.New(color.Bold, color.FgGreen).FprintlnFunc() success := color.New(color.Bold, color.FgGreen).FprintlnFunc()
success(myWriter, don't forget this...") success(myWriter, don't forget this...")
Or create SprintXxx functions to mix strings with other non-colorized strings: Or create SprintXxx functions to mix strings with other non-colorized strings:
yellow := New(FgYellow).SprintFunc() yellow := New(FgYellow).SprintFunc()
@ -80,7 +79,7 @@ Or create SprintXxx functions to mix strings with other non-colorized strings:
fmt.Printf("this %s rocks!\n", info("package")) fmt.Printf("this %s rocks!\n", info("package"))
Windows support is enabled by default. All Print functions work as intended. Windows support is enabled by default. All Print functions work as intended.
However only for color.SprintXXX functions, user should use fmt.FprintXXX and However, only for color.SprintXXX functions, user should use fmt.FprintXXX and
set the output to color.Output: set the output to color.Output:
fmt.Fprintf(color.Output, "Windows support: %s", color.GreenString("PASS")) fmt.Fprintf(color.Output, "Windows support: %s", color.GreenString("PASS"))

View file

@ -1,48 +1,82 @@
# Changelog # Changelog
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.5.0] - 2020-01-03 ## [0.6.0] - 2023-01-30
[0.6.0]: https://github.com/go-logfmt/logfmt/compare/v0.5.1...v0.6.0
### Added
- NewDecoderSize by [@alexanderjophus]
## [0.5.1] - 2021-08-18
[0.5.1]: https://github.com/go-logfmt/logfmt/compare/v0.5.0...v0.5.1
### Changed ### Changed
- Update the `go.mod` file for Go 1.17 as described in the [Go 1.17 release
notes](https://golang.org/doc/go1.17#go-command)
## [0.5.0] - 2020-01-03
[0.5.0]: https://github.com/go-logfmt/logfmt/compare/v0.4.0...v0.5.0
### Changed
- Remove the dependency on github.com/kr/logfmt by [@ChrisHines] - Remove the dependency on github.com/kr/logfmt by [@ChrisHines]
- Move fuzz code to github.com/go-logfmt/fuzzlogfmt by [@ChrisHines] - Move fuzz code to github.com/go-logfmt/fuzzlogfmt by [@ChrisHines]
## [0.4.0] - 2018-11-21 ## [0.4.0] - 2018-11-21
[0.4.0]: https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.4.0
### Added ### Added
- Go module support by [@ChrisHines] - Go module support by [@ChrisHines]
- CHANGELOG by [@ChrisHines] - CHANGELOG by [@ChrisHines]
### Changed ### Changed
- Drop invalid runes from keys instead of returning ErrInvalidKey by [@ChrisHines] - Drop invalid runes from keys instead of returning ErrInvalidKey by [@ChrisHines]
- On panic while printing, attempt to print panic value by [@bboreham] - On panic while printing, attempt to print panic value by [@bboreham]
## [0.3.0] - 2016-11-15 ## [0.3.0] - 2016-11-15
[0.3.0]: https://github.com/go-logfmt/logfmt/compare/v0.2.0...v0.3.0
### Added ### Added
- Pool buffers for quoted strings and byte slices by [@nussjustin] - Pool buffers for quoted strings and byte slices by [@nussjustin]
### Fixed ### Fixed
- Fuzz fix, quote invalid UTF-8 values by [@judwhite] - Fuzz fix, quote invalid UTF-8 values by [@judwhite]
## [0.2.0] - 2016-05-08 ## [0.2.0] - 2016-05-08
[0.2.0]: https://github.com/go-logfmt/logfmt/compare/v0.1.0...v0.2.0
### Added ### Added
- Encoder.EncodeKeyvals by [@ChrisHines] - Encoder.EncodeKeyvals by [@ChrisHines]
## [0.1.0] - 2016-03-28 ## [0.1.0] - 2016-03-28
[0.1.0]: https://github.com/go-logfmt/logfmt/commits/v0.1.0
### Added ### Added
- Encoder by [@ChrisHines] - Encoder by [@ChrisHines]
- Decoder by [@ChrisHines] - Decoder by [@ChrisHines]
- MarshalKeyvals by [@ChrisHines] - MarshalKeyvals by [@ChrisHines]
[0.5.0]: https://github.com/go-logfmt/logfmt/compare/v0.4.0...v0.5.0
[0.4.0]: https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.4.0
[0.3.0]: https://github.com/go-logfmt/logfmt/compare/v0.2.0...v0.3.0
[0.2.0]: https://github.com/go-logfmt/logfmt/compare/v0.1.0...v0.2.0
[0.1.0]: https://github.com/go-logfmt/logfmt/commits/v0.1.0
[@ChrisHines]: https://github.com/ChrisHines [@ChrisHines]: https://github.com/ChrisHines
[@bboreham]: https://github.com/bboreham [@bboreham]: https://github.com/bboreham
[@judwhite]: https://github.com/judwhite [@judwhite]: https://github.com/judwhite
[@nussjustin]: https://github.com/nussjustin [@nussjustin]: https://github.com/nussjustin
[@alexanderjophus]: https://github.com/alexanderjophus

View file

@ -1,20 +1,25 @@
# logfmt
[![Go Reference](https://pkg.go.dev/badge/github.com/go-logfmt/logfmt.svg)](https://pkg.go.dev/github.com/go-logfmt/logfmt) [![Go Reference](https://pkg.go.dev/badge/github.com/go-logfmt/logfmt.svg)](https://pkg.go.dev/github.com/go-logfmt/logfmt)
[![Go Report Card](https://goreportcard.com/badge/go-logfmt/logfmt)](https://goreportcard.com/report/go-logfmt/logfmt) [![Go Report Card](https://goreportcard.com/badge/go-logfmt/logfmt)](https://goreportcard.com/report/go-logfmt/logfmt)
[![Github Actions](https://github.com/go-logfmt/logfmt/actions/workflows/test.yml/badge.svg)](https://github.com/go-logfmt/logfmt/actions/workflows/test.yml) [![Github Actions](https://github.com/go-logfmt/logfmt/actions/workflows/test.yml/badge.svg)](https://github.com/go-logfmt/logfmt/actions/workflows/test.yml)
[![Coverage Status](https://coveralls.io/repos/github/go-logfmt/logfmt/badge.svg?branch=master)](https://coveralls.io/github/go-logfmt/logfmt?branch=master) [![Coverage Status](https://coveralls.io/repos/github/go-logfmt/logfmt/badge.svg?branch=master)](https://coveralls.io/github/go-logfmt/logfmt?branch=main)
# logfmt
Package logfmt implements utilities to marshal and unmarshal data in the [logfmt Package logfmt implements utilities to marshal and unmarshal data in the [logfmt
format](https://brandur.org/logfmt). It provides an API similar to format][fmt]. It provides an API similar to [encoding/json][json] and
[encoding/json](http://golang.org/pkg/encoding/json/) and [encoding/xml][xml].
[encoding/xml](http://golang.org/pkg/encoding/xml/).
[fmt]: https://brandur.org/logfmt
[json]: https://pkg.go.dev/encoding/json
[xml]: https://pkg.go.dev/encoding/xml
The logfmt format was first documented by Brandur Leach in [this The logfmt format was first documented by Brandur Leach in [this
article](https://brandur.org/logfmt). The format has not been formally article][origin]. The format has not been formally standardized. The most
standardized. The most authoritative public specification to date has been the authoritative public specification to date has been the documentation of a Go
documentation of a Go Language [package](http://godoc.org/github.com/kr/logfmt) Language [package][parser] written by Blake Mizerany and Keith Rarick.
written by Blake Mizerany and Keith Rarick.
[origin]: https://brandur.org/logfmt
[parser]: https://pkg.go.dev/github.com/kr/logfmt
## Goals ## Goals
@ -30,4 +35,7 @@ standard as a goal.
## Versioning ## Versioning
Package logfmt publishes releases via [semver](http://semver.org/) compatible Git tags prefixed with a single 'v'. This project publishes releases according to the Go language guidelines for
[developing and publishing modules][pub].
[pub]: https://go.dev/doc/modules/developing

View file

@ -29,6 +29,23 @@ func NewDecoder(r io.Reader) *Decoder {
return dec return dec
} }
// NewDecoderSize returns a new decoder that reads from r.
//
// The decoder introduces its own buffering and may read data from r beyond
// the logfmt records requested.
// The size argument specifies the size of the initial buffer that the
// Decoder will use to read records from r.
// If a log line is longer than the size argument, the Decoder will return
// a bufio.ErrTooLong error.
func NewDecoderSize(r io.Reader, size int) *Decoder {
scanner := bufio.NewScanner(r)
scanner.Buffer(make([]byte, 0, size), size)
dec := &Decoder{
s: scanner,
}
return dec
}
// ScanRecord advances the Decoder to the next record, which can then be // ScanRecord advances the Decoder to the next record, which can then be
// parsed with the ScanKeyval method. It returns false when decoding stops, // parsed with the ScanKeyval method. It returns false when decoding stops,
// either by reaching the end of the input or an error. After ScanRecord // either by reaching the end of the input or an error. After ScanRecord

View file

@ -1,6 +1,15 @@
// Copyright 2022 Google LLC. // Copyright 2022 Google LLC.
// Use of this source code is governed by a BSD-style // Licensed under the Apache License, Version 2.0 (the "License");
// license that can be found in the LICENSE file. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package client is a cross-platform client for the signer binary (a.k.a."EnterpriseCertSigner"). // Package client is a cross-platform client for the signer binary (a.k.a."EnterpriseCertSigner").
// //
@ -13,10 +22,9 @@ import (
"crypto/rsa" "crypto/rsa"
"crypto/x509" "crypto/x509"
"encoding/gob" "encoding/gob"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"log"
"net/rpc" "net/rpc"
"os" "os"
"os/exec" "os/exec"
@ -44,17 +52,6 @@ func (c *Connection) Close() error {
return werr return werr
} }
// If ECP Logging is enabled return true
// Otherwise return false
func enableECPLogging() bool {
if os.Getenv("ENABLE_ENTERPRISE_CERTIFICATE_LOGS") != "" {
return true
}
log.SetOutput(ioutil.Discard)
return false
}
func init() { func init() {
gob.Register(crypto.SHA256) gob.Register(crypto.SHA256)
gob.Register(&rsa.PSSOptions{}) gob.Register(&rsa.PSSOptions{})
@ -87,7 +84,7 @@ func (k *Key) Close() error {
} }
// Wait for cmd to exit and release resources. Since the process is forcefully killed, this // Wait for cmd to exit and release resources. Since the process is forcefully killed, this
// will return a non-nil error (varies by OS), which we will ignore. // will return a non-nil error (varies by OS), which we will ignore.
k.cmd.Wait() _ = k.cmd.Wait()
// The Pipes connecting the RPC client should have been closed when the signer subprocess was killed. // The Pipes connecting the RPC client should have been closed when the signer subprocess was killed.
// Calling `k.client.Close()` before `k.cmd.Process.Kill()` or `k.cmd.Wait()` _will_ cause a segfault. // Calling `k.client.Close()` before `k.cmd.Process.Kill()` or `k.cmd.Wait()` _will_ cause a segfault.
if err := k.client.Close(); err.Error() != "close |0: file already closed" { if err := k.client.Close(); err.Error() != "close |0: file already closed" {
@ -110,6 +107,10 @@ func (k *Key) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) (signed [
return return
} }
// ErrCredUnavailable is a sentinel error that indicates ECP Cred is unavailable,
// possibly due to missing config or missing binary path.
var ErrCredUnavailable = errors.New("Cred is unavailable")
// Cred spawns a signer subprocess that listens on stdin/stdout to perform certificate // Cred spawns a signer subprocess that listens on stdin/stdout to perform certificate
// related operations, including signing messages with the private key. // related operations, including signing messages with the private key.
// //
@ -118,12 +119,14 @@ func (k *Key) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) (signed [
// //
// The config file also specifies which certificate the signer should use. // The config file also specifies which certificate the signer should use.
func Cred(configFilePath string) (*Key, error) { func Cred(configFilePath string) (*Key, error) {
enableECPLogging()
if configFilePath == "" { if configFilePath == "" {
configFilePath = util.GetDefaultConfigFilePath() configFilePath = util.GetDefaultConfigFilePath()
} }
enterpriseCertSignerPath, err := util.LoadSignerBinaryPath(configFilePath) enterpriseCertSignerPath, err := util.LoadSignerBinaryPath(configFilePath)
if err != nil { if err != nil {
if errors.Is(err, util.ErrConfigUnavailable) {
return nil, ErrCredUnavailable
}
return nil, err return nil, err
} }
k := &Key{ k := &Key{

View file

@ -1,10 +1,23 @@
// Copyright 2022 Google LLC.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package util provides helper functions for the client. // Package util provides helper functions for the client.
package util package util
import ( import (
"encoding/json" "encoding/json"
"errors" "errors"
"io/ioutil" "io"
"os" "os"
"os/user" "os/user"
"path/filepath" "path/filepath"
@ -23,14 +36,21 @@ type Libs struct {
ECP string `json:"ecp"` ECP string `json:"ecp"`
} }
// ErrConfigUnavailable is a sentinel error that indicates ECP config is unavailable,
// possibly due to entire config missing or missing binary path.
var ErrConfigUnavailable = errors.New("Config is unavailable")
// LoadSignerBinaryPath retrieves the path of the signer binary from the config file. // LoadSignerBinaryPath retrieves the path of the signer binary from the config file.
func LoadSignerBinaryPath(configFilePath string) (path string, err error) { func LoadSignerBinaryPath(configFilePath string) (path string, err error) {
jsonFile, err := os.Open(configFilePath) jsonFile, err := os.Open(configFilePath)
if err != nil { if err != nil {
if errors.Is(err, os.ErrNotExist) {
return "", ErrConfigUnavailable
}
return "", err return "", err
} }
byteValue, err := ioutil.ReadAll(jsonFile) byteValue, err := io.ReadAll(jsonFile)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -41,7 +61,7 @@ func LoadSignerBinaryPath(configFilePath string) (path string, err error) {
} }
signerBinaryPath := config.Libs.ECP signerBinaryPath := config.Libs.ECP
if signerBinaryPath == "" { if signerBinaryPath == "" {
return "", errors.New("signer binary path is missing") return "", ErrConfigUnavailable
} }
return signerBinaryPath, nil return signerBinaryPath, nil
} }

View file

@ -16,6 +16,13 @@ This package provides various compression algorithms.
# changelog # changelog
* Jan 3rd, 2023 (v1.15.14)
* flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718
* zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720
* export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722
* s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723
* Dec 11, 2022 (v1.15.13) * Dec 11, 2022 (v1.15.13)
* zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691
* zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708

View file

@ -294,7 +294,6 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
} }
offset = 0 offset = 0
cGain := 0
if d.chain < 100 { if d.chain < 100 {
for i := prevHead; tries > 0; tries-- { for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] { if wEnd == win[i+length] {
@ -322,10 +321,14 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
return return
} }
// Minimum gain to accept a match.
cGain := 4
// Some like it higher (CSV), some like it lower (JSON) // Some like it higher (CSV), some like it lower (JSON)
const baseCost = 6 const baseCost = 3
// Base is 4 bytes at with an additional cost. // Base is 4 bytes at with an additional cost.
// Matches must be better than this. // Matches must be better than this.
for i := prevHead; tries > 0; tries-- { for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] { if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos) n := matchLen(win[i:i+minMatchLook], wPos)
@ -333,7 +336,7 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
// Calculate gain. Estimate // Calculate gain. Estimate
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]]) newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
//fmt.Println(n, "gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n])) //fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length)
if newGain > cGain { if newGain > cGain {
length = n length = n
offset = pos - i offset = pos - i
@ -490,27 +493,103 @@ func (d *compressor) deflateLazy() {
} }
if prevLength >= minMatchLength && s.length <= prevLength { if prevLength >= minMatchLength && s.length <= prevLength {
// Check for better match at end... // No better match, but check for better match at end...
// //
// checkOff must be >=2 since we otherwise risk checking s.index // Skip forward a number of bytes.
// Offset of 2 seems to yield best results. // Offset of 2 seems to yield best results. 3 is sometimes better.
const checkOff = 2 const checkOff = 2
// Check all, except full length
if prevLength < maxMatchLength-checkOff {
prevIndex := s.index - 1 prevIndex := s.index - 1
if prevIndex+prevLength+checkOff < s.maxInsertIndex { if prevIndex+prevLength < s.maxInsertIndex {
end := lookahead end := lookahead
if lookahead > maxMatchLength { if lookahead > maxMatchLength+checkOff {
end = maxMatchLength end = maxMatchLength + checkOff
} }
end += prevIndex end += prevIndex
idx := prevIndex + prevLength - (4 - checkOff)
h := hash4(d.window[idx:]) // Hash at match end.
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + (4 - checkOff) h := hash4(d.window[prevIndex+prevLength:])
if ch2 > minIndex { ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
length := matchLen(d.window[prevIndex:end], d.window[ch2:]) if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
// It seems like a pure length metric is best. // It seems like a pure length metric is best.
if length > prevLength { if length > prevLength {
prevLength = length prevLength = length
prevOffset = prevIndex - ch2 prevOffset = prevIndex - ch2
// Extend back...
for i := checkOff - 1; i >= 0; i-- {
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] {
// Emit tokens we "owe"
for j := 0; j <= i; j++ {
d.tokens.AddLiteral(d.window[prevIndex+j])
if d.tokens.n == maxFlateBlockTokens {
// The block includes the current character
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
return
}
d.tokens.Reset()
}
s.index++
if s.index < s.maxInsertIndex {
h := hash4(d.window[s.index:])
ch := s.hashHead[h]
s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch
s.hashHead[h] = uint32(s.index + s.hashOffset)
}
}
break
} else {
prevLength++
}
}
} else if false {
// Check one further ahead.
// Only rarely better, disabled for now.
prevIndex++
h := hash4(d.window[prevIndex+prevLength:])
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
// It seems like a pure length metric is best.
if length > prevLength+checkOff {
prevLength = length
prevOffset = prevIndex - ch2
prevIndex--
// Extend back...
for i := checkOff; i >= 0; i-- {
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] {
// Emit tokens we "owe"
for j := 0; j <= i; j++ {
d.tokens.AddLiteral(d.window[prevIndex+j])
if d.tokens.n == maxFlateBlockTokens {
// The block includes the current character
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
return
}
d.tokens.Reset()
}
s.index++
if s.index < s.maxInsertIndex {
h := hash4(d.window[s.index:])
ch := s.hashHead[h]
s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch
s.hashHead[h] = uint32(s.index + s.hashOffset)
}
}
break
} else {
prevLength++
}
}
}
}
}
} }
} }
} }

View file

@ -146,54 +146,51 @@ func (s *Scratch) compress(src []byte) error {
c1.encodeZero(tt[src[ip-2]]) c1.encodeZero(tt[src[ip-2]])
ip -= 2 ip -= 2
} }
src = src[:ip]
// Main compression loop. // Main compression loop.
switch { switch {
case !s.zeroBits && s.actualTableLog <= 8: case !s.zeroBits && s.actualTableLog <= 8:
// We can encode 4 symbols without requiring a flush. // We can encode 4 symbols without requiring a flush.
// We do not need to check if any output is 0 bits. // We do not need to check if any output is 0 bits.
for ip >= 4 { for ; len(src) >= 4; src = src[:len(src)-4] {
s.bw.flush32() s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
c2.encode(tt[v0]) c2.encode(tt[v0])
c1.encode(tt[v1]) c1.encode(tt[v1])
c2.encode(tt[v2]) c2.encode(tt[v2])
c1.encode(tt[v3]) c1.encode(tt[v3])
ip -= 4
} }
case !s.zeroBits: case !s.zeroBits:
// We do not need to check if any output is 0 bits. // We do not need to check if any output is 0 bits.
for ip >= 4 { for ; len(src) >= 4; src = src[:len(src)-4] {
s.bw.flush32() s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
c2.encode(tt[v0]) c2.encode(tt[v0])
c1.encode(tt[v1]) c1.encode(tt[v1])
s.bw.flush32() s.bw.flush32()
c2.encode(tt[v2]) c2.encode(tt[v2])
c1.encode(tt[v3]) c1.encode(tt[v3])
ip -= 4
} }
case s.actualTableLog <= 8: case s.actualTableLog <= 8:
// We can encode 4 symbols without requiring a flush // We can encode 4 symbols without requiring a flush
for ip >= 4 { for ; len(src) >= 4; src = src[:len(src)-4] {
s.bw.flush32() s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
c2.encodeZero(tt[v0]) c2.encodeZero(tt[v0])
c1.encodeZero(tt[v1]) c1.encodeZero(tt[v1])
c2.encodeZero(tt[v2]) c2.encodeZero(tt[v2])
c1.encodeZero(tt[v3]) c1.encodeZero(tt[v3])
ip -= 4
} }
default: default:
for ip >= 4 { for ; len(src) >= 4; src = src[:len(src)-4] {
s.bw.flush32() s.bw.flush32()
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
c2.encodeZero(tt[v0]) c2.encodeZero(tt[v0])
c1.encodeZero(tt[v1]) c1.encodeZero(tt[v1])
s.bw.flush32() s.bw.flush32()
c2.encodeZero(tt[v2]) c2.encodeZero(tt[v2])
c1.encodeZero(tt[v3]) c1.encodeZero(tt[v3])
ip -= 4
} }
} }
@ -459,15 +456,17 @@ func (s *Scratch) countSimple(in []byte) (max int) {
for _, v := range in { for _, v := range in {
s.count[v]++ s.count[v]++
} }
m := uint32(0) m, symlen := uint32(0), s.symbolLen
for i, v := range s.count[:] { for i, v := range s.count[:] {
if v == 0 {
continue
}
if v > m { if v > m {
m = v m = v
} }
if v > 0 { symlen = uint16(i) + 1
s.symbolLen = uint16(i) + 1
}
} }
s.symbolLen = symlen
return int(m) return int(m)
} }

View file

@ -67,7 +67,6 @@ func (b *bitReaderBytes) fillFast() {
// 2 bounds checks. // 2 bounds checks.
v := b.in[b.off-4 : b.off] v := b.in[b.off-4 : b.off]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value |= uint64(low) << (b.bitsRead - 32) b.value |= uint64(low) << (b.bitsRead - 32)
b.bitsRead -= 32 b.bitsRead -= 32
@ -88,8 +87,7 @@ func (b *bitReaderBytes) fill() {
return return
} }
if b.off > 4 { if b.off > 4 {
v := b.in[b.off-4:] v := b.in[b.off-4 : b.off]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value |= uint64(low) << (b.bitsRead - 32) b.value |= uint64(low) << (b.bitsRead - 32)
b.bitsRead -= 32 b.bitsRead -= 32
@ -179,7 +177,6 @@ func (b *bitReaderShifted) fillFast() {
// 2 bounds checks. // 2 bounds checks.
v := b.in[b.off-4 : b.off] v := b.in[b.off-4 : b.off]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
b.bitsRead -= 32 b.bitsRead -= 32
@ -200,8 +197,7 @@ func (b *bitReaderShifted) fill() {
return return
} }
if b.off > 4 { if b.off > 4 {
v := b.in[b.off-4:] v := b.in[b.off-4 : b.off]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
b.bitsRead -= 32 b.bitsRead -= 32

View file

@ -484,34 +484,35 @@ func (s *Scratch) buildCTable() error {
// Different from reference implementation. // Different from reference implementation.
huffNode0 := s.nodes[0 : huffNodesLen+1] huffNode0 := s.nodes[0 : huffNodesLen+1]
for huffNode[nonNullRank].count == 0 { for huffNode[nonNullRank].count() == 0 {
nonNullRank-- nonNullRank--
} }
lowS := int16(nonNullRank) lowS := int16(nonNullRank)
nodeRoot := nodeNb + lowS - 1 nodeRoot := nodeNb + lowS - 1
lowN := nodeNb lowN := nodeNb
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count huffNode[nodeNb].setCount(huffNode[lowS].count() + huffNode[lowS-1].count())
huffNode[lowS].parent, huffNode[lowS-1].parent = uint16(nodeNb), uint16(nodeNb) huffNode[lowS].setParent(nodeNb)
huffNode[lowS-1].setParent(nodeNb)
nodeNb++ nodeNb++
lowS -= 2 lowS -= 2
for n := nodeNb; n <= nodeRoot; n++ { for n := nodeNb; n <= nodeRoot; n++ {
huffNode[n].count = 1 << 30 huffNode[n].setCount(1 << 30)
} }
// fake entry, strong barrier // fake entry, strong barrier
huffNode0[0].count = 1 << 31 huffNode0[0].setCount(1 << 31)
// create parents // create parents
for nodeNb <= nodeRoot { for nodeNb <= nodeRoot {
var n1, n2 int16 var n1, n2 int16
if huffNode0[lowS+1].count < huffNode0[lowN+1].count { if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() {
n1 = lowS n1 = lowS
lowS-- lowS--
} else { } else {
n1 = lowN n1 = lowN
lowN++ lowN++
} }
if huffNode0[lowS+1].count < huffNode0[lowN+1].count { if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() {
n2 = lowS n2 = lowS
lowS-- lowS--
} else { } else {
@ -519,18 +520,19 @@ func (s *Scratch) buildCTable() error {
lowN++ lowN++
} }
huffNode[nodeNb].count = huffNode0[n1+1].count + huffNode0[n2+1].count huffNode[nodeNb].setCount(huffNode0[n1+1].count() + huffNode0[n2+1].count())
huffNode0[n1+1].parent, huffNode0[n2+1].parent = uint16(nodeNb), uint16(nodeNb) huffNode0[n1+1].setParent(nodeNb)
huffNode0[n2+1].setParent(nodeNb)
nodeNb++ nodeNb++
} }
// distribute weights (unlimited tree height) // distribute weights (unlimited tree height)
huffNode[nodeRoot].nbBits = 0 huffNode[nodeRoot].setNbBits(0)
for n := nodeRoot - 1; n >= startNode; n-- { for n := nodeRoot - 1; n >= startNode; n-- {
huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1)
} }
for n := uint16(0); n <= nonNullRank; n++ { for n := uint16(0); n <= nonNullRank; n++ {
huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1)
} }
s.actualTableLog = s.setMaxHeight(int(nonNullRank)) s.actualTableLog = s.setMaxHeight(int(nonNullRank))
maxNbBits := s.actualTableLog maxNbBits := s.actualTableLog
@ -542,7 +544,7 @@ func (s *Scratch) buildCTable() error {
var nbPerRank [tableLogMax + 1]uint16 var nbPerRank [tableLogMax + 1]uint16
var valPerRank [16]uint16 var valPerRank [16]uint16
for _, v := range huffNode[:nonNullRank+1] { for _, v := range huffNode[:nonNullRank+1] {
nbPerRank[v.nbBits]++ nbPerRank[v.nbBits()]++
} }
// determine stating value per rank // determine stating value per rank
{ {
@ -557,7 +559,7 @@ func (s *Scratch) buildCTable() error {
// push nbBits per symbol, symbol order // push nbBits per symbol, symbol order
for _, v := range huffNode[:nonNullRank+1] { for _, v := range huffNode[:nonNullRank+1] {
s.cTable[v.symbol].nBits = v.nbBits s.cTable[v.symbol()].nBits = v.nbBits()
} }
// assign value within rank, symbol order // assign value within rank, symbol order
@ -603,12 +605,12 @@ func (s *Scratch) huffSort() {
pos := rank[r].current pos := rank[r].current
rank[r].current++ rank[r].current++
prev := nodes[(pos-1)&huffNodesMask] prev := nodes[(pos-1)&huffNodesMask]
for pos > rank[r].base && c > prev.count { for pos > rank[r].base && c > prev.count() {
nodes[pos&huffNodesMask] = prev nodes[pos&huffNodesMask] = prev
pos-- pos--
prev = nodes[(pos-1)&huffNodesMask] prev = nodes[(pos-1)&huffNodesMask]
} }
nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)} nodes[pos&huffNodesMask] = makeNodeElt(c, byte(n))
} }
} }
@ -617,7 +619,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
huffNode := s.nodes[1 : huffNodesLen+1] huffNode := s.nodes[1 : huffNodesLen+1]
//huffNode = huffNode[: huffNodesLen] //huffNode = huffNode[: huffNodesLen]
largestBits := huffNode[lastNonNull].nbBits largestBits := huffNode[lastNonNull].nbBits()
// early exit : no elt > maxNbBits // early exit : no elt > maxNbBits
if largestBits <= maxNbBits { if largestBits <= maxNbBits {
@ -627,14 +629,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
baseCost := int(1) << (largestBits - maxNbBits) baseCost := int(1) << (largestBits - maxNbBits)
n := uint32(lastNonNull) n := uint32(lastNonNull)
for huffNode[n].nbBits > maxNbBits { for huffNode[n].nbBits() > maxNbBits {
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)) totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits()))
huffNode[n].nbBits = maxNbBits huffNode[n].setNbBits(maxNbBits)
n-- n--
} }
// n stops at huffNode[n].nbBits <= maxNbBits // n stops at huffNode[n].nbBits <= maxNbBits
for huffNode[n].nbBits == maxNbBits { for huffNode[n].nbBits() == maxNbBits {
n-- n--
} }
// n end at index of smallest symbol using < maxNbBits // n end at index of smallest symbol using < maxNbBits
@ -655,10 +657,10 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
{ {
currentNbBits := maxNbBits currentNbBits := maxNbBits
for pos := int(n); pos >= 0; pos-- { for pos := int(n); pos >= 0; pos-- {
if huffNode[pos].nbBits >= currentNbBits { if huffNode[pos].nbBits() >= currentNbBits {
continue continue
} }
currentNbBits = huffNode[pos].nbBits // < maxNbBits currentNbBits = huffNode[pos].nbBits() // < maxNbBits
rankLast[maxNbBits-currentNbBits] = uint32(pos) rankLast[maxNbBits-currentNbBits] = uint32(pos)
} }
} }
@ -675,8 +677,8 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
if lowPos == noSymbol { if lowPos == noSymbol {
break break
} }
highTotal := huffNode[highPos].count highTotal := huffNode[highPos].count()
lowTotal := 2 * huffNode[lowPos].count lowTotal := 2 * huffNode[lowPos].count()
if highTotal <= lowTotal { if highTotal <= lowTotal {
break break
} }
@ -692,13 +694,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
// this rank is no longer empty // this rank is no longer empty
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease] rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]
} }
huffNode[rankLast[nBitsToDecrease]].nbBits++ huffNode[rankLast[nBitsToDecrease]].setNbBits(1 +
huffNode[rankLast[nBitsToDecrease]].nbBits())
if rankLast[nBitsToDecrease] == 0 { if rankLast[nBitsToDecrease] == 0 {
/* special case, reached largest symbol */ /* special case, reached largest symbol */
rankLast[nBitsToDecrease] = noSymbol rankLast[nBitsToDecrease] = noSymbol
} else { } else {
rankLast[nBitsToDecrease]-- rankLast[nBitsToDecrease]--
if huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease { if huffNode[rankLast[nBitsToDecrease]].nbBits() != maxNbBits-nBitsToDecrease {
rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */ rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */
} }
} }
@ -706,15 +709,15 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
for totalCost < 0 { /* Sometimes, cost correction overshoot */ for totalCost < 0 { /* Sometimes, cost correction overshoot */
if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
for huffNode[n].nbBits == maxNbBits { for huffNode[n].nbBits() == maxNbBits {
n-- n--
} }
huffNode[n+1].nbBits-- huffNode[n+1].setNbBits(huffNode[n+1].nbBits() - 1)
rankLast[1] = n + 1 rankLast[1] = n + 1
totalCost++ totalCost++
continue continue
} }
huffNode[rankLast[1]+1].nbBits-- huffNode[rankLast[1]+1].setNbBits(huffNode[rankLast[1]+1].nbBits() - 1)
rankLast[1]++ rankLast[1]++
totalCost++ totalCost++
} }
@ -722,9 +725,26 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
return maxNbBits return maxNbBits
} }
type nodeElt struct { // A nodeElt is the fields
count uint32 //
parent uint16 // count uint32
symbol byte // parent uint16
nbBits uint8 // symbol byte
// nbBits uint8
//
// in some order, all squashed into an integer so that the compiler
// always loads and stores entire nodeElts instead of separate fields.
type nodeElt uint64
func makeNodeElt(count uint32, symbol byte) nodeElt {
return nodeElt(count) | nodeElt(symbol)<<48
} }
func (e *nodeElt) count() uint32 { return uint32(*e) }
func (e *nodeElt) parent() uint16 { return uint16(*e >> 32) }
func (e *nodeElt) symbol() byte { return byte(*e >> 48) }
func (e *nodeElt) nbBits() uint8 { return uint8(*e >> 56) }
func (e *nodeElt) setCount(c uint32) { *e = (*e)&0xffffffff00000000 | nodeElt(c) }
func (e *nodeElt) setParent(p int16) { *e = (*e)&0xffff0000ffffffff | nodeElt(uint16(p))<<32 }
func (e *nodeElt) setNbBits(n uint8) { *e = (*e)&0x00ffffffffffffff | nodeElt(n)<<56 }

View file

@ -4,360 +4,349 @@
// func decompress4x_main_loop_amd64(ctx *decompress4xContext) // func decompress4x_main_loop_amd64(ctx *decompress4xContext)
TEXT ·decompress4x_main_loop_amd64(SB), $0-8 TEXT ·decompress4x_main_loop_amd64(SB), $0-8
XORQ DX, DX
// Preload values // Preload values
MOVQ ctx+0(FP), AX MOVQ ctx+0(FP), AX
MOVBQZX 8(AX), DI MOVBQZX 8(AX), DI
MOVQ 16(AX), SI MOVQ 16(AX), BX
MOVQ 48(AX), BX MOVQ 48(AX), SI
MOVQ 24(AX), R9 MOVQ 24(AX), R8
MOVQ 32(AX), R10 MOVQ 32(AX), R9
MOVQ (AX), R11 MOVQ (AX), R10
// Main loop // Main loop
main_loop: main_loop:
MOVQ SI, R8 XORL DX, DX
CMPQ R8, BX CMPQ BX, SI
SETGE DL SETGE DL
// br0.fillFast32() // br0.fillFast32()
MOVQ 32(R11), R12 MOVQ 32(R10), R11
MOVBQZX 40(R11), R13 MOVBQZX 40(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill0 JBE skip_fill0
MOVQ 24(R11), AX MOVQ 24(R10), AX
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, AX SUBQ $0x04, AX
MOVQ (R11), R14 MOVQ (R10), R13
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (AX)(R14*1), R14 MOVL (AX)(R13*1), R13
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R14 SHLQ CL, R13
MOVQ AX, 24(R11) MOVQ AX, 24(R10)
ORQ R14, R12 ORQ R13, R11
// exhausted = exhausted || (br0.off < 4) // exhausted += (br0.off < 4)
CMPQ AX, $0x04 CMPQ AX, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill0: skip_fill0:
// val0 := br0.peekTopBits(peekBits) // val0 := br0.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v0.entry) // br0.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br0.peekTopBits(peekBits) // val1 := br0.peekTopBits(peekBits)
MOVQ DI, CX MOVQ DI, CX
MOVQ R12, R14 MOVQ R11, R13
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val1&mask] // v1 := table[val1&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v1.entry)) // br0.advance(uint8(v1.entry))
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// these two writes get coalesced // these two writes get coalesced
// out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 0] = uint8(v0.entry >> 8)
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
MOVW AX, (R8) MOVW AX, (BX)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 32(R11) MOVQ R11, 32(R10)
MOVB R13, 40(R11) MOVB R12, 40(R10)
ADDQ R9, R8
// br1.fillFast32() // br1.fillFast32()
MOVQ 80(R11), R12 MOVQ 80(R10), R11
MOVBQZX 88(R11), R13 MOVBQZX 88(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill1 JBE skip_fill1
MOVQ 72(R11), AX MOVQ 72(R10), AX
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, AX SUBQ $0x04, AX
MOVQ 48(R11), R14 MOVQ 48(R10), R13
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (AX)(R14*1), R14 MOVL (AX)(R13*1), R13
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R14 SHLQ CL, R13
MOVQ AX, 72(R11) MOVQ AX, 72(R10)
ORQ R14, R12 ORQ R13, R11
// exhausted = exhausted || (br1.off < 4) // exhausted += (br1.off < 4)
CMPQ AX, $0x04 CMPQ AX, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill1: skip_fill1:
// val0 := br1.peekTopBits(peekBits) // val0 := br1.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v0.entry) // br1.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br1.peekTopBits(peekBits) // val1 := br1.peekTopBits(peekBits)
MOVQ DI, CX MOVQ DI, CX
MOVQ R12, R14 MOVQ R11, R13
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val1&mask] // v1 := table[val1&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v1.entry)) // br1.advance(uint8(v1.entry))
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// these two writes get coalesced // these two writes get coalesced
// out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 0] = uint8(v0.entry >> 8)
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
MOVW AX, (R8) MOVW AX, (BX)(R8*1)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 80(R11) MOVQ R11, 80(R10)
MOVB R13, 88(R11) MOVB R12, 88(R10)
ADDQ R9, R8
// br2.fillFast32() // br2.fillFast32()
MOVQ 128(R11), R12 MOVQ 128(R10), R11
MOVBQZX 136(R11), R13 MOVBQZX 136(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill2 JBE skip_fill2
MOVQ 120(R11), AX MOVQ 120(R10), AX
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, AX SUBQ $0x04, AX
MOVQ 96(R11), R14 MOVQ 96(R10), R13
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (AX)(R14*1), R14 MOVL (AX)(R13*1), R13
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R14 SHLQ CL, R13
MOVQ AX, 120(R11) MOVQ AX, 120(R10)
ORQ R14, R12 ORQ R13, R11
// exhausted = exhausted || (br2.off < 4) // exhausted += (br2.off < 4)
CMPQ AX, $0x04 CMPQ AX, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill2: skip_fill2:
// val0 := br2.peekTopBits(peekBits) // val0 := br2.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v0.entry) // br2.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br2.peekTopBits(peekBits) // val1 := br2.peekTopBits(peekBits)
MOVQ DI, CX MOVQ DI, CX
MOVQ R12, R14 MOVQ R11, R13
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val1&mask] // v1 := table[val1&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v1.entry)) // br2.advance(uint8(v1.entry))
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// these two writes get coalesced // these two writes get coalesced
// out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 0] = uint8(v0.entry >> 8)
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
MOVW AX, (R8) MOVW AX, (BX)(R8*2)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 128(R11) MOVQ R11, 128(R10)
MOVB R13, 136(R11) MOVB R12, 136(R10)
ADDQ R9, R8
// br3.fillFast32() // br3.fillFast32()
MOVQ 176(R11), R12 MOVQ 176(R10), R11
MOVBQZX 184(R11), R13 MOVBQZX 184(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill3 JBE skip_fill3
MOVQ 168(R11), AX MOVQ 168(R10), AX
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, AX SUBQ $0x04, AX
MOVQ 144(R11), R14 MOVQ 144(R10), R13
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (AX)(R14*1), R14 MOVL (AX)(R13*1), R13
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R14 SHLQ CL, R13
MOVQ AX, 168(R11) MOVQ AX, 168(R10)
ORQ R14, R12 ORQ R13, R11
// exhausted = exhausted || (br3.off < 4) // exhausted += (br3.off < 4)
CMPQ AX, $0x04 CMPQ AX, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill3: skip_fill3:
// val0 := br3.peekTopBits(peekBits) // val0 := br3.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v0.entry) // br3.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br3.peekTopBits(peekBits) // val1 := br3.peekTopBits(peekBits)
MOVQ DI, CX MOVQ DI, CX
MOVQ R12, R14 MOVQ R11, R13
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val1&mask] // v1 := table[val1&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v1.entry)) // br3.advance(uint8(v1.entry))
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// these two writes get coalesced // these two writes get coalesced
// out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 0] = uint8(v0.entry >> 8)
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
MOVW AX, (R8) LEAQ (R8)(R8*2), CX
MOVW AX, (BX)(CX*1)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 176(R11) MOVQ R11, 176(R10)
MOVB R13, 184(R11) MOVB R12, 184(R10)
ADDQ $0x02, SI ADDQ $0x02, BX
TESTB DL, DL TESTB DL, DL
JZ main_loop JZ main_loop
MOVQ ctx+0(FP), AX MOVQ ctx+0(FP), AX
SUBQ 16(AX), SI SUBQ 16(AX), BX
SHLQ $0x02, SI SHLQ $0x02, BX
MOVQ SI, 40(AX) MOVQ BX, 40(AX)
RET RET
// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext)
TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8
XORQ DX, DX
// Preload values // Preload values
MOVQ ctx+0(FP), CX MOVQ ctx+0(FP), CX
MOVBQZX 8(CX), DI MOVBQZX 8(CX), DI
MOVQ 16(CX), BX MOVQ 16(CX), BX
MOVQ 48(CX), SI MOVQ 48(CX), SI
MOVQ 24(CX), R9 MOVQ 24(CX), R8
MOVQ 32(CX), R10 MOVQ 32(CX), R9
MOVQ (CX), R11 MOVQ (CX), R10
// Main loop // Main loop
main_loop: main_loop:
MOVQ BX, R8 XORL DX, DX
CMPQ R8, SI CMPQ BX, SI
SETGE DL SETGE DL
// br0.fillFast32() // br0.fillFast32()
MOVQ 32(R11), R12 MOVQ 32(R10), R11
MOVBQZX 40(R11), R13 MOVBQZX 40(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill0 JBE skip_fill0
MOVQ 24(R11), R14 MOVQ 24(R10), R13
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, R14 SUBQ $0x04, R13
MOVQ (R11), R15 MOVQ (R10), R14
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (R14)(R15*1), R15 MOVL (R13)(R14*1), R14
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R15 SHLQ CL, R14
MOVQ R14, 24(R11) MOVQ R13, 24(R10)
ORQ R15, R12 ORQ R14, R11
// exhausted = exhausted || (br0.off < 4) // exhausted += (br0.off < 4)
CMPQ R14, $0x04 CMPQ R13, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill0: skip_fill0:
// val0 := br0.peekTopBits(peekBits) // val0 := br0.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v0.entry) // br0.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br0.peekTopBits(peekBits) // val1 := br0.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val0&mask] // v1 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v1.entry) // br0.advance(uint8(v1.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// val2 := br0.peekTopBits(peekBits) // val2 := br0.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v2 := table[val0&mask] // v2 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v2.entry) // br0.advance(uint8(v2.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val3 := br0.peekTopBits(peekBits) // val3 := br0.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v3 := table[val0&mask] // v3 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br0.advance(uint8(v3.entry) // br0.advance(uint8(v3.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// these four writes get coalesced // these four writes get coalesced
@ -365,88 +354,86 @@ skip_fill0:
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
// out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8)
// out[id * dstEvery + 4] = uint8(v3.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8)
MOVL AX, (R8) MOVL AX, (BX)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 32(R11) MOVQ R11, 32(R10)
MOVB R13, 40(R11) MOVB R12, 40(R10)
ADDQ R9, R8
// br1.fillFast32() // br1.fillFast32()
MOVQ 80(R11), R12 MOVQ 80(R10), R11
MOVBQZX 88(R11), R13 MOVBQZX 88(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill1 JBE skip_fill1
MOVQ 72(R11), R14 MOVQ 72(R10), R13
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, R14 SUBQ $0x04, R13
MOVQ 48(R11), R15 MOVQ 48(R10), R14
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (R14)(R15*1), R15 MOVL (R13)(R14*1), R14
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R15 SHLQ CL, R14
MOVQ R14, 72(R11) MOVQ R13, 72(R10)
ORQ R15, R12 ORQ R14, R11
// exhausted = exhausted || (br1.off < 4) // exhausted += (br1.off < 4)
CMPQ R14, $0x04 CMPQ R13, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill1: skip_fill1:
// val0 := br1.peekTopBits(peekBits) // val0 := br1.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v0.entry) // br1.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br1.peekTopBits(peekBits) // val1 := br1.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val0&mask] // v1 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v1.entry) // br1.advance(uint8(v1.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// val2 := br1.peekTopBits(peekBits) // val2 := br1.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v2 := table[val0&mask] // v2 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v2.entry) // br1.advance(uint8(v2.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val3 := br1.peekTopBits(peekBits) // val3 := br1.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v3 := table[val0&mask] // v3 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br1.advance(uint8(v3.entry) // br1.advance(uint8(v3.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// these four writes get coalesced // these four writes get coalesced
@ -454,88 +441,86 @@ skip_fill1:
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
// out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8)
// out[id * dstEvery + 4] = uint8(v3.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8)
MOVL AX, (R8) MOVL AX, (BX)(R8*1)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 80(R11) MOVQ R11, 80(R10)
MOVB R13, 88(R11) MOVB R12, 88(R10)
ADDQ R9, R8
// br2.fillFast32() // br2.fillFast32()
MOVQ 128(R11), R12 MOVQ 128(R10), R11
MOVBQZX 136(R11), R13 MOVBQZX 136(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill2 JBE skip_fill2
MOVQ 120(R11), R14 MOVQ 120(R10), R13
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, R14 SUBQ $0x04, R13
MOVQ 96(R11), R15 MOVQ 96(R10), R14
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (R14)(R15*1), R15 MOVL (R13)(R14*1), R14
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R15 SHLQ CL, R14
MOVQ R14, 120(R11) MOVQ R13, 120(R10)
ORQ R15, R12 ORQ R14, R11
// exhausted = exhausted || (br2.off < 4) // exhausted += (br2.off < 4)
CMPQ R14, $0x04 CMPQ R13, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill2: skip_fill2:
// val0 := br2.peekTopBits(peekBits) // val0 := br2.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v0.entry) // br2.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br2.peekTopBits(peekBits) // val1 := br2.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val0&mask] // v1 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v1.entry) // br2.advance(uint8(v1.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// val2 := br2.peekTopBits(peekBits) // val2 := br2.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v2 := table[val0&mask] // v2 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v2.entry) // br2.advance(uint8(v2.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val3 := br2.peekTopBits(peekBits) // val3 := br2.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v3 := table[val0&mask] // v3 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br2.advance(uint8(v3.entry) // br2.advance(uint8(v3.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// these four writes get coalesced // these four writes get coalesced
@ -543,88 +528,86 @@ skip_fill2:
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
// out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8)
// out[id * dstEvery + 4] = uint8(v3.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8)
MOVL AX, (R8) MOVL AX, (BX)(R8*2)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 128(R11) MOVQ R11, 128(R10)
MOVB R13, 136(R11) MOVB R12, 136(R10)
ADDQ R9, R8
// br3.fillFast32() // br3.fillFast32()
MOVQ 176(R11), R12 MOVQ 176(R10), R11
MOVBQZX 184(R11), R13 MOVBQZX 184(R10), R12
CMPQ R13, $0x20 CMPQ R12, $0x20
JBE skip_fill3 JBE skip_fill3
MOVQ 168(R11), R14 MOVQ 168(R10), R13
SUBQ $0x20, R13 SUBQ $0x20, R12
SUBQ $0x04, R14 SUBQ $0x04, R13
MOVQ 144(R11), R15 MOVQ 144(R10), R14
// b.value |= uint64(low) << (b.bitsRead & 63) // b.value |= uint64(low) << (b.bitsRead & 63)
MOVL (R14)(R15*1), R15 MOVL (R13)(R14*1), R14
MOVQ R13, CX MOVQ R12, CX
SHLQ CL, R15 SHLQ CL, R14
MOVQ R14, 168(R11) MOVQ R13, 168(R10)
ORQ R15, R12 ORQ R14, R11
// exhausted = exhausted || (br3.off < 4) // exhausted += (br3.off < 4)
CMPQ R14, $0x04 CMPQ R13, $0x04
SETLT AL ADCB $+0, DL
ORB AL, DL
skip_fill3: skip_fill3:
// val0 := br3.peekTopBits(peekBits) // val0 := br3.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v0 := table[val0&mask] // v0 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v0.entry) // br3.advance(uint8(v0.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val1 := br3.peekTopBits(peekBits) // val1 := br3.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v1 := table[val0&mask] // v1 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v1.entry) // br3.advance(uint8(v1.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// val2 := br3.peekTopBits(peekBits) // val2 := br3.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v2 := table[val0&mask] // v2 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v2.entry) // br3.advance(uint8(v2.entry)
MOVB CH, AH MOVB CH, AH
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
// val3 := br3.peekTopBits(peekBits) // val3 := br3.peekTopBits(peekBits)
MOVQ R12, R14 MOVQ R11, R13
MOVQ DI, CX MOVQ DI, CX
SHRQ CL, R14 SHRQ CL, R13
// v3 := table[val0&mask] // v3 := table[val0&mask]
MOVW (R10)(R14*2), CX MOVW (R9)(R13*2), CX
// br3.advance(uint8(v3.entry) // br3.advance(uint8(v3.entry)
MOVB CH, AL MOVB CH, AL
SHLQ CL, R12 SHLQ CL, R11
ADDB CL, R13 ADDB CL, R12
BSWAPL AX BSWAPL AX
// these four writes get coalesced // these four writes get coalesced
@ -632,11 +615,12 @@ skip_fill3:
// out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8)
// out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8)
// out[id * dstEvery + 4] = uint8(v3.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8)
MOVL AX, (R8) LEAQ (R8)(R8*2), CX
MOVL AX, (BX)(CX*1)
// update the bitreader structure // update the bitreader structure
MOVQ R12, 176(R11) MOVQ R11, 176(R10)
MOVB R13, 184(R11) MOVB R12, 184(R10)
ADDQ $0x04, BX ADDQ $0x04, BX
TESTB DL, DL TESTB DL, DL
JZ main_loop JZ main_loop
@ -652,7 +636,7 @@ TEXT ·decompress1x_main_loop_amd64(SB), $0-8
MOVQ 16(CX), DX MOVQ 16(CX), DX
MOVQ 24(CX), BX MOVQ 24(CX), BX
CMPQ BX, $0x04 CMPQ BX, $0x04
JB error_max_decoded_size_exeeded JB error_max_decoded_size_exceeded
LEAQ (DX)(BX*1), BX LEAQ (DX)(BX*1), BX
MOVQ (CX), SI MOVQ (CX), SI
MOVQ (SI), R8 MOVQ (SI), R8
@ -667,7 +651,7 @@ main_loop:
// Check if we have room for 4 bytes in the output buffer // Check if we have room for 4 bytes in the output buffer
LEAQ 4(DX), CX LEAQ 4(DX), CX
CMPQ CX, BX CMPQ CX, BX
JGE error_max_decoded_size_exeeded JGE error_max_decoded_size_exceeded
// Decode 4 values // Decode 4 values
CMPQ R11, $0x20 CMPQ R11, $0x20
@ -744,7 +728,7 @@ loop_condition:
RET RET
// Report error // Report error
error_max_decoded_size_exeeded: error_max_decoded_size_exceeded:
MOVQ ctx+0(FP), AX MOVQ ctx+0(FP), AX
MOVQ $-1, CX MOVQ $-1, CX
MOVQ CX, 40(AX) MOVQ CX, 40(AX)
@ -757,7 +741,7 @@ TEXT ·decompress1x_main_loop_bmi2(SB), $0-8
MOVQ 16(CX), DX MOVQ 16(CX), DX
MOVQ 24(CX), BX MOVQ 24(CX), BX
CMPQ BX, $0x04 CMPQ BX, $0x04
JB error_max_decoded_size_exeeded JB error_max_decoded_size_exceeded
LEAQ (DX)(BX*1), BX LEAQ (DX)(BX*1), BX
MOVQ (CX), SI MOVQ (CX), SI
MOVQ (SI), R8 MOVQ (SI), R8
@ -772,7 +756,7 @@ main_loop:
// Check if we have room for 4 bytes in the output buffer // Check if we have room for 4 bytes in the output buffer
LEAQ 4(DX), CX LEAQ 4(DX), CX
CMPQ CX, BX CMPQ CX, BX
JGE error_max_decoded_size_exeeded JGE error_max_decoded_size_exceeded
// Decode 4 values // Decode 4 values
CMPQ R11, $0x20 CMPQ R11, $0x20
@ -839,7 +823,7 @@ loop_condition:
RET RET
// Report error // Report error
error_max_decoded_size_exeeded: error_max_decoded_size_exceeded:
MOVQ ctx+0(FP), AX MOVQ ctx+0(FP), AX
MOVQ $-1, CX MOVQ $-1, CX
MOVQ CX, 40(AX) MOVQ CX, 40(AX)

View file

@ -192,16 +192,14 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
} }
// Read block data. // Read block data.
if cap(b.dataStorage) < cSize { if _, ok := br.(*byteBuf); !ok && cap(b.dataStorage) < cSize {
// byteBuf doesn't need a destination buffer.
if b.lowMem || cSize > maxCompressedBlockSize { if b.lowMem || cSize > maxCompressedBlockSize {
b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc) b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc)
} else { } else {
b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc) b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc)
} }
} }
if cap(b.dst) <= maxSize {
b.dst = make([]byte, 0, maxSize+1)
}
b.data, err = br.readBig(cSize, b.dataStorage) b.data, err = br.readBig(cSize, b.dataStorage)
if err != nil { if err != nil {
if debugDecoder { if debugDecoder {
@ -210,6 +208,9 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
} }
return err return err
} }
if cap(b.dst) <= maxSize {
b.dst = make([]byte, 0, maxSize+1)
}
return nil return nil
} }

View file

@ -40,8 +40,7 @@ type Decoder struct {
frame *frameDec frame *frameDec
// Custom dictionaries. // Custom dictionaries.
// Always uses copies. dicts map[uint32]*dict
dicts map[uint32]dict
// streamWg is the waitgroup for all streams // streamWg is the waitgroup for all streams
streamWg sync.WaitGroup streamWg sync.WaitGroup
@ -103,7 +102,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
} }
// Transfer option dicts. // Transfer option dicts.
d.dicts = make(map[uint32]dict, len(d.o.dicts)) d.dicts = make(map[uint32]*dict, len(d.o.dicts))
for _, dc := range d.o.dicts { for _, dc := range d.o.dicts {
d.dicts[dc.id] = dc d.dicts[dc.id] = dc
} }
@ -341,15 +340,8 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
} }
return dst, err return dst, err
} }
if frame.DictionaryID != nil { if err = d.setDict(frame); err != nil {
dict, ok := d.dicts[*frame.DictionaryID] return nil, err
if !ok {
return nil, ErrUnknownDictionary
}
if debugDecoder {
println("setting dict", frame.DictionaryID)
}
frame.history.setDict(&dict)
} }
if frame.WindowSize > d.o.maxWindowSize { if frame.WindowSize > d.o.maxWindowSize {
if debugDecoder { if debugDecoder {
@ -495,18 +487,12 @@ func (d *Decoder) nextBlockSync() (ok bool) {
if !d.syncStream.inFrame { if !d.syncStream.inFrame {
d.frame.history.reset() d.frame.history.reset()
d.current.err = d.frame.reset(&d.syncStream.br) d.current.err = d.frame.reset(&d.syncStream.br)
if d.current.err == nil {
d.current.err = d.setDict(d.frame)
}
if d.current.err != nil { if d.current.err != nil {
return false return false
} }
if d.frame.DictionaryID != nil {
dict, ok := d.dicts[*d.frame.DictionaryID]
if !ok {
d.current.err = ErrUnknownDictionary
return false
} else {
d.frame.history.setDict(&dict)
}
}
if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize {
d.current.err = ErrDecoderSizeExceeded d.current.err = ErrDecoderSizeExceeded
return false return false
@ -865,13 +851,8 @@ decodeStream:
if debugDecoder && err != nil { if debugDecoder && err != nil {
println("Frame decoder returned", err) println("Frame decoder returned", err)
} }
if err == nil && frame.DictionaryID != nil { if err == nil {
dict, ok := d.dicts[*frame.DictionaryID] err = d.setDict(frame)
if !ok {
err = ErrUnknownDictionary
} else {
frame.history.setDict(&dict)
}
} }
if err == nil && d.frame.WindowSize > d.o.maxWindowSize { if err == nil && d.frame.WindowSize > d.o.maxWindowSize {
if debugDecoder { if debugDecoder {
@ -953,3 +934,20 @@ decodeStream:
hist.reset() hist.reset()
d.frame.history.b = frameHistCache d.frame.history.b = frameHistCache
} }
func (d *Decoder) setDict(frame *frameDec) (err error) {
dict, ok := d.dicts[frame.DictionaryID]
if ok {
if debugDecoder {
println("setting dict", frame.DictionaryID)
}
frame.history.setDict(dict)
} else if frame.DictionaryID != 0 {
// A zero or missing dictionary id is ambiguous:
// either dictionary zero, or no dictionary. In particular,
// zstd --patch-from uses this id for the source file,
// so only return an error if the dictionary id is not zero.
err = ErrUnknownDictionary
}
return err
}

View file

@ -6,6 +6,8 @@ package zstd
import ( import (
"errors" "errors"
"fmt"
"math/bits"
"runtime" "runtime"
) )
@ -18,7 +20,7 @@ type decoderOptions struct {
concurrent int concurrent int
maxDecodedSize uint64 maxDecodedSize uint64
maxWindowSize uint64 maxWindowSize uint64
dicts []dict dicts []*dict
ignoreChecksum bool ignoreChecksum bool
limitToCap bool limitToCap bool
decodeBufsBelow int decodeBufsBelow int
@ -85,7 +87,13 @@ func WithDecoderMaxMemory(n uint64) DOption {
} }
// WithDecoderDicts allows to register one or more dictionaries for the decoder. // WithDecoderDicts allows to register one or more dictionaries for the decoder.
// If several dictionaries with the same ID is provided the last one will be used. //
// Each slice in dict must be in the [dictionary format] produced by
// "zstd --train" from the Zstandard reference implementation.
//
// If several dictionaries with the same ID are provided, the last one will be used.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithDecoderDicts(dicts ...[]byte) DOption { func WithDecoderDicts(dicts ...[]byte) DOption {
return func(o *decoderOptions) error { return func(o *decoderOptions) error {
for _, b := range dicts { for _, b := range dicts {
@ -93,12 +101,24 @@ func WithDecoderDicts(dicts ...[]byte) DOption {
if err != nil { if err != nil {
return err return err
} }
o.dicts = append(o.dicts, *d) o.dicts = append(o.dicts, d)
} }
return nil return nil
} }
} }
// WithEncoderDictRaw registers a dictionary that may be used by the decoder.
// The slice content can be arbitrary data.
func WithDecoderDictRaw(id uint32, content []byte) DOption {
return func(o *decoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
}
o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}})
return nil
}
}
// WithDecoderMaxWindow allows to set a maximum window size for decodes. // WithDecoderMaxWindow allows to set a maximum window size for decodes.
// This allows rejecting packets that will cause big memory usage. // This allows rejecting packets that will cause big memory usage.
// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting.

View file

@ -21,6 +21,9 @@ type dict struct {
const dictMagic = "\x37\xa4\x30\xec" const dictMagic = "\x37\xa4\x30\xec"
// Maximum dictionary size for the reference implementation (1.5.3) is 2 GiB.
const dictMaxLength = 1 << 31
// ID returns the dictionary id or 0 if d is nil. // ID returns the dictionary id or 0 if d is nil.
func (d *dict) ID() uint32 { func (d *dict) ID() uint32 {
if d == nil { if d == nil {

View file

@ -4,6 +4,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"math" "math"
"math/bits"
"runtime" "runtime"
"strings" "strings"
) )
@ -305,7 +306,13 @@ func WithLowerEncoderMem(b bool) EOption {
} }
// WithEncoderDict allows to register a dictionary that will be used for the encode. // WithEncoderDict allows to register a dictionary that will be used for the encode.
//
// The slice dict must be in the [dictionary format] produced by
// "zstd --train" from the Zstandard reference implementation.
//
// The encoder *may* choose to use no dictionary instead for certain payloads. // The encoder *may* choose to use no dictionary instead for certain payloads.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithEncoderDict(dict []byte) EOption { func WithEncoderDict(dict []byte) EOption {
return func(o *encoderOptions) error { return func(o *encoderOptions) error {
d, err := loadDict(dict) d, err := loadDict(dict)
@ -316,3 +323,17 @@ func WithEncoderDict(dict []byte) EOption {
return nil return nil
} }
} }
// WithEncoderDictRaw registers a dictionary that may be used by the encoder.
//
// The slice content may contain arbitrary data. It will be used as an initial
// history.
func WithEncoderDictRaw(id uint32, content []byte) EOption {
return func(o *encoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
}
o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}
return nil
}
}

View file

@ -29,7 +29,7 @@ type frameDec struct {
FrameContentSize uint64 FrameContentSize uint64
DictionaryID *uint32 DictionaryID uint32
HasCheckSum bool HasCheckSum bool
SingleSegment bool SingleSegment bool
} }
@ -155,7 +155,7 @@ func (d *frameDec) reset(br byteBuffer) error {
// Read Dictionary_ID // Read Dictionary_ID
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id
d.DictionaryID = nil d.DictionaryID = 0
if size := fhd & 3; size != 0 { if size := fhd & 3; size != 0 {
if size == 3 { if size == 3 {
size = 4 size = 4
@ -178,11 +178,7 @@ func (d *frameDec) reset(br byteBuffer) error {
if debugDecoder { if debugDecoder {
println("Dict size", size, "ID:", id) println("Dict size", size, "ID:", id)
} }
if id > 0 { d.DictionaryID = id
// ID 0 means "sorry, no dictionary anyway".
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
d.DictionaryID = &id
}
} }
// Read Frame_Content_Size // Read Frame_Content_Size

View file

@ -72,7 +72,6 @@ var (
ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit") ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit")
// ErrUnknownDictionary is returned if the dictionary ID is unknown. // ErrUnknownDictionary is returned if the dictionary ID is unknown.
// For the time being dictionaries are not supported.
ErrUnknownDictionary = errors.New("unknown dictionary") ErrUnknownDictionary = errors.New("unknown dictionary")
// ErrFrameSizeExceeded is returned if the stated frame size is exceeded. // ErrFrameSizeExceeded is returned if the stated frame size is exceeded.

View file

@ -1,9 +1,9 @@
# cli # cli
[![GoDoc](https://godoc.org/github.com/urfave/cli?status.svg)](https://pkg.go.dev/github.com/urfave/cli/v2) [![Run Tests](https://github.com/urfave/cli/actions/workflows/cli.yml/badge.svg?branch=v2-maint)](https://github.com/urfave/cli/actions/workflows/cli.yml)
[![codebeat](https://codebeat.co/badges/0a8f30aa-f975-404b-b878-5fab3ae1cc5f)](https://codebeat.co/projects/github-com-urfave-cli) [![Go Reference](https://pkg.go.dev/badge/github.com/urfave/cli/v2.svg)](https://pkg.go.dev/github.com/urfave/cli/v2)
[![Go Report Card](https://goreportcard.com/badge/urfave/cli)](https://goreportcard.com/report/urfave/cli) [![Go Report Card](https://goreportcard.com/badge/github.com/urfave/cli/v2)](https://goreportcard.com/report/github.com/urfave/cli/v2)
[![codecov](https://codecov.io/gh/urfave/cli/branch/main/graph/badge.svg)](https://codecov.io/gh/urfave/cli) [![codecov](https://codecov.io/gh/urfave/cli/branch/v2-maint/graph/badge.svg?token=t9YGWLh05g)](https://app.codecov.io/gh/urfave/cli/tree/v2-maint)
cli is a simple, fast, and fun package for building command line apps in Go. The cli is a simple, fast, and fun package for building command line apps in Go. The
goal is to enable developers to write fast and distributable command line goal is to enable developers to write fast and distributable command line

View file

@ -122,6 +122,7 @@ type App struct {
SkipFlagParsing bool SkipFlagParsing bool
didSetup bool didSetup bool
separator separatorSpec
rootCommand *Command rootCommand *Command
} }
@ -216,6 +217,16 @@ func (a *App) Setup() {
}) })
} }
if len(a.SliceFlagSeparator) != 0 {
a.separator.customized = true
a.separator.sep = a.SliceFlagSeparator
}
if a.DisableSliceFlagSeparator {
a.separator.customized = true
a.separator.disabled = true
}
var newCommands []*Command var newCommands []*Command
for _, c := range a.Commands { for _, c := range a.Commands {
@ -223,8 +234,8 @@ func (a *App) Setup() {
if c.HelpName != "" { if c.HelpName != "" {
cname = c.HelpName cname = c.HelpName
} }
c.separator = a.separator
c.HelpName = fmt.Sprintf("%s %s", a.HelpName, cname) c.HelpName = fmt.Sprintf("%s %s", a.HelpName, cname)
c.flagCategories = newFlagCategoriesFromFlags(c.Flags) c.flagCategories = newFlagCategoriesFromFlags(c.Flags)
newCommands = append(newCommands, c) newCommands = append(newCommands, c)
} }
@ -250,24 +261,11 @@ func (a *App) Setup() {
} }
sort.Sort(a.categories.(*commandCategories)) sort.Sort(a.categories.(*commandCategories))
a.flagCategories = newFlagCategories() a.flagCategories = newFlagCategoriesFromFlags(a.Flags)
for _, fl := range a.Flags {
if cf, ok := fl.(CategorizableFlag); ok {
if cf.GetCategory() != "" {
a.flagCategories.AddFlag(cf.GetCategory(), cf)
}
}
}
if a.Metadata == nil { if a.Metadata == nil {
a.Metadata = make(map[string]interface{}) a.Metadata = make(map[string]interface{})
} }
if len(a.SliceFlagSeparator) != 0 {
defaultSliceFlagSeparator = a.SliceFlagSeparator
}
disableSliceFlagSeparator = a.DisableSliceFlagSeparator
} }
func (a *App) newRootCommand() *Command { func (a *App) newRootCommand() *Command {
@ -293,11 +291,12 @@ func (a *App) newRootCommand() *Command {
categories: a.categories, categories: a.categories,
SkipFlagParsing: a.SkipFlagParsing, SkipFlagParsing: a.SkipFlagParsing,
isRoot: true, isRoot: true,
separator: a.separator,
} }
} }
func (a *App) newFlagSet() (*flag.FlagSet, error) { func (a *App) newFlagSet() (*flag.FlagSet, error) {
return flagSet(a.Name, a.Flags) return flagSet(a.Name, a.Flags, a.separator)
} }
func (a *App) useShortOptionHandling() bool { func (a *App) useShortOptionHandling() bool {

View file

@ -100,10 +100,23 @@ func newFlagCategories() FlagCategories {
func newFlagCategoriesFromFlags(fs []Flag) FlagCategories { func newFlagCategoriesFromFlags(fs []Flag) FlagCategories {
fc := newFlagCategories() fc := newFlagCategories()
var categorized bool
for _, fl := range fs { for _, fl := range fs {
if cf, ok := fl.(CategorizableFlag); ok { if cf, ok := fl.(CategorizableFlag); ok {
if cf.GetCategory() != "" { if cat := cf.GetCategory(); cat != "" {
fc.AddFlag(cf.GetCategory(), cf) fc.AddFlag(cat, cf)
categorized = true
}
}
}
if categorized == true {
for _, fl := range fs {
if cf, ok := fl.(CategorizableFlag); ok {
if cf.GetCategory() == "" {
fc.AddFlag("", fl)
}
} }
} }
} }

View file

@ -69,6 +69,8 @@ type Command struct {
// if this is a root "special" command // if this is a root "special" command
isRoot bool isRoot bool
separator separatorSpec
} }
type Commands []*Command type Commands []*Command
@ -275,7 +277,7 @@ func (c *Command) Run(cCtx *Context, arguments ...string) (err error) {
} }
func (c *Command) newFlagSet() (*flag.FlagSet, error) { func (c *Command) newFlagSet() (*flag.FlagSet, error) {
return flagSet(c.Name, c.Flags) return flagSet(c.Name, c.Flags, c.separator)
} }
func (c *Command) useShortOptionHandling() bool { func (c *Command) useShortOptionHandling() bool {

View file

@ -7,6 +7,8 @@ flag_types:
- name: Count - name: Count
type: int type: int
pointer: true pointer: true
- name: DisableDefaultText
type: bool
- name: Action - name: Action
type: "func(*Context, bool) error" type: "func(*Context, bool) error"
float64: float64:
@ -18,6 +20,8 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: Action - name: Action
type: "func(*Context, []float64) error" type: "func(*Context, []float64) error"
int: int:
@ -31,6 +35,8 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: Action - name: Action
type: "func(*Context, []int) error" type: "func(*Context, []int) error"
int64: int64:
@ -44,6 +50,8 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: Action - name: Action
type: "func(*Context, []int64) error" type: "func(*Context, []int64) error"
uint: uint:
@ -57,6 +65,8 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: Action - name: Action
type: "func(*Context, []uint) error" type: "func(*Context, []uint) error"
uint64: uint64:
@ -70,6 +80,8 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: Action - name: Action
type: "func(*Context, []uint64) error" type: "func(*Context, []uint64) error"
string: string:
@ -83,10 +95,14 @@ flag_types:
skip_interfaces: skip_interfaces:
- fmt.Stringer - fmt.Stringer
struct_fields: struct_fields:
- name: separator
type: separatorSpec
- name: TakesFile - name: TakesFile
type: bool type: bool
- name: Action - name: Action
type: "func(*Context, []string) error" type: "func(*Context, []string) error"
- name: KeepSpace
type: bool
time.Duration: time.Duration:
struct_fields: struct_fields:
- name: Action - name: Action

View file

@ -4,7 +4,7 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"io/ioutil" "io"
"os" "os"
"regexp" "regexp"
"runtime" "runtime"
@ -15,7 +15,7 @@ import (
const defaultPlaceholder = "value" const defaultPlaceholder = "value"
var ( const (
defaultSliceFlagSeparator = "," defaultSliceFlagSeparator = ","
disableSliceFlagSeparator = false disableSliceFlagSeparator = false
) )
@ -37,6 +37,7 @@ var VersionFlag Flag = &BoolFlag{
Name: "version", Name: "version",
Aliases: []string{"v"}, Aliases: []string{"v"},
Usage: "print the version", Usage: "print the version",
DisableDefaultText: true,
} }
// HelpFlag prints the help for all commands and subcommands. // HelpFlag prints the help for all commands and subcommands.
@ -46,6 +47,7 @@ var HelpFlag Flag = &BoolFlag{
Name: "help", Name: "help",
Aliases: []string{"h"}, Aliases: []string{"h"},
Usage: "show help", Usage: "show help",
DisableDefaultText: true,
} }
// FlagStringer converts a flag definition to a string. This is used by help // FlagStringer converts a flag definition to a string. This is used by help
@ -165,15 +167,18 @@ type Countable interface {
Count() int Count() int
} }
func flagSet(name string, flags []Flag) (*flag.FlagSet, error) { func flagSet(name string, flags []Flag, spec separatorSpec) (*flag.FlagSet, error) {
set := flag.NewFlagSet(name, flag.ContinueOnError) set := flag.NewFlagSet(name, flag.ContinueOnError)
for _, f := range flags { for _, f := range flags {
if c, ok := f.(customizedSeparator); ok {
c.WithSeparatorSpec(spec)
}
if err := f.Apply(set); err != nil { if err := f.Apply(set); err != nil {
return nil, err return nil, err
} }
} }
set.SetOutput(ioutil.Discard) set.SetOutput(io.Discard)
return set, nil return set, nil
} }
@ -337,9 +342,14 @@ func stringifyFlag(f Flag) string {
defaultValueString := "" defaultValueString := ""
// set default text for all flags except bool flags
// for bool flags display default text if DisableDefaultText is not
// set
if bf, ok := f.(*BoolFlag); !ok || !bf.DisableDefaultText {
if s := df.GetDefaultText(); s != "" { if s := df.GetDefaultText(); s != "" {
defaultValueString = fmt.Sprintf(formatDefault("%s"), s) defaultValueString = fmt.Sprintf(formatDefault("%s"), s)
} }
}
usageWithDefault := strings.TrimSpace(usage + defaultValueString) usageWithDefault := strings.TrimSpace(usage + defaultValueString)
@ -374,7 +384,7 @@ func flagFromEnvOrFile(envVars []string, filePath string) (value string, fromWhe
} }
for _, fileVar := range strings.Split(filePath, ",") { for _, fileVar := range strings.Split(filePath, ",") {
if fileVar != "" { if fileVar != "" {
if data, err := ioutil.ReadFile(fileVar); err == nil { if data, err := os.ReadFile(fileVar); err == nil {
return string(data), fmt.Sprintf("file %q", filePath), true return string(data), fmt.Sprintf("file %q", filePath), true
} }
} }
@ -382,10 +392,28 @@ func flagFromEnvOrFile(envVars []string, filePath string) (value string, fromWhe
return "", "", false return "", "", false
} }
func flagSplitMultiValues(val string) []string { type customizedSeparator interface {
if disableSliceFlagSeparator { WithSeparatorSpec(separatorSpec)
}
type separatorSpec struct {
sep string
disabled bool
customized bool
}
func (s separatorSpec) flagSplitMultiValues(val string) []string {
var (
disabled bool = s.disabled
sep string = s.sep
)
if !s.customized {
disabled = disableSliceFlagSeparator
sep = defaultSliceFlagSeparator
}
if disabled {
return []string{val} return []string{val}
} }
return strings.Split(val, defaultSliceFlagSeparator) return strings.Split(val, sep)
} }

View file

@ -11,6 +11,7 @@ import (
// Float64Slice wraps []float64 to satisfy flag.Value // Float64Slice wraps []float64 to satisfy flag.Value
type Float64Slice struct { type Float64Slice struct {
slice []float64 slice []float64
separator separatorSpec
hasBeenSet bool hasBeenSet bool
} }
@ -29,6 +30,10 @@ func (f *Float64Slice) clone() *Float64Slice {
return n return n
} }
func (f *Float64Slice) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Set parses the value into a float64 and appends it to the list of values // Set parses the value into a float64 and appends it to the list of values
func (f *Float64Slice) Set(value string) error { func (f *Float64Slice) Set(value string) error {
if !f.hasBeenSet { if !f.hasBeenSet {
@ -43,7 +48,7 @@ func (f *Float64Slice) Set(value string) error {
return nil return nil
} }
for _, s := range flagSplitMultiValues(value) { for _, s := range f.separator.flagSplitMultiValues(value) {
tmp, err := strconv.ParseFloat(strings.TrimSpace(s), 64) tmp, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
if err != nil { if err != nil {
return err return err
@ -148,11 +153,12 @@ func (f *Float64SliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(Float64Slice) setValue = new(Float64Slice)
setValue.WithSeparatorSpec(f.separator)
} }
if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found { if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found {
if val != "" { if val != "" {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if err := setValue.Set(strings.TrimSpace(s)); err != nil {
return fmt.Errorf("could not parse %q as float64 slice value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as float64 slice value from %s for flag %s: %s", val, source, f.Name, err)
} }
@ -172,6 +178,10 @@ func (f *Float64SliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *Float64SliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *Float64SliceFlag) Get(ctx *Context) []float64 { func (f *Float64SliceFlag) Get(ctx *Context) []float64 {
return ctx.Float64Slice(f.Name) return ctx.Float64Slice(f.Name)

View file

@ -11,6 +11,7 @@ import (
// Int64Slice wraps []int64 to satisfy flag.Value // Int64Slice wraps []int64 to satisfy flag.Value
type Int64Slice struct { type Int64Slice struct {
slice []int64 slice []int64
separator separatorSpec
hasBeenSet bool hasBeenSet bool
} }
@ -29,6 +30,10 @@ func (i *Int64Slice) clone() *Int64Slice {
return n return n
} }
func (i *Int64Slice) WithSeparatorSpec(spec separatorSpec) {
i.separator = spec
}
// Set parses the value into an integer and appends it to the list of values // Set parses the value into an integer and appends it to the list of values
func (i *Int64Slice) Set(value string) error { func (i *Int64Slice) Set(value string) error {
if !i.hasBeenSet { if !i.hasBeenSet {
@ -43,7 +48,7 @@ func (i *Int64Slice) Set(value string) error {
return nil return nil
} }
for _, s := range flagSplitMultiValues(value) { for _, s := range i.separator.flagSplitMultiValues(value) {
tmp, err := strconv.ParseInt(strings.TrimSpace(s), 0, 64) tmp, err := strconv.ParseInt(strings.TrimSpace(s), 0, 64)
if err != nil { if err != nil {
return err return err
@ -149,10 +154,11 @@ func (f *Int64SliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(Int64Slice) setValue = new(Int64Slice)
setValue.WithSeparatorSpec(f.separator)
} }
if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" { if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if err := setValue.Set(strings.TrimSpace(s)); err != nil {
return fmt.Errorf("could not parse %q as int64 slice value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as int64 slice value from %s for flag %s: %s", val, source, f.Name, err)
} }
@ -171,6 +177,10 @@ func (f *Int64SliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *Int64SliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *Int64SliceFlag) Get(ctx *Context) []int64 { func (f *Int64SliceFlag) Get(ctx *Context) []int64 {
return ctx.Int64Slice(f.Name) return ctx.Int64Slice(f.Name)

View file

@ -11,6 +11,7 @@ import (
// IntSlice wraps []int to satisfy flag.Value // IntSlice wraps []int to satisfy flag.Value
type IntSlice struct { type IntSlice struct {
slice []int slice []int
separator separatorSpec
hasBeenSet bool hasBeenSet bool
} }
@ -40,6 +41,10 @@ func (i *IntSlice) SetInt(value int) {
i.slice = append(i.slice, value) i.slice = append(i.slice, value)
} }
func (i *IntSlice) WithSeparatorSpec(spec separatorSpec) {
i.separator = spec
}
// Set parses the value into an integer and appends it to the list of values // Set parses the value into an integer and appends it to the list of values
func (i *IntSlice) Set(value string) error { func (i *IntSlice) Set(value string) error {
if !i.hasBeenSet { if !i.hasBeenSet {
@ -54,7 +59,7 @@ func (i *IntSlice) Set(value string) error {
return nil return nil
} }
for _, s := range flagSplitMultiValues(value) { for _, s := range i.separator.flagSplitMultiValues(value) {
tmp, err := strconv.ParseInt(strings.TrimSpace(s), 0, 64) tmp, err := strconv.ParseInt(strings.TrimSpace(s), 0, 64)
if err != nil { if err != nil {
return err return err
@ -160,10 +165,11 @@ func (f *IntSliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(IntSlice) setValue = new(IntSlice)
setValue.WithSeparatorSpec(f.separator)
} }
if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" { if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if err := setValue.Set(strings.TrimSpace(s)); err != nil {
return fmt.Errorf("could not parse %q as int slice value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as int slice value from %s for flag %s: %s", val, source, f.Name, err)
} }
@ -182,6 +188,10 @@ func (f *IntSliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *IntSliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *IntSliceFlag) Get(ctx *Context) []int { func (f *IntSliceFlag) Get(ctx *Context) []int {
return ctx.IntSlice(f.Name) return ctx.IntSlice(f.Name)

View file

@ -11,7 +11,9 @@ import (
// StringSlice wraps a []string to satisfy flag.Value // StringSlice wraps a []string to satisfy flag.Value
type StringSlice struct { type StringSlice struct {
slice []string slice []string
separator separatorSpec
hasBeenSet bool hasBeenSet bool
keepSpace bool
} }
// NewStringSlice creates a *StringSlice with default values // NewStringSlice creates a *StringSlice with default values
@ -43,13 +45,20 @@ func (s *StringSlice) Set(value string) error {
return nil return nil
} }
for _, t := range flagSplitMultiValues(value) { for _, t := range s.separator.flagSplitMultiValues(value) {
s.slice = append(s.slice, strings.TrimSpace(t)) if !s.keepSpace {
t = strings.TrimSpace(t)
}
s.slice = append(s.slice, t)
} }
return nil return nil
} }
func (s *StringSlice) WithSeparatorSpec(spec separatorSpec) {
s.separator = spec
}
// String returns a readable representation of this value (for usage defaults) // String returns a readable representation of this value (for usage defaults)
func (s *StringSlice) String() string { func (s *StringSlice) String() string {
return fmt.Sprintf("%s", s.slice) return fmt.Sprintf("%s", s.slice)
@ -141,11 +150,17 @@ func (f *StringSliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(StringSlice) setValue = new(StringSlice)
setValue.WithSeparatorSpec(f.separator)
} }
setValue.keepSpace = f.KeepSpace
if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found { if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if !f.KeepSpace {
s = strings.TrimSpace(s)
}
if err := setValue.Set(s); err != nil {
return fmt.Errorf("could not parse %q as string value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as string value from %s for flag %s: %s", val, source, f.Name, err)
} }
} }
@ -163,6 +178,10 @@ func (f *StringSliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *StringSliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *StringSliceFlag) Get(ctx *Context) []string { func (f *StringSliceFlag) Get(ctx *Context) []string {
return ctx.StringSlice(f.Name) return ctx.StringSlice(f.Name)

View file

@ -11,6 +11,7 @@ import (
// Uint64Slice wraps []int64 to satisfy flag.Value // Uint64Slice wraps []int64 to satisfy flag.Value
type Uint64Slice struct { type Uint64Slice struct {
slice []uint64 slice []uint64
separator separatorSpec
hasBeenSet bool hasBeenSet bool
} }
@ -43,7 +44,7 @@ func (i *Uint64Slice) Set(value string) error {
return nil return nil
} }
for _, s := range flagSplitMultiValues(value) { for _, s := range i.separator.flagSplitMultiValues(value) {
tmp, err := strconv.ParseUint(strings.TrimSpace(s), 0, 64) tmp, err := strconv.ParseUint(strings.TrimSpace(s), 0, 64)
if err != nil { if err != nil {
return err return err
@ -55,6 +56,10 @@ func (i *Uint64Slice) Set(value string) error {
return nil return nil
} }
func (i *Uint64Slice) WithSeparatorSpec(spec separatorSpec) {
i.separator = spec
}
// String returns a readable representation of this value (for usage defaults) // String returns a readable representation of this value (for usage defaults)
func (i *Uint64Slice) String() string { func (i *Uint64Slice) String() string {
v := i.slice v := i.slice
@ -153,10 +158,11 @@ func (f *Uint64SliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(Uint64Slice) setValue = new(Uint64Slice)
setValue.WithSeparatorSpec(f.separator)
} }
if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" { if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if err := setValue.Set(strings.TrimSpace(s)); err != nil {
return fmt.Errorf("could not parse %q as uint64 slice value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as uint64 slice value from %s for flag %s: %s", val, source, f.Name, err)
} }
@ -175,6 +181,10 @@ func (f *Uint64SliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *Uint64SliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *Uint64SliceFlag) Get(ctx *Context) []uint64 { func (f *Uint64SliceFlag) Get(ctx *Context) []uint64 {
return ctx.Uint64Slice(f.Name) return ctx.Uint64Slice(f.Name)

View file

@ -11,6 +11,7 @@ import (
// UintSlice wraps []int to satisfy flag.Value // UintSlice wraps []int to satisfy flag.Value
type UintSlice struct { type UintSlice struct {
slice []uint slice []uint
separator separatorSpec
hasBeenSet bool hasBeenSet bool
} }
@ -54,7 +55,7 @@ func (i *UintSlice) Set(value string) error {
return nil return nil
} }
for _, s := range flagSplitMultiValues(value) { for _, s := range i.separator.flagSplitMultiValues(value) {
tmp, err := strconv.ParseUint(strings.TrimSpace(s), 0, 32) tmp, err := strconv.ParseUint(strings.TrimSpace(s), 0, 32)
if err != nil { if err != nil {
return err return err
@ -66,6 +67,10 @@ func (i *UintSlice) Set(value string) error {
return nil return nil
} }
func (i *UintSlice) WithSeparatorSpec(spec separatorSpec) {
i.separator = spec
}
// String returns a readable representation of this value (for usage defaults) // String returns a readable representation of this value (for usage defaults)
func (i *UintSlice) String() string { func (i *UintSlice) String() string {
v := i.slice v := i.slice
@ -164,10 +169,11 @@ func (f *UintSliceFlag) Apply(set *flag.FlagSet) error {
setValue = f.Value.clone() setValue = f.Value.clone()
default: default:
setValue = new(UintSlice) setValue = new(UintSlice)
setValue.WithSeparatorSpec(f.separator)
} }
if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" { if val, source, ok := flagFromEnvOrFile(f.EnvVars, f.FilePath); ok && val != "" {
for _, s := range flagSplitMultiValues(val) { for _, s := range f.separator.flagSplitMultiValues(val) {
if err := setValue.Set(strings.TrimSpace(s)); err != nil { if err := setValue.Set(strings.TrimSpace(s)); err != nil {
return fmt.Errorf("could not parse %q as uint slice value from %s for flag %s: %s", val, source, f.Name, err) return fmt.Errorf("could not parse %q as uint slice value from %s for flag %s: %s", val, source, f.Name, err)
} }
@ -186,6 +192,10 @@ func (f *UintSliceFlag) Apply(set *flag.FlagSet) error {
return nil return nil
} }
func (f *UintSliceFlag) WithSeparatorSpec(spec separatorSpec) {
f.separator = spec
}
// Get returns the flags value in the given Context. // Get returns the flags value in the given Context.
func (f *UintSliceFlag) Get(ctx *Context) []uint { func (f *UintSliceFlag) Get(ctx *Context) []uint {
return ctx.UintSlice(f.Name) return ctx.UintSlice(f.Name)

View file

@ -455,6 +455,8 @@ type BoolFlag struct {
Count *int Count *int
DisableDefaultText bool
Action func(*Context, bool) error Action func(*Context, bool) error
// Has unexported fields. // Has unexported fields.
} }
@ -884,6 +886,7 @@ var HelpFlag Flag = &BoolFlag{
Name: "help", Name: "help",
Aliases: []string{"h"}, Aliases: []string{"h"},
Usage: "show help", Usage: "show help",
DisableDefaultText: true,
} }
HelpFlag prints the help for all commands and subcommands. Set to nil to HelpFlag prints the help for all commands and subcommands. Set to nil to
disable the flag. The subcommand will still be added unless HideHelp or disable the flag. The subcommand will still be added unless HideHelp or
@ -893,6 +896,7 @@ var VersionFlag Flag = &BoolFlag{
Name: "version", Name: "version",
Aliases: []string{"v"}, Aliases: []string{"v"},
Usage: "print the version", Usage: "print the version",
DisableDefaultText: true,
} }
VersionFlag prints the version for the application VersionFlag prints the version for the application
@ -1034,6 +1038,8 @@ func (f *Float64Slice) String() string
func (f *Float64Slice) Value() []float64 func (f *Float64Slice) Value() []float64
Value returns the slice of float64s set by this flag Value returns the slice of float64s set by this flag
func (f *Float64Slice) WithSeparatorSpec(spec separatorSpec)
type Float64SliceFlag struct { type Float64SliceFlag struct {
Name string Name string
@ -1109,6 +1115,8 @@ func (f *Float64SliceFlag) String() string
func (f *Float64SliceFlag) TakesValue() bool func (f *Float64SliceFlag) TakesValue() bool
TakesValue returns true if the flag takes a value, otherwise false TakesValue returns true if the flag takes a value, otherwise false
func (f *Float64SliceFlag) WithSeparatorSpec(spec separatorSpec)
type Generic interface { type Generic interface {
Set(value string) error Set(value string) error
String() string String() string
@ -1275,6 +1283,8 @@ func (i *Int64Slice) String() string
func (i *Int64Slice) Value() []int64 func (i *Int64Slice) Value() []int64
Value returns the slice of ints set by this flag Value returns the slice of ints set by this flag
func (i *Int64Slice) WithSeparatorSpec(spec separatorSpec)
type Int64SliceFlag struct { type Int64SliceFlag struct {
Name string Name string
@ -1350,6 +1360,8 @@ func (f *Int64SliceFlag) String() string
func (f *Int64SliceFlag) TakesValue() bool func (f *Int64SliceFlag) TakesValue() bool
TakesValue returns true of the flag takes a value, otherwise false TakesValue returns true of the flag takes a value, otherwise false
func (f *Int64SliceFlag) WithSeparatorSpec(spec separatorSpec)
type IntFlag struct { type IntFlag struct {
Name string Name string
@ -1445,6 +1457,8 @@ func (i *IntSlice) String() string
func (i *IntSlice) Value() []int func (i *IntSlice) Value() []int
Value returns the slice of ints set by this flag Value returns the slice of ints set by this flag
func (i *IntSlice) WithSeparatorSpec(spec separatorSpec)
type IntSliceFlag struct { type IntSliceFlag struct {
Name string Name string
@ -1520,6 +1534,8 @@ func (f *IntSliceFlag) String() string
func (f *IntSliceFlag) TakesValue() bool func (f *IntSliceFlag) TakesValue() bool
TakesValue returns true of the flag takes a value, otherwise false TakesValue returns true of the flag takes a value, otherwise false
func (f *IntSliceFlag) WithSeparatorSpec(spec separatorSpec)
type InvalidFlagAccessFunc func(*Context, string) type InvalidFlagAccessFunc func(*Context, string)
InvalidFlagAccessFunc is executed when an invalid flag is accessed from the InvalidFlagAccessFunc is executed when an invalid flag is accessed from the
context. context.
@ -1788,6 +1804,8 @@ func (s *StringSlice) String() string
func (s *StringSlice) Value() []string func (s *StringSlice) Value() []string
Value returns the slice of strings set by this flag Value returns the slice of strings set by this flag
func (s *StringSlice) WithSeparatorSpec(spec separatorSpec)
type StringSliceFlag struct { type StringSliceFlag struct {
Name string Name string
@ -1809,6 +1827,8 @@ type StringSliceFlag struct {
TakesFile bool TakesFile bool
Action func(*Context, []string) error Action func(*Context, []string) error
KeepSpace bool
// Has unexported fields. // Has unexported fields.
} }
StringSliceFlag is a flag with type *StringSlice StringSliceFlag is a flag with type *StringSlice
@ -1865,6 +1885,8 @@ func (f *StringSliceFlag) String() string
func (f *StringSliceFlag) TakesValue() bool func (f *StringSliceFlag) TakesValue() bool
TakesValue returns true of the flag takes a value, otherwise false TakesValue returns true of the flag takes a value, otherwise false
func (f *StringSliceFlag) WithSeparatorSpec(spec separatorSpec)
type SuggestCommandFunc func(commands []*Command, provided string) string type SuggestCommandFunc func(commands []*Command, provided string) string
type SuggestFlagFunc func(flags []Flag, provided string, hideHelp bool) string type SuggestFlagFunc func(flags []Flag, provided string, hideHelp bool) string
@ -2059,6 +2081,8 @@ func (i *Uint64Slice) String() string
func (i *Uint64Slice) Value() []uint64 func (i *Uint64Slice) Value() []uint64
Value returns the slice of ints set by this flag Value returns the slice of ints set by this flag
func (i *Uint64Slice) WithSeparatorSpec(spec separatorSpec)
type Uint64SliceFlag struct { type Uint64SliceFlag struct {
Name string Name string
@ -2125,6 +2149,8 @@ func (f *Uint64SliceFlag) String() string
func (f *Uint64SliceFlag) TakesValue() bool func (f *Uint64SliceFlag) TakesValue() bool
TakesValue returns true of the flag takes a value, otherwise false TakesValue returns true of the flag takes a value, otherwise false
func (f *Uint64SliceFlag) WithSeparatorSpec(spec separatorSpec)
type UintFlag struct { type UintFlag struct {
Name string Name string
@ -2220,6 +2246,8 @@ func (i *UintSlice) String() string
func (i *UintSlice) Value() []uint func (i *UintSlice) Value() []uint
Value returns the slice of ints set by this flag Value returns the slice of ints set by this flag
func (i *UintSlice) WithSeparatorSpec(spec separatorSpec)
type UintSliceFlag struct { type UintSliceFlag struct {
Name string Name string
@ -2286,6 +2314,8 @@ func (f *UintSliceFlag) String() string
func (f *UintSliceFlag) TakesValue() bool func (f *UintSliceFlag) TakesValue() bool
TakesValue returns true of the flag takes a value, otherwise false TakesValue returns true of the flag takes a value, otherwise false
func (f *UintSliceFlag) WithSeparatorSpec(spec separatorSpec)
type VisibleFlag interface { type VisibleFlag interface {
Flag Flag
@ -2413,6 +2443,9 @@ func (f *Float64SliceFlag) Apply(set *flag.FlagSet) error
Apply saves the flagSet for later usage calls, then calls the wrapped Apply saves the flagSet for later usage calls, then calls the wrapped
Float64SliceFlag.Apply Float64SliceFlag.Apply
func (f *Float64SliceFlag) ApplyInputSourceValue(cCtx *cli.Context, isc InputSourceContext) error
ApplyInputSourceValue applies a Float64Slice value if required
type GenericFlag struct { type GenericFlag struct {
*cli.GenericFlag *cli.GenericFlag
// Has unexported fields. // Has unexported fields.
@ -2434,12 +2467,16 @@ type InputSourceContext interface {
Source() string Source() string
Int(name string) (int, error) Int(name string) (int, error)
Int64(name string) (int64, error)
Uint(name string) (uint, error)
Uint64(name string) (uint64, error)
Duration(name string) (time.Duration, error) Duration(name string) (time.Duration, error)
Float64(name string) (float64, error) Float64(name string) (float64, error)
String(name string) (string, error) String(name string) (string, error)
StringSlice(name string) ([]string, error) StringSlice(name string) ([]string, error)
IntSlice(name string) ([]int, error) IntSlice(name string) ([]int, error)
Int64Slice(name string) ([]int64, error) Int64Slice(name string) ([]int64, error)
Float64Slice(name string) ([]float64, error)
Generic(name string) (cli.Generic, error) Generic(name string) (cli.Generic, error)
Bool(name string) (bool, error) Bool(name string) (bool, error)
@ -2483,6 +2520,8 @@ func (f *Int64Flag) Apply(set *flag.FlagSet) error
Apply saves the flagSet for later usage calls, then calls the wrapped Apply saves the flagSet for later usage calls, then calls the wrapped
Int64Flag.Apply Int64Flag.Apply
func (f *Int64Flag) ApplyInputSourceValue(cCtx *cli.Context, isc InputSourceContext) error
type Int64SliceFlag struct { type Int64SliceFlag struct {
*cli.Int64SliceFlag *cli.Int64SliceFlag
// Has unexported fields. // Has unexported fields.
@ -2553,6 +2592,10 @@ func (fsm *MapInputSource) Duration(name string) (time.Duration, error)
func (fsm *MapInputSource) Float64(name string) (float64, error) func (fsm *MapInputSource) Float64(name string) (float64, error)
Float64 returns an float64 from the map if it exists otherwise returns 0 Float64 returns an float64 from the map if it exists otherwise returns 0
func (fsm *MapInputSource) Float64Slice(name string) ([]float64, error)
Float64Slice returns an []float64 from the map if it exists otherwise
returns nil
func (fsm *MapInputSource) Generic(name string) (cli.Generic, error) func (fsm *MapInputSource) Generic(name string) (cli.Generic, error)
Generic returns an cli.Generic from the map if it exists otherwise returns Generic returns an cli.Generic from the map if it exists otherwise returns
nil nil
@ -2560,6 +2603,9 @@ func (fsm *MapInputSource) Generic(name string) (cli.Generic, error)
func (fsm *MapInputSource) Int(name string) (int, error) func (fsm *MapInputSource) Int(name string) (int, error)
Int returns an int from the map if it exists otherwise returns 0 Int returns an int from the map if it exists otherwise returns 0
func (fsm *MapInputSource) Int64(name string) (int64, error)
Int64 returns an int64 from the map if it exists otherwise returns 0
func (fsm *MapInputSource) Int64Slice(name string) ([]int64, error) func (fsm *MapInputSource) Int64Slice(name string) ([]int64, error)
Int64Slice returns an []int64 from the map if it exists otherwise returns Int64Slice returns an []int64 from the map if it exists otherwise returns
nil nil
@ -2578,6 +2624,12 @@ func (fsm *MapInputSource) StringSlice(name string) ([]string, error)
StringSlice returns an []string from the map if it exists otherwise returns StringSlice returns an []string from the map if it exists otherwise returns
nil nil
func (fsm *MapInputSource) Uint(name string) (uint, error)
Int64 returns an int64 from the map if it exists otherwise returns 0
func (fsm *MapInputSource) Uint64(name string) (uint64, error)
UInt64 returns an uint64 from the map if it exists otherwise returns 0
type PathFlag struct { type PathFlag struct {
*cli.PathFlag *cli.PathFlag
// Has unexported fields. // Has unexported fields.
@ -2643,6 +2695,8 @@ func (f *Uint64Flag) Apply(set *flag.FlagSet) error
Apply saves the flagSet for later usage calls, then calls the wrapped Apply saves the flagSet for later usage calls, then calls the wrapped
Uint64Flag.Apply Uint64Flag.Apply
func (f *Uint64Flag) ApplyInputSourceValue(cCtx *cli.Context, isc InputSourceContext) error
type UintFlag struct { type UintFlag struct {
*cli.UintFlag *cli.UintFlag
// Has unexported fields. // Has unexported fields.
@ -2657,3 +2711,5 @@ func (f *UintFlag) Apply(set *flag.FlagSet) error
Apply saves the flagSet for later usage calls, then calls the wrapped Apply saves the flagSet for later usage calls, then calls the wrapped
UintFlag.Apply UintFlag.Apply
func (f *UintFlag) ApplyInputSourceValue(cCtx *cli.Context, isc InputSourceContext) error

View file

@ -25,6 +25,8 @@ type Float64SliceFlag struct {
defaultValue *Float64Slice defaultValue *Float64Slice
separator separatorSpec
Action func(*Context, []float64) error Action func(*Context, []float64) error
} }
@ -120,6 +122,8 @@ type Int64SliceFlag struct {
defaultValue *Int64Slice defaultValue *Int64Slice
separator separatorSpec
Action func(*Context, []int64) error Action func(*Context, []int64) error
} }
@ -164,6 +168,8 @@ type IntSliceFlag struct {
defaultValue *IntSlice defaultValue *IntSlice
separator separatorSpec
Action func(*Context, []int) error Action func(*Context, []int) error
} }
@ -259,9 +265,13 @@ type StringSliceFlag struct {
defaultValue *StringSlice defaultValue *StringSlice
separator separatorSpec
TakesFile bool TakesFile bool
Action func(*Context, []string) error Action func(*Context, []string) error
KeepSpace bool
} }
// IsSet returns whether or not the flag has been set through env or file // IsSet returns whether or not the flag has been set through env or file
@ -358,6 +368,8 @@ type Uint64SliceFlag struct {
defaultValue *Uint64Slice defaultValue *Uint64Slice
separator separatorSpec
Action func(*Context, []uint64) error Action func(*Context, []uint64) error
} }
@ -402,6 +414,8 @@ type UintSliceFlag struct {
defaultValue *UintSlice defaultValue *UintSlice
separator separatorSpec
Action func(*Context, []uint) error Action func(*Context, []uint) error
} }
@ -448,6 +462,8 @@ type BoolFlag struct {
Count *int Count *int
DisableDefaultText bool
Action func(*Context, bool) error Action func(*Context, bool) error
} }

View file

@ -3,8 +3,8 @@ GOARCH ?= $(shell go env GOARCH)
GOOS_GOARCH := $(GOOS)_$(GOARCH) GOOS_GOARCH := $(GOOS)_$(GOARCH)
GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH) GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH)
LIBZSTD_NAME := libzstd_$(GOOS_GOARCH).a LIBZSTD_NAME := libzstd_$(GOOS_GOARCH).a
ZSTD_VERSION ?= v1.5.2 ZSTD_VERSION ?= v1.5.4
MUSL_BUILDER_IMAGE=golang:1.8.1-alpine MUSL_BUILDER_IMAGE=golang:1.20.1-alpine
BUILDER_IMAGE := local/builder_musl:2.0.0-$(shell echo $(MUSL_BUILDER_IMAGE) | tr : _)-1 BUILDER_IMAGE := local/builder_musl:2.0.0-$(shell echo $(MUSL_BUILDER_IMAGE) | tr : _)-1
.PHONY: libzstd.a $(LIBZSTD_NAME) .PHONY: libzstd.a $(LIBZSTD_NAME)

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -15,7 +15,12 @@ package gozstd
// See https://github.com/golang/go/issues/24450 . // See https://github.com/golang/go/issues/24450 .
static size_t ZSTD_initDStream_usingDDict_wrapper(uintptr_t ds, uintptr_t dict) { static size_t ZSTD_initDStream_usingDDict_wrapper(uintptr_t ds, uintptr_t dict) {
return ZSTD_initDStream_usingDDict((ZSTD_DStream*)ds, (ZSTD_DDict*)dict); ZSTD_DStream *zds = (ZSTD_DStream *)ds;
size_t rv = ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
if (rv != 0) {
return rv;
}
return ZSTD_DCtx_refDDict(zds, (ZSTD_DDict *)dict);
} }
static size_t ZSTD_freeDStream_wrapper(uintptr_t ds) { static size_t ZSTD_freeDStream_wrapper(uintptr_t ds) {

View file

@ -1,5 +1,5 @@
/* /*
* Copyright (c) Yann Collet, Facebook, Inc. * Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved. * All rights reserved.
* *
* This source code is licensed under both the BSD-style license (found in the * This source code is licensed under both the BSD-style license (found in the
@ -8,32 +8,43 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
#ifndef DICTBUILDER_H_001
#define DICTBUILDER_H_001
#if defined (__cplusplus) #if defined (__cplusplus)
extern "C" { extern "C" {
#endif #endif
#ifndef ZSTD_ZDICT_H
#define ZSTD_ZDICT_H
/*====== Dependencies ======*/ /*====== Dependencies ======*/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/* ===== ZDICTLIB_API : control library symbols visibility ===== */ /* ===== ZDICTLIB_API : control library symbols visibility ===== */
#ifndef ZDICTLIB_VISIBILITY #ifndef ZDICTLIB_VISIBLE
# if defined(__GNUC__) && (__GNUC__ >= 4) /* Backwards compatibility with old macro name */
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) # ifdef ZDICTLIB_VISIBILITY
# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
# else # else
# define ZDICTLIB_VISIBILITY # define ZDICTLIB_VISIBLE
# endif # endif
#endif #endif
#ifndef ZDICTLIB_HIDDEN
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else
# define ZDICTLIB_HIDDEN
# endif
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY # define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ # define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else #else
# define ZDICTLIB_API ZDICTLIB_VISIBILITY # define ZDICTLIB_API ZDICTLIB_VISIBLE
#endif #endif
/******************************************************************************* /*******************************************************************************
@ -110,8 +121,8 @@ extern "C" {
* The zstd CLI defaults to a 110KB dictionary. You likely don't need a * The zstd CLI defaults to a 110KB dictionary. You likely don't need a
* dictionary larger than that. But, most use cases can get away with a * dictionary larger than that. But, most use cases can get away with a
* smaller dictionary. The advanced dictionary builders can automatically * smaller dictionary. The advanced dictionary builders can automatically
* shrink the dictionary for you, and select a the smallest size that * shrink the dictionary for you, and select the smallest size that doesn't
* doesn't hurt compression ratio too much. See the `shrinkDict` parameter. * hurt compression ratio too much. See the `shrinkDict` parameter.
* A smaller dictionary can save memory, and potentially speed up * A smaller dictionary can save memory, and potentially speed up
* compression. * compression.
* *
@ -201,9 +212,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
const size_t* samplesSizes, unsigned nbSamples); const size_t* samplesSizes, unsigned nbSamples);
typedef struct { typedef struct {
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value)
* NOTE: The zstd format reserves some dictionary IDs for future use. * NOTE: The zstd format reserves some dictionary IDs for future use.
* You may use them in private settings, but be warned that they * You may use them in private settings, but be warned that they
* may be used by zstd in a public dictionary registry in the future. * may be used by zstd in a public dictionary registry in the future.
@ -260,9 +271,21 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
#endif /* ZSTD_ZDICT_H */
#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
#define ZSTD_ZDICT_H_STATIC
#ifdef ZDICT_STATIC_LINKING_ONLY /* This can be overridden externally to hide static symbols. */
#ifndef ZDICTLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
# else
# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
# endif
#endif
/* ==================================================================================== /* ====================================================================================
* The definitions in this section are considered experimental. * The definitions in this section are considered experimental.
@ -318,7 +341,7 @@ typedef struct {
* In general, it's recommended to provide a few thousands samples, though this can vary a lot. * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/ */
ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
void *dictBuffer, size_t dictBufferCapacity, void *dictBuffer, size_t dictBufferCapacity,
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
ZDICT_cover_params_t parameters); ZDICT_cover_params_t parameters);
@ -340,7 +363,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
* See ZDICT_trainFromBuffer() for details on failure modes. * See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
*/ */
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
void* dictBuffer, size_t dictBufferCapacity, void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_cover_params_t* parameters); ZDICT_cover_params_t* parameters);
@ -361,7 +384,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
* In general, it's recommended to provide a few thousands samples, though this can vary a lot. * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/ */
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
size_t dictBufferCapacity, const void *samplesBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples, const size_t *samplesSizes, unsigned nbSamples,
ZDICT_fastCover_params_t parameters); ZDICT_fastCover_params_t parameters);
@ -384,7 +407,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
* See ZDICT_trainFromBuffer() for details on failure modes. * See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
*/ */
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
size_t dictBufferCapacity, const void* samplesBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
const size_t* samplesSizes, unsigned nbSamples, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_fastCover_params_t* parameters); ZDICT_fastCover_params_t* parameters);
@ -409,7 +432,7 @@ typedef struct {
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
* Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
*/ */
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
void* dictBuffer, size_t dictBufferCapacity, void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_legacy_params_t parameters); ZDICT_legacy_params_t parameters);
@ -421,32 +444,31 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
or _CRT_SECURE_NO_WARNINGS in Visual. or _CRT_SECURE_NO_WARNINGS in Visual.
Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ # define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
#else #else
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API # define ZDICT_DEPRECATED(message) [[deprecated(message)]]
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) # define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (ZDICT_GCC_VERSION >= 301) # elif (ZDICT_GCC_VERSION >= 301)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) # define ZDICT_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER) # elif defined(_MSC_VER)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) # define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
# else # else
# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
# define ZDICT_DEPRECATED(message) ZDICTLIB_API # define ZDICT_DEPRECATED(message)
# endif # endif
#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
ZDICTLIB_STATIC_API
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
#endif /* ZDICT_STATIC_LINKING_ONLY */ #endif /* ZSTD_ZDICT_H_STATIC */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
#endif /* DICTBUILDER_H_001 */

View file

@ -1,5 +1,5 @@
/* /*
* Copyright (c) Yann Collet, Facebook, Inc. * Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved. * All rights reserved.
* *
* This source code is licensed under both the BSD-style license (found in the * This source code is licensed under both the BSD-style license (found in the
@ -14,21 +14,31 @@ extern "C" {
#ifndef ZSTD_H_235446 #ifndef ZSTD_H_235446
#define ZSTD_H_235446 #define ZSTD_H_235446
/* ====== Dependency ======*/ /* ====== Dependencies ======*/
#include <limits.h> /* INT_MAX */ #include <limits.h> /* INT_MAX */
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/* ===== ZSTDLIB_API : control library symbols visibility ===== */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */
#ifndef ZSTDLIB_VISIBLE #ifndef ZSTDLIB_VISIBLE
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) /* Backwards compatibility with old macro name */
# ifdef ZSTDLIB_VISIBILITY
# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) # define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else # else
# define ZSTDLIB_VISIBLE # define ZSTDLIB_VISIBLE
# endif
#endif
#ifndef ZSTDLIB_HIDDEN
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else
# define ZSTDLIB_HIDDEN # define ZSTDLIB_HIDDEN
# endif # endif
#endif #endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
@ -37,6 +47,28 @@ extern "C" {
# define ZSTDLIB_API ZSTDLIB_VISIBLE # define ZSTDLIB_API ZSTDLIB_VISIBLE
#endif #endif
/* Deprecation warnings :
* Should these warnings be a problem, it is generally possible to disable them,
* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
*/
#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
# define ZSTD_DEPRECATED(message)
# endif
#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
/******************************************************************************* /*******************************************************************************
Introduction Introduction
@ -74,7 +106,7 @@ extern "C" {
/*------ Version ------*/ /*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 5 #define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 2 #define ZSTD_VERSION_RELEASE 4
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() : /*! ZSTD_versionNumber() :
@ -165,7 +197,9 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
* "empty", "unknown" and "error" results to the same return value (0), * "empty", "unknown" and "error" results to the same return value (0),
* while ZSTD_getFrameContentSize() gives them separate return values. * while ZSTD_getFrameContentSize() gives them separate return values.
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
ZSTDLIB_API
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
* `src` should point to the start of a ZSTD frame or skippable frame. * `src` should point to the start of a ZSTD frame or skippable frame.
@ -177,8 +211,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
/*====== Helper functions ======*/ /*====== Helper functions ======*/
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ /* ZSTD_compressBound() :
* maximum compressed size in worst case single-pass scenario.
* When invoking `ZSTD_compress()` or any other one-pass compression function,
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
* as it eliminates one potential failure scenario,
* aka not enough room in dst buffer to write the compressed frame.
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
* In which case, ZSTD_compressBound() will return an error code
* which can be tested using ZSTD_isError().
*
* ZSTD_COMPRESSBOUND() :
* same as ZSTD_compressBound(), but as a macro.
* It can be used to produce constants, which can be useful for static allocation,
* for example to size a static array on stack.
* Will produce constant value 0 if srcSize too large.
*/
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
/* ZSTD_isError() :
* Most ZSTD_* functions returning a size_t value can be tested for error,
* using ZSTD_isError().
* @return 1 if error, 0 otherwise
*/
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
@ -421,6 +477,9 @@ typedef enum {
* ZSTD_c_validateSequences * ZSTD_c_validateSequences
* ZSTD_c_useBlockSplitter * ZSTD_c_useBlockSplitter
* ZSTD_c_useRowMatchFinder * ZSTD_c_useRowMatchFinder
* ZSTD_c_prefetchCDictTables
* ZSTD_c_enableSeqProducerFallback
* ZSTD_c_maxBlockSize
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly; * note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change. * also, the enums values themselves are unstable and can still change.
@ -439,7 +498,11 @@ typedef enum {
ZSTD_c_experimentalParam12=1009, ZSTD_c_experimentalParam12=1009,
ZSTD_c_experimentalParam13=1010, ZSTD_c_experimentalParam13=1010,
ZSTD_c_experimentalParam14=1011, ZSTD_c_experimentalParam14=1011,
ZSTD_c_experimentalParam15=1012 ZSTD_c_experimentalParam15=1012,
ZSTD_c_experimentalParam16=1013,
ZSTD_c_experimentalParam17=1014,
ZSTD_c_experimentalParam18=1015,
ZSTD_c_experimentalParam19=1016
} ZSTD_cParameter; } ZSTD_cParameter;
typedef struct { typedef struct {
@ -502,7 +565,7 @@ typedef enum {
* They will be used to compress next frame. * They will be used to compress next frame.
* Resetting session never fails. * Resetting session never fails.
* - The parameters : changes all parameters back to "default". * - The parameters : changes all parameters back to "default".
* This removes any reference to any dictionary too. * This also removes any reference to any dictionary or external sequence producer.
* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
* - Both : similar to resetting the session, followed by resetting parameters. * - Both : similar to resetting the session, followed by resetting parameters.
@ -552,13 +615,15 @@ typedef enum {
* ZSTD_d_stableOutBuffer * ZSTD_d_stableOutBuffer
* ZSTD_d_forceIgnoreChecksum * ZSTD_d_forceIgnoreChecksum
* ZSTD_d_refMultipleDDicts * ZSTD_d_refMultipleDDicts
* ZSTD_d_disableHuffmanAssembly
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly * note : never ever use experimentalParam? names directly
*/ */
ZSTD_d_experimentalParam1=1000, ZSTD_d_experimentalParam1=1000,
ZSTD_d_experimentalParam2=1001, ZSTD_d_experimentalParam2=1001,
ZSTD_d_experimentalParam3=1002, ZSTD_d_experimentalParam3=1002,
ZSTD_d_experimentalParam4=1003 ZSTD_d_experimentalParam4=1003,
ZSTD_d_experimentalParam5=1004
} ZSTD_dParameter; } ZSTD_dParameter;
@ -737,8 +802,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* This following is a legacy streaming API, available since v1.0+ . * This following is a legacy streaming API, available since v1.0+ .
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
* It is redundant, but remains fully supported. * It is redundant, but remains fully supported.
* Streaming in combination with advanced parameters and dictionary compression
* can only be used through the new API.
******************************************************************************/ ******************************************************************************/
/*! /*!
@ -747,6 +810,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
*
* Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
* to compress with a dictionary.
*/ */
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
/*! /*!
@ -797,13 +863,31 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer
/*===== Streaming decompression functions =====*/ /*===== Streaming decompression functions =====*/
/* This function is redundant with the advanced API and equivalent to: /*! ZSTD_initDStream() :
* Initialize/reset DStream state for new decompression operation.
* Call before new decompression operation using same DStream.
* *
* Note : This function is redundant with the advanced API and equivalent to:
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* ZSTD_DCtx_refDDict(zds, NULL); * ZSTD_DCtx_refDDict(zds, NULL);
*/ */
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
/*! ZSTD_decompressStream() :
* Streaming decompression function.
* Call repetitively to consume full input updating it as necessary.
* Function will update both input and output `pos` fields exposing current state via these fields:
* - `input.pos < input.size`, some input remaining and caller should provide remaining input
* on the next call.
* - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
* - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
* call ZSTD_decompressStream() again to flush remaining data to output.
* Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
*
* @return : 0 when a frame is completely decoded and fully flushed,
* or an error code, which can be tested using ZSTD_isError(),
* or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
*/
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
@ -922,7 +1006,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
* If @return == 0, the dictID could not be decoded. * If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons : * This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded (most common case). * - The frame does not require a dictionary to be decoded (most common case).
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
* Note : this use case also happens when using a non-conformant dictionary. * Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame. * - This is not a Zstandard frame.
@ -946,8 +1030,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
* meaning "return to no-dictionary mode". * meaning "return to no-dictionary mode".
* Note 1 : Dictionary is sticky, it will be used for all future compressed frames. * Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
* To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). * until parameters are reset, a new dictionary is loaded, or the dictionary
* is explicitly invalidated by loading a NULL dictionary.
* Note 2 : Loading a dictionary involves building tables. * Note 2 : Loading a dictionary involves building tables.
* It's also a CPU consuming operation, with non-negligible impact on latency. * It's also a CPU consuming operation, with non-negligible impact on latency.
* Tables are dependent on compression parameters, and for this reason, * Tables are dependent on compression parameters, and for this reason,
@ -960,7 +1045,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
* Reference a prepared dictionary, to be used for all next compressed frames. * Reference a prepared dictionary, to be used for all future compressed frames.
* Note that compression parameters are enforced from within CDict, * Note that compression parameters are enforced from within CDict,
* and supersede any compression parameter previously set within CCtx. * and supersede any compression parameter previously set within CCtx.
* The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@ -995,9 +1080,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize); const void* prefix, size_t prefixSize);
/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
* Create an internal DDict from dict buffer, * Create an internal DDict from dict buffer, to be used to decompress all future frames.
* to be used to decompress next frames. * The dictionary remains valid for all future frames, until explicitly invalidated, or
* The dictionary remains valid for all future frames, until explicitly invalidated. * a new dictionary is loaded.
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
* meaning "return to no-dictionary mode". * meaning "return to no-dictionary mode".
@ -1021,9 +1106,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
* The memory for the table is allocated on the first call to refDDict, and can be * The memory for the table is allocated on the first call to refDDict, and can be
* freed with ZSTD_freeDCtx(). * freed with ZSTD_freeDCtx().
* *
* If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
* will be managed, and referencing a dictionary effectively "discards" any previous one.
*
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Note 1 : Currently, only one dictionary can be managed.
* Referencing a new dictionary effectively "discards" any previous one.
* Special: referencing a NULL DDict means "return to no-dictionary mode". * Special: referencing a NULL DDict means "return to no-dictionary mode".
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
*/ */
@ -1086,28 +1172,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
# endif # endif
#endif #endif
/* Deprecation warnings :
* Should these warnings be a problem, it is generally possible to disable them,
* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
*/
#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
# elif defined(_MSC_VER)
# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
# endif
#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
/**************************************************************************************** /****************************************************************************************
* experimental API (static linking only) * experimental API (static linking only)
**************************************************************************************** ****************************************************************************************
@ -1142,6 +1206,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
#define ZSTD_STRATEGY_MIN ZSTD_fast #define ZSTD_STRATEGY_MIN ZSTD_fast
#define ZSTD_STRATEGY_MAX ZSTD_btultra2 #define ZSTD_STRATEGY_MAX ZSTD_btultra2
#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
#define ZSTD_OVERLAPLOG_MIN 0 #define ZSTD_OVERLAPLOG_MIN 0
@ -1369,33 +1434,89 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
* or an error code (if srcSize is too small) */ * or an error code (if srcSize is too small) */
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
/*! ZSTD_decompressionMargin() :
* Zstd supports in-place decompression, where the input and output buffers overlap.
* In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
* and the input buffer must be at the end of the output buffer.
*
* _______________________ Output Buffer ________________________
* | |
* | ____ Input Buffer ____|
* | | |
* v v v
* |---------------------------------------|-----------|----------|
* ^ ^ ^
* |___________________ Output_Size ___________________|_ Margin _|
*
* NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
* NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
* ZSTD_decompressDCtx().
* NOTE: This function supports multi-frame input.
*
* @param src The compressed frame(s)
* @param srcSize The size of the compressed frame(s)
* @returns The decompression margin or an error that can be checked with ZSTD_isError().
*/
ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
/*! ZSTD_DECOMPRESS_MARGIN() :
* Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
* the compressed frame, compute it from the original size and the blockSizeLog.
* See ZSTD_decompressionMargin() for details.
*
* WARNING: This macro does not support multi-frame input, the input must be a single
* zstd frame. If you need that support use the function, or implement it yourself.
*
* @param originalSize The original uncompressed size of the data.
* @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
* Unless you explicitly set the windowLog smaller than
* ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
*/
#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
4 /* checksum */ + \
((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
(blockSize) /* One block of margin */ \
))
typedef enum { typedef enum {
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
} ZSTD_sequenceFormat_e; } ZSTD_sequenceFormat_e;
/*! ZSTD_sequenceBound() :
* `srcSize` : size of the input buffer
* @return : upper-bound for the number of sequences that can be generated
* from a buffer of srcSize bytes
*
* note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
*/
ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
/*! ZSTD_generateSequences() : /*! ZSTD_generateSequences() :
* Generate sequences using ZSTD_compress2, given a source buffer. * Generate sequences using ZSTD_compress2(), given a source buffer.
* *
* Each block will end with a dummy sequence * Each block will end with a dummy sequence
* with offset == 0, matchLength == 0, and litLength == length of last literals. * with offset == 0, matchLength == 0, and litLength == length of last literals.
* litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
* simply acts as a block delimiter. * simply acts as a block delimiter.
* *
* zc can be used to insert custom compression params. * @zc can be used to insert custom compression params.
* This function invokes ZSTD_compress2 * This function invokes ZSTD_compress2().
* *
* The output of this function can be fed into ZSTD_compressSequences() with CCtx * The output of this function can be fed into ZSTD_compressSequences() with CCtx
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
* @return : number of sequences generated * @return : number of sequences generated
*/ */
ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, ZSTDLIB_STATIC_API size_t
size_t outSeqsSize, const void* src, size_t srcSize); ZSTD_generateSequences( ZSTD_CCtx* zc,
ZSTD_Sequence* outSeqs, size_t outSeqsSize,
const void* src, size_t srcSize);
/*! ZSTD_mergeBlockDelimiters() : /*! ZSTD_mergeBlockDelimiters() :
* Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
* by merging them into into the literals of the next sequence. * by merging them into the literals of the next sequence.
* *
* As such, the final generated result has no explicit representation of block boundaries, * As such, the final generated result has no explicit representation of block boundaries,
* and the final last literals segment is not represented in the sequences. * and the final last literals segment is not represented in the sequences.
@ -1407,7 +1528,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* o
ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
/*! ZSTD_compressSequences() : /*! ZSTD_compressSequences() :
* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
* @src contains the entire input (not just the literals).
* If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
* The entire source is compressed into a single frame. * The entire source is compressed into a single frame.
* *
@ -1432,9 +1555,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
* and cannot emit an RLE block that disagrees with the repcode history * and cannot emit an RLE block that disagrees with the repcode history
* @return : final compressed size or a ZSTD error. * @return : final compressed size, or a ZSTD error code.
*/ */
ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, ZSTDLIB_STATIC_API size_t
ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize); const void* src, size_t srcSize);
@ -1442,7 +1566,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* ds
/*! ZSTD_writeSkippableFrame() : /*! ZSTD_writeSkippableFrame() :
* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
* *
* Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
@ -1500,8 +1624,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
* *
* Note 2 : only single-threaded compression is supported. * Note : only single-threaded compression is supported.
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
*
* Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
* Size estimates assume that no external sequence producer is registered.
*/ */
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@ -1520,7 +1647,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(), * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
* an internal ?Dict will be created, which additional size is not estimated here. * an internal ?Dict will be created, which additional size is not estimated here.
* In this case, get total size by adding ZSTD_estimate?DictSize */ * In this case, get total size by adding ZSTD_estimate?DictSize
* Note 2 : only single-threaded compression is supported.
* ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
* Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
* Size estimates assume that no external sequence producer is registered.
*/
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@ -1670,11 +1802,19 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
* This function never fails (wide contract) */ * This function never fails (wide contract) */
ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_CCtx_setCParams() :
* Set all parameters provided within @cparams into the working @cctx.
* Note : if modifying parameters during compression (MT mode only),
* note that changes to the .windowLog parameter will be ignored.
* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
/*! ZSTD_compress_advanced() : /*! ZSTD_compress_advanced() :
* Note : this function is now DEPRECATED. * Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
* This prototype will generate compilation warnings. */ * This prototype will generate compilation warnings. */
ZSTD_DEPRECATED("use ZSTD_compress2") ZSTD_DEPRECATED("use ZSTD_compress2")
ZSTDLIB_STATIC_API
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@ -1686,6 +1826,7 @@ size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
* This prototype will generate compilation warnings. */ * This prototype will generate compilation warnings. */
ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
ZSTDLIB_STATIC_API
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@ -1829,13 +1970,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* Experimental parameter. * Experimental parameter.
* Default is 0 == disabled. Set to 1 to enable. * Default is 0 == disabled. Set to 1 to enable.
* *
* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same * Tells the compressor that input data presented with ZSTD_inBuffer
* between calls, except for the modifications that zstd makes to pos (the * will ALWAYS be the same between calls.
* caller must not modify pos). This is checked by the compressor, and * Technically, the @src pointer must never be changed,
* compression will fail if it ever changes. This means the only flush * and the @pos field can only be updated by zstd.
* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end * However, it's possible to increase the @size field,
* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) * allowing scenarios where more data can be appended after compressions starts.
* MUST not be modified during compression or you will get data corruption. * These conditions are checked by the compressor,
* and compression will fail if they are not respected.
* Also, data in the ZSTD_inBuffer within the range [src, src + pos)
* MUST not be modified during compression or it will result in data corruption.
* *
* When this flag is enabled zstd won't allocate an input window buffer, * When this flag is enabled zstd won't allocate an input window buffer,
* because the user guarantees it can reference the ZSTD_inBuffer until * because the user guarantees it can reference the ZSTD_inBuffer until
@ -1843,18 +1987,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
* avoid the memcpy() from the input buffer to the input window buffer. * avoid the memcpy() from the input buffer to the input window buffer.
* *
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
* That means this flag cannot be used with ZSTD_compressStream().
*
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
* this flag is ALWAYS memory safe, and will never access out-of-bounds * this flag is ALWAYS memory safe, and will never access out-of-bounds
* memory. However, compression WILL fail if you violate the preconditions. * memory. However, compression WILL fail if conditions are not respected.
* *
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
* not be modified during compression or you will get data corruption. This * not be modified during compression or it will result in data corruption.
* is because zstd needs to reference data in the ZSTD_inBuffer to find * This is because zstd needs to reference data in the ZSTD_inBuffer to find
* matches. Normally zstd maintains its own window buffer for this purpose, * matches. Normally zstd maintains its own window buffer for this purpose,
* but passing this flag tells zstd to use the user provided buffer. * but passing this flag tells zstd to rely on user provided buffer instead.
*/ */
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
@ -1899,7 +2040,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* Without validation, providing a sequence that does not conform to the zstd spec will cause * Without validation, providing a sequence that does not conform to the zstd spec will cause
* undefined behavior, and may produce a corrupted block. * undefined behavior, and may produce a corrupted block.
* *
* With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) then the function will bail out and * specifics regarding offset/matchlength requirements) then the function will bail out and
* return an error. * return an error.
* *
@ -1949,6 +2090,79 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
*/ */
#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
/* ZSTD_c_prefetchCDictTables
* Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
*
* In some situations, zstd uses CDict tables in-place rather than copying them
* into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
* In such situations, compression speed is seriously impacted when CDict tables are
* "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
* when they are used in-place.
*
* For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
* For sufficiently large inputs, zstd will by default memcpy() CDict tables
* into the working context, so there is no need to prefetch. This parameter is
* targeted at a middle range of input sizes, where a prefetch is cheap enough to be
* useful but memcpy() is too expensive. The exact range of input sizes where this
* makes sense is best determined by careful experimentation.
*
* Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
* but in the future zstd may conditionally enable this feature via an auto-detection
* heuristic for cold CDicts.
* Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
*/
#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
/* ZSTD_c_enableSeqProducerFallback
* Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
*
* Controls whether zstd will fall back to an internal sequence producer if an
* external sequence producer is registered and returns an error code. This fallback
* is block-by-block: the internal sequence producer will only be called for blocks
* where the external sequence producer returns an error code. Fallback parsing will
* follow any other cParam settings, such as compression level, the same as in a
* normal (fully-internal) compression operation.
*
* The user is strongly encouraged to read the full Block-Level Sequence Producer API
* documentation (below) before setting this parameter. */
#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
/* ZSTD_c_maxBlockSize
* Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
* The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
*
* This parameter can be used to set an upper bound on the blocksize
* that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
* bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
* compressBound() innacurate). Only currently meant to be used for testing.
*
*/
#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
/* ZSTD_c_searchForExternalRepcodes
* This parameter affects how zstd parses external sequences, such as sequences
* provided through the compressSequences() API or from an external block-level
* sequence producer.
*
* If set to ZSTD_ps_enable, the library will check for repeated offsets in
* external sequences, even if those repcodes are not explicitly indicated in
* the "rep" field. Note that this is the only way to exploit repcode matches
* while using compressSequences() or an external sequence producer, since zstd
* currently ignores the "rep" field of external sequences.
*
* If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
* external sequences, regardless of whether the "rep" field has been set. This
* reduces sequence compression overhead by about 25% while sacrificing some
* compression ratio.
*
* The default value is ZSTD_ps_auto, for which the library will enable/disable
* based on compression level.
*
* Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
* set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
*/
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
/*! ZSTD_CCtx_getParameter() : /*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter, * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value. * and store it into int* value.
@ -2105,7 +2319,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
* in the range [dst, dst + pos) MUST not be modified during decompression * in the range [dst, dst + pos) MUST not be modified during decompression
* or you will get data corruption. * or you will get data corruption.
* *
* When this flags is enabled zstd won't allocate an output buffer, because * When this flag is enabled zstd won't allocate an output buffer, because
* it can write directly to the ZSTD_outBuffer, but it will still allocate * it can write directly to the ZSTD_outBuffer, but it will still allocate
* an input buffer large enough to fit any compressed block. This will also * an input buffer large enough to fit any compressed block. This will also
* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
@ -2158,6 +2372,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
*/ */
#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
/* ZSTD_d_disableHuffmanAssembly
* Set to 1 to disable the Huffman assembly implementation.
* The default value is 0, which allows zstd to use the Huffman assembly
* implementation if available.
*
* This parameter can be used to disable Huffman assembly at runtime.
* If you want to disable it at compile time you can define the macro
* ZSTD_DISABLE_ASM.
*/
#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
/*! ZSTD_DCtx_setFormat() : /*! ZSTD_DCtx_setFormat() :
* This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
@ -2166,6 +2391,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
* such ZSTD_f_zstd1_magicless for example. * such ZSTD_f_zstd1_magicless for example.
* @return : 0, or an error code (which can be tested using ZSTD_isError()). */ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
ZSTDLIB_STATIC_API
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
/*! ZSTD_decompressStream_simpleArgs() : /*! ZSTD_decompressStream_simpleArgs() :
@ -2202,6 +2428,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel, int compressionLevel,
unsigned long long pledgedSrcSize); unsigned long long pledgedSrcSize);
@ -2219,6 +2446,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
int compressionLevel); int compressionLevel);
@ -2239,6 +2467,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, ZSTD_parameters params,
@ -2253,6 +2482,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
/*! ZSTD_initCStream_usingCDict_advanced() : /*! ZSTD_initCStream_usingCDict_advanced() :
@ -2271,6 +2501,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams, ZSTD_frameParameters fParams,
@ -2295,6 +2526,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
* This prototype will generate compilation warnings. * This prototype will generate compilation warnings.
*/ */
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
@ -2340,8 +2572,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
* *
* note: no dictionary will be used if dict == NULL or dictSize < 8 * note: no dictionary will be used if dict == NULL or dictSize < 8
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/*! /*!
@ -2351,8 +2583,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo
* ZSTD_DCtx_refDDict(zds, ddict); * ZSTD_DCtx_refDDict(zds, ddict);
* *
* note : ddict is referenced, it must outlive decompression session * note : ddict is referenced, it must outlive decompression session
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
/*! /*!
@ -2361,8 +2593,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* *
* re-use decompression parameters from previous init; saves dictionary loading * re-use decompression parameters from previous init; saves dictionary loading
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
@ -2383,7 +2615,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
Start by initializing a context. Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
Then, consume your input using ZSTD_compressContinue(). Then, consume your input using ZSTD_compressContinue().
There are some important considerations to keep in mind when using this advanced function : There are some important considerations to keep in mind when using this advanced function :
@ -2408,15 +2639,20 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
ZSTD_DEPRECATED("use advanced API to access custom parameters") ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("use advanced API to access custom parameters") ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
/** /**
Buffer-less streaming decompression (synchronous mode) Buffer-less streaming decompression (synchronous mode)
@ -2429,8 +2665,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
Data fragment must be large enough to ensure successful decoding. Data fragment must be large enough to ensure successful decoding.
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt. >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError(). errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@ -2449,7 +2685,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
The most memory efficient way is to use a round buffer of sufficient size. The most memory efficient way is to use a round buffer of sufficient size.
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
which can @return an error code if required value is too large for current system (in 32-bits mode). which can return an error code if required value is too large for current system (in 32-bits mode).
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
up to the moment there is not enough room left in the buffer to guarantee decoding another full block, up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@ -2469,7 +2705,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
It can also be an error code, which can be tested with ZSTD_isError(). It can also be an error code, which can be tested with ZSTD_isError().
@ -2501,6 +2737,8 @@ typedef struct {
unsigned headerSize; unsigned headerSize;
unsigned dictID; unsigned dictID;
unsigned checksumFlag; unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader; } ZSTD_frameHeader;
/*! ZSTD_getFrameHeader() : /*! ZSTD_getFrameHeader() :
@ -2523,6 +2761,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* misc */ /* misc */
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
@ -2545,7 +2784,6 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
- It is necessary to init context before starting - It is necessary to init context before starting
+ compression : any ZSTD_compressBegin*() variant, including with dictionary + compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ copyCCtx() and copyDCtx() can be used too
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ If input is larger than a block size, it's necessary to split input data into multiple blocks + If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead. + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
@ -2568,6 +2806,167 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
*
* *** OVERVIEW ***
* The Block-Level Sequence Producer API allows users to provide their own custom
* sequence producer which libzstd invokes to process each block. The produced list
* of sequences (literals and matches) is then post-processed by libzstd to produce
* valid compressed blocks.
*
* This block-level offload API is a more granular complement of the existing
* frame-level offload API compressSequences() (introduced in v1.5.1). It offers
* an easier migration story for applications already integrated with libzstd: the
* user application continues to invoke the same compression functions
* ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
* from the specific advantages of the external sequence producer. For example,
* the sequence producer could be tuned to take advantage of known characteristics
* of the input, to offer better speed / ratio, or could leverage hardware
* acceleration not available within libzstd itself.
*
* See contrib/externalSequenceProducer for an example program employing the
* Block-Level Sequence Producer API.
*
* *** USAGE ***
* The user is responsible for implementing a function of type
* ZSTD_sequenceProducer_F. For each block, zstd will pass the following
* arguments to the user-provided function:
*
* - sequenceProducerState: a pointer to a user-managed state for the sequence
* producer.
*
* - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
* outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
* backing outSeqs is managed by the CCtx.
*
* - src, srcSize: an input buffer for the sequence producer to parse.
* srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
*
* - dict, dictSize: a history buffer, which may be empty, which the sequence
* producer may reference as it parses the src buffer. Currently, zstd will
* always pass dictSize == 0 into external sequence producers, but this will
* change in the future.
*
* - compressionLevel: a signed integer representing the zstd compression level
* set by the user for the current operation. The sequence producer may choose
* to use this information to change its compression strategy and speed/ratio
* tradeoff. Note: the compression level does not reflect zstd parameters set
* through the advanced API.
*
* - windowSize: a size_t representing the maximum allowed offset for external
* sequences. Note that sequence offsets are sometimes allowed to exceed the
* windowSize if a dictionary is present, see doc/zstd_compression_format.md
* for details.
*
* The user-provided function shall return a size_t representing the number of
* sequences written to outSeqs. This return value will be treated as an error
* code if it is greater than outSeqsCapacity. The return value must be non-zero
* if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
* for convenience, but any value greater than outSeqsCapacity will be treated as
* an error code.
*
* If the user-provided function does not return an error code, the sequences
* written to outSeqs must be a valid parse of the src buffer. Data corruption may
* occur if the parse is not valid. A parse is defined to be valid if the
* following conditions hold:
* - The sum of matchLengths and literalLengths must equal srcSize.
* - All sequences in the parse, except for the final sequence, must have
* matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
* matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
* - All offsets must respect the windowSize parameter as specified in
* doc/zstd_compression_format.md.
* - If the final sequence has matchLength == 0, it must also have offset == 0.
*
* zstd will only validate these conditions (and fail compression if they do not
* hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
* validation has a performance cost.
*
* If the user-provided function returns an error, zstd will either fall back
* to an internal sequence producer or fail the compression operation. The user can
* choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
* cParam. Fallback compression will follow any other cParam settings, such as
* compression level, the same as in a normal compression operation.
*
* The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
* function by calling
* ZSTD_registerSequenceProducer(cctx,
* sequenceProducerState,
* sequenceProducer)
* This setting will persist until the next parameter reset of the CCtx.
*
* The sequenceProducerState must be initialized by the user before calling
* ZSTD_registerSequenceProducer(). The user is responsible for destroying the
* sequenceProducerState.
*
* *** LIMITATIONS ***
* This API is compatible with all zstd compression APIs which respect advanced parameters.
* However, there are three limitations:
*
* First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
* COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
* external sequence producer.
* - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
* cases (see its documentation for details). Users must explicitly set
* ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
* sequence producer is registered.
* - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
* whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
* check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
* Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
*
* Second, history buffers are not currently supported. Concretely, zstd will always pass
* dictSize == 0 to the external sequence producer (for now). This has two implications:
* - Dictionaries are not currently supported. Compression will *not* fail if the user
* references a dictionary, but the dictionary won't have any effect.
* - Stream history is not currently supported. All advanced compression APIs, including
* streaming APIs, work with external sequence producers, but each block is treated as
* an independent chunk without history from previous blocks.
*
* Third, multi-threading within a single compression is not currently supported. In other words,
* COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
* Multi-threading across compressions is fine: simply create one CCtx per thread.
*
* Long-term, we plan to overcome all three limitations. There is no technical blocker to
* overcoming them. It is purely a question of engineering effort.
*/
#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
typedef size_t ZSTD_sequenceProducer_F (
void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel,
size_t windowSize
);
/*! ZSTD_registerSequenceProducer() :
* Instruct zstd to use a block-level external sequence producer function.
*
* The sequenceProducerState must be initialized by the caller, and the caller is
* responsible for managing its lifetime. This parameter is sticky across
* compressions. It will remain set until the user explicitly resets compression
* parameters.
*
* Sequence producer registration is considered to be an "advanced parameter",
* part of the "advanced API". This means it will only have an effect on compression
* APIs which respect advanced parameters, such as compress2() and compressStream2().
* Older compression APIs such as compressCCtx(), which predate the introduction of
* "advanced parameters", will ignore any external sequence producer setting.
*
* The sequence producer can be "cleared" by registering a NULL function pointer. This
* removes all limitations described above in the "LIMITATIONS" section of the API docs.
*
* The user is strongly encouraged to read the full API documentation (above) before
* calling this function. */
ZSTDLIB_STATIC_API void
ZSTD_registerSequenceProducer(
ZSTD_CCtx* cctx,
void* sequenceProducerState,
ZSTD_sequenceProducer_F* sequenceProducer
);
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)

View file

@ -1,5 +1,5 @@
/* /*
* Copyright (c) Yann Collet, Facebook, Inc. * Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved. * All rights reserved.
* *
* This source code is licensed under both the BSD-style license (found in the * This source code is licensed under both the BSD-style license (found in the
@ -20,19 +20,31 @@ extern "C" {
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#ifndef ZSTDERRORLIB_VISIBILITY #ifndef ZSTDERRORLIB_VISIBLE
# if defined(__GNUC__) && (__GNUC__ >= 4) /* Backwards compatibility with old macro name */
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) # ifdef ZSTDERRORLIB_VISIBILITY
# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
# else # else
# define ZSTDERRORLIB_VISIBILITY # define ZSTDERRORLIB_VISIBLE
# endif # endif
#endif #endif
#ifndef ZSTDERRORLIB_HIDDEN
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else
# define ZSTDERRORLIB_HIDDEN
# endif
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY # define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ # define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else #else
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY # define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
#endif #endif
/*-********************************************* /*-*********************************************
@ -58,14 +70,17 @@ typedef enum {
ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_frameParameter_windowTooLarge = 16,
ZSTD_error_corruption_detected = 20, ZSTD_error_corruption_detected = 20,
ZSTD_error_checksum_wrong = 22, ZSTD_error_checksum_wrong = 22,
ZSTD_error_literals_headerWrong = 24,
ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_corrupted = 30,
ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionary_wrong = 32,
ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_dictionaryCreation_failed = 34,
ZSTD_error_parameter_unsupported = 40, ZSTD_error_parameter_unsupported = 40,
ZSTD_error_parameter_combination_unsupported = 41,
ZSTD_error_parameter_outOfBound = 42, ZSTD_error_parameter_outOfBound = 42,
ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_tableLog_tooLarge = 44,
ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooLarge = 46,
ZSTD_error_maxSymbolValue_tooSmall = 48, ZSTD_error_maxSymbolValue_tooSmall = 48,
ZSTD_error_stabilityCondition_notRespected = 50,
ZSTD_error_stage_wrong = 60, ZSTD_error_stage_wrong = 60,
ZSTD_error_init_missing = 62, ZSTD_error_init_missing = 62,
ZSTD_error_memory_allocation = 64, ZSTD_error_memory_allocation = 64,
@ -73,11 +88,15 @@ typedef enum {
ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_dstSize_tooSmall = 70,
ZSTD_error_srcSize_wrong = 72, ZSTD_error_srcSize_wrong = 72,
ZSTD_error_dstBuffer_null = 74, ZSTD_error_dstBuffer_null = 74,
ZSTD_error_noForwardProgress_destFull = 80,
ZSTD_error_noForwardProgress_inputEmpty = 82,
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_frameIndex_tooLarge = 100,
ZSTD_error_seekableIO = 102, ZSTD_error_seekableIO = 102,
ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_dstBuffer_wrong = 104,
ZSTD_error_srcBuffer_wrong = 105, ZSTD_error_srcBuffer_wrong = 105,
ZSTD_error_sequenceProducer_failed = 106,
ZSTD_error_externalSequences_invalid = 107,
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
} ZSTD_ErrorCode; } ZSTD_ErrorCode;

View file

@ -1,71 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package ctxhttp provides helper functions for performing context-aware HTTP requests.
package ctxhttp // import "golang.org/x/net/context/ctxhttp"
import (
"context"
"io"
"net/http"
"net/url"
"strings"
)
// Do sends an HTTP request with the provided http.Client and returns
// an HTTP response.
//
// If the client is nil, http.DefaultClient is used.
//
// The provided ctx must be non-nil. If it is canceled or times out,
// ctx.Err() will be returned.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req.WithContext(ctx))
// If we got an error, and the context has been canceled,
// the context's error is probably more useful.
if err != nil {
select {
case <-ctx.Done():
err = ctx.Err()
default:
}
}
return resp, err
}
// Get issues a GET request via the Do function.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Head issues a HEAD request via the Do function.
func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Post issues a POST request via the Do function.
func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return Do(ctx, client, req)
}
// PostForm issues a POST request via the Do function.
func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) {
return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

View file

@ -18,7 +18,7 @@ type inflow struct {
unsent int32 unsent int32
} }
// set sets the initial window. // init sets the initial window.
func (f *inflow) init(n int32) { func (f *inflow) init(n int32) {
f.avail = n f.avail = n
} }

View file

@ -662,6 +662,15 @@ func (f *Framer) WriteData(streamID uint32, endStream bool, data []byte) error {
// It is the caller's responsibility not to violate the maximum frame size // It is the caller's responsibility not to violate the maximum frame size
// and to not call other Write methods concurrently. // and to not call other Write methods concurrently.
func (f *Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error { func (f *Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error {
if err := f.startWriteDataPadded(streamID, endStream, data, pad); err != nil {
return err
}
return f.endWrite()
}
// startWriteDataPadded is WriteDataPadded, but only writes the frame to the Framer's internal buffer.
// The caller should call endWrite to flush the frame to the underlying writer.
func (f *Framer) startWriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error {
if !validStreamID(streamID) && !f.AllowIllegalWrites { if !validStreamID(streamID) && !f.AllowIllegalWrites {
return errStreamID return errStreamID
} }
@ -691,7 +700,7 @@ func (f *Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []by
} }
f.wbuf = append(f.wbuf, data...) f.wbuf = append(f.wbuf, data...)
f.wbuf = append(f.wbuf, pad...) f.wbuf = append(f.wbuf, pad...)
return f.endWrite() return nil
} }
// A SettingsFrame conveys configuration parameters that affect how // A SettingsFrame conveys configuration parameters that affect how

View file

@ -211,7 +211,7 @@ func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) {
return dt.ents[dt.len()-(int(i)-staticTable.len())], true return dt.ents[dt.len()-(int(i)-staticTable.len())], true
} }
// Decode decodes an entire block. // DecodeFull decodes an entire block.
// //
// TODO: remove this method and make it incremental later? This is // TODO: remove this method and make it incremental later? This is
// easier for debugging now. // easier for debugging now.
@ -359,6 +359,7 @@ func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error {
var hf HeaderField var hf HeaderField
wantStr := d.emitEnabled || it.indexed() wantStr := d.emitEnabled || it.indexed()
var undecodedName undecodedString
if nameIdx > 0 { if nameIdx > 0 {
ihf, ok := d.at(nameIdx) ihf, ok := d.at(nameIdx)
if !ok { if !ok {
@ -366,15 +367,27 @@ func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error {
} }
hf.Name = ihf.Name hf.Name = ihf.Name
} else { } else {
hf.Name, buf, err = d.readString(buf, wantStr) undecodedName, buf, err = d.readString(buf)
if err != nil { if err != nil {
return err return err
} }
} }
hf.Value, buf, err = d.readString(buf, wantStr) undecodedValue, buf, err := d.readString(buf)
if err != nil { if err != nil {
return err return err
} }
if wantStr {
if nameIdx <= 0 {
hf.Name, err = d.decodeString(undecodedName)
if err != nil {
return err
}
}
hf.Value, err = d.decodeString(undecodedValue)
if err != nil {
return err
}
}
d.buf = buf d.buf = buf
if it.indexed() { if it.indexed() {
d.dynTab.add(hf) d.dynTab.add(hf)
@ -459,46 +472,52 @@ func readVarInt(n byte, p []byte) (i uint64, remain []byte, err error) {
return 0, origP, errNeedMore return 0, origP, errNeedMore
} }
// readString decodes an hpack string from p. // readString reads an hpack string from p.
// //
// wantStr is whether s will be used. If false, decompression and // It returns a reference to the encoded string data to permit deferring decode costs
// []byte->string garbage are skipped if s will be ignored // until after the caller verifies all data is present.
// anyway. This does mean that huffman decoding errors for non-indexed func (d *Decoder) readString(p []byte) (u undecodedString, remain []byte, err error) {
// strings past the MAX_HEADER_LIST_SIZE are ignored, but the server
// is returning an error anyway, and because they're not indexed, the error
// won't affect the decoding state.
func (d *Decoder) readString(p []byte, wantStr bool) (s string, remain []byte, err error) {
if len(p) == 0 { if len(p) == 0 {
return "", p, errNeedMore return u, p, errNeedMore
} }
isHuff := p[0]&128 != 0 isHuff := p[0]&128 != 0
strLen, p, err := readVarInt(7, p) strLen, p, err := readVarInt(7, p)
if err != nil { if err != nil {
return "", p, err return u, p, err
} }
if d.maxStrLen != 0 && strLen > uint64(d.maxStrLen) { if d.maxStrLen != 0 && strLen > uint64(d.maxStrLen) {
return "", nil, ErrStringLength // Returning an error here means Huffman decoding errors
// for non-indexed strings past the maximum string length
// are ignored, but the server is returning an error anyway
// and because the string is not indexed the error will not
// affect the decoding state.
return u, nil, ErrStringLength
} }
if uint64(len(p)) < strLen { if uint64(len(p)) < strLen {
return "", p, errNeedMore return u, p, errNeedMore
}
if !isHuff {
if wantStr {
s = string(p[:strLen])
}
return s, p[strLen:], nil
} }
u.isHuff = isHuff
u.b = p[:strLen]
return u, p[strLen:], nil
}
if wantStr { type undecodedString struct {
isHuff bool
b []byte
}
func (d *Decoder) decodeString(u undecodedString) (string, error) {
if !u.isHuff {
return string(u.b), nil
}
buf := bufPool.Get().(*bytes.Buffer) buf := bufPool.Get().(*bytes.Buffer)
buf.Reset() // don't trust others buf.Reset() // don't trust others
defer bufPool.Put(buf) var s string
if err := huffmanDecode(buf, d.maxStrLen, p[:strLen]); err != nil { err := huffmanDecode(buf, d.maxStrLen, u.b)
buf.Reset() if err == nil {
return "", nil, err
}
s = buf.String() s = buf.String()
buf.Reset() // be nice to GC
} }
return s, p[strLen:], nil buf.Reset() // be nice to GC
bufPool.Put(buf)
return s, err
} }

View file

@ -843,8 +843,13 @@ type frameWriteResult struct {
// and then reports when it's done. // and then reports when it's done.
// At most one goroutine can be running writeFrameAsync at a time per // At most one goroutine can be running writeFrameAsync at a time per
// serverConn. // serverConn.
func (sc *serverConn) writeFrameAsync(wr FrameWriteRequest) { func (sc *serverConn) writeFrameAsync(wr FrameWriteRequest, wd *writeData) {
err := wr.write.writeFrame(sc) var err error
if wd == nil {
err = wr.write.writeFrame(sc)
} else {
err = sc.framer.endWrite()
}
sc.wroteFrameCh <- frameWriteResult{wr: wr, err: err} sc.wroteFrameCh <- frameWriteResult{wr: wr, err: err}
} }
@ -1251,9 +1256,16 @@ func (sc *serverConn) startFrameWrite(wr FrameWriteRequest) {
sc.writingFrameAsync = false sc.writingFrameAsync = false
err := wr.write.writeFrame(sc) err := wr.write.writeFrame(sc)
sc.wroteFrame(frameWriteResult{wr: wr, err: err}) sc.wroteFrame(frameWriteResult{wr: wr, err: err})
} else if wd, ok := wr.write.(*writeData); ok {
// Encode the frame in the serve goroutine, to ensure we don't have
// any lingering asynchronous references to data passed to Write.
// See https://go.dev/issue/58446.
sc.framer.startWriteDataPadded(wd.streamID, wd.endStream, wd.p, nil)
sc.writingFrameAsync = true
go sc.writeFrameAsync(wr, wd)
} else { } else {
sc.writingFrameAsync = true sc.writingFrameAsync = true
go sc.writeFrameAsync(wr) go sc.writeFrameAsync(wr, nil)
} }
} }
@ -2192,7 +2204,7 @@ func (sc *serverConn) newWriterAndRequestNoBody(st *stream, rp requestParam) (*r
tlsState = sc.tlsState tlsState = sc.tlsState
} }
needsContinue := rp.header.Get("Expect") == "100-continue" needsContinue := httpguts.HeaderValuesContainsToken(rp.header["Expect"], "100-continue")
if needsContinue { if needsContinue {
rp.header.Del("Expect") rp.header.Del("Expect")
} }

View file

@ -1569,7 +1569,7 @@ func (cs *clientStream) cleanupWriteRequest(err error) {
close(cs.donec) close(cs.donec)
} }
// awaitOpenSlotForStream waits until len(streams) < maxConcurrentStreams. // awaitOpenSlotForStreamLocked waits until len(streams) < maxConcurrentStreams.
// Must hold cc.mu. // Must hold cc.mu.
func (cc *ClientConn) awaitOpenSlotForStreamLocked(cs *clientStream) error { func (cc *ClientConn) awaitOpenSlotForStreamLocked(cs *clientStream) error {
for { for {

View file

@ -32,7 +32,7 @@ type histogram struct {
valueCount int64 // number of values recorded for single value valueCount int64 // number of values recorded for single value
} }
// AddMeasurement records a value measurement observation to the histogram. // addMeasurement records a value measurement observation to the histogram.
func (h *histogram) addMeasurement(value int64) { func (h *histogram) addMeasurement(value int64) {
// TODO: assert invariant // TODO: assert invariant
h.sum += value h.sum += value

View file

@ -19,8 +19,6 @@ import (
"strings" "strings"
"sync" "sync"
"time" "time"
"golang.org/x/net/context/ctxhttp"
) )
// Token represents the credentials used to authorize // Token represents the credentials used to authorize
@ -229,7 +227,7 @@ func RetrieveToken(ctx context.Context, clientID, clientSecret, tokenURL string,
} }
func doTokenRoundTrip(ctx context.Context, req *http.Request) (*Token, error) { func doTokenRoundTrip(ctx context.Context, req *http.Request) (*Token, error) {
r, err := ctxhttp.Do(ctx, ContextClient(ctx), req) r, err := ContextClient(ctx).Do(req.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// +build gccgo,!hurd //go:build gccgo && !aix && !hurd
// +build !aix,!hurd // +build gccgo,!aix,!hurd
#include <errno.h> #include <errno.h>
#include <stdint.h> #include <stdint.h>

View file

@ -230,6 +230,7 @@ func direntNamlen(buf []byte) (uint64, bool) {
func PtraceAttach(pid int) (err error) { return ptrace(PT_ATTACH, pid, 0, 0) } func PtraceAttach(pid int) (err error) { return ptrace(PT_ATTACH, pid, 0, 0) }
func PtraceDetach(pid int) (err error) { return ptrace(PT_DETACH, pid, 0, 0) } func PtraceDetach(pid int) (err error) { return ptrace(PT_DETACH, pid, 0, 0) }
func PtraceDenyAttach() (err error) { return ptrace(PT_DENY_ATTACH, 0, 0, 0) }
//sysnb pipe(p *[2]int32) (err error) //sysnb pipe(p *[2]int32) (err error)

View file

@ -60,8 +60,13 @@ func PtraceGetFsBase(pid int, fsbase *int64) (err error) {
return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0) return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0)
} }
func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) {
ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint32(countin)} ioDesc := PtraceIoDesc{
Op: int32(req),
Offs: offs,
Addr: uintptr(unsafe.Pointer(&out[0])), // TODO(#58351): this is not safe.
Len: uint32(countin),
}
err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0)
return int(ioDesc.Len), err return int(ioDesc.Len), err
} }

View file

@ -60,8 +60,13 @@ func PtraceGetFsBase(pid int, fsbase *int64) (err error) {
return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0) return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0)
} }
func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) {
ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} ioDesc := PtraceIoDesc{
Op: int32(req),
Offs: offs,
Addr: uintptr(unsafe.Pointer(&out[0])), // TODO(#58351): this is not safe.
Len: uint64(countin),
}
err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0)
return int(ioDesc.Len), err return int(ioDesc.Len), err
} }

View file

@ -56,8 +56,13 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno)
func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) {
ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint32(countin)} ioDesc := PtraceIoDesc{
Op: int32(req),
Offs: offs,
Addr: uintptr(unsafe.Pointer(&out[0])), // TODO(#58351): this is not safe.
Len: uint32(countin),
}
err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0)
return int(ioDesc.Len), err return int(ioDesc.Len), err
} }

View file

@ -56,8 +56,13 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno)
func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) {
ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} ioDesc := PtraceIoDesc{
Op: int32(req),
Offs: offs,
Addr: uintptr(unsafe.Pointer(&out[0])), // TODO(#58351): this is not safe.
Len: uint64(countin),
}
err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0)
return int(ioDesc.Len), err return int(ioDesc.Len), err
} }

View file

@ -56,8 +56,13 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno)
func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) {
ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} ioDesc := PtraceIoDesc{
Op: int32(req),
Offs: offs,
Addr: uintptr(unsafe.Pointer(&out[0])), // TODO(#58351): this is not safe.
Len: uint64(countin),
}
err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0)
return int(ioDesc.Len), err return int(ioDesc.Len), err
} }

View file

@ -1800,6 +1800,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
//sysnb Capset(hdr *CapUserHeader, data *CapUserData) (err error) //sysnb Capset(hdr *CapUserHeader, data *CapUserData) (err error)
//sys Chdir(path string) (err error) //sys Chdir(path string) (err error)
//sys Chroot(path string) (err error) //sys Chroot(path string) (err error)
//sys ClockAdjtime(clockid int32, buf *Timex) (state int, err error)
//sys ClockGetres(clockid int32, res *Timespec) (err error) //sys ClockGetres(clockid int32, res *Timespec) (err error)
//sys ClockGettime(clockid int32, time *Timespec) (err error) //sys ClockGettime(clockid int32, time *Timespec) (err error)
//sys ClockNanosleep(clockid int32, flags int, request *Timespec, remain *Timespec) (err error) //sys ClockNanosleep(clockid int32, flags int, request *Timespec, remain *Timespec) (err error)
@ -1999,7 +2000,7 @@ func appendBytes(vecs []Iovec, bs [][]byte) []Iovec {
// offs2lohi splits offs into its low and high order bits. // offs2lohi splits offs into its low and high order bits.
func offs2lohi(offs int64) (lo, hi uintptr) { func offs2lohi(offs int64) (lo, hi uintptr) {
const longBits = SizeofLong * 8 const longBits = SizeofLong * 8
return uintptr(offs), uintptr(uint64(offs) >> longBits) return uintptr(offs), uintptr(uint64(offs) >> (longBits - 1) >> 1) // two shifts to avoid false positive in vet
} }
func Readv(fd int, iovs [][]byte) (n int, err error) { func Readv(fd int, iovs [][]byte) (n int, err error) {

View file

@ -578,7 +578,7 @@ func Lutimes(path string, tv []Timeval) error {
return UtimesNanoAt(AT_FDCWD, path, ts, AT_SYMLINK_NOFOLLOW) return UtimesNanoAt(AT_FDCWD, path, ts, AT_SYMLINK_NOFOLLOW)
} }
// emptyIovec reports whether there are no bytes in the slice of Iovec. // emptyIovecs reports whether there are no bytes in the slice of Iovec.
func emptyIovecs(iov []Iovec) bool { func emptyIovecs(iov []Iovec) bool {
for i := range iov { for i := range iov {
if iov[i].Len > 0 { if iov[i].Len > 0 {

View file

@ -9,7 +9,7 @@ package unix
import "time" import "time"
// TimespecToNSec returns the time stored in ts as nanoseconds. // TimespecToNsec returns the time stored in ts as nanoseconds.
func TimespecToNsec(ts Timespec) int64 { return ts.Nano() } func TimespecToNsec(ts Timespec) int64 { return ts.Nano() }
// NsecToTimespec converts a number of nanoseconds into a Timespec. // NsecToTimespec converts a number of nanoseconds into a Timespec.

View file

@ -36,9 +36,14 @@ func xattrnamespace(fullattr string) (ns int, attr string, err error) {
func initxattrdest(dest []byte, idx int) (d unsafe.Pointer) { func initxattrdest(dest []byte, idx int) (d unsafe.Pointer) {
if len(dest) > idx { if len(dest) > idx {
return unsafe.Pointer(&dest[idx]) return unsafe.Pointer(&dest[idx])
} else {
return unsafe.Pointer(_zero)
} }
if dest != nil {
// extattr_get_file and extattr_list_file treat NULL differently from
// a non-NULL pointer of length zero. Preserve the property of nilness,
// even if we can't use dest directly.
return unsafe.Pointer(&_zero)
}
return nil
} }
// FreeBSD and NetBSD implement their own syscalls to handle extended attributes // FreeBSD and NetBSD implement their own syscalls to handle extended attributes

View file

@ -457,7 +457,6 @@ const (
B600 = 0x8 B600 = 0x8
B75 = 0x2 B75 = 0x2
B9600 = 0xd B9600 = 0xd
BALLOON_KVM_MAGIC = 0x13661366
BDEVFS_MAGIC = 0x62646576 BDEVFS_MAGIC = 0x62646576
BINDERFS_SUPER_MAGIC = 0x6c6f6f70 BINDERFS_SUPER_MAGIC = 0x6c6f6f70
BINFMTFS_MAGIC = 0x42494e4d BINFMTFS_MAGIC = 0x42494e4d
@ -563,6 +562,7 @@ const (
BUS_USB = 0x3 BUS_USB = 0x3
BUS_VIRTUAL = 0x6 BUS_VIRTUAL = 0x6
CAN_BCM = 0x2 CAN_BCM = 0x2
CAN_BUS_OFF_THRESHOLD = 0x100
CAN_CTRLMODE_3_SAMPLES = 0x4 CAN_CTRLMODE_3_SAMPLES = 0x4
CAN_CTRLMODE_BERR_REPORTING = 0x10 CAN_CTRLMODE_BERR_REPORTING = 0x10
CAN_CTRLMODE_CC_LEN8_DLC = 0x100 CAN_CTRLMODE_CC_LEN8_DLC = 0x100
@ -577,9 +577,12 @@ const (
CAN_EFF_FLAG = 0x80000000 CAN_EFF_FLAG = 0x80000000
CAN_EFF_ID_BITS = 0x1d CAN_EFF_ID_BITS = 0x1d
CAN_EFF_MASK = 0x1fffffff CAN_EFF_MASK = 0x1fffffff
CAN_ERROR_PASSIVE_THRESHOLD = 0x80
CAN_ERROR_WARNING_THRESHOLD = 0x60
CAN_ERR_ACK = 0x20 CAN_ERR_ACK = 0x20
CAN_ERR_BUSERROR = 0x80 CAN_ERR_BUSERROR = 0x80
CAN_ERR_BUSOFF = 0x40 CAN_ERR_BUSOFF = 0x40
CAN_ERR_CNT = 0x200
CAN_ERR_CRTL = 0x4 CAN_ERR_CRTL = 0x4
CAN_ERR_CRTL_ACTIVE = 0x40 CAN_ERR_CRTL_ACTIVE = 0x40
CAN_ERR_CRTL_RX_OVERFLOW = 0x1 CAN_ERR_CRTL_RX_OVERFLOW = 0x1
@ -820,9 +823,9 @@ const (
DM_UUID_FLAG = 0x4000 DM_UUID_FLAG = 0x4000
DM_UUID_LEN = 0x81 DM_UUID_LEN = 0x81
DM_VERSION = 0xc138fd00 DM_VERSION = 0xc138fd00
DM_VERSION_EXTRA = "-ioctl (2022-02-22)" DM_VERSION_EXTRA = "-ioctl (2022-07-28)"
DM_VERSION_MAJOR = 0x4 DM_VERSION_MAJOR = 0x4
DM_VERSION_MINOR = 0x2e DM_VERSION_MINOR = 0x2f
DM_VERSION_PATCHLEVEL = 0x0 DM_VERSION_PATCHLEVEL = 0x0
DT_BLK = 0x6 DT_BLK = 0x6
DT_CHR = 0x2 DT_CHR = 0x2
@ -1049,6 +1052,7 @@ const (
ETH_P_CAIF = 0xf7 ETH_P_CAIF = 0xf7
ETH_P_CAN = 0xc ETH_P_CAN = 0xc
ETH_P_CANFD = 0xd ETH_P_CANFD = 0xd
ETH_P_CANXL = 0xe
ETH_P_CFM = 0x8902 ETH_P_CFM = 0x8902
ETH_P_CONTROL = 0x16 ETH_P_CONTROL = 0x16
ETH_P_CUST = 0x6006 ETH_P_CUST = 0x6006
@ -1060,6 +1064,7 @@ const (
ETH_P_DNA_RT = 0x6003 ETH_P_DNA_RT = 0x6003
ETH_P_DSA = 0x1b ETH_P_DSA = 0x1b
ETH_P_DSA_8021Q = 0xdadb ETH_P_DSA_8021Q = 0xdadb
ETH_P_DSA_A5PSW = 0xe001
ETH_P_ECONET = 0x18 ETH_P_ECONET = 0x18
ETH_P_EDSA = 0xdada ETH_P_EDSA = 0xdada
ETH_P_ERSPAN = 0x88be ETH_P_ERSPAN = 0x88be
@ -1194,8 +1199,10 @@ const (
FAN_MARK_EVICTABLE = 0x200 FAN_MARK_EVICTABLE = 0x200
FAN_MARK_FILESYSTEM = 0x100 FAN_MARK_FILESYSTEM = 0x100
FAN_MARK_FLUSH = 0x80 FAN_MARK_FLUSH = 0x80
FAN_MARK_IGNORE = 0x400
FAN_MARK_IGNORED_MASK = 0x20 FAN_MARK_IGNORED_MASK = 0x20
FAN_MARK_IGNORED_SURV_MODIFY = 0x40 FAN_MARK_IGNORED_SURV_MODIFY = 0x40
FAN_MARK_IGNORE_SURV = 0x440
FAN_MARK_INODE = 0x0 FAN_MARK_INODE = 0x0
FAN_MARK_MOUNT = 0x10 FAN_MARK_MOUNT = 0x10
FAN_MARK_ONLYDIR = 0x8 FAN_MARK_ONLYDIR = 0x8
@ -1253,6 +1260,7 @@ const (
FSCRYPT_MODE_AES_128_CBC = 0x5 FSCRYPT_MODE_AES_128_CBC = 0x5
FSCRYPT_MODE_AES_128_CTS = 0x6 FSCRYPT_MODE_AES_128_CTS = 0x6
FSCRYPT_MODE_AES_256_CTS = 0x4 FSCRYPT_MODE_AES_256_CTS = 0x4
FSCRYPT_MODE_AES_256_HCTR2 = 0xa
FSCRYPT_MODE_AES_256_XTS = 0x1 FSCRYPT_MODE_AES_256_XTS = 0x1
FSCRYPT_POLICY_FLAGS_PAD_16 = 0x2 FSCRYPT_POLICY_FLAGS_PAD_16 = 0x2
FSCRYPT_POLICY_FLAGS_PAD_32 = 0x3 FSCRYPT_POLICY_FLAGS_PAD_32 = 0x3
@ -1430,6 +1438,7 @@ const (
IFF_NOARP = 0x80 IFF_NOARP = 0x80
IFF_NOFILTER = 0x1000 IFF_NOFILTER = 0x1000
IFF_NOTRAILERS = 0x20 IFF_NOTRAILERS = 0x20
IFF_NO_CARRIER = 0x40
IFF_NO_PI = 0x1000 IFF_NO_PI = 0x1000
IFF_ONE_QUEUE = 0x2000 IFF_ONE_QUEUE = 0x2000
IFF_PERSIST = 0x800 IFF_PERSIST = 0x800
@ -1805,6 +1814,7 @@ const (
MADV_DONTDUMP = 0x10 MADV_DONTDUMP = 0x10
MADV_DONTFORK = 0xa MADV_DONTFORK = 0xa
MADV_DONTNEED = 0x4 MADV_DONTNEED = 0x4
MADV_DONTNEED_LOCKED = 0x18
MADV_FREE = 0x8 MADV_FREE = 0x8
MADV_HUGEPAGE = 0xe MADV_HUGEPAGE = 0xe
MADV_HWPOISON = 0x64 MADV_HWPOISON = 0x64
@ -1846,7 +1856,7 @@ const (
MFD_ALLOW_SEALING = 0x2 MFD_ALLOW_SEALING = 0x2
MFD_CLOEXEC = 0x1 MFD_CLOEXEC = 0x1
MFD_HUGETLB = 0x4 MFD_HUGETLB = 0x4
MFD_HUGE_16GB = -0x78000000 MFD_HUGE_16GB = 0x88000000
MFD_HUGE_16MB = 0x60000000 MFD_HUGE_16MB = 0x60000000
MFD_HUGE_1GB = 0x78000000 MFD_HUGE_1GB = 0x78000000
MFD_HUGE_1MB = 0x50000000 MFD_HUGE_1MB = 0x50000000
@ -2212,6 +2222,11 @@ const (
PERF_AUX_FLAG_PARTIAL = 0x4 PERF_AUX_FLAG_PARTIAL = 0x4
PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK = 0xff00 PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK = 0xff00
PERF_AUX_FLAG_TRUNCATED = 0x1 PERF_AUX_FLAG_TRUNCATED = 0x1
PERF_BR_ARM64_DEBUG_DATA = 0x7
PERF_BR_ARM64_DEBUG_EXIT = 0x5
PERF_BR_ARM64_DEBUG_HALT = 0x4
PERF_BR_ARM64_DEBUG_INST = 0x6
PERF_BR_ARM64_FIQ = 0x3
PERF_FLAG_FD_CLOEXEC = 0x8 PERF_FLAG_FD_CLOEXEC = 0x8
PERF_FLAG_FD_NO_GROUP = 0x1 PERF_FLAG_FD_NO_GROUP = 0x1
PERF_FLAG_FD_OUTPUT = 0x2 PERF_FLAG_FD_OUTPUT = 0x2
@ -2232,6 +2247,8 @@ const (
PERF_MEM_LOCK_NA = 0x1 PERF_MEM_LOCK_NA = 0x1
PERF_MEM_LOCK_SHIFT = 0x18 PERF_MEM_LOCK_SHIFT = 0x18
PERF_MEM_LVLNUM_ANY_CACHE = 0xb PERF_MEM_LVLNUM_ANY_CACHE = 0xb
PERF_MEM_LVLNUM_CXL = 0x9
PERF_MEM_LVLNUM_IO = 0xa
PERF_MEM_LVLNUM_L1 = 0x1 PERF_MEM_LVLNUM_L1 = 0x1
PERF_MEM_LVLNUM_L2 = 0x2 PERF_MEM_LVLNUM_L2 = 0x2
PERF_MEM_LVLNUM_L3 = 0x3 PERF_MEM_LVLNUM_L3 = 0x3
@ -2265,6 +2282,7 @@ const (
PERF_MEM_REMOTE_REMOTE = 0x1 PERF_MEM_REMOTE_REMOTE = 0x1
PERF_MEM_REMOTE_SHIFT = 0x25 PERF_MEM_REMOTE_SHIFT = 0x25
PERF_MEM_SNOOPX_FWD = 0x1 PERF_MEM_SNOOPX_FWD = 0x1
PERF_MEM_SNOOPX_PEER = 0x2
PERF_MEM_SNOOPX_SHIFT = 0x26 PERF_MEM_SNOOPX_SHIFT = 0x26
PERF_MEM_SNOOP_HIT = 0x4 PERF_MEM_SNOOP_HIT = 0x4
PERF_MEM_SNOOP_HITM = 0x10 PERF_MEM_SNOOP_HITM = 0x10
@ -2301,7 +2319,6 @@ const (
PERF_SAMPLE_BRANCH_PLM_ALL = 0x7 PERF_SAMPLE_BRANCH_PLM_ALL = 0x7
PERF_SAMPLE_WEIGHT_TYPE = 0x1004000 PERF_SAMPLE_WEIGHT_TYPE = 0x1004000
PIPEFS_MAGIC = 0x50495045 PIPEFS_MAGIC = 0x50495045
PPC_CMM_MAGIC = 0xc7571590
PPPIOCGNPMODE = 0xc008744c PPPIOCGNPMODE = 0xc008744c
PPPIOCNEWUNIT = 0xc004743e PPPIOCNEWUNIT = 0xc004743e
PRIO_PGRP = 0x1 PRIO_PGRP = 0x1
@ -2999,6 +3016,7 @@ const (
STATX_BLOCKS = 0x400 STATX_BLOCKS = 0x400
STATX_BTIME = 0x800 STATX_BTIME = 0x800
STATX_CTIME = 0x80 STATX_CTIME = 0x80
STATX_DIOALIGN = 0x2000
STATX_GID = 0x10 STATX_GID = 0x10
STATX_INO = 0x100 STATX_INO = 0x100
STATX_MNT_ID = 0x1000 STATX_MNT_ID = 0x1000
@ -3392,9 +3410,7 @@ const (
XDP_ZEROCOPY = 0x4 XDP_ZEROCOPY = 0x4
XENFS_SUPER_MAGIC = 0xabba1974 XENFS_SUPER_MAGIC = 0xabba1974
XFS_SUPER_MAGIC = 0x58465342 XFS_SUPER_MAGIC = 0x58465342
Z3FOLD_MAGIC = 0x33
ZONEFS_MAGIC = 0x5a4f4653 ZONEFS_MAGIC = 0x5a4f4653
ZSMALLOC_MAGIC = 0x58295829
_HIDIOCGRAWNAME_LEN = 0x80 _HIDIOCGRAWNAME_LEN = 0x80
_HIDIOCGRAWPHYS_LEN = 0x40 _HIDIOCGRAWPHYS_LEN = 0x40
_HIDIOCGRAWUNIQ_LEN = 0x40 _HIDIOCGRAWUNIQ_LEN = 0x40

View file

@ -133,6 +133,7 @@ const (
MEMGETREGIONCOUNT = 0x80044d07 MEMGETREGIONCOUNT = 0x80044d07
MEMISLOCKED = 0x80084d17 MEMISLOCKED = 0x80084d17
MEMLOCK = 0x40084d05 MEMLOCK = 0x40084d05
MEMREAD = 0xc03c4d1a
MEMREADOOB = 0xc00c4d04 MEMREADOOB = 0xc00c4d04
MEMSETBADBLOCK = 0x40084d0c MEMSETBADBLOCK = 0x40084d0c
MEMUNLOCK = 0x40084d06 MEMUNLOCK = 0x40084d06

View file

@ -133,6 +133,7 @@ const (
MEMGETREGIONCOUNT = 0x80044d07 MEMGETREGIONCOUNT = 0x80044d07
MEMISLOCKED = 0x80084d17 MEMISLOCKED = 0x80084d17
MEMLOCK = 0x40084d05 MEMLOCK = 0x40084d05
MEMREAD = 0xc0404d1a
MEMREADOOB = 0xc0104d04 MEMREADOOB = 0xc0104d04
MEMSETBADBLOCK = 0x40084d0c MEMSETBADBLOCK = 0x40084d0c
MEMUNLOCK = 0x40084d06 MEMUNLOCK = 0x40084d06

View file

@ -131,6 +131,7 @@ const (
MEMGETREGIONCOUNT = 0x80044d07 MEMGETREGIONCOUNT = 0x80044d07
MEMISLOCKED = 0x80084d17 MEMISLOCKED = 0x80084d17
MEMLOCK = 0x40084d05 MEMLOCK = 0x40084d05
MEMREAD = 0xc0404d1a
MEMREADOOB = 0xc00c4d04 MEMREADOOB = 0xc00c4d04
MEMSETBADBLOCK = 0x40084d0c MEMSETBADBLOCK = 0x40084d0c
MEMUNLOCK = 0x40084d06 MEMUNLOCK = 0x40084d06

View file

@ -134,6 +134,7 @@ const (
MEMGETREGIONCOUNT = 0x80044d07 MEMGETREGIONCOUNT = 0x80044d07
MEMISLOCKED = 0x80084d17 MEMISLOCKED = 0x80084d17
MEMLOCK = 0x40084d05 MEMLOCK = 0x40084d05
MEMREAD = 0xc0404d1a
MEMREADOOB = 0xc0104d04 MEMREADOOB = 0xc0104d04
MEMSETBADBLOCK = 0x40084d0c MEMSETBADBLOCK = 0x40084d0c
MEMUNLOCK = 0x40084d06 MEMUNLOCK = 0x40084d06

View file

@ -132,6 +132,7 @@ const (
MEMGETREGIONCOUNT = 0x80044d07 MEMGETREGIONCOUNT = 0x80044d07
MEMISLOCKED = 0x80084d17 MEMISLOCKED = 0x80084d17
MEMLOCK = 0x40084d05 MEMLOCK = 0x40084d05
MEMREAD = 0xc0404d1a
MEMREADOOB = 0xc0104d04 MEMREADOOB = 0xc0104d04
MEMSETBADBLOCK = 0x40084d0c MEMSETBADBLOCK = 0x40084d0c
MEMUNLOCK = 0x40084d06 MEMUNLOCK = 0x40084d06

Some files were not shown because too many files have changed in this diff Show more