vendor: make vendor-update

This commit is contained in:
Aliaksandr Valialkin 2022-09-19 15:12:22 +03:00
parent fd98ec8ba3
commit 310d0caec2
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
58 changed files with 1283 additions and 927 deletions

12
go.mod
View file

@ -11,7 +11,7 @@ require (
github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.22.2 github.com/VictoriaMetrics/metrics v1.22.2
github.com/VictoriaMetrics/metricsql v0.45.0 github.com/VictoriaMetrics/metricsql v0.45.0
github.com/aws/aws-sdk-go v1.44.96 github.com/aws/aws-sdk-go v1.44.100
github.com/cespare/xxhash/v2 v2.1.2 github.com/cespare/xxhash/v2 v2.1.2
// TODO: switch back to https://github.com/cheggaaa/pb/v3 when v3-pooling branch // TODO: switch back to https://github.com/cheggaaa/pb/v3 when v3-pooling branch
@ -20,7 +20,7 @@ require (
github.com/dmitryk-dk/pb/v3 v3.0.9 github.com/dmitryk-dk/pb/v3 v3.0.9
github.com/golang/snappy v0.0.4 github.com/golang/snappy v0.0.4
github.com/influxdata/influxdb v1.10.0 github.com/influxdata/influxdb v1.10.0
github.com/klauspost/compress v1.15.9 github.com/klauspost/compress v1.15.10
github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9 github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9
github.com/urfave/cli/v2 v2.16.3 github.com/urfave/cli/v2 v2.16.3
github.com/valyala/fastjson v1.6.3 github.com/valyala/fastjson v1.6.3
@ -30,14 +30,14 @@ require (
github.com/valyala/quicktemplate v1.7.0 github.com/valyala/quicktemplate v1.7.0
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 golang.org/x/net v0.0.0-20220909164309-bea034e7d591
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
golang.org/x/sys v0.0.0-20220913120320-3275c407cedc golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
google.golang.org/api v0.95.0 google.golang.org/api v0.96.0
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
) )
require ( require (
cloud.google.com/go v0.104.0 // indirect cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.9.0 // indirect cloud.google.com/go/compute v1.10.0 // indirect
cloud.google.com/go/iam v0.4.0 // indirect cloud.google.com/go/iam v0.4.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
@ -75,7 +75,7 @@ require (
golang.org/x/text v0.3.7 // indirect golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220909194730-69f6226f97e5 // indirect google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa // indirect
google.golang.org/grpc v1.49.0 // indirect google.golang.org/grpc v1.49.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
) )

26
go.sum
View file

@ -45,8 +45,8 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m
cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
cloud.google.com/go/compute v1.9.0 h1:ED/FP4xv8GJw63v556/ASNc1CeeLUO2Bs8nzaHchkHg= cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4=
cloud.google.com/go/compute v1.9.0/go.mod h1:lWv1h/zUWTm/LozzfTJhBSkd6ShQq8la8VeeuOEGxfY= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
@ -148,8 +148,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.96 h1:S9paaqnJ0AJ95t5AB+iK8RM6YNZN0W0Lek1gOVJsEr8= github.com/aws/aws-sdk-go v1.44.100 h1:7I86bWNQB+HGDT5z/dJy61J7qgbgLoZ7O51C9eL6hrA=
github.com/aws/aws-sdk-go v1.44.96/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.100/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -576,8 +576,8 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@ -1140,10 +1140,10 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220913120320-3275c407cedc h1:dpclq5m2YrqPGStKmtw7IcNbKLfbIqKXvNxDJKdIKYc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
golang.org/x/sys v0.0.0-20220913120320-3275c407cedc/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1290,8 +1290,8 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
google.golang.org/api v0.95.0 h1:d1c24AAS01DYqXreBeuVV7ewY/U8Mnhh47pwtsgVtYg= google.golang.org/api v0.96.0 h1:F60cuQPJq7K7FzsxMYHAUJSiXh2oKctHxBMbDygxhfM=
google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1383,8 +1383,8 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP
google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220909194730-69f6226f97e5 h1:ngtP8S8JkBWfJACT9cmj5eTkS9tIWPQI5leBz/7Bq/c= google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa h1:VWkrxnAx2C2hirAP+W5ADU7e/+93Yhk//ioKd2XFyDI=
google.golang.org/genproto v0.0.0-20220909194730-69f6226f97e5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

View file

@ -16671,6 +16671,70 @@ var awsPartition = partition{
}, },
}, },
}, },
"participant.connect": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "af-south-1",
}: endpoint{},
endpointKey{
Region: "ap-northeast-1",
}: endpoint{},
endpointKey{
Region: "ap-northeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "eu-west-2",
}: endpoint{},
endpointKey{
Region: "fips-us-east-1",
}: endpoint{
Hostname: "participant.connect-fips.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-us-west-2",
}: endpoint{
Hostname: "participant.connect-fips.us-west-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-2",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "participant.connect-fips.us-east-1.amazonaws.com",
},
endpointKey{
Region: "us-west-2",
}: endpoint{},
endpointKey{
Region: "us-west-2",
Variant: fipsVariant,
}: endpoint{
Hostname: "participant.connect-fips.us-west-2.amazonaws.com",
},
},
},
"personalize": service{ "personalize": service{
Endpoints: serviceEndpoints{ Endpoints: serviceEndpoints{
endpointKey{ endpointKey{
@ -18710,6 +18774,9 @@ var awsPartition = partition{
}, },
"rolesanywhere": service{ "rolesanywhere": service{
Endpoints: serviceEndpoints{ Endpoints: serviceEndpoints{
endpointKey{
Region: "af-south-1",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ap-east-1", Region: "ap-east-1",
}: endpoint{}, }: endpoint{},
@ -18731,6 +18798,9 @@ var awsPartition = partition{
endpointKey{ endpointKey{
Region: "ap-southeast-2", Region: "ap-southeast-2",
}: endpoint{}, }: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ca-central-1", Region: "ca-central-1",
}: endpoint{}, }: endpoint{},
@ -18740,6 +18810,9 @@ var awsPartition = partition{
endpointKey{ endpointKey{
Region: "eu-north-1", Region: "eu-north-1",
}: endpoint{}, }: endpoint{},
endpointKey{
Region: "eu-south-1",
}: endpoint{},
endpointKey{ endpointKey{
Region: "eu-west-1", Region: "eu-west-1",
}: endpoint{}, }: endpoint{},
@ -21533,6 +21606,9 @@ var awsPartition = partition{
}: endpoint{ }: endpoint{
Hostname: "snowball-fips.ap-southeast-2.amazonaws.com", Hostname: "snowball-fips.ap-southeast-2.amazonaws.com",
}, },
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ca-central-1", Region: "ca-central-1",
}: endpoint{}, }: endpoint{},
@ -25262,6 +25338,9 @@ var awsPartition = partition{
}, },
"workspaces": service{ "workspaces": service{
Endpoints: serviceEndpoints{ Endpoints: serviceEndpoints{
endpointKey{
Region: "af-south-1",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ap-northeast-1", Region: "ap-northeast-1",
}: endpoint{}, }: endpoint{},
@ -30495,6 +30574,18 @@ var awsusgovPartition = partition{
}, },
}, },
}, },
"participant.connect": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "us-gov-west-1",
}: endpoint{
Hostname: "participant.connect.us-gov-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-gov-west-1",
},
},
},
},
"pinpoint": service{ "pinpoint": service{
Defaults: endpointDefaults{ Defaults: endpointDefaults{
defaultKey{}: endpoint{ defaultKey{}: endpoint{

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go" const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK // SDKVersion is the version of this SDK
const SDKVersion = "1.44.96" const SDKVersion = "1.44.100"

View file

@ -17,6 +17,12 @@ This package provides various compression algorithms.
# changelog # changelog
* July 21, 2022 (v1.15.9)
* zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
* zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
* zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643
* July 13, 2022 (v1.15.8) * July 13, 2022 (v1.15.8)
* gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641

View file

@ -131,7 +131,8 @@ func (d *compressor) fillDeflate(b []byte) int {
s := d.state s := d.state
if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) { if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
// shift the window by windowSize // shift the window by windowSize
copy(d.window[:], d.window[windowSize:2*windowSize]) //copy(d.window[:], d.window[windowSize:2*windowSize])
*(*[windowSize]byte)(d.window) = *(*[windowSize]byte)(d.window[windowSize:])
s.index -= windowSize s.index -= windowSize
d.windowEnd -= windowSize d.windowEnd -= windowSize
if d.blockStart >= windowSize { if d.blockStart >= windowSize {

View file

@ -7,19 +7,19 @@ package flate
// dictDecoder implements the LZ77 sliding dictionary as used in decompression. // dictDecoder implements the LZ77 sliding dictionary as used in decompression.
// LZ77 decompresses data through sequences of two forms of commands: // LZ77 decompresses data through sequences of two forms of commands:
// //
// * Literal insertions: Runs of one or more symbols are inserted into the data // - Literal insertions: Runs of one or more symbols are inserted into the data
// stream as is. This is accomplished through the writeByte method for a // stream as is. This is accomplished through the writeByte method for a
// single symbol, or combinations of writeSlice/writeMark for multiple symbols. // single symbol, or combinations of writeSlice/writeMark for multiple symbols.
// Any valid stream must start with a literal insertion if no preset dictionary // Any valid stream must start with a literal insertion if no preset dictionary
// is used. // is used.
// //
// * Backward copies: Runs of one or more symbols are copied from previously // - Backward copies: Runs of one or more symbols are copied from previously
// emitted data. Backward copies come as the tuple (dist, length) where dist // emitted data. Backward copies come as the tuple (dist, length) where dist
// determines how far back in the stream to copy from and length determines how // determines how far back in the stream to copy from and length determines how
// many bytes to copy. Note that it is valid for the length to be greater than // many bytes to copy. Note that it is valid for the length to be greater than
// the distance. Since LZ77 uses forward copies, that situation is used to // the distance. Since LZ77 uses forward copies, that situation is used to
// perform a form of run-length encoding on repeated runs of symbols. // perform a form of run-length encoding on repeated runs of symbols.
// The writeCopy and tryWriteCopy are used to implement this command. // The writeCopy and tryWriteCopy are used to implement this command.
// //
// For performance reasons, this implementation performs little to no sanity // For performance reasons, this implementation performs little to no sanity
// checks about the arguments. As such, the invariants documented for each // checks about the arguments. As such, the invariants documented for each

View file

@ -104,7 +104,8 @@ func (e *fastGen) addBlock(src []byte) int32 {
} }
// Move down // Move down
offset := int32(len(e.hist)) - maxMatchOffset offset := int32(len(e.hist)) - maxMatchOffset
copy(e.hist[0:maxMatchOffset], e.hist[offset:]) // copy(e.hist[0:maxMatchOffset], e.hist[offset:])
*(*[maxMatchOffset]byte)(e.hist) = *(*[maxMatchOffset]byte)(e.hist[offset:])
e.cur += offset e.cur += offset
e.hist = e.hist[:maxMatchOffset] e.hist = e.hist[:maxMatchOffset]
} }

View file

@ -265,9 +265,9 @@ func (w *huffmanBitWriter) writeBytes(bytes []byte) {
// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
// information. Code badCode is an end marker // information. Code badCode is an end marker
// //
// numLiterals The number of literals in literalEncoding // numLiterals The number of literals in literalEncoding
// numOffsets The number of offsets in offsetEncoding // numOffsets The number of offsets in offsetEncoding
// litenc, offenc The literal and offset encoder to use // litenc, offenc The literal and offset encoder to use
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
for i := range w.codegenFreq { for i := range w.codegenFreq {
w.codegenFreq[i] = 0 w.codegenFreq[i] = 0
@ -460,9 +460,9 @@ func (w *huffmanBitWriter) writeOutBits() {
// Write the header of a dynamic Huffman block to the output stream. // Write the header of a dynamic Huffman block to the output stream.
// //
// numLiterals The number of literals specified in codegen // numLiterals The number of literals specified in codegen
// numOffsets The number of offsets specified in codegen // numOffsets The number of offsets specified in codegen
// numCodegens The number of codegens used in codegen // numCodegens The number of codegens used in codegen
func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
if w.err != nil { if w.err != nil {
return return
@ -790,9 +790,11 @@ func (w *huffmanBitWriter) fillTokens() {
// and offsetEncoding. // and offsetEncoding.
// The number of literal and offset tokens is returned. // The number of literal and offset tokens is returned.
func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) { func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) {
copy(w.literalFreq[:], t.litHist[:]) //copy(w.literalFreq[:], t.litHist[:])
copy(w.literalFreq[256:], t.extraHist[:]) *(*[256]uint16)(w.literalFreq[:]) = t.litHist
copy(w.offsetFreq[:], t.offHist[:offsetCodeCount]) //copy(w.literalFreq[256:], t.extraHist[:])
*(*[32]uint16)(w.literalFreq[256:]) = t.extraHist
w.offsetFreq = t.offHist
if t.n == 0 { if t.n == 0 {
return return

View file

@ -168,13 +168,18 @@ func (h *huffmanEncoder) canReuseBits(freq []uint16) int {
// The cases of 0, 1, and 2 literals are handled by special case code. // The cases of 0, 1, and 2 literals are handled by special case code.
// //
// list An array of the literals with non-zero frequencies // list An array of the literals with non-zero frequencies
// and their associated frequencies. The array is in order of increasing //
// frequency, and has as its last element a special element with frequency // and their associated frequencies. The array is in order of increasing
// MaxInt32 // frequency, and has as its last element a special element with frequency
// MaxInt32
//
// maxBits The maximum number of bits that should be used to encode any literal. // maxBits The maximum number of bits that should be used to encode any literal.
// Must be less than 16. //
// Must be less than 16.
//
// return An integer array in which array[i] indicates the number of literals // return An integer array in which array[i] indicates the number of literals
// that should be encoded in i bits. //
// that should be encoded in i bits.
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
if maxBits >= maxBitsLimit { if maxBits >= maxBitsLimit {
panic("flate: maxBits too large") panic("flate: maxBits too large")

View file

@ -191,14 +191,21 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) {
// Try to locate a better match by checking the end of best match... // Try to locate a better match by checking the end of best match...
if sAt := s + l; l < 30 && sAt < sLimit { if sAt := s + l; l < 30 && sAt < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is 2/3 bytes depending on input.
// 3 is only a little better when it is but sometimes a lot worse.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset
// Test current t2 := eLong - e.cur - l + skipBeginning
t2 := eLong - e.cur - l s2 := s + skipBeginning
off := s - t2 off := s2 - t2
if t2 >= 0 && off < maxMatchOffset && off > 0 { if t2 >= 0 && off < maxMatchOffset && off > 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l { if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2 t = t2
l = l2 l = l2
s = s2
} }
} }
} }

View file

@ -213,24 +213,33 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) {
// Try to locate a better match by checking the end-of-match... // Try to locate a better match by checking the end-of-match...
if sAt := s + l; sAt < sLimit { if sAt := s + l; sAt < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is 2/3 bytes depending on input.
// 3 is only a little better when it is but sometimes a lot worse.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)] eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)]
// Test current // Test current
t2 := eLong.Cur.offset - e.cur - l t2 := eLong.Cur.offset - e.cur - l + skipBeginning
off := s - t2 s2 := s + skipBeginning
off := s2 - t2
if off < maxMatchOffset { if off < maxMatchOffset {
if off > 0 && t2 >= 0 { if off > 0 && t2 >= 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l { if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2 t = t2
l = l2 l = l2
s = s2
} }
} }
// Test next: // Test next:
t2 = eLong.Prev.offset - e.cur - l t2 = eLong.Prev.offset - e.cur - l + skipBeginning
off := s - t2 off := s2 - t2
if off > 0 && off < maxMatchOffset && t2 >= 0 { if off > 0 && off < maxMatchOffset && t2 >= 0 {
if l2 := e.matchlenLong(s, t2, src); l2 > l { if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2 t = t2
l = l2 l = l2
s = s2
} }
} }
} }

View file

@ -763,17 +763,20 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1") return nil, errors.New("corruption detected: stream overrun 1")
} }
copy(out, buf[0][:])
copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[3][:])
out = out[bufoff:]
decoded += bufoff * 4
// There must at least be 3 buffers left. // There must at least be 3 buffers left.
if len(out) < dstEvery*3 { if len(out)-bufoff < dstEvery*3 {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2") return nil, errors.New("corruption detected: stream overrun 2")
} }
//copy(out, buf[0][:])
//copy(out[dstEvery:], buf[1][:])
//copy(out[dstEvery*2:], buf[2][:])
*(*[bufoff]byte)(out) = buf[0]
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
out = out[bufoff:]
decoded += bufoff * 4
} }
} }
if off > 0 { if off > 0 {
@ -997,17 +1000,22 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1") return nil, errors.New("corruption detected: stream overrun 1")
} }
copy(out, buf[0][:])
copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[3][:])
out = out[bufoff:]
decoded += bufoff * 4
// There must at least be 3 buffers left. // There must at least be 3 buffers left.
if len(out) < dstEvery*3 { if len(out)-bufoff < dstEvery*3 {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2") return nil, errors.New("corruption detected: stream overrun 2")
} }
//copy(out, buf[0][:])
//copy(out[dstEvery:], buf[1][:])
//copy(out[dstEvery*2:], buf[2][:])
// copy(out[dstEvery*3:], buf[3][:])
*(*[bufoff]byte)(out) = buf[0]
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
out = out[bufoff:]
decoded += bufoff * 4
} }
} }
if off > 0 { if off > 0 {

View file

@ -14,12 +14,14 @@ import (
// decompress4x_main_loop_x86 is an x86 assembler implementation // decompress4x_main_loop_x86 is an x86 assembler implementation
// of Decompress4X when tablelog > 8. // of Decompress4X when tablelog > 8.
//
//go:noescape //go:noescape
func decompress4x_main_loop_amd64(ctx *decompress4xContext) func decompress4x_main_loop_amd64(ctx *decompress4xContext)
// decompress4x_8b_loop_x86 is an x86 assembler implementation // decompress4x_8b_loop_x86 is an x86 assembler implementation
// of Decompress4X when tablelog <= 8 which decodes 4 entries // of Decompress4X when tablelog <= 8 which decodes 4 entries
// per loop. // per loop.
//
//go:noescape //go:noescape
func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext)
@ -145,11 +147,13 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
// decompress4x_main_loop_x86 is an x86 assembler implementation // decompress4x_main_loop_x86 is an x86 assembler implementation
// of Decompress1X when tablelog > 8. // of Decompress1X when tablelog > 8.
//
//go:noescape //go:noescape
func decompress1x_main_loop_amd64(ctx *decompress1xContext) func decompress1x_main_loop_amd64(ctx *decompress1xContext)
// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation // decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation
// of Decompress1X when tablelog > 8. // of Decompress1X when tablelog > 8.
//
//go:noescape //go:noescape
func decompress1x_main_loop_bmi2(ctx *decompress1xContext) func decompress1x_main_loop_bmi2(ctx *decompress1xContext)

View file

@ -1,7 +1,6 @@
// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. // Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT.
//go:build amd64 && !appengine && !noasm && gc //go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
// func decompress4x_main_loop_amd64(ctx *decompress4xContext) // func decompress4x_main_loop_amd64(ctx *decompress4xContext)
TEXT ·decompress4x_main_loop_amd64(SB), $0-8 TEXT ·decompress4x_main_loop_amd64(SB), $0-8

View file

@ -122,17 +122,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1") return nil, errors.New("corruption detected: stream overrun 1")
} }
copy(out, buf[0][:])
copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[3][:])
out = out[bufoff:]
decoded += bufoff * 4
// There must at least be 3 buffers left. // There must at least be 3 buffers left.
if len(out) < dstEvery*3 { if len(out)-bufoff < dstEvery*3 {
d.bufs.Put(buf) d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2") return nil, errors.New("corruption detected: stream overrun 2")
} }
//copy(out, buf[0][:])
//copy(out[dstEvery:], buf[1][:])
//copy(out[dstEvery*2:], buf[2][:])
//copy(out[dstEvery*3:], buf[3][:])
*(*[bufoff]byte)(out) = buf[0]
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
out = out[bufoff:]
decoded += bufoff * 4
} }
} }
if off > 0 { if off > 0 {

View file

@ -18,6 +18,7 @@ func load64(b []byte, i int) uint64 {
// emitLiteral writes a literal chunk and returns the number of bytes written. // emitLiteral writes a literal chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
//
// dst is long enough to hold the encoded bytes // dst is long enough to hold the encoded bytes
// 1 <= len(lit) && len(lit) <= 65536 // 1 <= len(lit) && len(lit) <= 65536
func emitLiteral(dst, lit []byte) int { func emitLiteral(dst, lit []byte) int {
@ -42,6 +43,7 @@ func emitLiteral(dst, lit []byte) int {
// emitCopy writes a copy chunk and returns the number of bytes written. // emitCopy writes a copy chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
//
// dst is long enough to hold the encoded bytes // dst is long enough to hold the encoded bytes
// 1 <= offset && offset <= 65535 // 1 <= offset && offset <= 65535
// 4 <= length && length <= 65535 // 4 <= length && length <= 65535
@ -89,6 +91,7 @@ func emitCopy(dst []byte, offset, length int) int {
// src[i:i+k-j] and src[j:k] have the same contents. // src[i:i+k-j] and src[j:k] have the same contents.
// //
// It assumes that: // It assumes that:
//
// 0 <= i && i < j && j <= len(src) // 0 <= i && i < j && j <= len(src)
func extendMatch(src []byte, i, j int) int { func extendMatch(src []byte, i, j int) int {
for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 {
@ -105,8 +108,9 @@ func hash(u, shift uint32) uint32 {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlock(dst, src []byte) (d int) { func encodeBlock(dst, src []byte) (d int) {
// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
// The table element type is uint16, as s < sLimit and sLimit < len(src) // The table element type is uint16, as s < sLimit and sLimit < len(src)

View file

@ -835,6 +835,13 @@ This is done using the regular "Skip" function:
This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset. This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset.
# Compact storage
For compact storage [RemoveIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RemoveIndexHeaders) can be used to remove any redundant info from
a serialized index. If you remove the header it must be restored before [Loading](https://pkg.go.dev/github.com/klauspost/compress/s2#Index.Load).
This is expected to save 20 bytes. These can be restored using [RestoreIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RestoreIndexHeaders). This removes a layer of security, but is the most compact representation. Returns nil if headers contains errors.
## Index Format: ## Index Format:
Each block is structured as a snappy skippable block, with the chunk ID 0x99. Each block is structured as a snappy skippable block, with the chunk ID 0x99.
@ -929,6 +936,7 @@ To decode from any given uncompressed offset `(wantOffset)`:
See [using indexes](https://github.com/klauspost/compress/tree/master/s2#using-indexes) for functions that perform the operations with a simpler interface. See [using indexes](https://github.com/klauspost/compress/tree/master/s2#using-indexes) for functions that perform the operations with a simpler interface.
# Format Extensions # Format Extensions
* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`. * Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
@ -951,10 +959,11 @@ The length is specified by reading the 3-bit length specified in the tag and dec
| 7 | 65540 + read 3 bytes | | 7 | 65540 + read 3 bytes |
This allows any repeat offset + length to be represented by 2 to 5 bytes. This allows any repeat offset + length to be represented by 2 to 5 bytes.
It also allows to emit matches longer than 64 bytes with one copy + one repeat instead of several 64 byte copies.
Lengths are stored as little endian values. Lengths are stored as little endian values.
The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams. The first copy of a block cannot be a repeat offset and the offset is reset on every block in streams.
Default streaming block size is 1MB. Default streaming block size is 1MB.

View file

@ -28,6 +28,9 @@ func s2Decode(dst, src []byte) int {
// As long as we can read at least 5 bytes... // As long as we can read at least 5 bytes...
for s < len(src)-5 { for s < len(src)-5 {
// Removing bounds checks is SLOWER, when if doing
// in := src[s:s+5]
// Checked on Go 1.18
switch src[s] & 0x03 { switch src[s] & 0x03 {
case tagLiteral: case tagLiteral:
x := uint32(src[s] >> 2) x := uint32(src[s] >> 2)
@ -38,14 +41,19 @@ func s2Decode(dst, src []byte) int {
s += 2 s += 2
x = uint32(src[s-1]) x = uint32(src[s-1])
case x == 61: case x == 61:
in := src[s : s+3]
x = uint32(in[1]) | uint32(in[2])<<8
s += 3 s += 3
x = uint32(src[s-2]) | uint32(src[s-1])<<8
case x == 62: case x == 62:
in := src[s : s+4]
// Load as 32 bit and shift down.
x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
x >>= 8
s += 4 s += 4
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
case x == 63: case x == 63:
in := src[s : s+5]
x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
s += 5 s += 5
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
} }
length = int(x) + 1 length = int(x) + 1
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) { if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
@ -62,8 +70,8 @@ func s2Decode(dst, src []byte) int {
case tagCopy1: case tagCopy1:
s += 2 s += 2
length = int(src[s-2]) >> 2 & 0x7
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
length = int(src[s-2]) >> 2 & 0x7
if toffset == 0 { if toffset == 0 {
if debug { if debug {
fmt.Print("(repeat) ") fmt.Print("(repeat) ")
@ -71,14 +79,16 @@ func s2Decode(dst, src []byte) int {
// keep last offset // keep last offset
switch length { switch length {
case 5: case 5:
length = int(src[s]) + 4
s += 1 s += 1
length = int(uint32(src[s-1])) + 4
case 6: case 6:
in := src[s : s+2]
length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
s += 2 s += 2
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
case 7: case 7:
in := src[s : s+3]
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
s += 3 s += 3
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
default: // 0-> 4 default: // 0-> 4
} }
} else { } else {
@ -86,14 +96,16 @@ func s2Decode(dst, src []byte) int {
} }
length += 4 length += 4
case tagCopy2: case tagCopy2:
in := src[s : s+3]
offset = int(uint32(in[1]) | uint32(in[2])<<8)
length = 1 + int(in[0])>>2
s += 3 s += 3
length = 1 + int(src[s-3])>>2
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
case tagCopy4: case tagCopy4:
in := src[s : s+5]
offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
length = 1 + int(in[0])>>2
s += 5 s += 5
length = 1 + int(src[s-5])>>2
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
} }
if offset <= 0 || d < offset || length > len(dst)-d { if offset <= 0 || d < offset || length > len(dst)-d {

View file

@ -58,8 +58,9 @@ func encodeGo(dst, src []byte) []byte {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockGo(dst, src []byte) (d int) { func encodeBlockGo(dst, src []byte) (d int) {
// Initialize the hash table. // Initialize the hash table.
const ( const (

View file

@ -8,8 +8,9 @@ package s2
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlock(dst, src []byte) (d int) { func encodeBlock(dst, src []byte) (d int) {
const ( const (
// Use 12 bit table when less than... // Use 12 bit table when less than...
@ -43,8 +44,9 @@ func encodeBlock(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetter(dst, src []byte) (d int) { func encodeBlockBetter(dst, src []byte) (d int) {
const ( const (
// Use 12 bit table when less than... // Use 12 bit table when less than...
@ -78,8 +80,9 @@ func encodeBlockBetter(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockSnappy(dst, src []byte) (d int) { func encodeBlockSnappy(dst, src []byte) (d int) {
const ( const (
// Use 12 bit table when less than... // Use 12 bit table when less than...
@ -112,8 +115,9 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetterSnappy(dst, src []byte) (d int) { func encodeBlockBetterSnappy(dst, src []byte) (d int) {
const ( const (
// Use 12 bit table when less than... // Use 12 bit table when less than...

View file

@ -15,8 +15,9 @@ import (
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBest(dst, src []byte) (d int) { func encodeBlockBest(dst, src []byte) (d int) {
// Initialize the hash tables. // Initialize the hash tables.
const ( const (
@ -176,14 +177,21 @@ func encodeBlockBest(dst, src []byte) (d int) {
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false)) best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
} }
// Search for a match at best match end, see if that is better. // Search for a match at best match end, see if that is better.
if sAt := best.s + best.length; sAt < sLimit { // Allow some bytes at the beginning to mismatch.
sBack := best.s // Sweet spot is around 1-2 bytes, but depends on input.
backL := best.length // The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
const skipEnd = 1
if sAt := best.s + best.length - skipEnd; sAt < sLimit {
sBack := best.s + skipBeginning - skipEnd
backL := best.length - skipBeginning
// Load initial values // Load initial values
cv = load64(src, sBack) cv = load64(src, sBack)
// Search for mismatch
// Grab candidates...
next := lTable[hash8(load64(src, sAt), lTableBits)] next := lTable[hash8(load64(src, sAt), lTableBits)]
//next := sTable[hash4(load64(src, sAt), sTableBits)]
if checkAt := getCur(next) - backL; checkAt > 0 { if checkAt := getCur(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
@ -191,6 +199,16 @@ func encodeBlockBest(dst, src []byte) (d int) {
if checkAt := getPrev(next) - backL; checkAt > 0 { if checkAt := getPrev(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
} }
// Disabled: Extremely small gain
if false {
next = sTable[hash4(load64(src, sAt), sTableBits)]
if checkAt := getCur(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
}
if checkAt := getPrev(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
}
}
} }
} }
} }
@ -288,8 +306,9 @@ emitRemainder:
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBestSnappy(dst, src []byte) (d int) { func encodeBlockBestSnappy(dst, src []byte) (d int) {
// Initialize the hash tables. // Initialize the hash tables.
const ( const (
@ -546,6 +565,7 @@ emitRemainder:
// emitCopySize returns the size to encode the offset+length // emitCopySize returns the size to encode the offset+length
// //
// It assumes that: // It assumes that:
//
// 1 <= offset && offset <= math.MaxUint32 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24 // 4 <= length && length <= 1 << 24
func emitCopySize(offset, length int) int { func emitCopySize(offset, length int) int {
@ -584,6 +604,7 @@ func emitCopySize(offset, length int) int {
// emitCopyNoRepeatSize returns the size to encode the offset+length // emitCopyNoRepeatSize returns the size to encode the offset+length
// //
// It assumes that: // It assumes that:
//
// 1 <= offset && offset <= math.MaxUint32 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24 // 4 <= length && length <= 1 << 24
func emitCopyNoRepeatSize(offset, length int) int { func emitCopyNoRepeatSize(offset, length int) int {

View file

@ -42,8 +42,9 @@ func hash8(u uint64, h uint8) uint32 {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetterGo(dst, src []byte) (d int) { func encodeBlockBetterGo(dst, src []byte) (d int) {
// sLimit is when to stop looking for offset/length copies. The inputMargin // sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are // lets us use a fast path for emitLiteral in the main loop, while we are
@ -56,7 +57,7 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
// Initialize the hash tables. // Initialize the hash tables.
const ( const (
// Long hash matches. // Long hash matches.
lTableBits = 16 lTableBits = 17
maxLTableSize = 1 << lTableBits maxLTableSize = 1 << lTableBits
// Short hash matches. // Short hash matches.
@ -97,9 +98,26 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
lTable[hashL] = uint32(s) lTable[hashL] = uint32(s)
sTable[hashS] = uint32(s) sTable[hashS] = uint32(s)
valLong := load64(src, candidateL)
valShort := load64(src, candidateS)
// If long matches at least 8 bytes, use that.
if cv == valLong {
break
}
if cv == valShort {
candidateL = candidateS
break
}
// Check repeat at offset checkRep. // Check repeat at offset checkRep.
const checkRep = 1 const checkRep = 1
if false && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { // Minimum length of a repeat. Tested with various values.
// While 4-5 offers improvements in some, 6 reduces
// regressions significantly.
const wantRepeatBytes = 6
const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep)
if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask {
base := s + checkRep base := s + checkRep
// Extend back // Extend back
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
@ -109,8 +127,8 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
d += emitLiteral(dst[d:], src[nextEmit:base]) d += emitLiteral(dst[d:], src[nextEmit:base])
// Extend forward // Extend forward
candidate := s - repeat + 4 + checkRep candidate := s - repeat + wantRepeatBytes + checkRep
s += 4 + checkRep s += wantRepeatBytes + checkRep
for s < len(src) { for s < len(src) {
if len(src)-s < 8 { if len(src)-s < 8 {
if src[s] == src[candidate] { if src[s] == src[candidate] {
@ -127,28 +145,40 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
s += 8 s += 8
candidate += 8 candidate += 8
} }
if nextEmit > 0 { // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. d += emitRepeat(dst[d:], repeat, s-base)
d += emitRepeat(dst[d:], repeat, s-base)
} else {
// First match, cannot be repeat.
d += emitCopy(dst[d:], repeat, s-base)
}
nextEmit = s nextEmit = s
if s >= sLimit { if s >= sLimit {
goto emitRemainder goto emitRemainder
} }
// Index in-between
index0 := base + 1
index1 := s - 2
cv = load64(src, s)
for index0 < index1 {
cv0 := load64(src, index0)
cv1 := load64(src, index1)
lTable[hash7(cv0, lTableBits)] = uint32(index0)
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
lTable[hash7(cv1, lTableBits)] = uint32(index1)
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
index0 += 2
index1 -= 2
}
cv = load64(src, s) cv = load64(src, s)
continue continue
} }
if uint32(cv) == load32(src, candidateL) { // Long likely matches 7, so take that.
if uint32(cv) == uint32(valLong) {
break break
} }
// Check our short candidate // Check our short candidate
if uint32(cv) == load32(src, candidateS) { if uint32(cv) == uint32(valShort) {
// Try a long candidate at s+1 // Try a long candidate at s+1
hashL = hash7(cv>>8, lTableBits) hashL = hash7(cv>>8, lTableBits)
candidateL = int(lTable[hashL]) candidateL = int(lTable[hashL])
@ -227,21 +257,29 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
// Do we have space for more, if not bail. // Do we have space for more, if not bail.
return 0 return 0
} }
// Index match start+1 (long) and start+2 (short)
// Index short & long
index0 := base + 1 index0 := base + 1
// Index match end-2 (long) and end-1 (short)
index1 := s - 2 index1 := s - 2
cv0 := load64(src, index0) cv0 := load64(src, index0)
cv1 := load64(src, index1) cv1 := load64(src, index1)
cv = load64(src, s)
lTable[hash7(cv0, lTableBits)] = uint32(index0) lTable[hash7(cv0, lTableBits)] = uint32(index0)
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
lTable[hash7(cv1, lTableBits)] = uint32(index1)
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
lTable[hash7(cv1, lTableBits)] = uint32(index1)
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
index0 += 1
index1 -= 1
cv = load64(src, s)
// index every second long in between.
for index0 < index1 {
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
index0 += 2
index1 -= 2
}
} }
emitRemainder: emitRemainder:
@ -260,8 +298,9 @@ emitRemainder:
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) && // len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize // minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetterSnappyGo(dst, src []byte) (d int) { func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
// sLimit is when to stop looking for offset/length copies. The inputMargin // sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are // lets us use a fast path for emitLiteral in the main loop, while we are
@ -402,21 +441,29 @@ func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
// Do we have space for more, if not bail. // Do we have space for more, if not bail.
return 0 return 0
} }
// Index match start+1 (long) and start+2 (short)
// Index short & long
index0 := base + 1 index0 := base + 1
// Index match end-2 (long) and end-1 (short)
index1 := s - 2 index1 := s - 2
cv0 := load64(src, index0) cv0 := load64(src, index0)
cv1 := load64(src, index1) cv1 := load64(src, index1)
cv = load64(src, s)
lTable[hash7(cv0, lTableBits)] = uint32(index0) lTable[hash7(cv0, lTableBits)] = uint32(index0)
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
lTable[hash7(cv1, lTableBits)] = uint32(index1)
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
lTable[hash7(cv1, lTableBits)] = uint32(index1)
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
index0 += 1
index1 -= 1
cv = load64(src, s)
// index every second long in between.
for index0 < index1 {
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
index0 += 2
index1 -= 2
}
} }
emitRemainder: emitRemainder:

View file

@ -12,6 +12,7 @@ import (
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) // len(dst) >= MaxEncodedLen(len(src))
func encodeBlock(dst, src []byte) (d int) { func encodeBlock(dst, src []byte) (d int) {
if len(src) < minNonLiteralBlockSize { if len(src) < minNonLiteralBlockSize {
@ -25,6 +26,7 @@ func encodeBlock(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) // len(dst) >= MaxEncodedLen(len(src))
func encodeBlockBetter(dst, src []byte) (d int) { func encodeBlockBetter(dst, src []byte) (d int) {
return encodeBlockBetterGo(dst, src) return encodeBlockBetterGo(dst, src)
@ -35,6 +37,7 @@ func encodeBlockBetter(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) // len(dst) >= MaxEncodedLen(len(src))
func encodeBlockBetterSnappy(dst, src []byte) (d int) { func encodeBlockBetterSnappy(dst, src []byte) (d int) {
return encodeBlockBetterSnappyGo(dst, src) return encodeBlockBetterSnappyGo(dst, src)
@ -45,6 +48,7 @@ func encodeBlockBetterSnappy(dst, src []byte) (d int) {
// been written. // been written.
// //
// It also assumes that: // It also assumes that:
//
// len(dst) >= MaxEncodedLen(len(src)) // len(dst) >= MaxEncodedLen(len(src))
func encodeBlockSnappy(dst, src []byte) (d int) { func encodeBlockSnappy(dst, src []byte) (d int) {
if len(src) < minNonLiteralBlockSize { if len(src) < minNonLiteralBlockSize {
@ -56,6 +60,7 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
// emitLiteral writes a literal chunk and returns the number of bytes written. // emitLiteral writes a literal chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
//
// dst is long enough to hold the encoded bytes // dst is long enough to hold the encoded bytes
// 0 <= len(lit) && len(lit) <= math.MaxUint32 // 0 <= len(lit) && len(lit) <= math.MaxUint32
func emitLiteral(dst, lit []byte) int { func emitLiteral(dst, lit []byte) int {
@ -146,6 +151,7 @@ func emitRepeat(dst []byte, offset, length int) int {
// emitCopy writes a copy chunk and returns the number of bytes written. // emitCopy writes a copy chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
//
// dst is long enough to hold the encoded bytes // dst is long enough to hold the encoded bytes
// 1 <= offset && offset <= math.MaxUint32 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24 // 4 <= length && length <= 1 << 24
@ -214,6 +220,7 @@ func emitCopy(dst []byte, offset, length int) int {
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written. // emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
//
// dst is long enough to hold the encoded bytes // dst is long enough to hold the encoded bytes
// 1 <= offset && offset <= math.MaxUint32 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24 // 4 <= length && length <= 1 << 24
@ -273,8 +280,8 @@ func emitCopyNoRepeat(dst []byte, offset, length int) int {
// matchLen returns how many bytes match in a and b // matchLen returns how many bytes match in a and b
// //
// It assumes that: // It assumes that:
// len(a) <= len(b)
// //
// len(a) <= len(b)
func matchLen(a []byte, b []byte) int { func matchLen(a []byte, b []byte) int {
b = b[:len(a)] b = b[:len(a)]
var checked int var checked int

View file

@ -1,7 +1,6 @@
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. // Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.
//go:build !appengine && !noasm && gc && !noasm //go:build !appengine && !noasm && gc && !noasm
// +build !appengine,!noasm,gc,!noasm
package s2 package s2
@ -150,8 +149,9 @@ func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
// emitLiteral writes a literal chunk and returns the number of bytes written. // emitLiteral writes a literal chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
// dst is long enough to hold the encoded bytes with margin of 0 bytes //
// 0 <= len(lit) && len(lit) <= math.MaxUint32 // dst is long enough to hold the encoded bytes with margin of 0 bytes
// 0 <= len(lit) && len(lit) <= math.MaxUint32
// //
//go:noescape //go:noescape
func emitLiteral(dst []byte, lit []byte) int func emitLiteral(dst []byte, lit []byte) int
@ -165,9 +165,10 @@ func emitRepeat(dst []byte, offset int, length int) int
// emitCopy writes a copy chunk and returns the number of bytes written. // emitCopy writes a copy chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
// dst is long enough to hold the encoded bytes //
// 1 <= offset && offset <= math.MaxUint32 // dst is long enough to hold the encoded bytes
// 4 <= length && length <= 1 << 24 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24
// //
//go:noescape //go:noescape
func emitCopy(dst []byte, offset int, length int) int func emitCopy(dst []byte, offset int, length int) int
@ -175,9 +176,10 @@ func emitCopy(dst []byte, offset int, length int) int
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written. // emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
// //
// It assumes that: // It assumes that:
// dst is long enough to hold the encoded bytes //
// 1 <= offset && offset <= math.MaxUint32 // dst is long enough to hold the encoded bytes
// 4 <= length && length <= 1 << 24 // 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24
// //
//go:noescape //go:noescape
func emitCopyNoRepeat(dst []byte, offset int, length int) int func emitCopyNoRepeat(dst []byte, offset int, length int) int
@ -185,7 +187,8 @@ func emitCopyNoRepeat(dst []byte, offset int, length int) int
// matchLen returns how many bytes match in a and b // matchLen returns how many bytes match in a and b
// //
// It assumes that: // It assumes that:
// len(a) <= len(b) //
// len(a) <= len(b)
// //
//go:noescape //go:noescape
func matchLen(a []byte, b []byte) int func matchLen(a []byte, b []byte) int

File diff suppressed because it is too large Load diff

View file

@ -12,6 +12,8 @@ The `zstd` package is provided as open source software using a Go standard licen
Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors. Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors.
For seekable zstd streams, see [this excellent package](https://github.com/SaveTheRbtz/zstd-seekable-format-go).
## Installation ## Installation
Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`. Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`.

View file

@ -10,7 +10,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"sync" "sync"
@ -651,7 +650,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse)) fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse))
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse)) fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse))
buf.Write(in) buf.Write(in)
ioutil.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm)
} }
return nil return nil

View file

@ -7,7 +7,6 @@ package zstd
import ( import (
"fmt" "fmt"
"io" "io"
"io/ioutil"
) )
type byteBuffer interface { type byteBuffer interface {
@ -124,7 +123,7 @@ func (r *readerWrapper) readByte() (byte, error) {
} }
func (r *readerWrapper) skipN(n int64) error { func (r *readerWrapper) skipN(n int64) error {
n2, err := io.CopyN(ioutil.Discard, r.r, n) n2, err := io.CopyN(io.Discard, r.r, n)
if n2 != n { if n2 != n {
err = io.ErrUnexpectedEOF err = io.ErrUnexpectedEOF
} }

View file

@ -312,6 +312,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
// Grab a block decoder and frame decoder. // Grab a block decoder and frame decoder.
block := <-d.decoders block := <-d.decoders
frame := block.localFrame frame := block.localFrame
initialSize := len(dst)
defer func() { defer func() {
if debugDecoder { if debugDecoder {
printf("re-adding decoder: %p", block) printf("re-adding decoder: %p", block)
@ -354,7 +355,16 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
return dst, ErrWindowSizeExceeded return dst, ErrWindowSizeExceeded
} }
if frame.FrameContentSize != fcsUnknown { if frame.FrameContentSize != fcsUnknown {
if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) {
if debugDecoder {
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst))
}
return dst, ErrDecoderSizeExceeded
}
if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) {
if debugDecoder {
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst))
}
return dst, ErrDecoderSizeExceeded return dst, ErrDecoderSizeExceeded
} }
if cap(dst)-len(dst) < int(frame.FrameContentSize) { if cap(dst)-len(dst) < int(frame.FrameContentSize) {
@ -364,7 +374,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
} }
} }
if cap(dst) == 0 { if cap(dst) == 0 && !d.o.limitToCap {
// Allocate len(input) * 2 by default if nothing is provided // Allocate len(input) * 2 by default if nothing is provided
// and we didn't get frame content size. // and we didn't get frame content size.
size := len(input) * 2 size := len(input) * 2
@ -382,6 +392,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
if err != nil { if err != nil {
return dst, err return dst, err
} }
if uint64(len(dst)-initialSize) > d.o.maxDecodedSize {
return dst, ErrDecoderSizeExceeded
}
if len(frame.bBuf) == 0 { if len(frame.bBuf) == 0 {
if debugDecoder { if debugDecoder {
println("frame dbuf empty") println("frame dbuf empty")
@ -852,6 +865,10 @@ decodeStream:
} }
} }
if err == nil && d.frame.WindowSize > d.o.maxWindowSize { if err == nil && d.frame.WindowSize > d.o.maxWindowSize {
if debugDecoder {
println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize)
}
err = ErrDecoderSizeExceeded err = ErrDecoderSizeExceeded
} }
if err != nil { if err != nil {

View file

@ -20,6 +20,7 @@ type decoderOptions struct {
maxWindowSize uint64 maxWindowSize uint64
dicts []dict dicts []dict
ignoreChecksum bool ignoreChecksum bool
limitToCap bool
} }
func (o *decoderOptions) setDefault() { func (o *decoderOptions) setDefault() {
@ -114,6 +115,17 @@ func WithDecoderMaxWindow(size uint64) DOption {
} }
} }
// WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes,
// or any size set in WithDecoderMaxMemory.
// This can be used to limit decoding to a specific maximum output size.
// Disabled by default.
func WithDecodeAllCapLimit(b bool) DOption {
return func(o *decoderOptions) error {
o.limitToCap = b
return nil
}
}
// IgnoreChecksum allows to forcibly ignore checksum checking. // IgnoreChecksum allows to forcibly ignore checksum checking.
func IgnoreChecksum(b bool) DOption { func IgnoreChecksum(b bool) DOption {
return func(o *decoderOptions) error { return func(o *decoderOptions) error {

View file

@ -416,15 +416,23 @@ encodeLoop:
// Try to find a better match by searching for a long match at the end of the current best match // Try to find a better match by searching for a long match at the end of the current best match
if s+matched < sLimit { if s+matched < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is around 3 bytes, but depends on input.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 3
nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen)
cv := load3232(src, s) s2 := s + skipBeginning
cv := load3232(src, s2)
candidateL := e.longTable[nextHashL] candidateL := e.longTable[nextHashL]
coffsetL := candidateL.offset - e.cur - matched coffsetL := candidateL.offset - e.cur - matched + skipBeginning
if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
// Found a long match, at least 4 bytes. // Found a long match, at least 4 bytes.
matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4
if matchedNext > matched { if matchedNext > matched {
t = coffsetL t = coffsetL
s = s2
matched = matchedNext matched = matchedNext
if debugMatches { if debugMatches {
println("long match at end-of-match") println("long match at end-of-match")
@ -434,12 +442,13 @@ encodeLoop:
// Check prev long... // Check prev long...
if true { if true {
coffsetL = candidateL.prev - e.cur - matched coffsetL = candidateL.prev - e.cur - matched + skipBeginning
if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
// Found a long match, at least 4 bytes. // Found a long match, at least 4 bytes.
matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4
if matchedNext > matched { if matchedNext > matched {
t = coffsetL t = coffsetL
s = s2
matched = matchedNext matched = matchedNext
if debugMatches { if debugMatches {
println("prev long match at end-of-match") println("prev long match at end-of-match")

View file

@ -1103,7 +1103,8 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
} }
if allDirty || dirtyShardCnt > dLongTableShardCnt/2 { if allDirty || dirtyShardCnt > dLongTableShardCnt/2 {
copy(e.longTable[:], e.dictLongTable) //copy(e.longTable[:], e.dictLongTable)
e.longTable = *(*[dFastLongTableSize]tableEntry)(e.dictLongTable)
for i := range e.longTableShardDirty { for i := range e.longTableShardDirty {
e.longTableShardDirty[i] = false e.longTableShardDirty[i] = false
} }
@ -1114,7 +1115,9 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
continue continue
} }
copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) // copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize])
*(*[dLongTableShardSize]tableEntry)(e.longTable[i*dLongTableShardSize:]) = *(*[dLongTableShardSize]tableEntry)(e.dictLongTable[i*dLongTableShardSize:])
e.longTableShardDirty[i] = false e.longTableShardDirty[i] = false
} }
} }

View file

@ -871,7 +871,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
const shardCnt = tableShardCnt const shardCnt = tableShardCnt
const shardSize = tableShardSize const shardSize = tableShardSize
if e.allDirty || dirtyShardCnt > shardCnt*4/6 { if e.allDirty || dirtyShardCnt > shardCnt*4/6 {
copy(e.table[:], e.dictTable) //copy(e.table[:], e.dictTable)
e.table = *(*[tableSize]tableEntry)(e.dictTable)
for i := range e.tableShardDirty { for i := range e.tableShardDirty {
e.tableShardDirty[i] = false e.tableShardDirty[i] = false
} }
@ -883,7 +884,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
continue continue
} }
copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) //copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize])
*(*[shardSize]tableEntry)(e.table[i*shardSize:]) = *(*[shardSize]tableEntry)(e.dictTable[i*shardSize:])
e.tableShardDirty[i] = false e.tableShardDirty[i] = false
} }
e.allDirty = false e.allDirty = false

View file

@ -353,12 +353,23 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
// Store input length, so we only check new data. // Store input length, so we only check new data.
crcStart := len(dst) crcStart := len(dst)
d.history.decoders.maxSyncLen = 0 d.history.decoders.maxSyncLen = 0
if d.o.limitToCap {
d.history.decoders.maxSyncLen = uint64(cap(dst) - len(dst))
}
if d.FrameContentSize != fcsUnknown { if d.FrameContentSize != fcsUnknown {
d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst)) if !d.o.limitToCap || d.FrameContentSize+uint64(len(dst)) < d.history.decoders.maxSyncLen {
d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst))
}
if d.history.decoders.maxSyncLen > d.o.maxDecodedSize { if d.history.decoders.maxSyncLen > d.o.maxDecodedSize {
if debugDecoder {
println("maxSyncLen:", d.history.decoders.maxSyncLen, "> maxDecodedSize:", d.o.maxDecodedSize)
}
return dst, ErrDecoderSizeExceeded return dst, ErrDecoderSizeExceeded
} }
if uint64(cap(dst)) < d.history.decoders.maxSyncLen { if debugDecoder {
println("maxSyncLen:", d.history.decoders.maxSyncLen)
}
if !d.o.limitToCap && uint64(cap(dst)-len(dst)) < d.history.decoders.maxSyncLen {
// Alloc for output // Alloc for output
dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc) dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc)
copy(dst2, dst) copy(dst2, dst)
@ -378,7 +389,13 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
if err != nil { if err != nil {
break break
} }
if uint64(len(d.history.b)) > d.o.maxDecodedSize { if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize {
println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize)
err = ErrDecoderSizeExceeded
break
}
if d.o.limitToCap && len(d.history.b) > cap(dst) {
println("runDecoder: cap exceeded", uint64(len(d.history.b)), ">", cap(dst))
err = ErrDecoderSizeExceeded err = ErrDecoderSizeExceeded
break break
} }

View file

@ -1,7 +1,6 @@
// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. // Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT.
//go:build !appengine && !noasm && gc && !noasm //go:build !appengine && !noasm && gc && !noasm
// +build !appengine,!noasm,gc,!noasm
// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int
TEXT ·buildDtable_asm(SB), $0-24 TEXT ·buildDtable_asm(SB), $0-24

View file

@ -32,18 +32,22 @@ type decodeSyncAsmContext struct {
// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. // sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm.
// //
// Please refer to seqdec_generic.go for the reference implementation. // Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape //go:noescape
func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. // sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions.
//
//go:noescape //go:noescape
func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. // sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer.
//
//go:noescape //go:noescape
func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. // sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer.
//
//go:noescape //go:noescape
func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
@ -201,20 +205,24 @@ const errorNotEnoughSpace = 5
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
// //
// Please refer to seqdec_generic.go for the reference implementation. // Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape //go:noescape
func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
// //
// Please refer to seqdec_generic.go for the reference implementation. // Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape //go:noescape
func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape //go:noescape
func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape //go:noescape
func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
@ -308,10 +316,12 @@ type executeAsmContext struct {
// Returns false if a match offset is too big. // Returns false if a match offset is too big.
// //
// Please refer to seqdec_generic.go for the reference implementation. // Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape //go:noescape
func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool
// Same as above, but with safe memcopies // Same as above, but with safe memcopies
//
//go:noescape //go:noescape
func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool

View file

@ -1,7 +1,6 @@
// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. // Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT.
//go:build !appengine && !noasm && gc && !noasm //go:build !appengine && !noasm && gc && !noasm
// +build !appengine,!noasm,gc,!noasm
// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: CMOV // Requires: CMOV

View file

@ -214,11 +214,6 @@ esac
if [ "$GOOSARCH" == "aix_ppc64" ]; then if [ "$GOOSARCH" == "aix_ppc64" ]; then
# aix/ppc64 script generates files instead of writing to stdin. # aix/ppc64 script generates files instead of writing to stdin.
echo "$mksyscall -tags $GOOS,$GOARCH $syscall_goos $GOOSARCH_in && gofmt -w zsyscall_$GOOSARCH.go && gofmt -w zsyscall_"$GOOSARCH"_gccgo.go && gofmt -w zsyscall_"$GOOSARCH"_gc.go " ; echo "$mksyscall -tags $GOOS,$GOARCH $syscall_goos $GOOSARCH_in && gofmt -w zsyscall_$GOOSARCH.go && gofmt -w zsyscall_"$GOOSARCH"_gccgo.go && gofmt -w zsyscall_"$GOOSARCH"_gc.go " ;
elif [ "$GOOS" == "darwin" ]; then
# 1.12 and later, syscalls via libSystem
echo "$mksyscall -tags $GOOS,$GOARCH,go1.12 $syscall_goos $GOOSARCH_in |gofmt >zsyscall_$GOOSARCH.go";
# 1.13 and later, syscalls via libSystem (including syscallPtr)
echo "$mksyscall -tags $GOOS,$GOARCH,go1.13 syscall_darwin.1_13.go |gofmt >zsyscall_$GOOSARCH.1_13.go";
elif [ "$GOOS" == "illumos" ]; then elif [ "$GOOS" == "illumos" ]; then
# illumos code generation requires a --illumos switch # illumos code generation requires a --illumos switch
echo "$mksyscall -illumos -tags illumos,$GOARCH syscall_illumos.go |gofmt > zsyscall_illumos_$GOARCH.go"; echo "$mksyscall -illumos -tags illumos,$GOARCH syscall_illumos.go |gofmt > zsyscall_illumos_$GOARCH.go";

View file

@ -80,8 +80,7 @@ func BytePtrToString(p *byte) string {
ptr = unsafe.Pointer(uintptr(ptr) + 1) ptr = unsafe.Pointer(uintptr(ptr) + 1)
} }
s := unsafe.Slice((*byte)(unsafe.Pointer(p)), n) return string(unsafe.Slice(p, n))
return string(s)
} }
// Single-word zero for use when we need a valid pointer to 0 bytes. // Single-word zero for use when we need a valid pointer to 0 bytes.

View file

@ -1,32 +0,0 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin && go1.12 && !go1.13
// +build darwin,go1.12,!go1.13
package unix
import (
"unsafe"
)
const _SYS_GETDIRENTRIES64 = 344
func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
// To implement this using libSystem we'd need syscall_syscallPtr for
// fdopendir. However, syscallPtr was only added in Go 1.13, so we fall
// back to raw syscalls for this func on Go 1.12.
var p unsafe.Pointer
if len(buf) > 0 {
p = unsafe.Pointer(&buf[0])
} else {
p = unsafe.Pointer(&_zero)
}
r0, _, e1 := Syscall6(_SYS_GETDIRENTRIES64, uintptr(fd), uintptr(p), uintptr(len(buf)), uintptr(unsafe.Pointer(basep)), 0, 0)
n = int(r0)
if e1 != 0 {
return n, errnoErr(e1)
}
return n, nil
}

View file

@ -1,100 +0,0 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin && go1.13
// +build darwin,go1.13
package unix
import "unsafe"
//sys closedir(dir uintptr) (err error)
//sys readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno)
func fdopendir(fd int) (dir uintptr, err error) {
r0, _, e1 := syscall_syscallPtr(libc_fdopendir_trampoline_addr, uintptr(fd), 0, 0)
dir = uintptr(r0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_fdopendir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_fdopendir fdopendir "/usr/lib/libSystem.B.dylib"
func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
// Simulate Getdirentries using fdopendir/readdir_r/closedir.
// We store the number of entries to skip in the seek
// offset of fd. See issue #31368.
// It's not the full required semantics, but should handle the case
// of calling Getdirentries or ReadDirent repeatedly.
// It won't handle assigning the results of lseek to *basep, or handle
// the directory being edited underfoot.
skip, err := Seek(fd, 0, 1 /* SEEK_CUR */)
if err != nil {
return 0, err
}
// We need to duplicate the incoming file descriptor
// because the caller expects to retain control of it, but
// fdopendir expects to take control of its argument.
// Just Dup'ing the file descriptor is not enough, as the
// result shares underlying state. Use Openat to make a really
// new file descriptor referring to the same directory.
fd2, err := Openat(fd, ".", O_RDONLY, 0)
if err != nil {
return 0, err
}
d, err := fdopendir(fd2)
if err != nil {
Close(fd2)
return 0, err
}
defer closedir(d)
var cnt int64
for {
var entry Dirent
var entryp *Dirent
e := readdir_r(d, &entry, &entryp)
if e != 0 {
return n, errnoErr(e)
}
if entryp == nil {
break
}
if skip > 0 {
skip--
cnt++
continue
}
reclen := int(entry.Reclen)
if reclen > len(buf) {
// Not enough room. Return for now.
// The counter will let us know where we should start up again.
// Note: this strategy for suspending in the middle and
// restarting is O(n^2) in the length of the directory. Oh well.
break
}
// Copy entry into return buffer.
s := unsafe.Slice((*byte)(unsafe.Pointer(&entry)), reclen)
copy(buf, s)
buf = buf[reclen:]
n += reclen
cnt++
}
// Set the seek offset of the input fd to record
// how many files we've already returned.
_, err = Seek(fd, cnt, 0 /* SEEK_SET */)
if err != nil {
return n, err
}
return n, nil
}

View file

@ -19,6 +19,96 @@ import (
"unsafe" "unsafe"
) )
//sys closedir(dir uintptr) (err error)
//sys readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno)
func fdopendir(fd int) (dir uintptr, err error) {
r0, _, e1 := syscall_syscallPtr(libc_fdopendir_trampoline_addr, uintptr(fd), 0, 0)
dir = uintptr(r0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_fdopendir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_fdopendir fdopendir "/usr/lib/libSystem.B.dylib"
func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
// Simulate Getdirentries using fdopendir/readdir_r/closedir.
// We store the number of entries to skip in the seek
// offset of fd. See issue #31368.
// It's not the full required semantics, but should handle the case
// of calling Getdirentries or ReadDirent repeatedly.
// It won't handle assigning the results of lseek to *basep, or handle
// the directory being edited underfoot.
skip, err := Seek(fd, 0, 1 /* SEEK_CUR */)
if err != nil {
return 0, err
}
// We need to duplicate the incoming file descriptor
// because the caller expects to retain control of it, but
// fdopendir expects to take control of its argument.
// Just Dup'ing the file descriptor is not enough, as the
// result shares underlying state. Use Openat to make a really
// new file descriptor referring to the same directory.
fd2, err := Openat(fd, ".", O_RDONLY, 0)
if err != nil {
return 0, err
}
d, err := fdopendir(fd2)
if err != nil {
Close(fd2)
return 0, err
}
defer closedir(d)
var cnt int64
for {
var entry Dirent
var entryp *Dirent
e := readdir_r(d, &entry, &entryp)
if e != 0 {
return n, errnoErr(e)
}
if entryp == nil {
break
}
if skip > 0 {
skip--
cnt++
continue
}
reclen := int(entry.Reclen)
if reclen > len(buf) {
// Not enough room. Return for now.
// The counter will let us know where we should start up again.
// Note: this strategy for suspending in the middle and
// restarting is O(n^2) in the length of the directory. Oh well.
break
}
// Copy entry into return buffer.
s := unsafe.Slice((*byte)(unsafe.Pointer(&entry)), reclen)
copy(buf, s)
buf = buf[reclen:]
n += reclen
cnt++
}
// Set the seek offset of the input fd to record
// how many files we've already returned.
_, err = Seek(fd, cnt, 0 /* SEEK_SET */)
if err != nil {
return n, err
}
return n, nil
}
// SockaddrDatalink implements the Sockaddr interface for AF_LINK type sockets. // SockaddrDatalink implements the Sockaddr interface for AF_LINK type sockets.
type SockaddrDatalink struct { type SockaddrDatalink struct {
Len uint8 Len uint8

View file

@ -160,13 +160,12 @@ func Lremovexattr(link string, attr string) (err error) {
} }
func Listxattr(file string, dest []byte) (sz int, err error) { func Listxattr(file string, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest) destsiz := len(dest)
// FreeBSD won't allow you to list xattrs from multiple namespaces // FreeBSD won't allow you to list xattrs from multiple namespaces
s := 0 s, pos := 0, 0
for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} { for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
stmp, e := ExtattrListFile(file, nsid, uintptr(d), destsiz) stmp, e := ListxattrNS(file, nsid, dest[pos:])
/* Errors accessing system attrs are ignored so that /* Errors accessing system attrs are ignored so that
* we can implement the Linux-like behavior of omitting errors that * we can implement the Linux-like behavior of omitting errors that
@ -175,66 +174,102 @@ func Listxattr(file string, dest []byte) (sz int, err error) {
* Linux will still error if we ask for user attributes on a file that * Linux will still error if we ask for user attributes on a file that
* we don't have read permissions on, so don't ignore those errors * we don't have read permissions on, so don't ignore those errors
*/ */
if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER { if e != nil {
continue if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
} else if e != nil { continue
}
return s, e return s, e
} }
s += stmp s += stmp
destsiz -= s pos = s
if destsiz < 0 { if pos > destsiz {
destsiz = 0 pos = destsiz
} }
d = initxattrdest(dest, s) }
return s, nil
}
func ListxattrNS(file string, nsid int, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest)
s, e := ExtattrListFile(file, nsid, uintptr(d), destsiz)
if e != nil {
return 0, err
} }
return s, nil return s, nil
} }
func Flistxattr(fd int, dest []byte) (sz int, err error) { func Flistxattr(fd int, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest) destsiz := len(dest)
s := 0 s, pos := 0, 0
for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} { for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
stmp, e := ExtattrListFd(fd, nsid, uintptr(d), destsiz) stmp, e := FlistxattrNS(fd, nsid, dest[pos:])
if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
continue if e != nil {
} else if e != nil { if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
continue
}
return s, e return s, e
} }
s += stmp s += stmp
destsiz -= s pos = s
if destsiz < 0 { if pos > destsiz {
destsiz = 0 pos = destsiz
} }
d = initxattrdest(dest, s) }
return s, nil
}
func FlistxattrNS(fd int, nsid int, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest)
s, e := ExtattrListFd(fd, nsid, uintptr(d), destsiz)
if e != nil {
return 0, err
} }
return s, nil return s, nil
} }
func Llistxattr(link string, dest []byte) (sz int, err error) { func Llistxattr(link string, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest) destsiz := len(dest)
s := 0 s, pos := 0, 0
for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} { for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
stmp, e := ExtattrListLink(link, nsid, uintptr(d), destsiz) stmp, e := LlistxattrNS(link, nsid, dest[pos:])
if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
continue if e != nil {
} else if e != nil { if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
continue
}
return s, e return s, e
} }
s += stmp s += stmp
destsiz -= s pos = s
if destsiz < 0 { if pos > destsiz {
destsiz = 0 pos = destsiz
} }
d = initxattrdest(dest, s) }
return s, nil
}
func LlistxattrNS(link string, nsid int, dest []byte) (sz int, err error) {
d := initxattrdest(dest, 0)
destsiz := len(dest)
s, e := ExtattrListLink(link, nsid, uintptr(d), destsiz)
if e != nil {
return 0, err
} }
return s, nil return s, nil

View file

@ -1,40 +0,0 @@
// go run mksyscall.go -tags darwin,amd64,go1.13 syscall_darwin.1_13.go
// Code generated by the command above; see README.md. DO NOT EDIT.
//go:build darwin && amd64 && go1.13
// +build darwin,amd64,go1.13
package unix
import (
"syscall"
"unsafe"
)
var _ syscall.Errno
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func closedir(dir uintptr) (err error) {
_, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_closedir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) {
r0, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result)))
res = Errno(r0)
return
}
var libc_readdir_r_trampoline_addr uintptr
//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib"

View file

@ -1,25 +0,0 @@
// go run mkasm.go darwin amd64
// Code generated by the command above; DO NOT EDIT.
//go:build go1.13
// +build go1.13
#include "textflag.h"
TEXT libc_fdopendir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fdopendir(SB)
GLOBL ·libc_fdopendir_trampoline_addr(SB), RODATA, $8
DATA ·libc_fdopendir_trampoline_addr(SB)/8, $libc_fdopendir_trampoline<>(SB)
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_closedir(SB)
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_readdir_r(SB)
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)

View file

@ -1,8 +1,8 @@
// go run mksyscall.go -tags darwin,amd64,go1.12 syscall_bsd.go syscall_darwin.go syscall_darwin_amd64.go // go run mksyscall.go -tags darwin,amd64 syscall_bsd.go syscall_darwin.go syscall_darwin_amd64.go
// Code generated by the command above; see README.md. DO NOT EDIT. // Code generated by the command above; see README.md. DO NOT EDIT.
//go:build darwin && amd64 && go1.12 //go:build darwin && amd64
// +build darwin,amd64,go1.12 // +build darwin,amd64
package unix package unix
@ -463,6 +463,32 @@ var libc_munlockall_trampoline_addr uintptr
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func closedir(dir uintptr) (err error) {
_, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_closedir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) {
r0, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result)))
res = Errno(r0)
return
}
var libc_readdir_r_trampoline_addr uintptr
//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func pipe(p *[2]int32) (err error) { func pipe(p *[2]int32) (err error) {
_, _, e1 := syscall_rawSyscall(libc_pipe_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0) _, _, e1 := syscall_rawSyscall(libc_pipe_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0)
if e1 != 0 { if e1 != 0 {

View file

@ -1,11 +1,14 @@
// go run mkasm.go darwin amd64 // go run mkasm.go darwin amd64
// Code generated by the command above; DO NOT EDIT. // Code generated by the command above; DO NOT EDIT.
//go:build go1.12
// +build go1.12
#include "textflag.h" #include "textflag.h"
TEXT libc_fdopendir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fdopendir(SB)
GLOBL ·libc_fdopendir_trampoline_addr(SB), RODATA, $8
DATA ·libc_fdopendir_trampoline_addr(SB)/8, $libc_fdopendir_trampoline<>(SB)
TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_getgroups(SB) JMP libc_getgroups(SB)
@ -174,6 +177,18 @@ TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0
GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8
DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB)
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_closedir(SB)
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_readdir_r(SB)
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)
TEXT libc_pipe_trampoline<>(SB),NOSPLIT,$0-0 TEXT libc_pipe_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_pipe(SB) JMP libc_pipe(SB)

View file

@ -1,40 +0,0 @@
// go run mksyscall.go -tags darwin,arm64,go1.13 syscall_darwin.1_13.go
// Code generated by the command above; see README.md. DO NOT EDIT.
//go:build darwin && arm64 && go1.13
// +build darwin,arm64,go1.13
package unix
import (
"syscall"
"unsafe"
)
var _ syscall.Errno
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func closedir(dir uintptr) (err error) {
_, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_closedir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) {
r0, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result)))
res = Errno(r0)
return
}
var libc_readdir_r_trampoline_addr uintptr
//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib"

View file

@ -1,25 +0,0 @@
// go run mkasm.go darwin arm64
// Code generated by the command above; DO NOT EDIT.
//go:build go1.13
// +build go1.13
#include "textflag.h"
TEXT libc_fdopendir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fdopendir(SB)
GLOBL ·libc_fdopendir_trampoline_addr(SB), RODATA, $8
DATA ·libc_fdopendir_trampoline_addr(SB)/8, $libc_fdopendir_trampoline<>(SB)
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_closedir(SB)
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_readdir_r(SB)
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)

View file

@ -1,8 +1,8 @@
// go run mksyscall.go -tags darwin,arm64,go1.12 syscall_bsd.go syscall_darwin.go syscall_darwin_arm64.go // go run mksyscall.go -tags darwin,arm64 syscall_bsd.go syscall_darwin.go syscall_darwin_arm64.go
// Code generated by the command above; see README.md. DO NOT EDIT. // Code generated by the command above; see README.md. DO NOT EDIT.
//go:build darwin && arm64 && go1.12 //go:build darwin && arm64
// +build darwin,arm64,go1.12 // +build darwin,arm64
package unix package unix
@ -463,6 +463,32 @@ var libc_munlockall_trampoline_addr uintptr
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func closedir(dir uintptr) (err error) {
_, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
var libc_closedir_trampoline_addr uintptr
//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) {
r0, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result)))
res = Errno(r0)
return
}
var libc_readdir_r_trampoline_addr uintptr
//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib"
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func pipe(p *[2]int32) (err error) { func pipe(p *[2]int32) (err error) {
_, _, e1 := syscall_rawSyscall(libc_pipe_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0) _, _, e1 := syscall_rawSyscall(libc_pipe_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0)
if e1 != 0 { if e1 != 0 {

View file

@ -1,11 +1,14 @@
// go run mkasm.go darwin arm64 // go run mkasm.go darwin arm64
// Code generated by the command above; DO NOT EDIT. // Code generated by the command above; DO NOT EDIT.
//go:build go1.12
// +build go1.12
#include "textflag.h" #include "textflag.h"
TEXT libc_fdopendir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fdopendir(SB)
GLOBL ·libc_fdopendir_trampoline_addr(SB), RODATA, $8
DATA ·libc_fdopendir_trampoline_addr(SB)/8, $libc_fdopendir_trampoline<>(SB)
TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_getgroups(SB) JMP libc_getgroups(SB)
@ -174,6 +177,18 @@ TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0
GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8
DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB)
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_closedir(SB)
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_readdir_r(SB)
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)
TEXT libc_pipe_trampoline<>(SB),NOSPLIT,$0-0 TEXT libc_pipe_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_pipe(SB) JMP libc_pipe(SB)

View file

@ -30,8 +30,6 @@ import (
"strings" "strings"
"syscall" "syscall"
"unsafe" "unsafe"
"golang.org/x/sys/internal/unsafeheader"
) )
// ByteSliceFromString returns a NUL-terminated slice of bytes // ByteSliceFromString returns a NUL-terminated slice of bytes
@ -83,13 +81,7 @@ func BytePtrToString(p *byte) string {
ptr = unsafe.Pointer(uintptr(ptr) + 1) ptr = unsafe.Pointer(uintptr(ptr) + 1)
} }
var s []byte return string(unsafe.Slice(p, n))
h := (*unsafeheader.Slice)(unsafe.Pointer(&s))
h.Data = unsafe.Pointer(p)
h.Len = n
h.Cap = n
return string(s)
} }
// Single-word zero for use when we need a valid pointer to 0 bytes. // Single-word zero for use when we need a valid pointer to 0 bytes.

View file

@ -138,13 +138,7 @@ func UTF16PtrToString(p *uint16) string {
ptr = unsafe.Pointer(uintptr(ptr) + unsafe.Sizeof(*p)) ptr = unsafe.Pointer(uintptr(ptr) + unsafe.Sizeof(*p))
} }
var s []uint16 return string(utf16.Decode(unsafe.Slice(p, n)))
h := (*unsafeheader.Slice)(unsafe.Pointer(&s))
h.Data = unsafe.Pointer(p)
h.Len = n
h.Cap = n
return string(utf16.Decode(s))
} }
func Getpagesize() int { return 4096 } func Getpagesize() int { return 4096 }
@ -448,6 +442,10 @@ func NewCallbackCDecl(fn interface{}) uintptr {
//sys RtlAddFunctionTable(functionTable *RUNTIME_FUNCTION, entryCount uint32, baseAddress uintptr) (ret bool) = ntdll.RtlAddFunctionTable //sys RtlAddFunctionTable(functionTable *RUNTIME_FUNCTION, entryCount uint32, baseAddress uintptr) (ret bool) = ntdll.RtlAddFunctionTable
//sys RtlDeleteFunctionTable(functionTable *RUNTIME_FUNCTION) (ret bool) = ntdll.RtlDeleteFunctionTable //sys RtlDeleteFunctionTable(functionTable *RUNTIME_FUNCTION) (ret bool) = ntdll.RtlDeleteFunctionTable
// Desktop Window Manager API (Dwmapi)
//sys DwmGetWindowAttribute(hwnd HWND, attribute uint32, value unsafe.Pointer, size uint32) (ret error) = dwmapi.DwmGetWindowAttribute
//sys DwmSetWindowAttribute(hwnd HWND, attribute uint32, value unsafe.Pointer, size uint32) (ret error) = dwmapi.DwmSetWindowAttribute
// syscall interface implementation for other packages // syscall interface implementation for other packages
// GetCurrentProcess returns the handle for the current process. // GetCurrentProcess returns the handle for the current process.

View file

@ -3232,3 +3232,29 @@ type GUIThreadInfo struct {
CaretHandle HWND CaretHandle HWND
CaretRect Rect CaretRect Rect
} }
const (
DWMWA_NCRENDERING_ENABLED = 1
DWMWA_NCRENDERING_POLICY = 2
DWMWA_TRANSITIONS_FORCEDISABLED = 3
DWMWA_ALLOW_NCPAINT = 4
DWMWA_CAPTION_BUTTON_BOUNDS = 5
DWMWA_NONCLIENT_RTL_LAYOUT = 6
DWMWA_FORCE_ICONIC_REPRESENTATION = 7
DWMWA_FLIP3D_POLICY = 8
DWMWA_EXTENDED_FRAME_BOUNDS = 9
DWMWA_HAS_ICONIC_BITMAP = 10
DWMWA_DISALLOW_PEEK = 11
DWMWA_EXCLUDED_FROM_PEEK = 12
DWMWA_CLOAK = 13
DWMWA_CLOAKED = 14
DWMWA_FREEZE_REPRESENTATION = 15
DWMWA_PASSIVE_UPDATE_MODE = 16
DWMWA_USE_HOSTBACKDROPBRUSH = 17
DWMWA_USE_IMMERSIVE_DARK_MODE = 20
DWMWA_WINDOW_CORNER_PREFERENCE = 33
DWMWA_BORDER_COLOR = 34
DWMWA_CAPTION_COLOR = 35
DWMWA_TEXT_COLOR = 36
DWMWA_VISIBLE_FRAME_BORDER_THICKNESS = 37
)

View file

@ -40,6 +40,7 @@ var (
modadvapi32 = NewLazySystemDLL("advapi32.dll") modadvapi32 = NewLazySystemDLL("advapi32.dll")
modcrypt32 = NewLazySystemDLL("crypt32.dll") modcrypt32 = NewLazySystemDLL("crypt32.dll")
moddnsapi = NewLazySystemDLL("dnsapi.dll") moddnsapi = NewLazySystemDLL("dnsapi.dll")
moddwmapi = NewLazySystemDLL("dwmapi.dll")
modiphlpapi = NewLazySystemDLL("iphlpapi.dll") modiphlpapi = NewLazySystemDLL("iphlpapi.dll")
modkernel32 = NewLazySystemDLL("kernel32.dll") modkernel32 = NewLazySystemDLL("kernel32.dll")
modmswsock = NewLazySystemDLL("mswsock.dll") modmswsock = NewLazySystemDLL("mswsock.dll")
@ -175,6 +176,8 @@ var (
procDnsNameCompare_W = moddnsapi.NewProc("DnsNameCompare_W") procDnsNameCompare_W = moddnsapi.NewProc("DnsNameCompare_W")
procDnsQuery_W = moddnsapi.NewProc("DnsQuery_W") procDnsQuery_W = moddnsapi.NewProc("DnsQuery_W")
procDnsRecordListFree = moddnsapi.NewProc("DnsRecordListFree") procDnsRecordListFree = moddnsapi.NewProc("DnsRecordListFree")
procDwmGetWindowAttribute = moddwmapi.NewProc("DwmGetWindowAttribute")
procDwmSetWindowAttribute = moddwmapi.NewProc("DwmSetWindowAttribute")
procGetAdaptersAddresses = modiphlpapi.NewProc("GetAdaptersAddresses") procGetAdaptersAddresses = modiphlpapi.NewProc("GetAdaptersAddresses")
procGetAdaptersInfo = modiphlpapi.NewProc("GetAdaptersInfo") procGetAdaptersInfo = modiphlpapi.NewProc("GetAdaptersInfo")
procGetBestInterfaceEx = modiphlpapi.NewProc("GetBestInterfaceEx") procGetBestInterfaceEx = modiphlpapi.NewProc("GetBestInterfaceEx")
@ -1534,6 +1537,22 @@ func DnsRecordListFree(rl *DNSRecord, freetype uint32) {
return return
} }
func DwmGetWindowAttribute(hwnd HWND, attribute uint32, value unsafe.Pointer, size uint32) (ret error) {
r0, _, _ := syscall.Syscall6(procDwmGetWindowAttribute.Addr(), 4, uintptr(hwnd), uintptr(attribute), uintptr(value), uintptr(size), 0, 0)
if r0 != 0 {
ret = syscall.Errno(r0)
}
return
}
func DwmSetWindowAttribute(hwnd HWND, attribute uint32, value unsafe.Pointer, size uint32) (ret error) {
r0, _, _ := syscall.Syscall6(procDwmSetWindowAttribute.Addr(), 4, uintptr(hwnd), uintptr(attribute), uintptr(value), uintptr(size), 0, 0)
if r0 != 0 {
ret = syscall.Errno(r0)
}
return
}
func GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) { func GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) {
r0, _, _ := syscall.Syscall6(procGetAdaptersAddresses.Addr(), 5, uintptr(family), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(adapterAddresses)), uintptr(unsafe.Pointer(sizePointer)), 0) r0, _, _ := syscall.Syscall6(procGetAdaptersAddresses.Addr(), 5, uintptr(family), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(adapterAddresses)), uintptr(unsafe.Pointer(sizePointer)), 0)
if r0 != 0 { if r0 != 0 {

View file

@ -5,4 +5,4 @@
package internal package internal
// Version is the current tagged release of the library. // Version is the current tagged release of the library.
const Version = "0.95.0" const Version = "0.96.0"

14
vendor/modules.txt vendored
View file

@ -4,7 +4,7 @@ cloud.google.com/go/internal
cloud.google.com/go/internal/optional cloud.google.com/go/internal/optional
cloud.google.com/go/internal/trace cloud.google.com/go/internal/trace
cloud.google.com/go/internal/version cloud.google.com/go/internal/version
# cloud.google.com/go/compute v1.9.0 # cloud.google.com/go/compute v1.10.0
## explicit; go 1.17 ## explicit; go 1.17
cloud.google.com/go/compute/metadata cloud.google.com/go/compute/metadata
# cloud.google.com/go/iam v0.4.0 # cloud.google.com/go/iam v0.4.0
@ -34,7 +34,7 @@ github.com/VictoriaMetrics/metricsql/binaryop
# github.com/VividCortex/ewma v1.2.0 # github.com/VividCortex/ewma v1.2.0
## explicit; go 1.12 ## explicit; go 1.12
github.com/VividCortex/ewma github.com/VividCortex/ewma
# github.com/aws/aws-sdk-go v1.44.96 # github.com/aws/aws-sdk-go v1.44.100
## explicit; go 1.11 ## explicit; go 1.11
github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/arn
@ -157,8 +157,8 @@ github.com/influxdata/influxdb/pkg/escape
# github.com/jmespath/go-jmespath v0.4.0 # github.com/jmespath/go-jmespath v0.4.0
## explicit; go 1.14 ## explicit; go 1.14
github.com/jmespath/go-jmespath github.com/jmespath/go-jmespath
# github.com/klauspost/compress v1.15.9 # github.com/klauspost/compress v1.15.10
## explicit; go 1.16 ## explicit; go 1.17
github.com/klauspost/compress github.com/klauspost/compress
github.com/klauspost/compress/flate github.com/klauspost/compress/flate
github.com/klauspost/compress/fse github.com/klauspost/compress/fse
@ -305,7 +305,7 @@ golang.org/x/oauth2/jwt
# golang.org/x/sync v0.0.0-20220907140024-f12130a52804 # golang.org/x/sync v0.0.0-20220907140024-f12130a52804
## explicit ## explicit
golang.org/x/sync/errgroup golang.org/x/sync/errgroup
# golang.org/x/sys v0.0.0-20220913120320-3275c407cedc # golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
## explicit; go 1.17 ## explicit; go 1.17
golang.org/x/sys/internal/unsafeheader golang.org/x/sys/internal/unsafeheader
golang.org/x/sys/unix golang.org/x/sys/unix
@ -320,7 +320,7 @@ golang.org/x/text/unicode/norm
## explicit; go 1.17 ## explicit; go 1.17
golang.org/x/xerrors golang.org/x/xerrors
golang.org/x/xerrors/internal golang.org/x/xerrors/internal
# google.golang.org/api v0.95.0 # google.golang.org/api v0.96.0
## explicit; go 1.15 ## explicit; go 1.15
google.golang.org/api/googleapi google.golang.org/api/googleapi
google.golang.org/api/googleapi/transport google.golang.org/api/googleapi/transport
@ -353,7 +353,7 @@ google.golang.org/appengine/internal/socket
google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/internal/urlfetch
google.golang.org/appengine/socket google.golang.org/appengine/socket
google.golang.org/appengine/urlfetch google.golang.org/appengine/urlfetch
# google.golang.org/genproto v0.0.0-20220909194730-69f6226f97e5 # google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa
## explicit; go 1.19 ## explicit; go 1.19
google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/api/annotations
google.golang.org/genproto/googleapis/iam/v1 google.golang.org/genproto/googleapis/iam/v1