Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2020-12-21 08:44:04 +02:00
commit fb006fc6c0
49 changed files with 1539 additions and 469 deletions

View file

@ -10,6 +10,11 @@ Release process guidance
5. Push release tag to https://github.com/VictoriaMetrics/VictoriaMetrics : `git push origin v1.xx.y`.
6. Go to https://github.com/VictoriaMetrics/VictoriaMetrics/releases , create new release from the pushed tag on step 5 and upload `*.tar.gz` archive with the corresponding `_checksums.txt` from step 2.
### Public Announcement
1. Publish message in slack (victoriametrics.slack.com, general channel)
2. Post twit with release notes URL
3. Post in subreddit https://www.reddit.com/r/VictoriaMetrics/
## Helm Charts

12
go.mod
View file

@ -10,11 +10,12 @@ require (
github.com/VictoriaMetrics/fasthttp v1.0.9
github.com/VictoriaMetrics/metrics v1.12.3
github.com/VictoriaMetrics/metricsql v0.9.1
github.com/aws/aws-sdk-go v1.36.7
github.com/aws/aws-sdk-go v1.36.12
github.com/cespare/xxhash/v2 v2.1.1
github.com/go-kit/kit v0.10.0
github.com/golang/snappy v0.0.2
github.com/klauspost/compress v1.11.3
github.com/klauspost/compress v1.11.4
github.com/prometheus/client_golang v1.9.0 // indirect
github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9
github.com/valyala/fastjson v1.6.3
github.com/valyala/fastrand v1.0.0
@ -22,12 +23,13 @@ require (
github.com/valyala/gozstd v1.9.0
github.com/valyala/histogram v1.1.2
github.com/valyala/quicktemplate v1.6.3
golang.org/x/net v0.0.0-20201216054612-986b41b23924 // indirect
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f
golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58 // indirect
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742
golang.org/x/tools v0.0.0-20201218024724-ae774e9781d2 // indirect
google.golang.org/api v0.36.0
google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a // indirect
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect
gopkg.in/yaml.v2 v2.4.0
)

26
go.sum
View file

@ -120,8 +120,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.36.7 h1:XoJPAjKoqvdL531XGWxKYn5eGX/xMoXzMN5fBtoyfSY=
github.com/aws/aws-sdk-go v1.36.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -491,8 +491,9 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@ -631,8 +632,9 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw=
github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM=
github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU=
github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -873,8 +875,9 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -958,8 +961,9 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f h1:QdHQnPce6K4XQewki9WNbG5KOROuDzqO3NaYjI1cXJ0=
golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 h1:+CBz4km/0KPU3RGTwARGh/noP3bEwtHcq+0YcBQM2JQ=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1043,8 +1047,8 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20201119054027-25dc3e1ccc3c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58 h1:1Bs6RVeBFtLZ8Yi1Hk07DiOqzvwLD/4hln4iahvFlag=
golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201218024724-ae774e9781d2 h1:lHDhNNs7asPT3p01mm8EP3B+bNyyVfg0bcYjhJUYgxw=
golang.org/x/tools v0.0.0-20201218024724-ae774e9781d2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1127,8 +1131,8 @@ google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a h1:GnJAhasbD8HiT8DZMvsEx3QLVy/X0icq/MGr0MqRJ2M=
google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

View file

@ -827,6 +827,30 @@ var awsPartition = partition{
"eu-west-1": endpoint{},
"eu-west-2": endpoint{},
"eu-west-3": endpoint{},
"fips-us-east-1": endpoint{
Hostname: "athena-fips.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
},
"fips-us-east-2": endpoint{
Hostname: "athena-fips.us-east-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-2",
},
},
"fips-us-west-1": endpoint{
Hostname: "athena-fips.us-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-1",
},
},
"fips-us-west-2": endpoint{
Hostname: "athena-fips.us-west-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-2",
},
},
"me-south-1": endpoint{},
"sa-east-1": endpoint{},
"us-east-1": endpoint{},
@ -1355,6 +1379,21 @@ var awsPartition = partition{
},
},
},
"codeguru-reviewer": service{
Endpoints: endpoints{
"ap-northeast-1": endpoint{},
"ap-southeast-1": endpoint{},
"ap-southeast-2": endpoint{},
"eu-central-1": endpoint{},
"eu-north-1": endpoint{},
"eu-west-1": endpoint{},
"eu-west-2": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
},
},
"codepipeline": service{
Endpoints: endpoints{
@ -3520,6 +3559,23 @@ var awsPartition = partition{
"us-west-2": endpoint{},
},
},
"iotwireless": service{
Endpoints: endpoints{
"eu-west-1": endpoint{
Hostname: "api.iotwireless.eu-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "eu-west-1",
},
},
"us-east-1": endpoint{
Hostname: "api.iotwireless.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
},
},
},
"kafka": service{
Endpoints: endpoints{
@ -5636,6 +5692,7 @@ var awsPartition = partition{
"servicediscovery": service{
Endpoints: endpoints{
"af-south-1": endpoint{},
"ap-east-1": endpoint{},
"ap-northeast-1": endpoint{},
"ap-northeast-2": endpoint{},
@ -5645,6 +5702,7 @@ var awsPartition = partition{
"ca-central-1": endpoint{},
"eu-central-1": endpoint{},
"eu-north-1": endpoint{},
"eu-south-1": endpoint{},
"eu-west-1": endpoint{},
"eu-west-2": endpoint{},
"eu-west-3": endpoint{},
@ -9751,6 +9809,12 @@ var awsisoPartition = partition{
},
},
},
"secretsmanager": service{
Endpoints: endpoints{
"us-iso-east-1": endpoint{},
},
},
"snowball": service{
Endpoints: endpoints{

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.36.7"
const SDKVersion = "1.36.12"

View file

@ -24,22 +24,21 @@ Godoc Documentation: https://godoc.org/github.com/klauspost/compress/zstd
### Status:
STABLE - there may always be subtle bugs, a wide variety of content has been tested and the library is actively
used by several projects. This library is being continuously [fuzz-tested](https://github.com/klauspost/compress-fuzz),
kindly supplied by [fuzzit.dev](https://fuzzit.dev/).
used by several projects. This library is being [fuzz-tested](https://github.com/klauspost/compress-fuzz) for all updates.
There may still be specific combinations of data types/size/settings that could lead to edge cases,
so as always, testing is recommended.
For now, a high speed (fastest) and medium-fast (default) compressor has been implemented.
The "Fastest" compression ratio is roughly equivalent to zstd level 1.
The "Default" compression ratio is roughly equivalent to zstd level 3 (default).
* The "Fastest" compression ratio is roughly equivalent to zstd level 1.
* The "Default" compression ratio is roughly equivalent to zstd level 3 (default).
* The "Better" compression ratio is roughly equivalent to zstd level 7.
* The "Best" compression ratio is roughly equivalent to zstd level 11.
In terms of speed, it is typically 2x as fast as the stdlib deflate/gzip in its fastest mode.
The compression ratio compared to stdlib is around level 3, but usually 3x as fast.
Compared to cgo zstd, the speed is around level 3 (default), but compression slightly worse, between level 1&2.
### Usage
@ -140,7 +139,7 @@ I have collected some speed examples to compare speed and compression against ot
* `file` is the input file.
* `out` is the compressor used. `zskp` is this package. `zstd` is the Datadog cgo library. `gzstd/gzkp` is gzip standard and this library.
* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default".
* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default"; 3 is "better", 4 is "best".
* `insize`/`outsize` is the input/output size.
* `millis` is the number of milliseconds used for compression.
* `mb/s` is megabytes (2^20 bytes) per second.
@ -154,11 +153,13 @@ file out level insize outsize millis mb/s
silesia.tar zskp 1 211947520 73101992 643 313.87
silesia.tar zskp 2 211947520 67504318 969 208.38
silesia.tar zskp 3 211947520 65177448 1899 106.44
silesia.tar zskp 4 211947520 61381950 8115 24.91
cgo zstd:
silesia.tar zstd 1 211947520 73605392 543 371.56
silesia.tar zstd 3 211947520 66793289 864 233.68
silesia.tar zstd 6 211947520 62916450 1913 105.66
silesia.tar zstd 9 211947520 60212393 5063 39.92
gzip, stdlib/this package:
silesia.tar gzstd 1 211947520 80007735 1654 122.21
@ -171,9 +172,11 @@ file out level insize outsize millis mb/s
gob-stream zskp 1 1911399616 235022249 3088 590.30
gob-stream zskp 2 1911399616 205669791 3786 481.34
gob-stream zskp 3 1911399616 185792019 9324 195.48
gob-stream zskp 4 1911399616 171537212 32113 56.76
gob-stream zstd 1 1911399616 249810424 2637 691.26
gob-stream zstd 3 1911399616 208192146 3490 522.31
gob-stream zstd 6 1911399616 193632038 6687 272.56
gob-stream zstd 9 1911399616 177620386 16175 112.70
gob-stream gzstd 1 1911399616 357382641 10251 177.82
gob-stream gzkp 1 1911399616 362156523 5695 320.08
@ -185,9 +188,11 @@ file out level insize outsize millis mb/s
enwik9 zskp 1 1000000000 343848582 3609 264.18
enwik9 zskp 2 1000000000 317276632 5746 165.97
enwik9 zskp 3 1000000000 294540704 11725 81.34
enwik9 zskp 4 1000000000 276609671 44029 21.66
enwik9 zstd 1 1000000000 358072021 3110 306.65
enwik9 zstd 3 1000000000 313734672 4784 199.35
enwik9 zstd 6 1000000000 295138875 10290 92.68
enwik9 zstd 9 1000000000 278348700 28549 33.40
enwik9 gzstd 1 1000000000 382578136 9604 99.30
enwik9 gzkp 1 1000000000 383825945 6544 145.73
@ -198,9 +203,11 @@ file out level insize outsize millis mb/s
github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40
github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96
github-june-2days-2019.json zskp 3 6273951764 537511906 29252 204.54
github-june-2days-2019.json zskp 4 6273951764 512796117 97791 61.18
github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00
github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57
github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18
github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16
github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79
github-june-2days-2019.json gzkp 1 6273951764 1128755542 19236 311.03
@ -211,9 +218,11 @@ file out level insize outsize millis mb/s
rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84
rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07
rawstudio-mint14.tar zskp 3 8558382592 3224594213 71751 113.75
rawstudio-mint14.tar zskp 4 8558382592 3027332295 486243 16.79
rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27
rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92
rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77
rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91
rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40
rawstudio-mint14.tar gzkp 1 8558382592 3970463184 41749 195.49
@ -224,9 +233,11 @@ file out level insize outsize millis mb/s
nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35
nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44
nyc-taxi-data-10M.csv zskp 3 3325605752 538490114 19880 159.53
nyc-taxi-data-10M.csv zskp 4 3325605752 495986829 89368 35.49
nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18
nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07
nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27
nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12
nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83
nyc-taxi-data-10M.csv gzkp 1 3325605752 924718719 16388 193.53
```

View file

@ -613,7 +613,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
// Decode treeless literal block.
if litType == literalsBlockTreeless {
// TODO: We could send the history early WITHOUT the stream history.
// This would allow decoding treeless literials before the byte history is available.
// This would allow decoding treeless literals before the byte history is available.
// Silencia stats: Treeless 4393, with: 32775, total: 37168, 11% treeless.
// So not much obvious gain here.

View file

@ -76,6 +76,7 @@ func (b *blockEnc) reset(prev *blockEnc) {
if prev != nil {
b.recentOffsets = prev.prevRecentOffsets
}
b.dictLitEnc = nil
}
// reset will reset the block for a new encode, but in the same stream,

View file

@ -0,0 +1,202 @@
// Copyright 2020+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
package zstd
import (
"bytes"
"errors"
"io"
)
// HeaderMaxSize is the maximum size of a Frame and Block Header.
// If less is sent to Header.Decode it *may* still contain enough information.
const HeaderMaxSize = 14 + 3
// Header contains information about the first frame and block within that.
type Header struct {
// Window Size the window of data to keep while decoding.
// Will only be set if HasFCS is false.
WindowSize uint64
// Frame content size.
// Expected size of the entire frame.
FrameContentSize uint64
// Dictionary ID.
// If 0, no dictionary.
DictionaryID uint32
// First block information.
FirstBlock struct {
// OK will be set if first block could be decoded.
OK bool
// Is this the last block of a frame?
Last bool
// Is the data compressed?
// If true CompressedSize will be populated.
// Unfortunately DecompressedSize cannot be determined
// without decoding the blocks.
Compressed bool
// DecompressedSize is the expected decompressed size of the block.
// Will be 0 if it cannot be determined.
DecompressedSize int
// CompressedSize of the data in the block.
// Does not include the block header.
// Will be equal to DecompressedSize if not Compressed.
CompressedSize int
}
// Skippable will be true if the frame is meant to be skipped.
// No other information will be populated.
Skippable bool
// If set there is a checksum present for the block content.
HasCheckSum bool
// If this is true FrameContentSize will have a valid value
HasFCS bool
SingleSegment bool
}
// Decode the header from the beginning of the stream.
// This will decode the frame header and the first block header if enough bytes are provided.
// It is recommended to provide at least HeaderMaxSize bytes.
// If the frame header cannot be read an error will be returned.
// If there isn't enough input, io.ErrUnexpectedEOF is returned.
// The FirstBlock.OK will indicate if enough information was available to decode the first block header.
func (h *Header) Decode(in []byte) error {
if len(in) < 4 {
return io.ErrUnexpectedEOF
}
b, in := in[:4], in[4:]
if !bytes.Equal(b, frameMagic) {
if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 {
return ErrMagicMismatch
}
*h = Header{Skippable: true}
return nil
}
if len(in) < 1 {
return io.ErrUnexpectedEOF
}
// Clear output
*h = Header{}
fhd, in := in[0], in[1:]
h.SingleSegment = fhd&(1<<5) != 0
h.HasCheckSum = fhd&(1<<2) != 0
if fhd&(1<<3) != 0 {
return errors.New("Reserved bit set on frame header")
}
// Read Window_Descriptor
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor
if !h.SingleSegment {
if len(in) < 1 {
return io.ErrUnexpectedEOF
}
var wd byte
wd, in = in[0], in[1:]
windowLog := 10 + (wd >> 3)
windowBase := uint64(1) << windowLog
windowAdd := (windowBase / 8) * uint64(wd&0x7)
h.WindowSize = windowBase + windowAdd
}
// Read Dictionary_ID
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id
if size := fhd & 3; size != 0 {
if size == 3 {
size = 4
}
if len(in) < int(size) {
return io.ErrUnexpectedEOF
}
b, in = in[:size], in[size:]
if b == nil {
return io.ErrUnexpectedEOF
}
switch size {
case 1:
h.DictionaryID = uint32(b[0])
case 2:
h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8)
case 4:
h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)
}
}
// Read Frame_Content_Size
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size
var fcsSize int
v := fhd >> 6
switch v {
case 0:
if h.SingleSegment {
fcsSize = 1
}
default:
fcsSize = 1 << v
}
if fcsSize > 0 {
h.HasFCS = true
if len(in) < fcsSize {
return io.ErrUnexpectedEOF
}
b, in = in[:fcsSize], in[fcsSize:]
if b == nil {
return io.ErrUnexpectedEOF
}
switch fcsSize {
case 1:
h.FrameContentSize = uint64(b[0])
case 2:
// When FCS_Field_Size is 2, the offset of 256 is added.
h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) + 256
case 4:
h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) | (uint64(b[2]) << 16) | (uint64(b[3]) << 24)
case 8:
d1 := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)
d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24)
h.FrameContentSize = uint64(d1) | (uint64(d2) << 32)
}
}
// Frame Header done, we will not fail from now on.
if len(in) < 3 {
return nil
}
tmp, in := in[:3], in[3:]
bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16)
h.FirstBlock.Last = bh&1 != 0
blockType := blockType((bh >> 1) & 3)
// find size.
cSize := int(bh >> 3)
switch blockType {
case blockTypeReserved:
return nil
case blockTypeRLE:
h.FirstBlock.Compressed = true
h.FirstBlock.DecompressedSize = cSize
h.FirstBlock.CompressedSize = 1
case blockTypeCompressed:
h.FirstBlock.Compressed = true
h.FirstBlock.CompressedSize = cSize
case blockTypeRaw:
h.FirstBlock.DecompressedSize = cSize
h.FirstBlock.CompressedSize = cSize
default:
panic("Invalid block type")
}
h.FirstBlock.OK = true
return nil
}

484
vendor/github.com/klauspost/compress/zstd/enc_best.go generated vendored Normal file
View file

@ -0,0 +1,484 @@
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.
package zstd
import (
"fmt"
"math/bits"
)
const (
bestLongTableBits = 20 // Bits used in the long match table
bestLongTableSize = 1 << bestLongTableBits // Size of the table
// Note: Increasing the short table bits or making the hash shorter
// can actually lead to compression degradation since it will 'steal' more from the
// long match table and match offsets are quite big.
// This greatly depends on the type of input.
bestShortTableBits = 16 // Bits used in the short match table
bestShortTableSize = 1 << bestShortTableBits // Size of the table
)
// bestFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches.
// The long match table contains the previous entry with the same hash,
// effectively making it a "chain" of length 2.
// When we find a long match we choose between the two values and select the longest.
// When we find a short match, after checking the long, we check if we can find a long at n+1
// and that it is longer (lazy matching).
type bestFastEncoder struct {
fastBase
table [bestShortTableSize]prevEntry
longTable [bestLongTableSize]prevEntry
dictTable []prevEntry
dictLongTable []prevEntry
}
// Encode improves compression...
func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
const (
// Input margin is the number of bytes we read (8)
// and the maximum we will read ahead (2)
inputMargin = 8 + 4
minNonLiteralBlockSize = 16
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = prevEntry{}
}
for i := range e.longTable[:] {
e.longTable[i] = prevEntry{}
}
e.cur = e.maxMatchOff
break
}
// Shift down everything in the table that isn't already too far away.
minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff
for i := range e.table[:] {
v := e.table[i].offset
v2 := e.table[i].prev
if v < minOff {
v = 0
v2 = 0
} else {
v = v - e.cur + e.maxMatchOff
if v2 < minOff {
v2 = 0
} else {
v2 = v2 - e.cur + e.maxMatchOff
}
}
e.table[i] = prevEntry{
offset: v,
prev: v2,
}
}
for i := range e.longTable[:] {
v := e.longTable[i].offset
v2 := e.longTable[i].prev
if v < minOff {
v = 0
v2 = 0
} else {
v = v - e.cur + e.maxMatchOff
if v2 < minOff {
v2 = 0
} else {
v2 = v2 - e.cur + e.maxMatchOff
}
}
e.longTable[i] = prevEntry{
offset: v,
prev: v2,
}
}
e.cur = e.maxMatchOff
break
}
s := e.addBlock(src)
blk.size = len(src)
if len(src) < minNonLiteralBlockSize {
blk.extraLits = len(src)
blk.literals = blk.literals[:len(src)]
copy(blk.literals, src)
return
}
// Override src
src = e.hist
sLimit := int32(len(src)) - inputMargin
const kSearchStrength = 12
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := s
cv := load6432(src, s)
// Relative offsets
offset1 := int32(blk.recentOffsets[0])
offset2 := int32(blk.recentOffsets[1])
offset3 := int32(blk.recentOffsets[2])
addLiterals := func(s *seq, until int32) {
if until == nextEmit {
return
}
blk.literals = append(blk.literals, src[nextEmit:until]...)
s.litLen = uint32(until - nextEmit)
}
_ = addLiterals
if debug {
println("recent offsets:", blk.recentOffsets)
}
encodeLoop:
for {
// We allow the encoder to optionally turn off repeat offsets across blocks
canRepeat := len(blk.sequences) > 2
if debugAsserts && canRepeat && offset1 == 0 {
panic("offset0 was 0")
}
type match struct {
offset int32
s int32
length int32
rep int32
}
matchAt := func(offset int32, s int32, first uint32, rep int32) match {
if s-offset >= e.maxMatchOff || load3232(src, offset) != first {
return match{offset: offset, s: s}
}
return match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep}
}
bestOf := func(a, b match) match {
aScore := b.s - a.s + a.length
bScore := a.s - b.s + b.length
if a.rep < 0 {
aScore = aScore - int32(bits.Len32(uint32(a.offset)))/8
}
if b.rep < 0 {
bScore = bScore - int32(bits.Len32(uint32(b.offset)))/8
}
if aScore >= bScore {
return a
}
return b
}
const goodEnough = 100
nextHashL := hash8(cv, bestLongTableBits)
nextHashS := hash4x64(cv, bestShortTableBits)
candidateL := e.longTable[nextHashL]
candidateS := e.table[nextHashS]
best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1))
if canRepeat && best.length < goodEnough {
best = bestOf(best, matchAt(s-offset1+1, s+1, uint32(cv>>8), 1))
best = bestOf(best, matchAt(s-offset2+1, s+1, uint32(cv>>8), 2))
best = bestOf(best, matchAt(s-offset3+1, s+1, uint32(cv>>8), 3))
best = bestOf(best, matchAt(s-offset1+3, s+3, uint32(cv>>24), 1))
best = bestOf(best, matchAt(s-offset2+3, s+3, uint32(cv>>24), 2))
best = bestOf(best, matchAt(s-offset3+3, s+3, uint32(cv>>24), 3))
}
// Load next and check...
e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: candidateL.offset}
e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: candidateS.offset}
// Look far ahead, unless we have a really long match already...
if best.length < goodEnough {
// No match found, move forward on input, no need to check forward...
if best.length < 4 {
s += 1 + (s-nextEmit)>>(kSearchStrength-1)
if s >= sLimit {
break encodeLoop
}
cv = load6432(src, s)
continue
}
s++
candidateS = e.table[hash4x64(cv>>8, bestShortTableBits)]
cv = load6432(src, s)
cv2 := load6432(src, s+1)
candidateL = e.longTable[hash8(cv, bestLongTableBits)]
candidateL2 := e.longTable[hash8(cv2, bestLongTableBits)]
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1))
best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1))
}
// We have a match, we can store the forward value
if best.rep > 0 {
s = best.s
var seq seq
seq.matchLen = uint32(best.length - zstdMinMatch)
// We might be able to match backwards.
// Extend as long as we can.
start := best.s
// We end the search early, so we don't risk 0 literals
// and have to do special offset treatment.
startLimit := nextEmit + 1
tMin := s - e.maxMatchOff
if tMin < 0 {
tMin = 0
}
repIndex := best.offset
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
repIndex--
start--
seq.matchLen++
}
addLiterals(&seq, start)
// rep 0
seq.offset = uint32(best.rep)
if debugSequences {
println("repeat sequence", seq, "next s:", s)
}
blk.sequences = append(blk.sequences, seq)
// Index match start+1 (long) -> s - 1
index0 := s
s = best.s + best.length
nextEmit = s
if s >= sLimit {
if debug {
println("repeat ended", s, best.length)
}
break encodeLoop
}
// Index skipped...
off := index0 + e.cur
for index0 < s-1 {
cv0 := load6432(src, index0)
h0 := hash8(cv0, bestLongTableBits)
h1 := hash4x64(cv0, bestShortTableBits)
e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset}
e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset}
off++
index0++
}
switch best.rep {
case 2:
offset1, offset2 = offset2, offset1
case 3:
offset1, offset2, offset3 = offset3, offset1, offset2
}
cv = load6432(src, s)
continue
}
// A 4-byte match has been found. Update recent offsets.
// We'll later see if more than 4 bytes.
s = best.s
t := best.offset
offset1, offset2, offset3 = s-t, offset1, offset2
if debugAsserts && s <= t {
panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
}
if debugAsserts && canRepeat && int(offset1) > len(src) {
panic("invalid offset")
}
// Extend the n-byte match as long as possible.
l := best.length
// Extend backwards
tMin := s - e.maxMatchOff
if tMin < 0 {
tMin = 0
}
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
s--
t--
l++
}
// Write our sequence
var seq seq
seq.litLen = uint32(s - nextEmit)
seq.matchLen = uint32(l - zstdMinMatch)
if seq.litLen > 0 {
blk.literals = append(blk.literals, src[nextEmit:s]...)
}
seq.offset = uint32(s-t) + 3
s += l
if debugSequences {
println("sequence", seq, "next s:", s)
}
blk.sequences = append(blk.sequences, seq)
nextEmit = s
if s >= sLimit {
break encodeLoop
}
// Index match start+1 (long) -> s - 1
index0 := s - l + 1
// every entry
for index0 < s-1 {
cv0 := load6432(src, index0)
h0 := hash8(cv0, bestLongTableBits)
h1 := hash4x64(cv0, bestShortTableBits)
off := index0 + e.cur
e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset}
e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset}
index0++
}
cv = load6432(src, s)
if !canRepeat {
continue
}
// Check offset 2
for {
o2 := s - offset2
if load3232(src, o2) != uint32(cv) {
// Do regular search
break
}
// Store this, since we have it.
nextHashS := hash4x64(cv, bestShortTableBits)
nextHashL := hash8(cv, bestLongTableBits)
// We have at least 4 byte match.
// No need to check backwards. We come straight from a match
l := 4 + e.matchlen(s+4, o2+4, src)
e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset}
e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: e.table[nextHashS].offset}
seq.matchLen = uint32(l) - zstdMinMatch
seq.litLen = 0
// Since litlen is always 0, this is offset 1.
seq.offset = 1
s += l
nextEmit = s
if debugSequences {
println("sequence", seq, "next s:", s)
}
blk.sequences = append(blk.sequences, seq)
// Swap offset 1 and 2.
offset1, offset2 = offset2, offset1
if s >= sLimit {
// Finished
break encodeLoop
}
cv = load6432(src, s)
}
}
if int(nextEmit) < len(src) {
blk.literals = append(blk.literals, src[nextEmit:]...)
blk.extraLits = len(src) - int(nextEmit)
}
blk.recentOffsets[0] = uint32(offset1)
blk.recentOffsets[1] = uint32(offset2)
blk.recentOffsets[2] = uint32(offset3)
if debug {
println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
}
}
// EncodeNoHist will encode a block with no history and no following blocks.
// Most notable difference is that src will not be copied for history and
// we do not need to check for max match length.
func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
e.Encode(blk, src)
}
// ResetDict will reset and set a dictionary if not nil
func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) {
e.resetBase(d, singleBlock)
if d == nil {
return
}
// Init or copy dict table
if len(e.dictTable) != len(e.table) || d.id != e.lastDictID {
if len(e.dictTable) != len(e.table) {
e.dictTable = make([]prevEntry, len(e.table))
}
end := int32(len(d.content)) - 8 + e.maxMatchOff
for i := e.maxMatchOff; i < end; i += 4 {
const hashLog = bestShortTableBits
cv := load6432(d.content, i-e.maxMatchOff)
nextHash := hash4x64(cv, hashLog) // 0 -> 4
nextHash1 := hash4x64(cv>>8, hashLog) // 1 -> 5
nextHash2 := hash4x64(cv>>16, hashLog) // 2 -> 6
nextHash3 := hash4x64(cv>>24, hashLog) // 3 -> 7
e.dictTable[nextHash] = prevEntry{
prev: e.dictTable[nextHash].offset,
offset: i,
}
e.dictTable[nextHash1] = prevEntry{
prev: e.dictTable[nextHash1].offset,
offset: i + 1,
}
e.dictTable[nextHash2] = prevEntry{
prev: e.dictTable[nextHash2].offset,
offset: i + 2,
}
e.dictTable[nextHash3] = prevEntry{
prev: e.dictTable[nextHash3].offset,
offset: i + 3,
}
}
e.lastDictID = d.id
}
// Init or copy dict table
if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID {
if len(e.dictLongTable) != len(e.longTable) {
e.dictLongTable = make([]prevEntry, len(e.longTable))
}
if len(d.content) >= 8 {
cv := load6432(d.content, 0)
h := hash8(cv, bestLongTableBits)
e.dictLongTable[h] = prevEntry{
offset: e.maxMatchOff,
prev: e.dictLongTable[h].offset,
}
end := int32(len(d.content)) - 8 + e.maxMatchOff
off := 8 // First to read
for i := e.maxMatchOff + 1; i < end; i++ {
cv = cv>>8 | (uint64(d.content[off]) << 56)
h := hash8(cv, bestLongTableBits)
e.dictLongTable[h] = prevEntry{
offset: i,
prev: e.dictLongTable[h].offset,
}
off++
}
}
e.lastDictID = d.id
}
// Reset table to initial state
copy(e.longTable[:], e.dictLongTable)
e.cur = e.maxMatchOff
// Reset table to initial state
copy(e.table[:], e.dictTable)
}

View file

@ -47,6 +47,8 @@ func (o encoderOptions) encoder() encoder {
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}}
case SpeedBetterCompression:
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
case SpeedBestCompression:
return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
case SpeedFastest:
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
}
@ -143,20 +145,20 @@ const (
// By using this, notice that CPU usage may go up in the future.
SpeedBetterCompression
// SpeedBestCompression will choose the best available compression option.
// This will offer the best compression no matter the CPU cost.
SpeedBestCompression
// speedLast should be kept as the last actual compression option.
// The is not for external usage, but is used to keep track of the valid options.
speedLast
// SpeedBestCompression will choose the best available compression option.
// For now this is not implemented.
SpeedBestCompression = SpeedBetterCompression
)
// EncoderLevelFromString will convert a string representation of an encoding level back
// to a compression level. The compare is not case sensitive.
// If the string wasn't recognized, (false, SpeedDefault) will be returned.
func EncoderLevelFromString(s string) (bool, EncoderLevel) {
for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ {
for l := speedNotSet + 1; l < speedLast; l++ {
if strings.EqualFold(s, l.String()) {
return true, l
}
@ -173,7 +175,9 @@ func EncoderLevelFromZstd(level int) EncoderLevel {
return SpeedFastest
case level >= 3 && level < 6:
return SpeedDefault
case level > 5:
case level >= 6 && level < 10:
return SpeedBetterCompression
case level >= 10:
return SpeedBetterCompression
}
return SpeedDefault
@ -188,6 +192,8 @@ func (e EncoderLevel) String() string {
return "default"
case SpeedBetterCompression:
return "better"
case SpeedBestCompression:
return "best"
default:
return "invalid"
}
@ -209,6 +215,8 @@ func WithEncoderLevel(l EncoderLevel) EOption {
o.windowSize = 8 << 20
case SpeedBetterCompression:
o.windowSize = 16 << 20
case SpeedBestCompression:
o.windowSize = 32 << 20
}
}
if !o.customALEntropy {

View file

@ -15,7 +15,11 @@ package prometheus
import (
"errors"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
)
type processCollector struct {
@ -149,3 +153,20 @@ func (c *processCollector) reportError(ch chan<- Metric, desc *Desc, err error)
}
ch <- NewInvalidMetric(desc, err)
}
// NewPidFileFn returns a function that retrieves a pid from the specified file.
// It is meant to be used for the PidFn field in ProcessCollectorOpts.
func NewPidFileFn(pidFilePath string) func() (int, error) {
return func() (int, error) {
content, err := ioutil.ReadFile(pidFilePath)
if err != nil {
return 0, fmt.Errorf("can't read pid file %q: %+v", pidFilePath, err)
}
pid, err := strconv.Atoi(strings.TrimSpace(string(content)))
if err != nil {
return 0, fmt.Errorf("can't parse pid file %q: %+v", pidFilePath, err)
}
return pid, nil
}
}

View file

@ -215,6 +215,8 @@ func (err AlreadyRegisteredError) Error() string {
// by a Gatherer to report multiple errors during MetricFamily gathering.
type MultiError []error
// Error formats the contained errors as a bullet point list, preceded by the
// total number of errors. Note that this results in a multi-line string.
func (errs MultiError) Error() string {
if len(errs) == 0 {
return ""

View file

@ -561,6 +561,7 @@ ccflags="$@"
$2 ~ /^(HDIO|WIN|SMART)_/ ||
$2 ~ /^CRYPTO_/ ||
$2 ~ /^TIPC_/ ||
$2 !~ "DEVLINK_RELOAD_LIMITS_VALID_MASK" &&
$2 ~ /^DEVLINK_/ ||
$2 ~ /^LWTUNNEL_IP/ ||
$2 !~ "WMESGLEN" &&

View file

@ -8,12 +8,10 @@ package unix
import "time"
// TimespecToNsec converts a Timespec value into a number of
// nanoseconds since the Unix epoch.
func TimespecToNsec(ts Timespec) int64 { return int64(ts.Sec)*1e9 + int64(ts.Nsec) }
// TimespecToNSec returns the time stored in ts as nanoseconds.
func TimespecToNsec(ts Timespec) int64 { return ts.Nano() }
// NsecToTimespec takes a number of nanoseconds since the Unix epoch
// and returns the corresponding Timespec value.
// NsecToTimespec converts a number of nanoseconds into a Timespec.
func NsecToTimespec(nsec int64) Timespec {
sec := nsec / 1e9
nsec = nsec % 1e9
@ -42,12 +40,10 @@ func TimeToTimespec(t time.Time) (Timespec, error) {
return ts, nil
}
// TimevalToNsec converts a Timeval value into a number of nanoseconds
// since the Unix epoch.
func TimevalToNsec(tv Timeval) int64 { return int64(tv.Sec)*1e9 + int64(tv.Usec)*1e3 }
// TimevalToNsec returns the time stored in tv as nanoseconds.
func TimevalToNsec(tv Timeval) int64 { return tv.Nano() }
// NsecToTimeval takes a number of nanoseconds since the Unix epoch
// and returns the corresponding Timeval value.
// NsecToTimeval converts a number of nanoseconds into a Timeval.
func NsecToTimeval(nsec int64) Timeval {
nsec += 999 // round up to microsecond
usec := nsec % 1e9 / 1e3
@ -59,24 +55,22 @@ func NsecToTimeval(nsec int64) Timeval {
return setTimeval(sec, usec)
}
// Unix returns ts as the number of seconds and nanoseconds elapsed since the
// Unix epoch.
// Unix returns the time stored in ts as seconds plus nanoseconds.
func (ts *Timespec) Unix() (sec int64, nsec int64) {
return int64(ts.Sec), int64(ts.Nsec)
}
// Unix returns tv as the number of seconds and nanoseconds elapsed since the
// Unix epoch.
// Unix returns the time stored in tv as seconds plus nanoseconds.
func (tv *Timeval) Unix() (sec int64, nsec int64) {
return int64(tv.Sec), int64(tv.Usec) * 1000
}
// Nano returns ts as the number of nanoseconds elapsed since the Unix epoch.
// Nano returns the time stored in ts as nanoseconds.
func (ts *Timespec) Nano() int64 {
return int64(ts.Sec)*1e9 + int64(ts.Nsec)
}
// Nano returns tv as the number of nanoseconds elapsed since the Unix epoch.
// Nano returns the time stored in tv as nanoseconds.
func (tv *Timeval) Nano() int64 {
return int64(tv.Sec)*1e9 + int64(tv.Usec)*1000
}

View file

@ -65,6 +65,7 @@ const (
ALG_OP_ENCRYPT = 0x1
ALG_SET_AEAD_ASSOCLEN = 0x4
ALG_SET_AEAD_AUTHSIZE = 0x5
ALG_SET_DRBG_ENTROPY = 0x6
ALG_SET_IV = 0x2
ALG_SET_KEY = 0x1
ALG_SET_OP = 0x3
@ -179,8 +180,10 @@ const (
BPF_F_ANY_ALIGNMENT = 0x2
BPF_F_QUERY_EFFECTIVE = 0x1
BPF_F_REPLACE = 0x4
BPF_F_SLEEPABLE = 0x10
BPF_F_STRICT_ALIGNMENT = 0x1
BPF_F_TEST_RND_HI32 = 0x4
BPF_F_TEST_RUN_ON_CPU = 0x1
BPF_F_TEST_STATE_FREQ = 0x8
BPF_H = 0x8
BPF_IMM = 0x0
@ -219,6 +222,7 @@ const (
BPF_NET_OFF = -0x100000
BPF_OBJ_NAME_LEN = 0x10
BPF_OR = 0x40
BPF_PSEUDO_BTF_ID = 0x3
BPF_PSEUDO_CALL = 0x1
BPF_PSEUDO_MAP_FD = 0x1
BPF_PSEUDO_MAP_VALUE = 0x2
@ -429,10 +433,13 @@ const (
DEBUGFS_MAGIC = 0x64626720
DEVLINK_CMD_ESWITCH_MODE_GET = 0x1d
DEVLINK_CMD_ESWITCH_MODE_SET = 0x1e
DEVLINK_FLASH_OVERWRITE_IDENTIFIERS = 0x2
DEVLINK_FLASH_OVERWRITE_SETTINGS = 0x1
DEVLINK_GENL_MCGRP_CONFIG_NAME = "config"
DEVLINK_GENL_NAME = "devlink"
DEVLINK_GENL_VERSION = 0x1
DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX = 0x14
DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS = 0x3
DEVMEM_MAGIC = 0x454d444d
DEVPTS_SUPER_MAGIC = 0x1cd1
DMA_BUF_MAGIC = 0x444d4142
@ -477,9 +484,9 @@ const (
DM_UUID_FLAG = 0x4000
DM_UUID_LEN = 0x81
DM_VERSION = 0xc138fd00
DM_VERSION_EXTRA = "-ioctl (2020-02-27)"
DM_VERSION_EXTRA = "-ioctl (2020-10-01)"
DM_VERSION_MAJOR = 0x4
DM_VERSION_MINOR = 0x2a
DM_VERSION_MINOR = 0x2b
DM_VERSION_PATCHLEVEL = 0x0
DT_BLK = 0x6
DT_CHR = 0x2
@ -1331,6 +1338,7 @@ const (
MS_NOREMOTELOCK = 0x8000000
MS_NOSEC = 0x10000000
MS_NOSUID = 0x2
MS_NOSYMFOLLOW = 0x100
MS_NOUSER = -0x80000000
MS_POSIXACL = 0x10000
MS_PRIVATE = 0x40000
@ -1572,7 +1580,7 @@ const (
PERF_MEM_REMOTE_REMOTE = 0x1
PERF_MEM_REMOTE_SHIFT = 0x25
PERF_MEM_SNOOPX_FWD = 0x1
PERF_MEM_SNOOPX_SHIFT = 0x25
PERF_MEM_SNOOPX_SHIFT = 0x26
PERF_MEM_SNOOP_HIT = 0x4
PERF_MEM_SNOOP_HITM = 0x10
PERF_MEM_SNOOP_MISS = 0x8
@ -1672,6 +1680,13 @@ const (
PR_MCE_KILL_SET = 0x1
PR_MPX_DISABLE_MANAGEMENT = 0x2c
PR_MPX_ENABLE_MANAGEMENT = 0x2b
PR_MTE_TAG_MASK = 0x7fff8
PR_MTE_TAG_SHIFT = 0x3
PR_MTE_TCF_ASYNC = 0x4
PR_MTE_TCF_MASK = 0x6
PR_MTE_TCF_NONE = 0x0
PR_MTE_TCF_SHIFT = 0x1
PR_MTE_TCF_SYNC = 0x2
PR_PAC_APDAKEY = 0x4
PR_PAC_APDBKEY = 0x8
PR_PAC_APGAKEY = 0x10
@ -2206,7 +2221,7 @@ const (
STATX_ATTR_APPEND = 0x20
STATX_ATTR_AUTOMOUNT = 0x1000
STATX_ATTR_COMPRESSED = 0x4
STATX_ATTR_DAX = 0x2000
STATX_ATTR_DAX = 0x200000
STATX_ATTR_ENCRYPTED = 0x800
STATX_ATTR_IMMUTABLE = 0x10
STATX_ATTR_MOUNT_ROOT = 0x2000
@ -2390,6 +2405,7 @@ const (
TIPC_NODE_STATE = 0x0
TIPC_OK = 0x0
TIPC_PUBLISHED = 0x1
TIPC_REKEYING_NOW = 0xffffffff
TIPC_RESERVED_TYPES = 0x40
TIPC_RETDATA = 0x2
TIPC_SERVICE_ADDR = 0x2

View file

@ -196,6 +196,8 @@ const (
PPPIOCXFERUNIT = 0x744e
PROT_BTI = 0x10
PR_SET_PTRACER_ANY = 0xffffffffffffffff
PTRACE_PEEKMTETAGS = 0x21
PTRACE_POKEMTETAGS = 0x22
PTRACE_SYSEMU = 0x1f
PTRACE_SYSEMU_SINGLESTEP = 0x20
RLIMIT_AS = 0x9

View file

@ -435,4 +435,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -357,4 +357,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -399,4 +399,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -302,4 +302,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -420,4 +420,5 @@ const (
SYS_OPENAT2 = 4437
SYS_PIDFD_GETFD = 4438
SYS_FACCESSAT2 = 4439
SYS_PROCESS_MADVISE = 4440
)

View file

@ -350,4 +350,5 @@ const (
SYS_OPENAT2 = 5437
SYS_PIDFD_GETFD = 5438
SYS_FACCESSAT2 = 5439
SYS_PROCESS_MADVISE = 5440
)

View file

@ -350,4 +350,5 @@ const (
SYS_OPENAT2 = 5437
SYS_PIDFD_GETFD = 5438
SYS_FACCESSAT2 = 5439
SYS_PROCESS_MADVISE = 5440
)

View file

@ -420,4 +420,5 @@ const (
SYS_OPENAT2 = 4437
SYS_PIDFD_GETFD = 4438
SYS_FACCESSAT2 = 4439
SYS_PROCESS_MADVISE = 4440
)

View file

@ -399,4 +399,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -399,4 +399,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -301,4 +301,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -364,4 +364,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -378,4 +378,5 @@ const (
SYS_OPENAT2 = 437
SYS_PIDFD_GETFD = 438
SYS_FACCESSAT2 = 439
SYS_PROCESS_MADVISE = 440
)

View file

@ -219,6 +219,7 @@ const (
SizeofSockaddrUnix = 0x401
SizeofSockaddrDatalink = 0x80
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofIPv6MTUInfo = 0x20

View file

@ -223,6 +223,7 @@ const (
SizeofSockaddrUnix = 0x401
SizeofSockaddrDatalink = 0x80
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofIPv6MTUInfo = 0x20

View file

@ -234,6 +234,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x36
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -313,6 +313,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x36
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPMreqn = 0xc
SizeofIPv6Mreq = 0x14

View file

@ -309,6 +309,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x36
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPMreqn = 0xc
SizeofIPv6Mreq = 0x14

View file

@ -311,6 +311,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x36
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPMreqn = 0xc
SizeofIPv6Mreq = 0x14

View file

@ -309,6 +309,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x36
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPMreqn = 0xc
SizeofIPv6Mreq = 0x14

View file

@ -1381,6 +1381,11 @@ const (
IFLA_PROP_LIST = 0x34
IFLA_ALT_IFNAME = 0x35
IFLA_PERM_ADDRESS = 0x36
IFLA_PROTO_DOWN_REASON = 0x37
IFLA_PROTO_DOWN_REASON_UNSPEC = 0x0
IFLA_PROTO_DOWN_REASON_MASK = 0x1
IFLA_PROTO_DOWN_REASON_VALUE = 0x2
IFLA_PROTO_DOWN_REASON_MAX = 0x2
IFLA_INET_UNSPEC = 0x0
IFLA_INET_CONF = 0x1
IFLA_INET6_UNSPEC = 0x0
@ -1475,6 +1480,7 @@ const (
IFLA_BRPORT_ISOLATED = 0x21
IFLA_BRPORT_BACKUP_PORT = 0x22
IFLA_BRPORT_MRP_RING_OPEN = 0x23
IFLA_BRPORT_MRP_IN_OPEN = 0x24
IFLA_INFO_UNSPEC = 0x0
IFLA_INFO_KIND = 0x1
IFLA_INFO_DATA = 0x2
@ -1673,6 +1679,7 @@ const (
IFLA_HSR_SUPERVISION_ADDR = 0x4
IFLA_HSR_SEQ_NR = 0x5
IFLA_HSR_VERSION = 0x6
IFLA_HSR_PROTOCOL = 0x7
IFLA_STATS_UNSPEC = 0x0
IFLA_STATS_LINK_64 = 0x1
IFLA_STATS_LINK_XSTATS = 0x2
@ -2221,6 +2228,8 @@ const (
NETNSA_NSID = 0x1
NETNSA_PID = 0x2
NETNSA_FD = 0x3
NETNSA_TARGET_NSID = 0x4
NETNSA_CURRENT_NSID = 0x5
)
type XDPRingOffset struct {
@ -2415,6 +2424,8 @@ const (
BPF_LINK_GET_NEXT_ID = 0x1f
BPF_ENABLE_STATS = 0x20
BPF_ITER_CREATE = 0x21
BPF_LINK_DETACH = 0x22
BPF_PROG_BIND_MAP = 0x23
BPF_MAP_TYPE_UNSPEC = 0x0
BPF_MAP_TYPE_HASH = 0x1
BPF_MAP_TYPE_ARRAY = 0x2
@ -2443,6 +2454,7 @@ const (
BPF_MAP_TYPE_DEVMAP_HASH = 0x19
BPF_MAP_TYPE_STRUCT_OPS = 0x1a
BPF_MAP_TYPE_RINGBUF = 0x1b
BPF_MAP_TYPE_INODE_STORAGE = 0x1c
BPF_PROG_TYPE_UNSPEC = 0x0
BPF_PROG_TYPE_SOCKET_FILTER = 0x1
BPF_PROG_TYPE_KPROBE = 0x2
@ -2473,6 +2485,7 @@ const (
BPF_PROG_TYPE_STRUCT_OPS = 0x1b
BPF_PROG_TYPE_EXT = 0x1c
BPF_PROG_TYPE_LSM = 0x1d
BPF_PROG_TYPE_SK_LOOKUP = 0x1e
BPF_CGROUP_INET_INGRESS = 0x0
BPF_CGROUP_INET_EGRESS = 0x1
BPF_CGROUP_INET_SOCK_CREATE = 0x2
@ -2507,12 +2520,17 @@ const (
BPF_CGROUP_INET4_GETSOCKNAME = 0x1f
BPF_CGROUP_INET6_GETSOCKNAME = 0x20
BPF_XDP_DEVMAP = 0x21
BPF_CGROUP_INET_SOCK_RELEASE = 0x22
BPF_XDP_CPUMAP = 0x23
BPF_SK_LOOKUP = 0x24
BPF_XDP = 0x25
BPF_LINK_TYPE_UNSPEC = 0x0
BPF_LINK_TYPE_RAW_TRACEPOINT = 0x1
BPF_LINK_TYPE_TRACING = 0x2
BPF_LINK_TYPE_CGROUP = 0x3
BPF_LINK_TYPE_ITER = 0x4
BPF_LINK_TYPE_NETNS = 0x5
BPF_LINK_TYPE_XDP = 0x6
BPF_ANY = 0x0
BPF_NOEXIST = 0x1
BPF_EXIST = 0x2
@ -2528,6 +2546,8 @@ const (
BPF_F_WRONLY_PROG = 0x100
BPF_F_CLONE = 0x200
BPF_F_MMAPABLE = 0x400
BPF_F_PRESERVE_ELEMS = 0x800
BPF_F_INNER_MAP = 0x1000
BPF_STATS_RUN_TIME = 0x0
BPF_STACK_BUILD_ID_EMPTY = 0x0
BPF_STACK_BUILD_ID_VALID = 0x1
@ -2565,6 +2585,7 @@ const (
BPF_ADJ_ROOM_ENCAP_L2_MASK = 0xff
BPF_ADJ_ROOM_ENCAP_L2_SHIFT = 0x38
BPF_F_SYSCTL_BASE_NAME = 0x1
BPF_LOCAL_STORAGE_GET_F_CREATE = 0x1
BPF_SK_STORAGE_GET_F_CREATE = 0x1
BPF_F_GET_BRANCH_RECORDS_SIZE = 0x1
BPF_RB_NO_WAKEUP = 0x1
@ -2576,6 +2597,8 @@ const (
BPF_RINGBUF_BUSY_BIT = 0x80000000
BPF_RINGBUF_DISCARD_BIT = 0x40000000
BPF_RINGBUF_HDR_SZ = 0x8
BPF_SK_LOOKUP_F_REPLACE = 0x1
BPF_SK_LOOKUP_F_NO_REUSEPORT = 0x2
BPF_ADJ_ROOM_NET = 0x0
BPF_ADJ_ROOM_MAC = 0x1
BPF_HDR_START_MAC = 0x0
@ -2591,7 +2614,10 @@ const (
BPF_SOCK_OPS_RETRANS_CB_FLAG = 0x2
BPF_SOCK_OPS_STATE_CB_FLAG = 0x4
BPF_SOCK_OPS_RTT_CB_FLAG = 0x8
BPF_SOCK_OPS_ALL_CB_FLAGS = 0xf
BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG = 0x10
BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG = 0x20
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG = 0x40
BPF_SOCK_OPS_ALL_CB_FLAGS = 0x7f
BPF_SOCK_OPS_VOID = 0x0
BPF_SOCK_OPS_TIMEOUT_INIT = 0x1
BPF_SOCK_OPS_RWND_INIT = 0x2
@ -2605,6 +2631,9 @@ const (
BPF_SOCK_OPS_STATE_CB = 0xa
BPF_SOCK_OPS_TCP_LISTEN_CB = 0xb
BPF_SOCK_OPS_RTT_CB = 0xc
BPF_SOCK_OPS_PARSE_HDR_OPT_CB = 0xd
BPF_SOCK_OPS_HDR_OPT_LEN_CB = 0xe
BPF_SOCK_OPS_WRITE_HDR_OPT_CB = 0xf
BPF_TCP_ESTABLISHED = 0x1
BPF_TCP_SYN_SENT = 0x2
BPF_TCP_SYN_RECV = 0x3
@ -2620,6 +2649,14 @@ const (
BPF_TCP_MAX_STATES = 0xd
TCP_BPF_IW = 0x3e9
TCP_BPF_SNDCWND_CLAMP = 0x3ea
TCP_BPF_DELACK_MAX = 0x3eb
TCP_BPF_RTO_MIN = 0x3ec
TCP_BPF_SYN = 0x3ed
TCP_BPF_SYN_IP = 0x3ee
TCP_BPF_SYN_MAC = 0x3ef
BPF_LOAD_HDR_OPT_TCP_SYN = 0x1
BPF_WRITE_HDR_TCP_CURRENT_MSS = 0x1
BPF_WRITE_HDR_TCP_SYNACK_COOKIE = 0x2
BPF_DEVCG_ACC_MKNOD = 0x1
BPF_DEVCG_ACC_READ = 0x2
BPF_DEVCG_ACC_WRITE = 0x4
@ -2681,6 +2718,7 @@ const (
RTNLGRP_IPV4_MROUTE_R = 0x1e
RTNLGRP_IPV6_MROUTE_R = 0x1f
RTNLGRP_NEXTHOP = 0x20
RTNLGRP_BRVLAN = 0x21
)
type CapUserHeader struct {
@ -2810,7 +2848,46 @@ const (
DEVLINK_CMD_DPIPE_ENTRIES_GET = 0x20
DEVLINK_CMD_DPIPE_HEADERS_GET = 0x21
DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET = 0x22
DEVLINK_CMD_MAX = 0x48
DEVLINK_CMD_RESOURCE_SET = 0x23
DEVLINK_CMD_RESOURCE_DUMP = 0x24
DEVLINK_CMD_RELOAD = 0x25
DEVLINK_CMD_PARAM_GET = 0x26
DEVLINK_CMD_PARAM_SET = 0x27
DEVLINK_CMD_PARAM_NEW = 0x28
DEVLINK_CMD_PARAM_DEL = 0x29
DEVLINK_CMD_REGION_GET = 0x2a
DEVLINK_CMD_REGION_SET = 0x2b
DEVLINK_CMD_REGION_NEW = 0x2c
DEVLINK_CMD_REGION_DEL = 0x2d
DEVLINK_CMD_REGION_READ = 0x2e
DEVLINK_CMD_PORT_PARAM_GET = 0x2f
DEVLINK_CMD_PORT_PARAM_SET = 0x30
DEVLINK_CMD_PORT_PARAM_NEW = 0x31
DEVLINK_CMD_PORT_PARAM_DEL = 0x32
DEVLINK_CMD_INFO_GET = 0x33
DEVLINK_CMD_HEALTH_REPORTER_GET = 0x34
DEVLINK_CMD_HEALTH_REPORTER_SET = 0x35
DEVLINK_CMD_HEALTH_REPORTER_RECOVER = 0x36
DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE = 0x37
DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET = 0x38
DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR = 0x39
DEVLINK_CMD_FLASH_UPDATE = 0x3a
DEVLINK_CMD_FLASH_UPDATE_END = 0x3b
DEVLINK_CMD_FLASH_UPDATE_STATUS = 0x3c
DEVLINK_CMD_TRAP_GET = 0x3d
DEVLINK_CMD_TRAP_SET = 0x3e
DEVLINK_CMD_TRAP_NEW = 0x3f
DEVLINK_CMD_TRAP_DEL = 0x40
DEVLINK_CMD_TRAP_GROUP_GET = 0x41
DEVLINK_CMD_TRAP_GROUP_SET = 0x42
DEVLINK_CMD_TRAP_GROUP_NEW = 0x43
DEVLINK_CMD_TRAP_GROUP_DEL = 0x44
DEVLINK_CMD_TRAP_POLICER_GET = 0x45
DEVLINK_CMD_TRAP_POLICER_SET = 0x46
DEVLINK_CMD_TRAP_POLICER_NEW = 0x47
DEVLINK_CMD_TRAP_POLICER_DEL = 0x48
DEVLINK_CMD_HEALTH_REPORTER_TEST = 0x49
DEVLINK_CMD_MAX = 0x49
DEVLINK_PORT_TYPE_NOTSET = 0x0
DEVLINK_PORT_TYPE_AUTO = 0x1
DEVLINK_PORT_TYPE_ETH = 0x2
@ -2827,6 +2904,47 @@ const (
DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT = 0x3
DEVLINK_ESWITCH_ENCAP_MODE_NONE = 0x0
DEVLINK_ESWITCH_ENCAP_MODE_BASIC = 0x1
DEVLINK_PORT_FLAVOUR_PHYSICAL = 0x0
DEVLINK_PORT_FLAVOUR_CPU = 0x1
DEVLINK_PORT_FLAVOUR_DSA = 0x2
DEVLINK_PORT_FLAVOUR_PCI_PF = 0x3
DEVLINK_PORT_FLAVOUR_PCI_VF = 0x4
DEVLINK_PORT_FLAVOUR_VIRTUAL = 0x5
DEVLINK_PORT_FLAVOUR_UNUSED = 0x6
DEVLINK_PARAM_CMODE_RUNTIME = 0x0
DEVLINK_PARAM_CMODE_DRIVERINIT = 0x1
DEVLINK_PARAM_CMODE_PERMANENT = 0x2
DEVLINK_PARAM_CMODE_MAX = 0x2
DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DRIVER = 0x0
DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_FLASH = 0x1
DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DISK = 0x2
DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_UNKNOWN = 0x3
DEVLINK_PARAM_RESET_DEV_ON_DRV_PROBE_VALUE_UNKNOWN = 0x0
DEVLINK_PARAM_RESET_DEV_ON_DRV_PROBE_VALUE_ALWAYS = 0x1
DEVLINK_PARAM_RESET_DEV_ON_DRV_PROBE_VALUE_NEVER = 0x2
DEVLINK_PARAM_RESET_DEV_ON_DRV_PROBE_VALUE_DISK = 0x3
DEVLINK_ATTR_STATS_RX_PACKETS = 0x0
DEVLINK_ATTR_STATS_RX_BYTES = 0x1
DEVLINK_ATTR_STATS_RX_DROPPED = 0x2
DEVLINK_ATTR_STATS_MAX = 0x2
DEVLINK_FLASH_OVERWRITE_SETTINGS_BIT = 0x0
DEVLINK_FLASH_OVERWRITE_IDENTIFIERS_BIT = 0x1
DEVLINK_FLASH_OVERWRITE_MAX_BIT = 0x1
DEVLINK_TRAP_ACTION_DROP = 0x0
DEVLINK_TRAP_ACTION_TRAP = 0x1
DEVLINK_TRAP_ACTION_MIRROR = 0x2
DEVLINK_TRAP_TYPE_DROP = 0x0
DEVLINK_TRAP_TYPE_EXCEPTION = 0x1
DEVLINK_TRAP_TYPE_CONTROL = 0x2
DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT = 0x0
DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE = 0x1
DEVLINK_RELOAD_ACTION_UNSPEC = 0x0
DEVLINK_RELOAD_ACTION_DRIVER_REINIT = 0x1
DEVLINK_RELOAD_ACTION_FW_ACTIVATE = 0x2
DEVLINK_RELOAD_ACTION_MAX = 0x2
DEVLINK_RELOAD_LIMIT_UNSPEC = 0x0
DEVLINK_RELOAD_LIMIT_NO_RESET = 0x1
DEVLINK_RELOAD_LIMIT_MAX = 0x1
DEVLINK_ATTR_UNSPEC = 0x0
DEVLINK_ATTR_BUS_NAME = 0x1
DEVLINK_ATTR_DEV_NAME = 0x2
@ -2890,7 +3008,108 @@ const (
DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c
DEVLINK_ATTR_PAD = 0x3d
DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e
DEVLINK_ATTR_MAX = 0x94
DEVLINK_ATTR_RESOURCE_LIST = 0x3f
DEVLINK_ATTR_RESOURCE = 0x40
DEVLINK_ATTR_RESOURCE_NAME = 0x41
DEVLINK_ATTR_RESOURCE_ID = 0x42
DEVLINK_ATTR_RESOURCE_SIZE = 0x43
DEVLINK_ATTR_RESOURCE_SIZE_NEW = 0x44
DEVLINK_ATTR_RESOURCE_SIZE_VALID = 0x45
DEVLINK_ATTR_RESOURCE_SIZE_MIN = 0x46
DEVLINK_ATTR_RESOURCE_SIZE_MAX = 0x47
DEVLINK_ATTR_RESOURCE_SIZE_GRAN = 0x48
DEVLINK_ATTR_RESOURCE_UNIT = 0x49
DEVLINK_ATTR_RESOURCE_OCC = 0x4a
DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID = 0x4b
DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS = 0x4c
DEVLINK_ATTR_PORT_FLAVOUR = 0x4d
DEVLINK_ATTR_PORT_NUMBER = 0x4e
DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER = 0x4f
DEVLINK_ATTR_PARAM = 0x50
DEVLINK_ATTR_PARAM_NAME = 0x51
DEVLINK_ATTR_PARAM_GENERIC = 0x52
DEVLINK_ATTR_PARAM_TYPE = 0x53
DEVLINK_ATTR_PARAM_VALUES_LIST = 0x54
DEVLINK_ATTR_PARAM_VALUE = 0x55
DEVLINK_ATTR_PARAM_VALUE_DATA = 0x56
DEVLINK_ATTR_PARAM_VALUE_CMODE = 0x57
DEVLINK_ATTR_REGION_NAME = 0x58
DEVLINK_ATTR_REGION_SIZE = 0x59
DEVLINK_ATTR_REGION_SNAPSHOTS = 0x5a
DEVLINK_ATTR_REGION_SNAPSHOT = 0x5b
DEVLINK_ATTR_REGION_SNAPSHOT_ID = 0x5c
DEVLINK_ATTR_REGION_CHUNKS = 0x5d
DEVLINK_ATTR_REGION_CHUNK = 0x5e
DEVLINK_ATTR_REGION_CHUNK_DATA = 0x5f
DEVLINK_ATTR_REGION_CHUNK_ADDR = 0x60
DEVLINK_ATTR_REGION_CHUNK_LEN = 0x61
DEVLINK_ATTR_INFO_DRIVER_NAME = 0x62
DEVLINK_ATTR_INFO_SERIAL_NUMBER = 0x63
DEVLINK_ATTR_INFO_VERSION_FIXED = 0x64
DEVLINK_ATTR_INFO_VERSION_RUNNING = 0x65
DEVLINK_ATTR_INFO_VERSION_STORED = 0x66
DEVLINK_ATTR_INFO_VERSION_NAME = 0x67
DEVLINK_ATTR_INFO_VERSION_VALUE = 0x68
DEVLINK_ATTR_SB_POOL_CELL_SIZE = 0x69
DEVLINK_ATTR_FMSG = 0x6a
DEVLINK_ATTR_FMSG_OBJ_NEST_START = 0x6b
DEVLINK_ATTR_FMSG_PAIR_NEST_START = 0x6c
DEVLINK_ATTR_FMSG_ARR_NEST_START = 0x6d
DEVLINK_ATTR_FMSG_NEST_END = 0x6e
DEVLINK_ATTR_FMSG_OBJ_NAME = 0x6f
DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE = 0x70
DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA = 0x71
DEVLINK_ATTR_HEALTH_REPORTER = 0x72
DEVLINK_ATTR_HEALTH_REPORTER_NAME = 0x73
DEVLINK_ATTR_HEALTH_REPORTER_STATE = 0x74
DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT = 0x75
DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT = 0x76
DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS = 0x77
DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD = 0x78
DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER = 0x79
DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME = 0x7a
DEVLINK_ATTR_FLASH_UPDATE_COMPONENT = 0x7b
DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG = 0x7c
DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE = 0x7d
DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL = 0x7e
DEVLINK_ATTR_PORT_PCI_PF_NUMBER = 0x7f
DEVLINK_ATTR_PORT_PCI_VF_NUMBER = 0x80
DEVLINK_ATTR_STATS = 0x81
DEVLINK_ATTR_TRAP_NAME = 0x82
DEVLINK_ATTR_TRAP_ACTION = 0x83
DEVLINK_ATTR_TRAP_TYPE = 0x84
DEVLINK_ATTR_TRAP_GENERIC = 0x85
DEVLINK_ATTR_TRAP_METADATA = 0x86
DEVLINK_ATTR_TRAP_GROUP_NAME = 0x87
DEVLINK_ATTR_RELOAD_FAILED = 0x88
DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS = 0x89
DEVLINK_ATTR_NETNS_FD = 0x8a
DEVLINK_ATTR_NETNS_PID = 0x8b
DEVLINK_ATTR_NETNS_ID = 0x8c
DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP = 0x8d
DEVLINK_ATTR_TRAP_POLICER_ID = 0x8e
DEVLINK_ATTR_TRAP_POLICER_RATE = 0x8f
DEVLINK_ATTR_TRAP_POLICER_BURST = 0x90
DEVLINK_ATTR_PORT_FUNCTION = 0x91
DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER = 0x92
DEVLINK_ATTR_PORT_LANES = 0x93
DEVLINK_ATTR_PORT_SPLITTABLE = 0x94
DEVLINK_ATTR_PORT_EXTERNAL = 0x95
DEVLINK_ATTR_PORT_CONTROLLER_NUMBER = 0x96
DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT = 0x97
DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK = 0x98
DEVLINK_ATTR_RELOAD_ACTION = 0x99
DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED = 0x9a
DEVLINK_ATTR_RELOAD_LIMITS = 0x9b
DEVLINK_ATTR_DEV_STATS = 0x9c
DEVLINK_ATTR_RELOAD_STATS = 0x9d
DEVLINK_ATTR_RELOAD_STATS_ENTRY = 0x9e
DEVLINK_ATTR_RELOAD_STATS_LIMIT = 0x9f
DEVLINK_ATTR_RELOAD_STATS_VALUE = 0xa0
DEVLINK_ATTR_REMOTE_RELOAD_STATS = 0xa1
DEVLINK_ATTR_RELOAD_ACTION_INFO = 0xa2
DEVLINK_ATTR_RELOAD_ACTION_STATS = 0xa3
DEVLINK_ATTR_MAX = 0xa3
DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0
DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1
DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0
@ -2901,6 +3120,10 @@ const (
DEVLINK_DPIPE_HEADER_ETHERNET = 0x0
DEVLINK_DPIPE_HEADER_IPV4 = 0x1
DEVLINK_DPIPE_HEADER_IPV6 = 0x2
DEVLINK_RESOURCE_UNIT_ENTRY = 0x0
DEVLINK_PORT_FUNCTION_ATTR_UNSPEC = 0x0
DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR = 0x1
DEVLINK_PORT_FUNCTION_ATTR_MAX = 0x1
)
type FsverityDigest struct {

View file

@ -248,6 +248,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x14
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x1c

View file

@ -255,6 +255,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x14
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -253,6 +253,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x14
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x1c

View file

@ -255,6 +255,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x14
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -231,6 +231,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x20
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x1c

View file

@ -235,6 +235,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x20
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -235,6 +235,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x20
SizeofLinger = 0x8
SizeofIovec = 0x8
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x1c

View file

@ -231,6 +231,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x20
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -231,6 +231,7 @@ const (
SizeofSockaddrUnix = 0x6a
SizeofSockaddrDatalink = 0x20
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

View file

@ -234,6 +234,7 @@ const (
SizeofSockaddrUnix = 0x6e
SizeofSockaddrDatalink = 0xfc
SizeofLinger = 0x8
SizeofIovec = 0x10
SizeofIPMreq = 0x8
SizeofIPv6Mreq = 0x14
SizeofMsghdr = 0x30

14
vendor/modules.txt vendored
View file

@ -19,7 +19,7 @@ github.com/VictoriaMetrics/metrics
# github.com/VictoriaMetrics/metricsql v0.9.1
github.com/VictoriaMetrics/metricsql
github.com/VictoriaMetrics/metricsql/binaryop
# github.com/aws/aws-sdk-go v1.36.7
# github.com/aws/aws-sdk-go v1.36.12
github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/arn
github.com/aws/aws-sdk-go/aws/awserr
@ -95,7 +95,7 @@ github.com/jmespath/go-jmespath
github.com/jstemmer/go-junit-report
github.com/jstemmer/go-junit-report/formatter
github.com/jstemmer/go-junit-report/parser
# github.com/klauspost/compress v1.11.3
# github.com/klauspost/compress v1.11.4
github.com/klauspost/compress/flate
github.com/klauspost/compress/fse
github.com/klauspost/compress/gzip
@ -110,7 +110,7 @@ github.com/matttproud/golang_protobuf_extensions/pbutil
github.com/oklog/ulid
# github.com/pkg/errors v0.9.1
github.com/pkg/errors
# github.com/prometheus/client_golang v1.8.0
# github.com/prometheus/client_golang v1.9.0
github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_golang/prometheus/internal
# github.com/prometheus/client_model v0.2.0
@ -179,7 +179,7 @@ golang.org/x/lint/golint
# golang.org/x/mod v0.4.0
golang.org/x/mod/module
golang.org/x/mod/semver
# golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
# golang.org/x/net v0.0.0-20201216054612-986b41b23924
golang.org/x/net/context
golang.org/x/net/context/ctxhttp
golang.org/x/net/http/httpguts
@ -196,7 +196,7 @@ golang.org/x/oauth2/jws
golang.org/x/oauth2/jwt
# golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
golang.org/x/sync/errgroup
# golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f
# golang.org/x/sys v0.0.0-20201218084310-7d0127a74742
golang.org/x/sys/internal/unsafeheader
golang.org/x/sys/unix
golang.org/x/sys/windows
@ -205,7 +205,7 @@ golang.org/x/text/secure/bidirule
golang.org/x/text/transform
golang.org/x/text/unicode/bidi
golang.org/x/text/unicode/norm
# golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58
# golang.org/x/tools v0.0.0-20201218024724-ae774e9781d2
golang.org/x/tools/cmd/goimports
golang.org/x/tools/go/ast/astutil
golang.org/x/tools/go/gcexportdata
@ -247,7 +247,7 @@ google.golang.org/appengine/internal/modules
google.golang.org/appengine/internal/remote_api
google.golang.org/appengine/internal/urlfetch
google.golang.org/appengine/urlfetch
# google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a
# google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d
google.golang.org/genproto/googleapis/api/annotations
google.golang.org/genproto/googleapis/iam/v1
google.golang.org/genproto/googleapis/rpc/code