From 78ff5f2aa5d5292af4f6f1eeafe57b65f96be133 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 20 Apr 2020 23:03:29 +0300 Subject: [PATCH] vendor: update github.com/valyala/gozstd from v1.6.4 to v1.7.0 --- go.mod | 2 +- go.sum | 4 +- vendor/github.com/valyala/gozstd/README.md | 16 ++++ vendor/github.com/valyala/gozstd/writer.go | 94 ++++++++++++++++++---- vendor/modules.txt | 2 +- 5 files changed, 100 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 8aaf7ada5c..70057bf963 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/valyala/fasthttp v1.10.0 github.com/valyala/fastjson v1.5.0 github.com/valyala/fastrand v1.0.0 - github.com/valyala/gozstd v1.6.4 + github.com/valyala/gozstd v1.7.0 github.com/valyala/histogram v1.0.1 github.com/valyala/quicktemplate v1.4.1 golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 diff --git a/go.sum b/go.sum index 042b5f86e0..12ec6201ec 100644 --- a/go.sum +++ b/go.sum @@ -150,8 +150,8 @@ github.com/valyala/fastjson v1.5.0 h1:DGrb4wEYso2HdGLyLmNoyNCQnCWfjd8yhghPv5/5YQ github.com/valyala/fastjson v1.5.0/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/valyala/gozstd v1.6.4 h1:nFLddjEf90SFl5cVWyElSHozQDsbvLljPK703/skBS0= -github.com/valyala/gozstd v1.6.4/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ= +github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg= github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto= github.com/valyala/quicktemplate v1.4.1 h1:tEtkSN6mTCJlYVT7As5x4wjtkk2hj2thsb0M+AcAVeM= diff --git a/vendor/github.com/valyala/gozstd/README.md b/vendor/github.com/valyala/gozstd/README.md index 08e567f6d6..1b42390f58 100644 --- a/vendor/github.com/valyala/gozstd/README.md +++ b/vendor/github.com/valyala/gozstd/README.md @@ -55,6 +55,22 @@ The easiest way is just to use [Decompress](https://godoc.org/github.com/valyala There is also [StreamDecompress](https://godoc.org/github.com/valyala/gozstd#StreamDecompress) and [Reader](https://godoc.org/github.com/valyala/gozstd#Reader) for stream decompression. +### How to cross-compile gozstd? + +If you're cross-compiling some code that uses gozstd and you stumble upon the following error: +``` +# github.com/valyala/gozstd +/go/pkg/mod/github.com/valyala/gozstd@v1.6.2/stream.go:31:59: undefined: CDict +/go/pkg/mod/github.com/valyala/gozstd@v1.6.2/stream.go:35:64: undefined: CDict +/go/pkg/mod/github.com/valyala/gozstd@v1.6.2/stream.go:47:20: undefined: Writer +``` + +You can easily fix it by enabling [CGO](https://golang.org/cmd/cgo/) and using a cross-compiler (e.g. `arm-linux-gnueabi-gcc`): +```bash +env CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 go build ./main.go +``` + +**NOTE**: Check [#21](https://github.com/valyala/gozstd/issues/21) for more info. ### Who uses gozstd? diff --git a/vendor/github.com/valyala/gozstd/writer.go b/vendor/github.com/valyala/gozstd/writer.go index c8a07eb27b..39f4a5947b 100644 --- a/vendor/github.com/valyala/gozstd/writer.go +++ b/vendor/github.com/valyala/gozstd/writer.go @@ -26,6 +26,7 @@ type cMemPtr *[1 << 30]byte type Writer struct { w io.Writer compressionLevel int + wlog int cs *C.ZSTD_CStream cd *CDict @@ -43,7 +44,7 @@ type Writer struct { // // Call Release when the Writer is no longer needed. func NewWriter(w io.Writer) *Writer { - return newWriterDictLevel(w, nil, DefaultCompressionLevel) + return NewWriterParams(w, nil) } // NewWriterLevel returns new zstd writer writing compressed data to w @@ -54,7 +55,10 @@ func NewWriter(w io.Writer) *Writer { // // Call Release when the Writer is no longer needed. func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { - return newWriterDictLevel(w, nil, compressionLevel) + params := &WriterParams{ + CompressionLevel: compressionLevel, + } + return NewWriterParams(w, params) } // NewWriterDict returns new zstd writer writing compressed data to w @@ -65,12 +69,59 @@ func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { // // Call Release when the Writer is no longer needed. func NewWriterDict(w io.Writer, cd *CDict) *Writer { - return newWriterDictLevel(w, cd, 0) + params := &WriterParams{ + Dict: cd, + } + return NewWriterParams(w, params) } -func newWriterDictLevel(w io.Writer, cd *CDict, compressionLevel int) *Writer { +const ( + // WindowLogMin is the minimum value of the windowLog parameter. + WindowLogMin = 10 // from zstd.h + // WindowLogMax32 is the maximum value of the windowLog parameter on 32-bit architectures. + WindowLogMax32 = 30 // from zstd.h + // WindowLogMax64 is the maximum value of the windowLog parameter on 64-bit architectures. + WindowLogMax64 = 31 // from zstd.h + + // DefaultWindowLog is the default value of the windowLog parameter. + DefaultWindowLog = 0 +) + +// A WriterParams allows users to specify compression parameters by calling +// NewWriterParams. +// +// Calling NewWriterParams with a nil WriterParams is equivalent to calling +// NewWriter. +type WriterParams struct { + // Compression level. Special value 0 means 'default compression level'. + CompressionLevel int + + // WindowLog. Must be clamped between WindowLogMin and WindowLogMin32/64. + // Special value 0 means 'use default windowLog'. + // + // Note: enabling log distance matching increases memory usage for both + // compressor and decompressor. When set to a value greater than 27, the + // decompressor requires special treatment. + WindowLog int + + // Dict is optional dictionary used for compression. + Dict *CDict +} + +// NewWriterParams returns new zstd writer writing compressed data to w +// using the given set of parameters. +// +// The returned writer must be closed with Close call in order +// to finalize the compressed stream. +// +// Call Release when the Writer is no longer needed. +func NewWriterParams(w io.Writer, params *WriterParams) *Writer { + if params == nil { + params = &WriterParams{} + } + cs := C.ZSTD_createCStream() - initCStream(cs, cd, compressionLevel) + initCStream(cs, *params) inBuf := (*C.ZSTD_inBuffer)(C.malloc(C.sizeof_ZSTD_inBuffer)) inBuf.src = C.malloc(cstreamInBufSize) @@ -84,9 +135,10 @@ func newWriterDictLevel(w io.Writer, cd *CDict, compressionLevel int) *Writer { zw := &Writer{ w: w, - compressionLevel: compressionLevel, + compressionLevel: params.CompressionLevel, + wlog: params.WindowLog, cs: cs, - cd: cd, + cd: params.Dict, inBuf: inBuf, outBuf: outBuf, } @@ -99,27 +151,41 @@ func newWriterDictLevel(w io.Writer, cd *CDict, compressionLevel int) *Writer { } // Reset resets zw to write to w using the given dictionary cd and the given -// compressionLevel. +// compressionLevel. Use ResetWriterParams if you wish to change other +// parameters that were set via WriterParams. func (zw *Writer) Reset(w io.Writer, cd *CDict, compressionLevel int) { + params := WriterParams{ + CompressionLevel: compressionLevel, + WindowLog: zw.wlog, + Dict: cd, + } + zw.ResetWriterParams(w, ¶ms) +} + +// ResetWriterParams resets zw to write to w using the given set of parameters. +func (zw *Writer) ResetWriterParams(w io.Writer, params *WriterParams) { zw.inBuf.size = 0 zw.inBuf.pos = 0 zw.outBuf.size = cstreamOutBufSize zw.outBuf.pos = 0 - zw.cd = cd - initCStream(zw.cs, zw.cd, compressionLevel) + zw.cd = params.Dict + initCStream(zw.cs, *params) zw.w = w } -func initCStream(cs *C.ZSTD_CStream, cd *CDict, compressionLevel int) { - if cd != nil { - result := C.ZSTD_initCStream_usingCDict(cs, cd.p) +func initCStream(cs *C.ZSTD_CStream, params WriterParams) { + if params.Dict != nil { + result := C.ZSTD_initCStream_usingCDict(cs, params.Dict.p) ensureNoError("ZSTD_initCStream_usingCDict", result) } else { - result := C.ZSTD_initCStream(cs, C.int(compressionLevel)) + result := C.ZSTD_initCStream(cs, C.int(params.CompressionLevel)) ensureNoError("ZSTD_initCStream", result) } + + result := C.ZSTD_CCtx_setParameter(cs, C.ZSTD_cParameter(C.ZSTD_c_windowLog), C.int(params.WindowLog)) + ensureNoError("ZSTD_CCtx_setParameter", result) } func freeCStream(v interface{}) { diff --git a/vendor/modules.txt b/vendor/modules.txt index f4138e44d2..0fff5f7193 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -102,7 +102,7 @@ github.com/valyala/fastjson github.com/valyala/fastjson/fastfloat # github.com/valyala/fastrand v1.0.0 github.com/valyala/fastrand -# github.com/valyala/gozstd v1.6.4 +# github.com/valyala/gozstd v1.7.0 github.com/valyala/gozstd # github.com/valyala/histogram v1.0.1 github.com/valyala/histogram