diff --git a/go.mod b/go.mod index 4c313b1529..d6bd726030 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,9 @@ require ( github.com/valyala/fastjson v1.5.4 github.com/valyala/fastrand v1.0.0 github.com/valyala/fasttemplate v1.2.1 - github.com/valyala/gozstd v1.8.1 + + // Do not upgrade gozstd version until https://github.com/facebook/zstd/issues/2222 is fixed + github.com/valyala/gozstd v1.7.1 github.com/valyala/histogram v1.1.2 github.com/valyala/quicktemplate v1.6.2 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect diff --git a/go.sum b/go.sum index fd9d1031c9..e56cb0b23e 100644 --- a/go.sum +++ b/go.sum @@ -174,8 +174,8 @@ github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2Obdk github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/gozstd v1.8.1 h1:voJLmQRBT5uiTwRfRr//va1TLTiCqtOczKp+TkWYJ+I= -github.com/valyala/gozstd v1.8.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.7.1 h1:2zWgJWU1859UbFoUnheDXnIK35MjPjFmMFhQYFz8brA= +github.com/valyala/gozstd v1.7.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8= github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg= github.com/valyala/quicktemplate v1.6.2 h1:k0vgK7zlmFzqAoIBIOrhrfmZ6JoTGJlLRPLbkPGr2/M= diff --git a/vendor/github.com/valyala/gozstd/.travis.yml b/vendor/github.com/valyala/gozstd/.travis.yml index b1d0fa5d61..a7419f6b9b 100644 --- a/vendor/github.com/valyala/gozstd/.travis.yml +++ b/vendor/github.com/valyala/gozstd/.travis.yml @@ -6,7 +6,7 @@ os: - freebsd go: - - 1.14 + - 1.12 script: # build test for supported platforms diff --git a/vendor/github.com/valyala/gozstd/libzstd_linux_amd64.a b/vendor/github.com/valyala/gozstd/libzstd_linux_amd64.a index c189376737..498c9a530e 100644 Binary files a/vendor/github.com/valyala/gozstd/libzstd_linux_amd64.a and b/vendor/github.com/valyala/gozstd/libzstd_linux_amd64.a differ diff --git a/vendor/github.com/valyala/gozstd/libzstd_linux_arm.a b/vendor/github.com/valyala/gozstd/libzstd_linux_arm.a index b6d601f423..38705b0c38 100644 Binary files a/vendor/github.com/valyala/gozstd/libzstd_linux_arm.a and b/vendor/github.com/valyala/gozstd/libzstd_linux_arm.a differ diff --git a/vendor/github.com/valyala/gozstd/libzstd_linux_arm64.a b/vendor/github.com/valyala/gozstd/libzstd_linux_arm64.a index 1465f419a7..d43968beda 100644 Binary files a/vendor/github.com/valyala/gozstd/libzstd_linux_arm64.a and b/vendor/github.com/valyala/gozstd/libzstd_linux_arm64.a differ diff --git a/vendor/github.com/valyala/gozstd/zdict.h b/vendor/github.com/valyala/gozstd/zdict.h index ff2e77faf7..37978ecdfb 100644 --- a/vendor/github.com/valyala/gozstd/zdict.h +++ b/vendor/github.com/valyala/gozstd/zdict.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -61,57 +61,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); -typedef struct { - int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ - unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */ -} ZDICT_params_t; - -/*! ZDICT_finalizeDictionary(): - * Given a custom content as a basis for dictionary, and a set of samples, - * finalize dictionary by adding headers and statistics according to the zstd - * dictionary format. - * - * Samples must be stored concatenated in a flat buffer `samplesBuffer`, - * supplied with an array of sizes `samplesSizes`, providing the size of each - * sample in order. The samples are used to construct the statistics, so they - * should be representative of what you will compress with this dictionary. - * - * The compression level can be set in `parameters`. You should pass the - * compression level you expect to use in production. The statistics for each - * compression level differ, so tuning the dictionary for the compression level - * can help quite a bit. - * - * You can set an explicit dictionary ID in `parameters`, or allow us to pick - * a random dictionary ID for you, but we can't guarantee no collisions. - * - * The dstDictBuffer and the dictContent may overlap, and the content will be - * appended to the end of the header. If the header + the content doesn't fit in - * maxDictSize the beginning of the content is truncated to make room, since it - * is presumed that the most profitable content is at the end of the dictionary, - * since that is the cheapest to reference. - * - * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes. - * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). - * - * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), - * or an error code, which can be tested by ZDICT_isError(). - * Note: ZDICT_finalizeDictionary() will push notifications into stderr if - * instructed to, using notificationLevel>0. - * NOTE: This function currently may fail in several edge cases including: - * * Not enough samples - * * Samples are uncompressible - * * Samples are all exactly the same - */ -ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, - const void* dictContent, size_t dictContentSize, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, - ZDICT_params_t parameters); - /*====== Helper functions ======*/ ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ -ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); @@ -126,8 +78,11 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); * Use them only in association with static linking. * ==================================================================================== */ -#define ZDICT_CONTENTSIZE_MIN 128 -#define ZDICT_DICTSIZE_MIN 256 +typedef struct { + int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */ +} ZDICT_params_t; /*! ZDICT_cover_params_t: * k and d are the only required parameters. @@ -243,6 +198,28 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t* parameters); +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. + * + * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes. + * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0. + * Note 2: dictBuffer and dictContent can overlap + */ +#define ZDICT_CONTENTSIZE_MIN 128 +#define ZDICT_DICTSIZE_MIN 256 +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + typedef struct { unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ ZDICT_params_t zParams; diff --git a/vendor/github.com/valyala/gozstd/zstd.h b/vendor/github.com/valyala/gozstd/zstd.h index 8c6fc6ae90..72080ea87e 100644 --- a/vendor/github.com/valyala/gozstd/zstd.h +++ b/vendor/github.com/valyala/gozstd/zstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -72,7 +72,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 5 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -274,10 +274,7 @@ typedef enum { * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level does not automatically set all other compression parameters - * to default. Setting this will however eventually dynamically impact the compression - * parameters which have not been manually set. The manually set - * ones will 'stick'. */ + * Note 2 : setting a level resets all other compression parameters to default */ /* Advanced compression parameters : * It's possible to pin down compression parameters to some specific values. * In which case, these values are no longer dynamically selected by the compressor */ @@ -522,13 +519,11 @@ typedef enum { /* note : additional experimental parameters are also available * within the experimental section of the API. * At the time of this writing, they include : - * ZSTD_d_format - * ZSTD_d_stableOutBuffer + * ZSTD_c_format * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ - ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001 + ZSTD_d_experimentalParam1=1000 } ZSTD_dParameter; @@ -768,7 +763,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* This function is redundant with the advanced API and equivalent to: * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_reset(zds); * ZSTD_DCtx_refDDict(zds, NULL); */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); @@ -1268,28 +1263,23 @@ ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, ***************************************/ /*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. + * These functions make it possible to estimate memory usage of a future + * {D,C}Ctx, before its creation. * - * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. - * The estimate will assume the input may be arbitrarily large, - * which is the worst case. + * ZSTD_estimateCCtxSize() will provide a budget large enough for any + * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(), + * this estimate does not include space for a window buffer, so this estimate + * is guaranteed to be enough for single-shot compressions, but not streaming + * compressions. It will however assume the input may be arbitrarily large, + * which is the worst case. If srcSize is known to always be small, + * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with + * ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with + * ZSTD_CCtxParams_setParameter(). * - * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions - * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), - * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). - * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. - * - * Note 2 : only single-threaded compression is supported. - * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. - */ + * Note: only single-threaded compression is supported. This function will + * return an error code if ZSTD_c_nbWorkers is >= 1. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); @@ -1652,37 +1642,6 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS * allowing selection between ZSTD_format_e input compression formats */ #define ZSTD_d_format ZSTD_d_experimentalParam1 -/* ZSTD_d_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the decompressor, and - * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer - * MUST be large enough to fit the entire decompressed frame. This will be - * checked when the frame content size is known. The data in the ZSTD_outBuffer - * in the range [dst, dst + pos) MUST not be modified during decompression - * or you will get data corruption. - * - * When this flags is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer, but it will still allocate - * an input buffer large enough to fit any compressed block. This will also - * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. - * If you need to avoid the input buffer allocation use the buffer-less - * streaming API. - * - * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, decompression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST - * not be modified during decompression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate - * matches. Normally zstd maintains its own buffer for this purpose, but passing - * this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 /*! ZSTD_DCtx_setFormat() : * Instruct the decoder context about what kind of data to decode next. diff --git a/vendor/github.com/valyala/gozstd/zstd_errors.h b/vendor/github.com/valyala/gozstd/zstd_errors.h index 998398e7e5..92a3433896 100644 --- a/vendor/github.com/valyala/gozstd/zstd_errors.h +++ b/vendor/github.com/valyala/gozstd/zstd_errors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -76,7 +76,6 @@ typedef enum { /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, - ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; diff --git a/vendor/modules.txt b/vendor/modules.txt index b9184ce6f4..80ed7e6546 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -108,7 +108,7 @@ github.com/valyala/fastjson/fastfloat github.com/valyala/fastrand # github.com/valyala/fasttemplate v1.2.1 github.com/valyala/fasttemplate -# github.com/valyala/gozstd v1.8.1 +# github.com/valyala/gozstd v1.7.1 github.com/valyala/gozstd # github.com/valyala/histogram v1.1.2 github.com/valyala/histogram