From 7bd420cbfe3d9513bb93a346e532dc1ad7567da3 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 7 Jan 2021 21:33:53 +0000 Subject: [PATCH 1/5] docker-compose: add blackhole receiver for alertmanager (#999) Currently, alertmanager spams logs with `Notify attempt failed, will retry later` message because default receiver is unreachable. The change updates default configuration with blackhole receiver which means alertmanager will continue to accept alerts but won't make attempts to send them anywhere. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/995 --- deployment/docker/alertmanager.yml | 5 +++++ deployment/docker/docker-compose.yml | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 deployment/docker/alertmanager.yml diff --git a/deployment/docker/alertmanager.yml b/deployment/docker/alertmanager.yml new file mode 100644 index 000000000..6cf304a51 --- /dev/null +++ b/deployment/docker/alertmanager.yml @@ -0,0 +1,5 @@ +route: + receiver: blackhole + +receivers: + - name: blackhole diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index fc1e4f115..736d3f4bd 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -77,6 +77,10 @@ services: alertmanager: container_name: alertmanager image: prom/alertmanager + volumes: + - ./alertmanager.yml:/config/alertmanager.yml + command: + - '--config.file=/config/alertmanager.yml' ports: - 9093:9093 networks: From 39b11b3ff46c258590b838db53d82bcbe1e892ec Mon Sep 17 00:00:00 2001 From: Dan Dascalescu Date: Thu, 7 Jan 2021 11:35:46 -1000 Subject: [PATCH 2/5] Tiny typo fix (#997) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6489dc581..67db1496c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ## VictoriaMetrics -VictoriaMetrics is fast, cost-effective and scalable monitoring solution and time series database. +VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database. It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), [docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap package](https://snapcraft.io/victoriametrics) From b1f9b39c4b2aa9bfca82179b587d610f01358d89 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 7 Jan 2021 23:37:31 +0200 Subject: [PATCH 3/5] docs/Single-server-VictoriaMetrics.md: sync with upstream --- docs/Single-server-VictoriaMetrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 6489dc581..67db1496c 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -10,7 +10,7 @@ ## VictoriaMetrics -VictoriaMetrics is fast, cost-effective and scalable monitoring solution and time series database. +VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database. It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), [docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap package](https://snapcraft.io/victoriametrics) From 0477991b4d51c4cc4ff62690d15430f6a802a668 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 7 Jan 2021 23:55:02 +0200 Subject: [PATCH 4/5] vendor: make vendor-update --- go.mod | 10 ++-- go.sum | 20 ++++---- .../aws/aws-sdk-go/aws/endpoints/defaults.go | 46 +++++++++++++++++-- .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- .../klauspost/compress/zstd/decoder.go | 20 ++++++-- .../klauspost/compress/zstd/seqdec.go | 15 ++++-- .../klauspost/compress/zstd/zstd.go | 4 ++ .../golang.org/x/sys/unix/syscall_darwin.go | 7 ++- .../x/sys/unix/zsyscall_darwin_386.go | 6 +-- .../x/sys/unix/zsyscall_darwin_amd64.go | 6 +-- .../x/sys/unix/zsyscall_darwin_arm.go | 6 +-- .../x/sys/unix/zsyscall_darwin_arm64.go | 6 +-- .../golang.org/x/tools/cmd/goimports/doc.go | 4 ++ .../golang.org/x/tools/go/ast/astutil/util.go | 4 ++ .../x/tools/internal/gocommand/version.go | 13 +++++- .../x/tools/internal/imports/mod.go | 4 ++ .../x/tools/internal/imports/mod_cache.go | 4 ++ vendor/modules.txt | 10 ++-- 18 files changed, 133 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 0b6c5f24e..04862cc14 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,10 @@ require ( github.com/VictoriaMetrics/fasthttp v1.0.9 github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metricsql v0.9.1 - github.com/aws/aws-sdk-go v1.36.15 + github.com/aws/aws-sdk-go v1.36.23 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 - github.com/klauspost/compress v1.11.4 + github.com/klauspost/compress v1.11.6 github.com/valyala/fastjson v1.6.3 github.com/valyala/fastrand v1.0.0 github.com/valyala/fasttemplate v1.2.1 @@ -22,10 +22,10 @@ require ( github.com/valyala/quicktemplate v1.6.3 golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 - golang.org/x/sys v0.0.0-20201223074533-0d417f636930 - golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e // indirect + golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 + golang.org/x/tools v0.0.0-20210107193943-4ed967dd8eff // indirect google.golang.org/api v0.36.0 - google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect + google.golang.org/genproto v0.0.0-20210106152847-07624b53cd92 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 5f0af6bd3..1961d66ca 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/VictoriaMetrics/metricsql v0.9.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/aws/aws-sdk-go v1.36.15 h1:nGqgPlXegCKPZOKXvWnYCLvLPJPRoSOHHn9d0N0DG7Y= -github.com/aws/aws-sdk-go v1.36.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.36.23 h1:umM44ptMKImsUWLtjGBv/4Ut7Nd99DfqoZDkO0j0/Kc= +github.com/aws/aws-sdk-go v1.36.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -150,8 +150,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.6 h1:EgWPCW6O3n1D5n99Zq3xXBt9uCwRGvpwGOusOLNBRSQ= +github.com/klauspost/compress v1.11.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -316,8 +316,8 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 h1:/dSxr6gT0FNI1MO5WLJo8mTmItROeOKTkDn+7OwWBos= +golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -376,8 +376,8 @@ golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210107193943-4ed967dd8eff h1:6EkB024TP1fu6cmQqeCNw685zYDVt5g8N1BXh755SQM= +golang.org/x/tools v0.0.0-20210107193943-4ed967dd8eff/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -448,8 +448,8 @@ google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210106152847-07624b53cd92 h1:jOTk2Z6KYaWoptUFqZ167cS8peoUPjFEXrsqfVkkCGc= +google.golang.org/genproto v0.0.0-20210106152847-07624b53cd92/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 35db88d99..34cb9eb62 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -3059,11 +3059,41 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-prod-ca-central-1": endpoint{ + Hostname: "fsx-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-prod-us-east-1": endpoint{ + Hostname: "fsx-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-prod-us-east-2": endpoint{ + Hostname: "fsx-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-prod-us-west-1": endpoint{ + Hostname: "fsx-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-prod-us-west-2": endpoint{ + Hostname: "fsx-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "gamelift": service{ @@ -3523,6 +3553,12 @@ var awsPartition = partition{ }, }, }, + "iotfleethub": service{ + + Endpoints: endpoints{ + "us-east-1": endpoint{}, + }, + }, "iotsecuredtunneling": service{ Endpoints: endpoints{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 919084d52..f13b119cf 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.36.15" +const SDKVersion = "1.36.23" diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index cdda0de58..62fd37324 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -85,6 +85,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { d.current.output = make(chan decodeOutput, d.o.concurrent) d.current.flushed = true + if r == nil { + d.current.err = ErrDecoderNilInput + } + // Transfer option dicts. d.dicts = make(map[uint32]dict, len(d.o.dicts)) for _, dc := range d.o.dicts { @@ -111,7 +115,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { // When the stream is done, io.EOF will be returned. func (d *Decoder) Read(p []byte) (int, error) { if d.stream == nil { - return 0, errors.New("no input has been initialized") + return 0, ErrDecoderNilInput } var n int for { @@ -152,12 +156,20 @@ func (d *Decoder) Read(p []byte) (int, error) { // Reset will reset the decoder the supplied stream after the current has finished processing. // Note that this functionality cannot be used after Close has been called. +// Reset can be called with a nil reader to release references to the previous reader. +// After being called with a nil reader, no other operations than Reset or DecodeAll or Close +// should be used. func (d *Decoder) Reset(r io.Reader) error { if d.current.err == ErrDecoderClosed { return d.current.err } + + d.drainOutput() + if r == nil { - return errors.New("nil Reader sent as input") + d.current.err = ErrDecoderNilInput + d.current.flushed = true + return nil } if d.stream == nil { @@ -166,8 +178,6 @@ func (d *Decoder) Reset(r io.Reader) error { go d.startStreamDecoder(d.stream) } - d.drainOutput() - // If bytes buffer and < 1MB, do sync decoding anyway. if bb, ok := r.(*bytes.Buffer); ok && bb.Len() < 1<<20 { if debug { @@ -249,7 +259,7 @@ func (d *Decoder) drainOutput() { // Any error encountered during the write is also returned. func (d *Decoder) WriteTo(w io.Writer) (int64, error) { if d.stream == nil { - return 0, errors.New("no input has been initialized") + return 0, ErrDecoderNilInput } var n int64 for { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index b5c8ef133..1dd39e63b 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -181,11 +181,18 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { return fmt.Errorf("output (%d) bigger than max block size", size) } if size > cap(s.out) { - // Not enough size, will be extremely rarely triggered, + // Not enough size, which can happen under high volume block streaming conditions // but could be if destination slice is too small for sync operations. - // We add maxBlockSize to the capacity. - s.out = append(s.out, make([]byte, maxBlockSize)...) - s.out = s.out[:len(s.out)-maxBlockSize] + // over-allocating here can create a large amount of GC pressure so we try to keep + // it as contained as possible + used := len(s.out) - startSize + addBytes := 256 + ll + ml + used>>2 + // Clamp to max block size. + if used+addBytes > maxBlockSize { + addBytes = maxBlockSize - used + } + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] } if ml > maxMatchLen { return fmt.Errorf("match len (%d) bigger than max allowed length", ml) diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 0807719c8..0c761dd62 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -73,6 +73,10 @@ var ( // ErrDecoderClosed will be returned if the Decoder was used after // Close has been called. ErrDecoderClosed = errors.New("decoder used after Close") + + // ErrDecoderNilInput is returned when a nil Reader was provided + // and an operation other than Reset/DecodeAll/Close was attempted. + ErrDecoderNilInput = errors.New("nil input provided as reader") ) func println(a ...interface{}) { diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go index b62573890..16f9c226b 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go @@ -119,13 +119,16 @@ type attrList struct { Forkattr uint32 } -//sysnb pipe() (r int, w int, err error) +//sysnb pipe(p *[2]int32) (err error) func Pipe(p []int) (err error) { if len(p) != 2 { return EINVAL } - p[0], p[1], err = pipe() + var x [2]int32 + err = pipe(&x) + p[0] = int(x[0]) + p[1] = int(x[1]) return } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go index 36ab7eab8..387718348 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go @@ -462,10 +462,8 @@ func libc_munlockall_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe() (r int, w int, err error) { - r0, r1, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), 0, 0, 0) - r = int(r0) - w = int(r1) +func pipe(p *[2]int32) (err error) { + _, _, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), uintptr(unsafe.Pointer(p)), 0, 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go index 7b854cc29..508e5639b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go @@ -462,10 +462,8 @@ func libc_munlockall_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe() (r int, w int, err error) { - r0, r1, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), 0, 0, 0) - r = int(r0) - w = int(r1) +func pipe(p *[2]int32) (err error) { + _, _, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), uintptr(unsafe.Pointer(p)), 0, 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go index 8e79ad377..c0c771f40 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go @@ -462,10 +462,8 @@ func libc_munlockall_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe() (r int, w int, err error) { - r0, r1, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), 0, 0, 0) - r = int(r0) - w = int(r1) +func pipe(p *[2]int32) (err error) { + _, _, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), uintptr(unsafe.Pointer(p)), 0, 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go index 99509b12d..9b01a79c4 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go @@ -462,10 +462,8 @@ func libc_munlockall_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe() (r int, w int, err error) { - r0, r1, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), 0, 0, 0) - r = int(r0) - w = int(r1) +func pipe(p *[2]int32) (err error) { + _, _, e1 := syscall_rawSyscall(funcPC(libc_pipe_trampoline), uintptr(unsafe.Pointer(p)), 0, 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/tools/cmd/goimports/doc.go b/vendor/golang.org/x/tools/cmd/goimports/doc.go index 7033e4d4c..f344d8014 100644 --- a/vendor/golang.org/x/tools/cmd/goimports/doc.go +++ b/vendor/golang.org/x/tools/cmd/goimports/doc.go @@ -1,3 +1,7 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + /* Command goimports updates your Go import lines, diff --git a/vendor/golang.org/x/tools/go/ast/astutil/util.go b/vendor/golang.org/x/tools/go/ast/astutil/util.go index 763062982..919d5305a 100644 --- a/vendor/golang.org/x/tools/go/ast/astutil/util.go +++ b/vendor/golang.org/x/tools/go/ast/astutil/util.go @@ -1,3 +1,7 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package astutil import "go/ast" diff --git a/vendor/golang.org/x/tools/internal/gocommand/version.go b/vendor/golang.org/x/tools/internal/gocommand/version.go index 60d45ac0e..0cebac6e6 100644 --- a/vendor/golang.org/x/tools/internal/gocommand/version.go +++ b/vendor/golang.org/x/tools/internal/gocommand/version.go @@ -16,9 +16,20 @@ func GoVersion(ctx context.Context, inv Invocation, r *Runner) (int, error) { inv.Verb = "list" inv.Args = []string{"-e", "-f", `{{context.ReleaseTags}}`} inv.Env = append(append([]string{}, inv.Env...), "GO111MODULE=off") - // Unset any unneeded flags. + // Unset any unneeded flags, and remove them from BuildFlags, if they're + // present. inv.ModFile = "" inv.ModFlag = "" + var buildFlags []string + for _, flag := range inv.BuildFlags { + // Flags can be prefixed by one or two dashes. + f := strings.TrimPrefix(strings.TrimPrefix(flag, "-"), "-") + if strings.HasPrefix(f, "mod=") || strings.HasPrefix(f, "modfile=") { + continue + } + buildFlags = append(buildFlags, flag) + } + inv.BuildFlags = buildFlags stdoutBytes, err := r.Run(ctx, inv) if err != nil { return 0, err diff --git a/vendor/golang.org/x/tools/internal/imports/mod.go b/vendor/golang.org/x/tools/internal/imports/mod.go index ce3269a43..901449a82 100644 --- a/vendor/golang.org/x/tools/internal/imports/mod.go +++ b/vendor/golang.org/x/tools/internal/imports/mod.go @@ -1,3 +1,7 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package imports import ( diff --git a/vendor/golang.org/x/tools/internal/imports/mod_cache.go b/vendor/golang.org/x/tools/internal/imports/mod_cache.go index 5b4f03acc..18dada495 100644 --- a/vendor/golang.org/x/tools/internal/imports/mod_cache.go +++ b/vendor/golang.org/x/tools/internal/imports/mod_cache.go @@ -1,3 +1,7 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package imports import ( diff --git a/vendor/modules.txt b/vendor/modules.txt index 772107b03..c07ffa30f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.9.1 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop -# github.com/aws/aws-sdk-go v1.36.15 +# github.com/aws/aws-sdk-go v1.36.23 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/awserr @@ -88,7 +88,7 @@ github.com/jmespath/go-jmespath github.com/jstemmer/go-junit-report github.com/jstemmer/go-junit-report/formatter github.com/jstemmer/go-junit-report/parser -# github.com/klauspost/compress v1.11.4 +# github.com/klauspost/compress v1.11.6 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/gzip @@ -150,7 +150,7 @@ golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt -# golang.org/x/sys v0.0.0-20201223074533-0d417f636930 +# golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix # golang.org/x/text v0.3.4 @@ -158,7 +158,7 @@ golang.org/x/text/secure/bidirule golang.org/x/text/transform golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm -# golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e +# golang.org/x/tools v0.0.0-20210107193943-4ed967dd8eff golang.org/x/tools/cmd/goimports golang.org/x/tools/go/ast/astutil golang.org/x/tools/go/gcexportdata @@ -200,7 +200,7 @@ google.golang.org/appengine/internal/modules google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch -# google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d +# google.golang.org/genproto v0.0.0-20210106152847-07624b53cd92 google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/iam/v1 google.golang.org/genproto/googleapis/rpc/code From 9dcb18e03d1ef04901feca2232b609ca2803adb7 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 8 Jan 2021 00:09:00 +0200 Subject: [PATCH 5/5] app/vmstorage: disable final merge by default, since it may result in high disk IO and CPU usage without measurable benefits such as increased query performance and reduced disk space usage --- app/vmstorage/main.go | 6 +++--- docs/CHANGELOG.md | 2 ++ lib/storage/partition.go | 11 ++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 920708422..2ced7346b 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -30,9 +30,9 @@ var ( // DataPath is a path to storage data. DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data") - finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+ - "Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+ - "disk IO usage and CPU usage") + finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+ + "Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+ + "Zero value disables final merge") bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0") smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0") diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c9aa52fbe..4ef7a1d49 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,8 @@ # tip +* FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration. + # [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0) diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 472c532c8..8c05d9f81 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -976,7 +976,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error { if !errors.Is(err, errNothingToMerge) { return err } - if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { + if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { // We have free time for merging into bigger parts. // This should improve select performance. lastMergeTime = fasttime.UnixTimestamp() @@ -998,17 +998,18 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error { } } -var finalMergeDelaySeconds = uint64(30) +// Disable final merge by default, since it may lead to high disk IO and CPU usage +// at the beginning of every month when merging data for the previous month. +var finalMergeDelaySeconds = uint64(0) // SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data. // // This function may be called only before Storage initialization. func SetFinalMergeDelay(delay time.Duration) { - delaySeconds := int(delay.Seconds() + 0.5) - if delaySeconds <= 0 { + if delay <= 0 { return } - finalMergeDelaySeconds = uint64(delaySeconds) + finalMergeDelaySeconds = uint64(delay.Seconds() + 1) } func maxRowsByPath(path string) uint64 {