diff --git a/LICENSE b/LICENSE index b0d2a9fb5..84f8ad8b8 100644 --- a/LICENSE +++ b/LICENSE @@ -175,7 +175,7 @@ END OF TERMS AND CONDITIONS - Copyright 2019-2020 VictoriaMetrics, Inc. + Copyright 2019-2021 VictoriaMetrics, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile index cd02309f8..9d7ffaad2 100644 --- a/Makefile +++ b/Makefile @@ -64,28 +64,54 @@ release: \ release-victoria-metrics \ release-vmutils -release-victoria-metrics: victoria-metrics-prod - cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod && \ - sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt +release-victoria-metrics: \ + release-victoria-metrics-amd64 \ + release-victoria-metrics-arm64 + +release-victoria-metrics-amd64: + GOARCH=amd64 $(MAKE) release-victoria-metrics-generic + +release-victoria-metrics-arm64: + GOARCH=arm64 $(MAKE) release-victoria-metrics-generic + +release-victoria-metrics-generic: victoria-metrics-$(GOARCH)-prod + cd bin && \ + tar --transform="flags=r;s|-$(GOARCH)||" -czf victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \ + victoria-metrics-$(GOARCH)-prod \ + && sha256sum victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \ + victoria-metrics-$(GOARCH)-prod \ + | sed s/-$(GOARCH)// > victoria-metrics-$(GOARCH)-$(PKG_TAG)_checksums.txt release-vmutils: \ - vmagent-prod \ - vmalert-prod \ - vmauth-prod \ - vmbackup-prod \ - vmrestore-prod - cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmauth-prod vmbackup-prod vmrestore-prod && \ - sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt + release-vmutils-amd64 \ + release-vmutils-arm64 -release-vmutils-arm64: \ - vmagent-arm64-prod \ - vmalert-arm64-prod \ - vmauth-arm64-prod \ - vmbackup-arm64-prod \ - vmrestore-arm64-prod - cd bin && tar czf vmutils-arm64-$(PKG_TAG).tar.gz vmagent-arm64-prod vmalert-arm64-prod vmauth-arm64-prod vmbackup-arm64-prod vmrestore-arm64-prod && \ - sha256sum vmutils-arm64-$(PKG_TAG).tar.gz > vmutils-arm64-$(PKG_TAG)_checksums.txt +release-vmutils-amd64: + GOARCH=amd64 $(MAKE) release-vmutils-generic +release-vmutils-arm64: + GOARCH=arm64 $(MAKE) release-vmutils-generic + +release-vmutils-generic: \ + vmagent-$(GOARCH)-prod \ + vmalert-$(GOARCH)-prod \ + vmauth-$(GOARCH)-prod \ + vmbackup-$(GOARCH)-prod \ + vmrestore-$(GOARCH)-prod + cd bin && \ + tar --transform="flags=r;s|-$(GOARCH)||" -czf vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \ + vmagent-$(GOARCH)-prod \ + vmalert-$(GOARCH)-prod \ + vmauth-$(GOARCH)-prod \ + vmbackup-$(GOARCH)-prod \ + vmrestore-$(GOARCH)-prod \ + && sha256sum vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \ + vmagent-$(GOARCH)-prod \ + vmalert-$(GOARCH)-prod \ + vmauth-$(GOARCH)-prod \ + vmbackup-$(GOARCH)-prod \ + vmrestore-$(GOARCH)-prod \ + | sed s/-$(GOARCH)// > vmutils-$(GOARCH)-$(PKG_TAG)_checksums.txt pprof-cpu: go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE) diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index ba46aeb2d..e1728932f 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -357,26 +357,29 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R func getCanonicalPaths(mns []storage.MetricName) []string { paths := make([]string, 0, len(mns)) - var b []byte - var tags []storage.Tag for _, mn := range mns { - b = append(b[:0], mn.MetricGroup...) - tags = append(tags[:0], mn.Tags...) - sort.Slice(tags, func(i, j int) bool { - return string(tags[i].Key) < string(tags[j].Key) - }) - for _, tag := range tags { - b = append(b, ';') - b = append(b, tag.Key...) - b = append(b, '=') - b = append(b, tag.Value...) - } - paths = append(paths, string(b)) + path := getCanonicalPath(&mn) + paths = append(paths, path) } sort.Strings(paths) return paths } +func getCanonicalPath(mn *storage.MetricName) string { + b := append([]byte{}, mn.MetricGroup...) + tags := append([]storage.Tag{}, mn.Tags...) + sort.Slice(tags, func(i, j int) bool { + return string(tags[i].Key) < string(tags[j].Key) + }) + for _, tag := range tags { + b = append(b, ';') + b = append(b, tag.Key...) + b = append(b, '=') + b = append(b, tag.Value...) + } + return string(b) +} + var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/findSeries"}`) // TagValuesHandler implements /tags/ endpoint from Graphite Tags API. diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index c11f7e771..bd62b3cc6 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -41,6 +41,8 @@ var ( "By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+ "Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+ "See also '-search.maxLookback' flag, which has the same meaning due to historical reasons") + maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+ + "points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data") ) // Default step used if not set. @@ -1132,9 +1134,11 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string, if err != nil { return fmt.Errorf("cannot execute query: %w", err) } - queryOffset := getLatencyOffsetMilliseconds() - if ct-queryOffset < end { - result = adjustLastPoints(result, ct-queryOffset, ct+step) + if step < maxStepForPointsAdjustment.Milliseconds() { + queryOffset := getLatencyOffsetMilliseconds() + if ct-queryOffset < end { + result = adjustLastPoints(result, ct-queryOffset, ct+step) + } } // Remove NaN values as Prometheus does. diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 41cf242fa..3c62fbdc7 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -391,7 +391,7 @@ type rollupConfig struct { // Whether window may be adjusted to 2 x interval between data points. // This is needed for functions which have dt in the denominator // such as rate, deriv, etc. - // Without the adjustement their value would jump in unexpected directions + // Without the adjustment their value would jump in unexpected directions // when using window smaller than 2 x scrape_interval. MayAdjustWindow bool diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 07e37748e..dd28c0590 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -2,9 +2,9 @@ DOCKER_NAMESPACE := victoriametrics -ROOT_IMAGE ?= alpine:3.12.3 -CERTS_IMAGE := alpine:3.12.3 -GO_BUILDER_IMAGE := golang:1.15.6 +ROOT_IMAGE ?= alpine:3.13.0 +CERTS_IMAGE := alpine:3.13.0 +GO_BUILDER_IMAGE := golang:1.15.7 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _) BASE_IMAGE := local/base:1.1.1-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 43d0c23d6..640e1a5c6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ # tip +* FEATURE: added `search.maxStepForPointsAdjustment` command-line flag, which can be used for disabling adjustment for points returned `/api/v1/query_range` handler if such points have timestamps closer than `-search.latencyOffset` to the current time. Such points may contain incomplete data, so they are substituted by the previous values for `step` query args smaller than one minute by default. + +* BUGFIX: vmagent: reduce the HTTP reconnection rate to scrape targets. Previously vmagent could errorneusly close HTTP keep-alive connections more often than needed. + # [v1.52.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.52.0) @@ -306,7 +310,7 @@ * BUGFIX: properly apply `-search.maxStalenessInterval` command-line flag value. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 . * BUGFIX: fix displaying data in Grafana tables. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 . * BUGFIX: do not adjust the number of detected CPU cores found at `/sys/devices/system/cpu/online`. - The adjustement was increasing the resulting GOMAXPROC by 1, which looked confusing to users. + The adjustment was increasing the resulting GOMAXPROC by 1, which looked confusing to users. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-698595309 . * BUGFIX: vmagent: do not show `-remoteWrite.url` in initial logs if `-remoteWrite.showURL` isn't set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/773 . * BUGFIX: properly handle case when [/metrics/find](https://victoriametrics.github.io/#graphite-metrics-api-usage) finds both a leaf and a node for the given `query=prefix.*`. diff --git a/go.mod b/go.mod index 9f17874af..120fb499f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b - github.com/VictoriaMetrics/fasthttp v1.0.11 + github.com/VictoriaMetrics/fasthttp v1.0.12 github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metricsql v0.10.0 github.com/aws/aws-sdk-go v1.36.25 diff --git a/go.sum b/go.sum index 5d16b8949..f5fb720ae 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= -github.com/VictoriaMetrics/fasthttp v1.0.11 h1:6XOvE1pF/EhW8qoi7V5qJQJ2rhNV+UGrb1/a9vMbTiw= -github.com/VictoriaMetrics/fasthttp v1.0.11/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU= +github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE= +github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU= github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= diff --git a/lib/mergeset/part_search.go b/lib/mergeset/part_search.go index 79ed0968f..c2ef73de2 100644 --- a/lib/mergeset/part_search.go +++ b/lib/mergeset/part_search.go @@ -77,6 +77,7 @@ func (ps *partSearch) Init(p *part, shouldCacheBlock func(item []byte) bool) { ps.p = p ps.idxbCache = p.idxbCache ps.ibCache = p.ibCache + ps.shouldCacheBlock = shouldCacheBlock } // Seek seeks for the first item greater or equal to k in ps. diff --git a/vendor/github.com/VictoriaMetrics/fasthttp/client.go b/vendor/github.com/VictoriaMetrics/fasthttp/client.go index 7ff9f9cbf..5f1f433d6 100644 --- a/vendor/github.com/VictoriaMetrics/fasthttp/client.go +++ b/vendor/github.com/VictoriaMetrics/fasthttp/client.go @@ -571,6 +571,14 @@ type clientConn struct { lastWriteDeadlineTime time.Time } +func (cc *clientConn) reset() { + cc.c = nil + cc.createdTime = zeroTime + cc.lastUseTime = zeroTime + cc.lastReadDeadlineTime = zeroTime + cc.lastWriteDeadlineTime = zeroTime +} + var startTimeUnix = time.Now().Unix() // LastUseTime returns time the client was last used @@ -1252,7 +1260,7 @@ func acquireClientConn(conn net.Conn) *clientConn { } func releaseClientConn(cc *clientConn) { - cc.c = nil + cc.reset() clientConnPool.Put(cc) } diff --git a/vendor/modules.txt b/vendor/modules.txt index db483be9d..d13bfe521 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -10,7 +10,7 @@ cloud.google.com/go/internal/version cloud.google.com/go/storage # github.com/VictoriaMetrics/fastcache v1.5.7 github.com/VictoriaMetrics/fastcache -# github.com/VictoriaMetrics/fasthttp v1.0.11 +# github.com/VictoriaMetrics/fasthttp v1.0.12 github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless