github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-11 14:53:49 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

Browse source
This commit is contained in:
Aliaksandr Valialkin 2021-01-22 12:10:57 +02:00
commit b68264b4f5
12 changed files with 92 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
LICENSE
View file

@ -175,7 +175,7 @@
END OF TERMS AND CONDITIONS
Copyright 2019-2020 VictoriaMetrics, Inc.
Copyright 2019-2021 VictoriaMetrics, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

62
Makefile
View file

@ -64,28 +64,54 @@ release: \
release-victoria-metrics \
release-vmutils
release-victoria-metrics: victoria-metrics-prod
cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod && \
sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
release-victoria-metrics: \
release-victoria-metrics-amd64 \
release-victoria-metrics-arm64
release-victoria-metrics-amd64:
GOARCH=amd64 $(MAKE) release-victoria-metrics-generic
release-victoria-metrics-arm64:
GOARCH=arm64 $(MAKE) release-victoria-metrics-generic
release-victoria-metrics-generic: victoria-metrics-$(GOARCH)-prod
cd bin && \
tar --transform="flags=r;s|-$(GOARCH)||" -czf victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOARCH)-prod \
&& sha256sum victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOARCH)-prod \
| sed s/-$(GOARCH)// > victoria-metrics-$(GOARCH)-$(PKG_TAG)_checksums.txt
release-vmutils: \
vmagent-prod \
vmalert-prod \
vmauth-prod \
vmbackup-prod \
vmrestore-prod
cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmauth-prod vmbackup-prod vmrestore-prod && \
sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
release-vmutils-amd64 \
release-vmutils-arm64
release-vmutils-arm64: \
vmagent-arm64-prod \
vmalert-arm64-prod \
vmauth-arm64-prod \
vmbackup-arm64-prod \
vmrestore-arm64-prod
cd bin && tar czf vmutils-arm64-$(PKG_TAG).tar.gz vmagent-arm64-prod vmalert-arm64-prod vmauth-arm64-prod vmbackup-arm64-prod vmrestore-arm64-prod && \
sha256sum vmutils-arm64-$(PKG_TAG).tar.gz > vmutils-arm64-$(PKG_TAG)_checksums.txt
release-vmutils-amd64:
GOARCH=amd64 $(MAKE) release-vmutils-generic
release-vmutils-arm64:
GOARCH=arm64 $(MAKE) release-vmutils-generic
release-vmutils-generic: \
vmagent-$(GOARCH)-prod \
vmalert-$(GOARCH)-prod \
vmauth-$(GOARCH)-prod \
vmbackup-$(GOARCH)-prod \
vmrestore-$(GOARCH)-prod
cd bin && \
tar --transform="flags=r;s|-$(GOARCH)||" -czf vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOARCH)-prod \
vmalert-$(GOARCH)-prod \
vmauth-$(GOARCH)-prod \
vmbackup-$(GOARCH)-prod \
vmrestore-$(GOARCH)-prod \
&& sha256sum vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOARCH)-prod \
vmalert-$(GOARCH)-prod \
vmauth-$(GOARCH)-prod \
vmbackup-$(GOARCH)-prod \
vmrestore-$(GOARCH)-prod \
| sed s/-$(GOARCH)// > vmutils-$(GOARCH)-$(PKG_TAG)_checksums.txt
pprof-cpu:
go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)

31
app/vmselect/graphite/tags_api.go
View file

@ -357,26 +357,29 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R
func getCanonicalPaths(mns []storage.MetricName) []string {
paths := make([]string, 0, len(mns))
var b []byte
var tags []storage.Tag
for _, mn := range mns {
b = append(b[:0], mn.MetricGroup...)
tags = append(tags[:0], mn.Tags...)
sort.Slice(tags, func(i, j int) bool {
return string(tags[i].Key) < string(tags[j].Key)
})
for _, tag := range tags {
b = append(b, ';')
b = append(b, tag.Key...)
b = append(b, '=')
b = append(b, tag.Value...)
}
paths = append(paths, string(b))
path := getCanonicalPath(&mn)
paths = append(paths, path)
}
sort.Strings(paths)
return paths
}
func getCanonicalPath(mn *storage.MetricName) string {
b := append([]byte{}, mn.MetricGroup...)
tags := append([]storage.Tag{}, mn.Tags...)
sort.Slice(tags, func(i, j int) bool {
return string(tags[i].Key) < string(tags[j].Key)
})
for _, tag := range tags {
b = append(b, ';')
b = append(b, tag.Key...)
b = append(b, '=')
b = append(b, tag.Value...)
}
return string(b)
}
var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/findSeries"}`)
// TagValuesHandler implements /tags/<tag_name> endpoint from Graphite Tags API.

10
app/vmselect/prometheus/prometheus.go
View file

@ -41,6 +41,8 @@ var (
"By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+
"Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+
"See also '-search.maxLookback' flag, which has the same meaning due to historical reasons")
maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
)
// Default step used if not set.
@ -1132,9 +1134,11 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
if err != nil {
return fmt.Errorf("cannot execute query: %w", err)
}
queryOffset := getLatencyOffsetMilliseconds()
if ct-queryOffset < end {
result = adjustLastPoints(result, ct-queryOffset, ct+step)
if step < maxStepForPointsAdjustment.Milliseconds() {
queryOffset := getLatencyOffsetMilliseconds()
if ct-queryOffset < end {
result = adjustLastPoints(result, ct-queryOffset, ct+step)
}
}
// Remove NaN values as Prometheus does.

2
app/vmselect/promql/rollup.go
View file

@ -391,7 +391,7 @@ type rollupConfig struct {
// Whether window may be adjusted to 2 x interval between data points.
// This is needed for functions which have dt in the denominator
// such as rate, deriv, etc.
// Without the adjustement their value would jump in unexpected directions
// Without the adjustment their value would jump in unexpected directions
// when using window smaller than 2 x scrape_interval.
MayAdjustWindow bool

6
deployment/docker/Makefile
View file

@ -2,9 +2,9 @@
DOCKER_NAMESPACE := victoriametrics
ROOT_IMAGE ?= alpine:3.12.3
CERTS_IMAGE := alpine:3.12.3
GO_BUILDER_IMAGE := golang:1.15.6
ROOT_IMAGE ?= alpine:3.13.0
CERTS_IMAGE := alpine:3.13.0
GO_BUILDER_IMAGE := golang:1.15.7
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _)
BASE_IMAGE := local/base:1.1.1-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _)

6
docs/CHANGELOG.md
View file

@ -2,6 +2,10 @@
# tip
* FEATURE: added `search.maxStepForPointsAdjustment` command-line flag, which can be used for disabling adjustment for points returned `/api/v1/query_range` handler if such points have timestamps closer than `-search.latencyOffset` to the current time. Such points may contain incomplete data, so they are substituted by the previous values for `step` query args smaller than one minute by default.
* BUGFIX: vmagent: reduce the HTTP reconnection rate to scrape targets. Previously vmagent could errorneusly close HTTP keep-alive connections more often than needed.
# [v1.52.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.52.0)
@ -306,7 +310,7 @@
* BUGFIX: properly apply `-search.maxStalenessInterval` command-line flag value. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 .
* BUGFIX: fix displaying data in Grafana tables. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 .
* BUGFIX: do not adjust the number of detected CPU cores found at `/sys/devices/system/cpu/online`.
The adjustement was increasing the resulting GOMAXPROC by 1, which looked confusing to users.
The adjustment was increasing the resulting GOMAXPROC by 1, which looked confusing to users.
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-698595309 .
* BUGFIX: vmagent: do not show `-remoteWrite.url` in initial logs if `-remoteWrite.showURL` isn't set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/773 .
* BUGFIX: properly handle case when [/metrics/find](https://victoriametrics.github.io/#graphite-metrics-api-usage) finds both a leaf and a node for the given `query=prefix.*`.

2
go.mod
View file

@ -7,7 +7,7 @@ require (
// Do not use the original github.com/valyala/fasthttp because of issues
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.0.11
github.com/VictoriaMetrics/fasthttp v1.0.12
github.com/VictoriaMetrics/metrics v1.12.3
github.com/VictoriaMetrics/metricsql v0.10.0
github.com/aws/aws-sdk-go v1.36.25

4
go.sum
View file

@ -81,8 +81,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
github.com/VictoriaMetrics/fasthttp v1.0.11 h1:6XOvE1pF/EhW8qoi7V5qJQJ2rhNV+UGrb1/a9vMbTiw=
github.com/VictoriaMetrics/fasthttp v1.0.11/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU=
github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE=
github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU=
github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=

1
lib/mergeset/part_search.go
View file

@ -77,6 +77,7 @@ func (ps *partSearch) Init(p *part, shouldCacheBlock func(item []byte) bool) {
ps.p = p
ps.idxbCache = p.idxbCache
ps.ibCache = p.ibCache
ps.shouldCacheBlock = shouldCacheBlock
}
// Seek seeks for the first item greater or equal to k in ps.

10
vendor/github.com/VictoriaMetrics/fasthttp/client.go generated vendored
View file

@ -571,6 +571,14 @@ type clientConn struct {
lastWriteDeadlineTime time.Time
}
func (cc *clientConn) reset() {
cc.c = nil
cc.createdTime = zeroTime
cc.lastUseTime = zeroTime
cc.lastReadDeadlineTime = zeroTime
cc.lastWriteDeadlineTime = zeroTime
}
var startTimeUnix = time.Now().Unix()
// LastUseTime returns time the client was last used
@ -1252,7 +1260,7 @@ func acquireClientConn(conn net.Conn) *clientConn {
}
func releaseClientConn(cc *clientConn) {
cc.c = nil
cc.reset()
clientConnPool.Put(cc)
}

2
vendor/modules.txt vendored
View file

@ -10,7 +10,7 @@ cloud.google.com/go/internal/version
cloud.google.com/go/storage
# github.com/VictoriaMetrics/fastcache v1.5.7
github.com/VictoriaMetrics/fastcache
# github.com/VictoriaMetrics/fasthttp v1.0.11
# github.com/VictoriaMetrics/fasthttp v1.0.12
github.com/VictoriaMetrics/fasthttp
github.com/VictoriaMetrics/fasthttp/fasthttputil
github.com/VictoriaMetrics/fasthttp/stackless