From c03e4ef9d61fe2c0771634b0b6c8ed8e87e10372 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 19 Nov 2020 19:21:05 +0200 Subject: [PATCH 1/6] vendor: `make vendor-update` --- go.mod | 9 ++++----- go.sum | 16 ++++++++-------- .../aws/aws-sdk-go/aws/endpoints/defaults.go | 9 ++++++++- .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- vendor/golang.org/x/sys/unix/mkerrors.sh | 2 ++ .../golang.org/x/sys/unix/syscall_dragonfly.go | 13 +++++++++++++ vendor/golang.org/x/sys/unix/zerrors_linux.go | 6 ++++++ .../x/sys/unix/zsyscall_dragonfly_amd64.go | 10 ++++++++++ vendor/golang.org/x/sys/unix/ztypes_linux.go | 18 ++++++++++++++++++ vendor/modules.txt | 8 ++++---- 10 files changed, 74 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 7d2113309..8b886db80 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/VictoriaMetrics/fasthttp v1.0.7 github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metricsql v0.7.2 - github.com/aws/aws-sdk-go v1.35.28 + github.com/aws/aws-sdk-go v1.35.31 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 github.com/klauspost/compress v1.11.3 @@ -21,13 +21,12 @@ require ( github.com/valyala/gozstd v1.8.3 github.com/valyala/histogram v1.1.2 github.com/valyala/quicktemplate v1.6.3 - golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 - golang.org/x/sys v0.0.0-20201116161645-c061ba923fbb - golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf // indirect + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 + golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 // indirect google.golang.org/api v0.35.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20201116144945-7adebfbe6a3f // indirect + google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 // indirect gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 4a117db35..b14e0bfb2 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/VictoriaMetrics/metricsql v0.7.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/aws/aws-sdk-go v1.35.28 h1:S2LuRnfC8X05zgZLC8gy/Sb82TGv2Cpytzbzz7tkeHc= -github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go v1.35.31 h1:6tlaYq4Q311qfhft/fIaND33XI27aW3zIdictcHxifE= +github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -303,8 +303,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201116161645-c061ba923fbb h1:+EHGEcgeA7ESswi5i4ojbo7sRzlz7vWoxFGcMuEZtu8= -golang.org/x/sys v0.0.0-20201116161645-c061ba923fbb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -360,8 +360,8 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf h1:sDQg8i3k24bqfv1V4MugOhRCHMRzkrHdLJX5QraRSt4= -golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 h1:26vapYZ9m+DJd68m3DCFP/swNNErXAU7tOMFG7NyUuM= +golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -429,8 +429,8 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201116144945-7adebfbe6a3f h1:YZKfGrT39pgYIg+3cfyIdK1z4VLjUPVboS1Ob49DyDA= -google.golang.org/genproto v0.0.0-20201116144945-7adebfbe6a3f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 h1:Rt0FRalMgdSlXAVJvX4pr65KfqaxHXSLkSJRD9pw6g0= +google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 1d1070a55..1b447aea5 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -787,6 +787,7 @@ var awsPartition = partition{ "athena": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, @@ -970,7 +971,7 @@ var awsPartition = partition{ }, Endpoints: endpoints{ "aws-global": endpoint{ - Hostname: "service.chime.aws.amazon.com", + Hostname: "chime.us-east-1.amazonaws.com", Protocols: []string{"https"}, CredentialScope: credentialScope{ Region: "us-east-1", @@ -9768,6 +9769,12 @@ var awsisobPartition = partition{ "us-isob-east-1": endpoint{}, }, }, + "es": service{ + + Endpoints: endpoints{ + "us-isob-east-1": endpoint{}, + }, + }, "events": service{ Endpoints: endpoints{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 1c4ff26bd..a849fbc75 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.35.28" +const SDKVersion = "1.35.31" diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index 0c9a5c44b..c0f9f2d52 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -225,6 +225,7 @@ struct ltchars { #include #include #include +#include #include #include #include @@ -561,6 +562,7 @@ ccflags="$@" $2 ~ /^CRYPTO_/ || $2 ~ /^TIPC_/ || $2 ~ /^DEVLINK_/ || + $2 ~ /^LWTUNNEL_IP/ || $2 !~ "WMESGLEN" && $2 ~ /^W[A-Z0-9]+$/ || $2 ~/^PPPIOC/ || diff --git a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go index 842ab5acd..a4f2944a2 100644 --- a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go +++ b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go @@ -105,6 +105,19 @@ func Pipe(p []int) (err error) { return } +//sysnb pipe2(p *[2]_C_int, flags int) (err error) + +func Pipe2(p []int, flags int) error { + if len(p) != 2 { + return EINVAL + } + var pp [2]_C_int + err := pipe2(&pp, flags) + p[0] = int(pp[0]) + p[1] = int(pp[1]) + return err +} + //sys extpread(fd int, p []byte, flags int, offset int64) (n int, err error) func Pread(fd int, p []byte, offset int64) (n int, err error) { return extpread(fd, p, 0, offset) diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index 2069fb861..b46110354 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -1217,6 +1217,12 @@ const ( LOOP_SET_STATUS_SETTABLE_FLAGS = 0xc LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + LWTUNNEL_IP6_MAX = 0x8 + LWTUNNEL_IP_MAX = 0x8 + LWTUNNEL_IP_OPTS_MAX = 0x3 + LWTUNNEL_IP_OPT_ERSPAN_MAX = 0x4 + LWTUNNEL_IP_OPT_GENEVE_MAX = 0x3 + LWTUNNEL_IP_OPT_VXLAN_MAX = 0x1 MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb diff --git a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go index aebfe511a..1aaccd361 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go @@ -362,6 +362,16 @@ func pipe() (r int, w int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func pipe2(p *[2]_C_int, flags int) (err error) { + _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func extpread(fd int, p []byte, flags int, offset int64) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index a96ad4c29..504ef131f 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -2981,3 +2981,21 @@ type PPSKTime struct { Nsec int32 Flags uint32 } + +const ( + LWTUNNEL_ENCAP_NONE = 0x0 + LWTUNNEL_ENCAP_MPLS = 0x1 + LWTUNNEL_ENCAP_IP = 0x2 + LWTUNNEL_ENCAP_ILA = 0x3 + LWTUNNEL_ENCAP_IP6 = 0x4 + LWTUNNEL_ENCAP_SEG6 = 0x5 + LWTUNNEL_ENCAP_BPF = 0x6 + LWTUNNEL_ENCAP_SEG6_LOCAL = 0x7 + LWTUNNEL_ENCAP_RPL = 0x8 + LWTUNNEL_ENCAP_MAX = 0x8 + + MPLS_IPTUNNEL_UNSPEC = 0x0 + MPLS_IPTUNNEL_DST = 0x1 + MPLS_IPTUNNEL_TTL = 0x2 + MPLS_IPTUNNEL_MAX = 0x2 +) diff --git a/vendor/modules.txt b/vendor/modules.txt index f5cbe0609..2affcefa9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.7.2 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop -# github.com/aws/aws-sdk-go v1.35.28 +# github.com/aws/aws-sdk-go v1.35.31 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/awserr @@ -150,7 +150,7 @@ golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt -# golang.org/x/sys v0.0.0-20201116161645-c061ba923fbb +# golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix # golang.org/x/text v0.3.4 @@ -158,7 +158,7 @@ golang.org/x/text/secure/bidirule golang.org/x/text/transform golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm -# golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf +# golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 golang.org/x/tools/cmd/goimports golang.org/x/tools/go/ast/astutil golang.org/x/tools/go/gcexportdata @@ -200,7 +200,7 @@ google.golang.org/appengine/internal/modules google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch -# google.golang.org/genproto v0.0.0-20201116144945-7adebfbe6a3f +# google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/iam/v1 google.golang.org/genproto/googleapis/rpc/code From 093a8917626c62c0b655e6bef66905dd47157920 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Fri, 20 Nov 2020 18:42:22 +0900 Subject: [PATCH 2/6] MetricsQL docs: parameter consistency (#915) * MetricsQL docs: parameter consistency if I understand correctly: * `fun(q)` - fun takes instant vector * `fun(m[d])` - fun takes range vector * Update docs/MetricsQL.md Co-authored-by: Aliaksandr Valialkin --- docs/MetricsQL.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 281511a23..f424f16b7 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -68,15 +68,15 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `step()` function for returning the step in seconds used in the query. - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query. - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`. -- `ideriv(m)` - for calculating `instant` derivative for `m`. +- `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`. - `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`. - `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range. - `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details. - `smooth_exponential(q, sf)` - smooths `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) with the given smooth factor `sf`. - `remove_resets(q)` - removes counter resets from `q`. -- `lag(q[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `q` over `d`. -- `lifetime(q[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `q`. -- `scrape_interval(q[d])` - returns the average interval in seconds between data points of `q` over `d` aka `scrape interval`. +- `lag(m[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `m` over `d`. +- `lifetime(m[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `m`. +- `scrape_interval(m[d])` - returns the average interval in seconds between data points of `m` over `d` aka `scrape interval`. - Trigonometric functions - `sin(q)`, `cos(q)`, `asin(q)`, `acos(q)` and `pi()`. - `range_over_time(m[d])` - returns value range for `m` over `d` time window, i.e. `max_over_time(m[d])-min_over_time(m[d])`. - `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`. From eec76718e9bddd6ce1903449e6b1844252734133 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Fri, 20 Nov 2020 14:38:12 +0300 Subject: [PATCH 3/6] Adds eureka service discovery (#913) * Adds eureka service discovery https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851 Netflix service discovery for AWS * Apply suggestions from code review Co-authored-by: Aliaksandr Valialkin --- README.md | 2 + app/vmagent/README.md | 2 + docs/CHANGELOG.md | 2 + docs/Single-server-VictoriaMetrics.md | 2 + docs/vmagent.md | 2 + lib/promscrape/config.go | 39 +++++ lib/promscrape/discovery/eureka/api.go | 81 ++++++++++ lib/promscrape/discovery/eureka/api_test.go | 107 +++++++++++++ lib/promscrape/discovery/eureka/eureka.go | 148 ++++++++++++++++++ .../discovery/eureka/eureka_test.go | 83 ++++++++++ lib/promscrape/scraper.go | 4 + 11 files changed, 472 insertions(+) create mode 100644 lib/promscrape/discovery/eureka/api.go create mode 100644 lib/promscrape/discovery/eureka/api_test.go create mode 100644 lib/promscrape/discovery/eureka/eureka.go create mode 100644 lib/promscrape/discovery/eureka/eureka_test.go diff --git a/README.md b/README.md index 529638b7e..85675eb54 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) * [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) +* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) + Other `*_sd_config` types will be supported in the future. diff --git a/app/vmagent/README.md b/app/vmagent/README.md index d9ad2772e..b54eac26c 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -169,6 +169,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. * `dockerswarm_sd_configs` - for scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details. +* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). + See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ea23e8a56..dca1d89b3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ snap install victoriametrics ``` * FEATURE: vminsert: export `vm_rpc_vmstorage_is_reachable` metric, which can be used for monitoring reachability of vmstorage nodes from vminsert nodes. +* FEATURE: vmagent: add Netflix Eureka service discovery (aka [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)). +See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851 * FEATURE: add `-loggerWarnsPerSecondLimit` command-line flag for rate limiting of WARN messages in logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905 * FEATURE: apply `loggerErrorsPerSecondLimit` and `-loggerWarnsPerSecondLimit` rate limit per caller. I.e. log messages are suppressed if the same caller logs the same message at the rate exceeding the given limit. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905#issuecomment-729395855 diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 529638b7e..85675eb54 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -306,6 +306,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) * [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) +* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) + Other `*_sd_config` types will be supported in the future. diff --git a/docs/vmagent.md b/docs/vmagent.md index d9ad2772e..b54eac26c 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -169,6 +169,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. * `dockerswarm_sd_configs` - for scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details. +* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). + See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 9ada106c6..1a8b1fdd2 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -20,6 +20,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack" @@ -76,6 +77,7 @@ type ScrapeConfig struct { KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"` OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"` ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"` DockerSwarmConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"` DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"` EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"` @@ -293,6 +295,34 @@ func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork { return dst } +// getEurekaSDScrapeWork returns `eureka_sd_configs` ScrapeWork from cfg. +func (cfg *Config) getEurekaSDScrapeWork(prev []ScrapeWork) []ScrapeWork { + swsPrevByJob := getSWSByJob(prev) + dst := make([]ScrapeWork, 0, len(prev)) + for i := range cfg.ScrapeConfigs { + sc := &cfg.ScrapeConfigs[i] + dstLen := len(dst) + ok := true + for j := range sc.EurekaSDConfigs { + sdc := &sc.EurekaSDConfigs[j] + var okLocal bool + dst, okLocal = appendEurekaScrapeWork(dst, sdc, cfg.baseDir, sc.swc) + if ok { + ok = okLocal + } + } + if ok { + continue + } + swsPrev := swsPrevByJob[sc.swc.jobName] + if len(swsPrev) > 0 { + logger.Errorf("there were errors when discovering eureka targets for job %q, so preserving the previous targets", sc.swc.jobName) + dst = append(dst[:dstLen], swsPrev...) + } + } + return dst +} + // getDNSSDScrapeWork returns `dns_sd_configs` ScrapeWork from cfg. func (cfg *Config) getDNSSDScrapeWork(prev []ScrapeWork) []ScrapeWork { swsPrevByJob := getSWSByJob(prev) @@ -537,6 +567,15 @@ func appendConsulScrapeWork(dst []ScrapeWork, sdc *consul.SDConfig, baseDir stri return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "consul_sd_config"), true } +func appendEurekaScrapeWork(dst []ScrapeWork, sdc *eureka.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) { + targetLabels, err := eureka.GetLabels(sdc, baseDir) + if err != nil { + logger.Errorf("error when discovering eureka targets for `job_name` %q: %s; skipping it", swc.jobName, err) + return dst, false + } + return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "eureka_sd_config"), true +} + func appendDNSScrapeWork(dst []ScrapeWork, sdc *dns.SDConfig, swc *scrapeWorkConfig) ([]ScrapeWork, bool) { targetLabels, err := dns.GetLabels(sdc) if err != nil { diff --git a/lib/promscrape/discovery/eureka/api.go b/lib/promscrape/discovery/eureka/api.go new file mode 100644 index 000000000..e623bd323 --- /dev/null +++ b/lib/promscrape/discovery/eureka/api.go @@ -0,0 +1,81 @@ +package eureka + +import ( + "encoding/xml" + "fmt" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +var configMap = discoveryutils.NewConfigMap() + +type apiConfig struct { + client *discoveryutils.Client +} + +func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + var ( + ba *promauth.BasicAuthConfig + token string + ) + if sdc.Token != nil { + token = *sdc.Token + } + port := 80 + if sdc.Port == nil { + sdc.Port = &port + } + if len(sdc.Username) > 0 { + ba = &promauth.BasicAuthConfig{ + Username: sdc.Username, + Password: sdc.Password, + } + token = "" + } + ac, err := promauth.NewConfig(baseDir, ba, token, "", sdc.TLSConfig) + if err != nil { + return nil, fmt.Errorf("cannot parse auth config: %w", err) + } + apiServer := sdc.Server + if apiServer == "" { + apiServer = "localhost:8080/eureka/v2" + } + if !strings.Contains(apiServer, "://") { + scheme := sdc.Scheme + if scheme == "" { + scheme = "http" + } + apiServer = scheme + "://" + apiServer + } + client, err := discoveryutils.NewClient(apiServer, ac) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err) + } + cfg := &apiConfig{ + client: client, + } + return cfg, nil +} + +func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) }) + if err != nil { + return nil, err + } + return v.(*apiConfig), nil + +} + +func getAPIResponse(cfg *apiConfig, path string) ([]byte, error) { + return cfg.client.GetAPIResponse(path) +} + +func parseAPIResponse(data []byte) (*applications, error) { + var apps applications + if err := xml.Unmarshal(data, &apps); err != nil { + return nil, fmt.Errorf("failed parse eureka api response: %q, err: %w", data, err) + } + return &apps, nil +} diff --git a/lib/promscrape/discovery/eureka/api_test.go b/lib/promscrape/discovery/eureka/api_test.go new file mode 100644 index 000000000..648476e22 --- /dev/null +++ b/lib/promscrape/discovery/eureka/api_test.go @@ -0,0 +1,107 @@ +package eureka + +import ( + "reflect" + "testing" +) + +func Test_parseAPIResponse(t *testing.T) { + type args struct { + data []byte + } + tests := []struct { + name string + args args + want *applications + wantErr bool + }{ + { + name: "parse ok 1 app with instance", + args: args{ + data: []byte(` + 1 + UP_1_ + + HELLO-NETFLIX-OSS + + 98de25ebef42 + HELLO-NETFLIX-OSS + 10.10.0.3 + UP + UNKNOWN + 8080 + 443 + 1 + + MyOwn + + + 30 + 90 + 1605757726477 + 1605759135484 + 0 + 1605757725913 + + + UNKNOWN + http://98de25ebef42:8080/ + http://98de25ebef42:8080/Status + http://98de25ebef42:8080/healthcheck + HELLO-NETFLIX-OSS + false + 1605757726478 + 1605757725753 + ADDED + + +`), + }, + want: &applications{ + Applications: []Application{ + { + Name: "HELLO-NETFLIX-OSS", + Instances: []Instance{ + { + HostName: "98de25ebef42", + HomePageURL: "http://98de25ebef42:8080/", + StatusPageURL: "http://98de25ebef42:8080/Status", + HealthCheckURL: "http://98de25ebef42:8080/healthcheck", + App: "HELLO-NETFLIX-OSS", + IPAddr: "10.10.0.3", + VipAddress: "HELLO-NETFLIX-OSS", + SecureVipAddress: "", + Status: "UP", + Port: Port{ + Enabled: true, + Port: 8080, + }, + SecurePort: Port{ + Port: 443, + }, + DataCenterInfo: DataCenterInfo{ + Name: "MyOwn", + }, + Metadata: MetaData{}, + CountryID: 1, + InstanceID: "", + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseAPIResponse(tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("parseAPIResponse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unxpected response for parseAPIResponse() \ngot = %v, \nwant %v", got, tt.want) + } + }) + } +} diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go new file mode 100644 index 000000000..df5324d97 --- /dev/null +++ b/lib/promscrape/discovery/eureka/eureka.go @@ -0,0 +1,148 @@ +package eureka + +import ( + "encoding/xml" + "fmt" + "strconv" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" +) + +const appsAPIPath = "/apps" + +// SDConfig represents service discovery config for eureka. +// +// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka +type SDConfig struct { + Server string `yaml:"server,omitempty"` + Token *string `yaml:"token"` + Datacenter string `yaml:"datacenter"` + Scheme string `yaml:"scheme,omitempty"` + Username string `yaml:"username"` + Password string `yaml:"password"` + TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"` + // RefreshInterval time.Duration `yaml:"refresh_interval"` + // refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option. + Port *int `yaml:"port,omitempty"` +} +type applications struct { + Applications []Application `xml:"application"` +} + +// Application - eureka application https://github.com/Netflix/eureka/wiki/Eureka-REST-operations/ +type Application struct { + Name string `xml:"name"` + Instances []Instance `xml:"instance"` +} + +// Port - eureka instance port. +type Port struct { + Port int `xml:",chardata"` + Enabled bool `xml:"enabled,attr"` +} + +// Instance - eureka instance https://github.com/Netflix/eureka/wiki/Eureka-REST-operations +type Instance struct { + HostName string `xml:"hostName"` + HomePageURL string `xml:"homePageUrl"` + StatusPageURL string `xml:"statusPageUrl"` + HealthCheckURL string `xml:"healthCheckUrl"` + App string `xml:"app"` + IPAddr string `xml:"ipAddr"` + VipAddress string `xml:"vipAddress"` + SecureVipAddress string `xml:"secureVipAddress"` + Status string `xml:"status"` + Port Port `xml:"port"` + SecurePort Port `xml:"securePort"` + DataCenterInfo DataCenterInfo `xml:"dataCenterInfo"` + Metadata MetaData `xml:"metadata"` + CountryID int `xml:"countryId"` + InstanceID string `xml:"instanceId"` +} + +// MetaData - eureka objects metadata. +type MetaData struct { + Items []Tag `xml:",any"` +} + +// Tag - eureka metadata tag - list of k/v values. +type Tag struct { + XMLName xml.Name + Content string `xml:",innerxml"` +} + +// DataCenterInfo -eureka datacentre metadata +type DataCenterInfo struct { + Name string `xml:"name"` + Metadata MetaData `xml:"metadata"` +} + +// GetLabels returns Eureka labels according to sdc. +func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) { + cfg, err := getAPIConfig(sdc, baseDir) + if err != nil { + return nil, fmt.Errorf("cannot get API config: %w", err) + } + data, err := getAPIResponse(cfg, appsAPIPath) + if err != nil { + return nil, err + } + apps, err := parseAPIResponse(data) + if err != nil { + return nil, err + } + return addInstanceLabels(apps, *sdc.Port), nil +} + +func addInstanceLabels(applications *applications, port int) []map[string]string { + var ms []map[string]string + for _, app := range applications.Applications { + for _, instance := range app.Instances { + var targetAddress string + if instance.Port.Port > 0 { + targetAddress = discoveryutils.JoinHostPort(instance.HostName, instance.Port.Port) + } else { + targetAddress = discoveryutils.JoinHostPort(instance.HostName, port) + } + m := map[string]string{ + "__address__": targetAddress, + "instance": instance.InstanceID, + "__meta_eureka_app_instance_app_nanem": app.Name, + "__meta_eureka_app_instance_hostname": instance.HostName, + "__meta_eureka_app_instance_homepage_url": instance.HomePageURL, + "__meta_eureka_app_instance_statuspage_url": instance.StatusPageURL, + "__meta_eureka_app_instance_healthcheck_url": instance.HealthCheckURL, + "__meta_eureka_app_instance_ip_addr": instance.IPAddr, + "__meta_eureka_app_instance_vip_address": instance.VipAddress, + "__meta_eureka_app_instance_status": instance.Status, + "__meta_eureka_app_instance_country_id": strconv.Itoa(instance.CountryID), + "__meta_eureka_app_instance_id": instance.InstanceID, + } + if len(instance.SecureVipAddress) > 0 { + m["__meta_eureka_app_instance_secure_vip_address"] = instance.SecureVipAddress + } + if instance.Port.Port > 0 { + m["__meta_eureka_app_instance_port"] = strconv.Itoa(instance.Port.Port) + m["__meta_eureka_app_instance_port_enabled"] = strconv.FormatBool(instance.Port.Enabled) + } + if instance.SecurePort.Port > 0 { + m["__meta_eureka_app_instance_secure_port"] = strconv.Itoa(instance.SecurePort.Port) + m["__meta_eureka_app_instance_secure_port_enabled"] = strconv.FormatBool(instance.SecurePort.Enabled) + + } + if len(instance.DataCenterInfo.Name) > 0 { + m["__meta_eureka_app_instance_datacenterinfo_name"] = instance.DataCenterInfo.Name + for _, tag := range instance.DataCenterInfo.Metadata.Items { + m["__meta_eureka_app_instance_datacenterinfo_metadata_"+discoveryutils.SanitizeLabelName(tag.XMLName.Local)] = tag.Content + } + } + for _, tag := range instance.Metadata.Items { + m["__meta_eureka_app_instance_metadata_"+discoveryutils.SanitizeLabelName(tag.XMLName.Local)] = tag.Content + } + ms = append(ms, m) + } + } + return ms +} diff --git a/lib/promscrape/discovery/eureka/eureka_test.go b/lib/promscrape/discovery/eureka/eureka_test.go new file mode 100644 index 000000000..9974118ef --- /dev/null +++ b/lib/promscrape/discovery/eureka/eureka_test.go @@ -0,0 +1,83 @@ +package eureka + +import ( + "reflect" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +func Test_addInstanceLabels(t *testing.T) { + type args struct { + applications *applications + port int + } + tests := []struct { + name string + args args + want [][]prompbmarshal.Label + }{ + { + name: "1 application", + args: args{ + port: 9100, + applications: &applications{ + Applications: []Application{ + { + Name: "test-app", + Instances: []Instance{ + { + Status: "Ok", + HealthCheckURL: "some-url", + HomePageURL: "some-home-url", + StatusPageURL: "some-status-url", + HostName: "host-1", + IPAddr: "10.15.11.11", + CountryID: 5, + VipAddress: "10.15.11.11", + InstanceID: "some-id", + Metadata: MetaData{Items: []Tag{ + { + Content: "value-1", + XMLName: struct{ Space, Local string }{Local: "key-1"}, + }, + }}, + }, + }, + }, + }, + }, + }, + want: [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "host-1:9100", + "instance": "some-id", + "__meta_eureka_app_instance_hostname": "host-1", + "__meta_eureka_app_instance_app_nanem": "test-app", + "__meta_eureka_app_instance_healthcheck_url": "some-url", + "__meta_eureka_app_instance_ip_addr": "10.15.11.11", + "__meta_eureka_app_instance_vip_address": "10.15.11.11", + "__meta_eureka_app_instance_country_id": "5", + "__meta_eureka_app_instance_homepage_url": "some-home-url", + "__meta_eureka_app_instance_statuspage_url": "some-status-url", + "__meta_eureka_app_instance_id": "some-id", + "__meta_eureka_app_instance_metadata_key_1": "value-1", + "__meta_eureka_app_instance_status": "Ok", + }), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := addInstanceLabels(tt.args.applications, tt.args.port) + var sortedLabelss [][]prompbmarshal.Label + for _, labels := range got { + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) + } + if !reflect.DeepEqual(sortedLabelss, tt.want) { + t.Fatalf("unexpected labels \ngot : %v, \nwant: %v", got, tt.want) + } + }) + } +} diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index 9e964caa2..40f82860a 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -28,6 +28,9 @@ var ( consulSDCheckInterval = flag.Duration("promscrape.consulSDCheckInterval", 30*time.Second, "Interval for checking for changes in consul. "+ "This works only if `consul_sd_configs` is configured in '-promscrape.config' file. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details") + eurekaSDCheckInterval = flag.Duration("promscrape.eurekaSDCheckInterval", 30*time.Second, "Interval for checking for changes in eureka. "+ + "This works only if `eureka_sd_configs` is configured in '-promscrape.config' file. "+ + "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details") dnsSDCheckInterval = flag.Duration("promscrape.dnsSDCheckInterval", 30*time.Second, "Interval for checking for changes in dns. "+ "This works only if `dns_sd_configs` is configured in '-promscrape.config' file. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details") @@ -99,6 +102,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) }) scs.add("openstack_sd_configs", *openstackSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) }) scs.add("consul_sd_configs", *consulSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getConsulSDScrapeWork(swsPrev) }) + scs.add("eureka_sd_configs", *eurekaSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) }) scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) }) scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) }) scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) }) From d7932775cc42a1c41f870d16d5c16c7cadab748a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 20 Nov 2020 14:00:12 +0200 Subject: [PATCH 4/6] lib/promscrape/discovery/eureka: follow-up after eec76718e9bddd6ce1903449e6b1844252734133 --- lib/promscrape/discovery/eureka/api.go | 10 +--- lib/promscrape/discovery/eureka/eureka.go | 52 ++++++++++--------- .../discovery/eureka/eureka_test.go | 27 +++++----- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/lib/promscrape/discovery/eureka/api.go b/lib/promscrape/discovery/eureka/api.go index e623bd323..f1a319bc3 100644 --- a/lib/promscrape/discovery/eureka/api.go +++ b/lib/promscrape/discovery/eureka/api.go @@ -16,17 +16,11 @@ type apiConfig struct { } func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { - var ( - ba *promauth.BasicAuthConfig - token string - ) + token := "" if sdc.Token != nil { token = *sdc.Token } - port := 80 - if sdc.Port == nil { - sdc.Port = &port - } + var ba *promauth.BasicAuthConfig if len(sdc.Username) > 0 { ba = &promauth.BasicAuthConfig{ Username: sdc.Username, diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go index df5324d97..5dca81937 100644 --- a/lib/promscrape/discovery/eureka/eureka.go +++ b/lib/promscrape/discovery/eureka/eureka.go @@ -27,6 +27,7 @@ type SDConfig struct { // refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option. Port *int `yaml:"port,omitempty"` } + type applications struct { Applications []Application `xml:"application"` } @@ -93,41 +94,42 @@ func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) { if err != nil { return nil, err } - return addInstanceLabels(apps, *sdc.Port), nil + port := 80 + if sdc.Port != nil { + port = *sdc.Port + } + return addInstanceLabels(apps, port), nil } -func addInstanceLabels(applications *applications, port int) []map[string]string { +func addInstanceLabels(apps *applications, port int) []map[string]string { var ms []map[string]string - for _, app := range applications.Applications { + for _, app := range apps.Applications { for _, instance := range app.Instances { - var targetAddress string - if instance.Port.Port > 0 { - targetAddress = discoveryutils.JoinHostPort(instance.HostName, instance.Port.Port) - } else { - targetAddress = discoveryutils.JoinHostPort(instance.HostName, port) + instancePort := port + if instance.Port.Port != 0 { + instancePort = instance.Port.Port } + targetAddress := discoveryutils.JoinHostPort(instance.HostName, instancePort) m := map[string]string{ - "__address__": targetAddress, - "instance": instance.InstanceID, - "__meta_eureka_app_instance_app_nanem": app.Name, - "__meta_eureka_app_instance_hostname": instance.HostName, - "__meta_eureka_app_instance_homepage_url": instance.HomePageURL, - "__meta_eureka_app_instance_statuspage_url": instance.StatusPageURL, - "__meta_eureka_app_instance_healthcheck_url": instance.HealthCheckURL, - "__meta_eureka_app_instance_ip_addr": instance.IPAddr, - "__meta_eureka_app_instance_vip_address": instance.VipAddress, - "__meta_eureka_app_instance_status": instance.Status, - "__meta_eureka_app_instance_country_id": strconv.Itoa(instance.CountryID), - "__meta_eureka_app_instance_id": instance.InstanceID, + "__address__": targetAddress, + "instance": instance.InstanceID, + "__meta_eureka_app_name": app.Name, + "__meta_eureka_app_instance_hostname": instance.HostName, + "__meta_eureka_app_instance_homepage_url": instance.HomePageURL, + "__meta_eureka_app_instance_statuspage_url": instance.StatusPageURL, + "__meta_eureka_app_instance_healthcheck_url": instance.HealthCheckURL, + "__meta_eureka_app_instance_ip_addr": instance.IPAddr, + "__meta_eureka_app_instance_vip_address": instance.VipAddress, + "__meta_eureka_app_instance_secure_vip_address": instance.SecureVipAddress, + "__meta_eureka_app_instance_status": instance.Status, + "__meta_eureka_app_instance_country_id": strconv.Itoa(instance.CountryID), + "__meta_eureka_app_instance_id": instance.InstanceID, } - if len(instance.SecureVipAddress) > 0 { - m["__meta_eureka_app_instance_secure_vip_address"] = instance.SecureVipAddress - } - if instance.Port.Port > 0 { + if instance.Port.Port != 0 { m["__meta_eureka_app_instance_port"] = strconv.Itoa(instance.Port.Port) m["__meta_eureka_app_instance_port_enabled"] = strconv.FormatBool(instance.Port.Enabled) } - if instance.SecurePort.Port > 0 { + if instance.SecurePort.Port != 0 { m["__meta_eureka_app_instance_secure_port"] = strconv.Itoa(instance.SecurePort.Port) m["__meta_eureka_app_instance_secure_port_enabled"] = strconv.FormatBool(instance.SecurePort.Enabled) diff --git a/lib/promscrape/discovery/eureka/eureka_test.go b/lib/promscrape/discovery/eureka/eureka_test.go index 9974118ef..8b5091648 100644 --- a/lib/promscrape/discovery/eureka/eureka_test.go +++ b/lib/promscrape/discovery/eureka/eureka_test.go @@ -51,19 +51,20 @@ func Test_addInstanceLabels(t *testing.T) { }, want: [][]prompbmarshal.Label{ discoveryutils.GetSortedLabels(map[string]string{ - "__address__": "host-1:9100", - "instance": "some-id", - "__meta_eureka_app_instance_hostname": "host-1", - "__meta_eureka_app_instance_app_nanem": "test-app", - "__meta_eureka_app_instance_healthcheck_url": "some-url", - "__meta_eureka_app_instance_ip_addr": "10.15.11.11", - "__meta_eureka_app_instance_vip_address": "10.15.11.11", - "__meta_eureka_app_instance_country_id": "5", - "__meta_eureka_app_instance_homepage_url": "some-home-url", - "__meta_eureka_app_instance_statuspage_url": "some-status-url", - "__meta_eureka_app_instance_id": "some-id", - "__meta_eureka_app_instance_metadata_key_1": "value-1", - "__meta_eureka_app_instance_status": "Ok", + "__address__": "host-1:9100", + "instance": "some-id", + "__meta_eureka_app_instance_hostname": "host-1", + "__meta_eureka_app_name": "test-app", + "__meta_eureka_app_instance_healthcheck_url": "some-url", + "__meta_eureka_app_instance_ip_addr": "10.15.11.11", + "__meta_eureka_app_instance_vip_address": "10.15.11.11", + "__meta_eureka_app_instance_secure_vip_address": "", + "__meta_eureka_app_instance_country_id": "5", + "__meta_eureka_app_instance_homepage_url": "some-home-url", + "__meta_eureka_app_instance_statuspage_url": "some-status-url", + "__meta_eureka_app_instance_id": "some-id", + "__meta_eureka_app_instance_metadata_key_1": "value-1", + "__meta_eureka_app_instance_status": "Ok", }), }, }, From 9d160f90489d10b3b6d5c427cd05c5f8d349f246 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 21 Nov 2020 14:38:54 +0200 Subject: [PATCH 5/6] lib/promscrape: hint that -enableTCP6 command-line flag can be used for connecting to IPv6 addresses --- lib/promscrape/statconn.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/promscrape/statconn.go b/lib/promscrape/statconn.go index 420b8b624..d62cefb57 100644 --- a/lib/promscrape/statconn.go +++ b/lib/promscrape/statconn.go @@ -2,6 +2,7 @@ package promscrape import ( "context" + "fmt" "net" "sync" "sync/atomic" @@ -52,6 +53,9 @@ func statDial(addr string) (conn net.Conn, err error) { dialsTotal.Inc() if err != nil { dialErrors.Inc() + if !netutil.TCP6Enabled() { + err = fmt.Errorf("%w; try -enableTCP6 command-line flag", err) + } return nil, err } conns.Inc() From e614a14b218ca06a5663127e154457a9335be77f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 23 Nov 2020 00:42:00 +0200 Subject: [PATCH 6/6] docs: sync with cluster branch --- docs/CHANGELOG.md | 2 ++ docs/Cluster-VictoriaMetrics.md | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dca1d89b3..44de82a62 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,8 @@ ```bash snap install victoriametrics ``` +* FEATURE: vmselect: add `-replicationFactor` command-line flag for reducing query duration when replication is enabled and a part of vmstorage nodes + are temporarily slow and/or temporarily unavailable. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711 * FEATURE: vminsert: export `vm_rpc_vmstorage_is_reachable` metric, which can be used for monitoring reachability of vmstorage nodes from vminsert nodes. * FEATURE: vmagent: add Netflix Eureka service discovery (aka [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851 diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 0433f4668..fed3173d3 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -335,8 +335,9 @@ In order to enable application-level replication, `-replicationFactor=N` command This guarantees that all the data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable. For example, when `-replicationFactor=3` is passed to `vminsert`, then it replicates all the ingested data to 3 distinct `vmstorage` nodes. -When the replication is enabled, `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect` -in order to de-duplicate replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms +When the replication is enabled, `-replicationFactor=N` and `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect` nodes. +The `-replicationFactor=N` improves query performance when a part of vmstorage nodes respond slowly and/or temporarily unavailable. +The `-dedup.minScrapeInterval=1ms` de-duplicates replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms when [deduplication](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#deduplication) is used additionally to replication. Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883),