From 1005d5a18615e107ac7a152c7355076a4483f169 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 5 Aug 2022 18:50:00 +0300 Subject: [PATCH] lib/promscrape/discovery/ec2: properly handle custom `endpoint` option in ec2_sd_configs This option was ignored since d289ecded1ae93300b8ab40ae1db89048d3e6871 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1287 --- app/vmagent/remotewrite/client.go | 14 +++++++++----- docs/CHANGELOG.md | 1 + lib/awsapi/config.go | 8 ++++---- lib/promscrape/discovery/ec2/api.go | 6 +++++- lib/promscrape/discovery/ec2/ec2.go | 11 ++++++----- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index 54cb61ea8..c9b5bcae8 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -56,10 +56,12 @@ var ( awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+ "It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled") - awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") - awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") - awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") - awsService = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+ + awsEC2Endpoint = flagutil.NewArray("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") + awsSTSEndpoint = flagutil.NewArray("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") + awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") + awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") + awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") + awsService = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+ "Defaults to \"aps\"") awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") ) @@ -231,12 +233,14 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) { if !awsUseSigv4.GetOptionalArg(argIdx) { return nil, nil } + ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx) + stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx) region := awsRegion.GetOptionalArg(argIdx) roleARN := awsRoleARN.GetOptionalArg(argIdx) accessKey := awsAccessKey.GetOptionalArg(argIdx) secretKey := awsSecretKey.GetOptionalArg(argIdx) service := awsService.GetOptionalArg(argIdx) - cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, service) + cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service) if err != nil { return nil, err } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dcb8115d0..2dffdc232 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## v1.79.x long-time support release (LTS) * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): fix potential panic in [multi-level cluster setup](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup) when top-level `vmselect` is configured with `-replicationFactor` bigger than 1. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2961). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly handle custom `endpoint` value in [ec2_sd_configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config). It was ignored since [v1.77.0](https://docs.victoriametrics.com/CHANGELOG.html#v1770) because of a bug in the implementation of [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1287). ## [v1.79.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.1) diff --git a/lib/awsapi/config.go b/lib/awsapi/config.go index f183bbe4d..f111b7b64 100644 --- a/lib/awsapi/config.go +++ b/lib/awsapi/config.go @@ -43,8 +43,8 @@ type credentials struct { Expiration time.Time } -// NewConfig returns new AWS Config. -func NewConfig(region, roleARN, accessKey, secretKey, service string) (*Config, error) { +// NewConfig returns new AWS Config from the given args. +func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service string) (*Config, error) { cfg := &Config{ client: http.DefaultClient, region: region, @@ -65,8 +65,8 @@ func NewConfig(region, roleARN, accessKey, secretKey, service string) (*Config, } cfg.region = r } - cfg.ec2Endpoint = buildAPIEndpoint(cfg.ec2Endpoint, cfg.region, "ec2") - cfg.stsEndpoint = buildAPIEndpoint(cfg.stsEndpoint, cfg.region, "sts") + cfg.ec2Endpoint = buildAPIEndpoint(ec2Endpoint, cfg.region, "ec2") + cfg.stsEndpoint = buildAPIEndpoint(stsEndpoint, cfg.region, "sts") if cfg.roleARN == "" { cfg.roleARN = os.Getenv("AWS_ROLE_ARN") } diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go index 652199fb3..def9d0a17 100644 --- a/lib/promscrape/discovery/ec2/api.go +++ b/lib/promscrape/discovery/ec2/api.go @@ -33,7 +33,11 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { if sdc.Port != nil { port = *sdc.Port } - awsCfg, err := awsapi.NewConfig(sdc.Region, sdc.RoleARN, sdc.AccessKey, sdc.SecretKey.String(), "ec2") + stsEndpoint := sdc.STSEndpoint + if stsEndpoint == "" { + stsEndpoint = sdc.Endpoint + } + awsCfg, err := awsapi.NewConfig(sdc.Endpoint, stsEndpoint, sdc.Region, sdc.RoleARN, sdc.AccessKey, sdc.SecretKey.String(), "ec2") if err != nil { return nil, err } diff --git a/lib/promscrape/discovery/ec2/ec2.go b/lib/promscrape/discovery/ec2/ec2.go index dd229ae89..e0b2a169b 100644 --- a/lib/promscrape/discovery/ec2/ec2.go +++ b/lib/promscrape/discovery/ec2/ec2.go @@ -18,12 +18,13 @@ var SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config type SDConfig struct { - Region string `yaml:"region,omitempty"` - Endpoint string `yaml:"endpoint,omitempty"` - AccessKey string `yaml:"access_key,omitempty"` - SecretKey *promauth.Secret `yaml:"secret_key,omitempty"` + Region string `yaml:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + STSEndpoint string `yaml:"sts_endpoint,omitempty"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey *promauth.Secret `yaml:"secret_key,omitempty"` // TODO add support for Profile, not working atm - Profile string `yaml:"profile,omitempty"` + // Profile string `yaml:"profile,omitempty"` RoleARN string `yaml:"role_arn,omitempty"` // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.