mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/promscrape/discovery/yandexcloud: follow-up after 6e5ac32fba
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1386
This commit is contained in:
Aliaksandr Valialkin
parent
6e5ac32fba
commit
83a4abda3f
11 changed files with 215 additions and 363 deletions
21
README.md
21
README.md
|
|
@ -310,27 +310,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](
|
|||
|
||||
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported:
|
||||
|
||||
* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
|
||||
* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
|
||||
* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config)
|
||||
* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
|
||||
* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
|
||||
* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config)
|
||||
* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
|
||||
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
|
||||
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
|
||||
* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config)
|
||||
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
|
||||
* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
|
||||
* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)
|
||||
* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
|
||||
|
||||
File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types.
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
|
||||
|
||||
The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
|
||||
|
||||
See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html).
|
||||
|
||||
VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
|
||||
|
||||
See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus.
|
||||
|
|
|
|||
|
|
@ -155,33 +155,8 @@ Use `-remoteWrite.*` command-line flag instead for configuring remote write sett
|
|||
|
||||
The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
|
||||
|
||||
The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
|
||||
See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html).
|
||||
|
||||
* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
|
||||
* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
|
||||
* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
|
||||
* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet.
|
||||
* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`:
|
||||
* if `project` arg is missing then `vmagent` uses the project for the instance where it runs;
|
||||
* if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs;
|
||||
* if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
|
||||
* `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
|
||||
* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud.
|
||||
See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details.
|
||||
* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
|
||||
* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
|
||||
* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details.
|
||||
* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
|
||||
* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
|
||||
* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
|
||||
* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
|
||||
|
||||
Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
|
||||
command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
|
||||
entries to 60s. Run `vmagent -help` in order to see default values for the `-promscrape.*CheckInterval` flags.
|
||||
|
||||
Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
|
||||
|
||||
## scrape_config enhancements
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
* FEATURE: add ability to push internal metrics (e.g. metrics exposed at `/metrics` page) to the configured remote storage from all the VictoriaMetrics components. See [these docs](https://docs.victoriametrics.com/#push-metrics).
|
||||
* FEATURE: improve performance for heavy queries over big number of time series on systems with big number of CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2896). Thanks to @zqyzyq for [the idea](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b596ac3745314fcc170a14e3ded062971cf7ced2).
|
||||
* FEATURE: improve performance for registering new time series in `indexdb` by up to 50%. Thanks to @ahfuzhang for [the issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2249).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add service discovery for [Yandex Cloud](https://cloud.yandex.com/en/). See [these docs](https://docs.victoriametrics.com/sd_configs.html#yandexcloud-sd-configs) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1386).
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): set `up` metric to `0` for partial scrapes in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). Previously the `up` metric was set to `1` when at least a single metric has been scraped before the error. This aligns the behaviour of `vmselect` with Prometheus.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): restart all the scrape jobs during [config reload](https://docs.victoriametrics.com/vmagent.html#configuration-update) after `global` section is changed inside `-promscrape.config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884).
|
||||
|
|
|
|||
|
|
@ -310,28 +310,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](
|
|||
|
||||
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported:
|
||||
|
||||
* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
|
||||
* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
|
||||
* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config)
|
||||
* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
|
||||
* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
|
||||
* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config)
|
||||
* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
|
||||
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
|
||||
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
|
||||
* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config)
|
||||
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
|
||||
* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
|
||||
* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)
|
||||
* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
|
||||
* [yandexcloud_sd_config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/sd_configs.md#yandex_cloud_service_discovery_config)
|
||||
|
||||
File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types.
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
|
||||
|
||||
The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
|
||||
|
||||
See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html).
|
||||
|
||||
VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
|
||||
|
||||
See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus.
|
||||
|
|
@ -2118,6 +2102,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
|
||||
-promscrape.suppressScrapeErrorsDelay duration
|
||||
The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
|
||||
-promscrape.yandexcloudSDCheckInterval duration
|
||||
Interval for checking for changes in Yandex Cloud. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. (default 30s)
|
||||
-pushmetrics.extraLabel array
|
||||
Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
|
|
|
|||
|
|
@ -314,27 +314,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](
|
|||
|
||||
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported:
|
||||
|
||||
* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
|
||||
* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
|
||||
* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config)
|
||||
* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
|
||||
* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
|
||||
* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config)
|
||||
* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
|
||||
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
|
||||
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
|
||||
* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config)
|
||||
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
|
||||
* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
|
||||
* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)
|
||||
* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
|
||||
|
||||
File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types.
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
|
||||
|
||||
The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
|
||||
|
||||
See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html).
|
||||
|
||||
VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
|
||||
|
||||
See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus.
|
||||
|
|
@ -2121,6 +2106,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
|
||||
-promscrape.suppressScrapeErrorsDelay duration
|
||||
The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
|
||||
-promscrape.yandexcloudSDCheckInterval duration
|
||||
Interval for checking for changes in Yandex Cloud. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. (default 30s)
|
||||
-pushmetrics.extraLabel array
|
||||
Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,36 @@
|
|||
## Yandex Cloud Service Discovery Configs
|
||||
# Prometheus service discovery
|
||||
|
||||
[vmagent](https://docs.victoriametrics.com/vmagent.html) and [single-node VictoriaMetrics](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter) supports the following Prometheus-compatible service discovery options for Prometheus-compatible scrape targets in the file pointed by `-promscrape.config` command-line flag.
|
||||
|
||||
* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud.
|
||||
See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details.
|
||||
* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
|
||||
* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
|
||||
* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
|
||||
* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details.
|
||||
* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
|
||||
* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet.
|
||||
* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
|
||||
* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
|
||||
* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`:
|
||||
* if `project` arg is missing then `vmagent` uses the project for the instance where it runs;
|
||||
* if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs;
|
||||
* if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
|
||||
* `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
|
||||
* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
|
||||
* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
|
||||
* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
|
||||
* `yandexcloud_sd_configs` is for discoverying and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud-sd-configs) for details.
|
||||
|
||||
Note that the `refresh_interval` option isn't supported for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
|
||||
command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
|
||||
entries to 60s. Run `vmagent -help` or `victoria-metrics -help` in order to see default values for the `-promscrape.*CheckInterval` flags.
|
||||
|
||||
Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by VictoriaMetrics and `vmagent`.
|
||||
|
||||
|
||||
## yandexcloud_sd_configs
|
||||
|
||||
Yandex Cloud SD configurations allow retrieving scrape targets from accessible folders.
|
||||
|
||||
|
|
|
|||
|
|
@ -159,34 +159,8 @@ Use `-remoteWrite.*` command-line flag instead for configuring remote write sett
|
|||
|
||||
The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
|
||||
|
||||
The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
|
||||
See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html).
|
||||
|
||||
* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
|
||||
* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
|
||||
* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
|
||||
* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet.
|
||||
* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`:
|
||||
* if `project` arg is missing then `vmagent` uses the project for the instance where it runs;
|
||||
* if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs;
|
||||
* if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
|
||||
* `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
|
||||
* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud.
|
||||
See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details.
|
||||
* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
|
||||
* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
|
||||
* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details.
|
||||
* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
|
||||
* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
|
||||
* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
|
||||
* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
|
||||
* `yandexcloud_sd_configs` is for discovering and scraping targets registered in [Yandex Cloud](https://cloud.yandex.ru/). See [yandexcloud_sd_configs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/sd_configs.md#yandex_cloud_service_discovery_configs) for details.
|
||||
|
||||
Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
|
||||
command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
|
||||
entries to 60s. Run `vmagent -help` in order to see default values for the `-promscrape.*CheckInterval` flags.
|
||||
|
||||
Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
|
||||
|
||||
## scrape_config enhancements
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -16,12 +15,6 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultInstanceCredsEndpoint = "http://169.254.169.254/latest/meta-data/iam/security-credentials/default"
|
||||
defaultAPIEndpoint = "https://api.cloud.yandex.net"
|
||||
defaultAPIVersion = "v1"
|
||||
)
|
||||
|
||||
var configMap = discoveryutils.NewConfigMap()
|
||||
|
||||
type apiCredentials struct {
|
||||
|
|
@ -35,19 +28,14 @@ type yandexPassportOAuth struct {
|
|||
YandexPassportOAuthToken string `json:"yandexPassportOauthToken"`
|
||||
}
|
||||
|
||||
// iamToken Yandex Cloud IAM token response
|
||||
// https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create
|
||||
type iamToken struct {
|
||||
IAMToken string `json:"iamToken"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
}
|
||||
|
||||
type apiConfig struct {
|
||||
client *http.Client
|
||||
tokenLock sync.Mutex
|
||||
creds *apiCredentials
|
||||
yandexPassportOAuth *yandexPassportOAuth
|
||||
serviceEndpoints map[string]*url.URL
|
||||
serviceEndpoints map[string]string
|
||||
|
||||
// credsLock protects the refresh of creds
|
||||
credsLock sync.Mutex
|
||||
creds *apiCredentials
|
||||
}
|
||||
|
||||
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
|
|
@ -59,12 +47,8 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
|||
}
|
||||
|
||||
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
cfg := &apiConfig{
|
||||
client: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConnsPerHost: 100,
|
||||
},
|
||||
},
|
||||
transport := &http.Transport{
|
||||
MaxIdleConnsPerHost: 100,
|
||||
}
|
||||
if sdc.TLSConfig != nil {
|
||||
opts := &promauth.Options{
|
||||
|
|
@ -73,47 +57,49 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
|||
}
|
||||
ac, err := opts.NewConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.client.Transport = &http.Transport{
|
||||
TLSClientConfig: ac.NewTLSConfig(),
|
||||
MaxIdleConnsPerHost: 100,
|
||||
return nil, fmt.Errorf("cannot initialize TLS config: %w", err)
|
||||
}
|
||||
transport.TLSClientConfig = ac.NewTLSConfig()
|
||||
}
|
||||
|
||||
if err := cfg.getEndpoints(sdc.APIEndpoint); err != nil {
|
||||
return nil, err
|
||||
cfg := &apiConfig{
|
||||
client: &http.Client{
|
||||
Transport: transport,
|
||||
},
|
||||
}
|
||||
|
||||
apiEndpoint := sdc.APIEndpoint
|
||||
if apiEndpoint == "" {
|
||||
apiEndpoint = "https://api.cloud.yandex.net"
|
||||
}
|
||||
serviceEndpoints, err := cfg.getServiceEndpoints(apiEndpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot obtain endpoints for yandex services: %w", err)
|
||||
}
|
||||
cfg.serviceEndpoints = serviceEndpoints
|
||||
if sdc.YandexPassportOAuthToken != nil {
|
||||
logger.Infof("Using yandex passport OAuth token")
|
||||
|
||||
logger.Infof("yandexcloud_sd: using yandex passport OAuth token")
|
||||
cfg.yandexPassportOAuth = &yandexPassportOAuth{
|
||||
YandexPassportOAuthToken: sdc.YandexPassportOAuthToken.String(),
|
||||
}
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// getFreshAPICredentials checks token lifetime and update if needed
|
||||
func (cfg *apiConfig) getFreshAPICredentials() (*apiCredentials, error) {
|
||||
cfg.tokenLock.Lock()
|
||||
defer cfg.tokenLock.Unlock()
|
||||
cfg.credsLock.Lock()
|
||||
defer cfg.credsLock.Unlock()
|
||||
|
||||
if cfg.creds != nil && time.Until(cfg.creds.Expiration) > 10*time.Second {
|
||||
// Credentials aren't expired yet.
|
||||
return cfg.creds, nil
|
||||
}
|
||||
|
||||
// Refresh credentials.
|
||||
newCreds, err := getCreds(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot refresh service account api token: %w", err)
|
||||
}
|
||||
cfg.creds = newCreds
|
||||
|
||||
logger.Infof("successfully refreshed service account api token; expiration: %.3f seconds", time.Until(newCreds.Expiration).Seconds())
|
||||
|
||||
logger.Infof("yandexcloud_sd: successfully refreshed service account api token; expiration: %.3f seconds", time.Until(newCreds.Expiration).Seconds())
|
||||
return newCreds, nil
|
||||
}
|
||||
|
||||
|
|
@ -122,12 +108,10 @@ func getCreds(cfg *apiConfig) (*apiCredentials, error) {
|
|||
if cfg.yandexPassportOAuth == nil {
|
||||
return getInstanceCreds(cfg)
|
||||
}
|
||||
|
||||
it, err := getIAMToken(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot get IAM token: %w", err)
|
||||
}
|
||||
|
||||
return &apiCredentials{
|
||||
Token: it.IAMToken,
|
||||
Expiration: it.ExpiresAt,
|
||||
|
|
@ -135,104 +119,118 @@ func getCreds(cfg *apiConfig) (*apiCredentials, error) {
|
|||
}
|
||||
|
||||
// getInstanceCreds gets Yandex Cloud IAM token using instance Service Account
|
||||
// https://cloud.yandex.com/en-ru/docs/compute/operations/vm-connect/auth-inside-vm
|
||||
//
|
||||
// See https://cloud.yandex.com/en-ru/docs/compute/operations/vm-connect/auth-inside-vm
|
||||
func getInstanceCreds(cfg *apiConfig) (*apiCredentials, error) {
|
||||
resp, err := cfg.client.Get(defaultInstanceCredsEndpoint)
|
||||
endpoint := "http://169.254.169.254/latest/meta-data/iam/security-credentials/default"
|
||||
resp, err := cfg.client.Get(endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed query security credentials api, url: %s, err: %w", defaultInstanceCredsEndpoint, err)
|
||||
return nil, fmt.Errorf("cannot read instance creds from %s: %w", endpoint, err)
|
||||
}
|
||||
r, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
data, err := readResponseBody(resp, endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read response from %q: %w", defaultInstanceCredsEndpoint, err)
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var ac apiCredentials
|
||||
if err := json.Unmarshal(r, &ac); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse auth credentials response: %w", err)
|
||||
if err := json.Unmarshal(data, &ac); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse auth credentials response from %s: %w", endpoint, err)
|
||||
}
|
||||
|
||||
return &ac, nil
|
||||
}
|
||||
|
||||
// getIAMToken gets Yandex Cloud IAM token using OAuth:
|
||||
// https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create
|
||||
// getIAMToken gets Yandex Cloud IAM token using OAuth
|
||||
//
|
||||
// See https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create
|
||||
func getIAMToken(cfg *apiConfig) (*iamToken, error) {
|
||||
iamURL := *cfg.serviceEndpoints["iam"]
|
||||
iamURL.Path = path.Join(iamURL.Path, "iam", defaultAPIVersion, "tokens")
|
||||
|
||||
iamURL := cfg.serviceEndpoints["iam"] + "/iam/v1/tokens"
|
||||
passport, err := json.Marshal(cfg.yandexPassportOAuth)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed marshall yandex passport OAuth token, err: %w", err)
|
||||
logger.Panicf("BUG: cannot marshal yandex passport OAuth token: %s", err)
|
||||
}
|
||||
|
||||
resp, err := cfg.client.Post(iamURL.String(), "application/json", bytes.NewBuffer(passport))
|
||||
body := bytes.NewBuffer(passport)
|
||||
resp, err := cfg.client.Post(iamURL, "application/json", body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed query yandex cloud iam api, url: %s, err: %w", iamURL.String(), err)
|
||||
logger.Panicf("BUG: cannot create request to yandex cloud iam api %q: %s", iamURL, err)
|
||||
}
|
||||
|
||||
r, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
data, err := readResponseBody(resp, iamURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read response from %q: %w", iamURL.String(), err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode)
|
||||
var it iamToken
|
||||
if err := json.Unmarshal(data, &it); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse iam token: %w; data: %s", err, data)
|
||||
}
|
||||
|
||||
it := iamToken{}
|
||||
if err := json.Unmarshal(r, &it); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse auth credentials response: %w", err)
|
||||
}
|
||||
|
||||
return &it, nil
|
||||
}
|
||||
|
||||
// getEndpoints makes services endpoints map:
|
||||
// https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints
|
||||
func (cfg *apiConfig) getEndpoints(apiEndpoint string) error {
|
||||
if apiEndpoint == "" {
|
||||
apiEndpoint = defaultAPIEndpoint
|
||||
}
|
||||
// iamToken represents Yandex Cloud IAM token response
|
||||
//
|
||||
// See https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create
|
||||
type iamToken struct {
|
||||
IAMToken string `json:"iamToken"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
}
|
||||
|
||||
// getServiceEndpoints returns services endpoints map
|
||||
//
|
||||
// See https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints
|
||||
func (cfg *apiConfig) getServiceEndpoints(apiEndpoint string) (map[string]string, error) {
|
||||
apiEndpointURL, err := url.Parse(apiEndpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse api_endpoint: %s as url, err: %w", apiEndpoint, err)
|
||||
return nil, fmt.Errorf("cannot parse api_endpoint %q: %w", apiEndpoint, err)
|
||||
}
|
||||
|
||||
apiEndpointURL.Path = path.Join(apiEndpointURL.Path, "endpoints")
|
||||
|
||||
resp, err := cfg.client.Get(apiEndpointURL.String())
|
||||
scheme := apiEndpointURL.Scheme
|
||||
if scheme == "" {
|
||||
return nil, fmt.Errorf("missing scheme in api_endpoint %q", apiEndpoint)
|
||||
}
|
||||
if apiEndpointURL.Host == "" {
|
||||
return nil, fmt.Errorf("missing host in api_endpoint %q", apiEndpoint)
|
||||
}
|
||||
endpointsURL := apiEndpoint + "/endpoints"
|
||||
resp, err := cfg.client.Get(endpointsURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed query endpoints, url: %s, err: %w", apiEndpointURL.String(), err)
|
||||
return nil, fmt.Errorf("cannot query %q: %w", endpointsURL, err)
|
||||
}
|
||||
r, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
data, err := readResponseBody(resp, endpointsURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read response from %q: %w", apiEndpointURL.String(), err)
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode)
|
||||
var eps endpoints
|
||||
if err := json.Unmarshal(data, &eps); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse API endpoints list: %w; data=%s", err, data)
|
||||
}
|
||||
m := make(map[string]string, len(eps.Endpoints))
|
||||
for _, endpoint := range eps.Endpoints {
|
||||
m[endpoint.ID] = scheme + "://" + endpoint.Address
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
endpoints, err := parseEndpoints(r)
|
||||
type endpoint struct {
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
type endpoints struct {
|
||||
Endpoints []endpoint `json:"endpoints"`
|
||||
}
|
||||
|
||||
// getAPIResponse calls Yandex Cloud apiURL and returns response body.
|
||||
func getAPIResponse(apiURL string, cfg *apiConfig) ([]byte, error) {
|
||||
creds, err := cfg.getFreshAPICredentials()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.serviceEndpoints = make(map[string]*url.URL, len(endpoints.Endpoints))
|
||||
for _, endpoint := range endpoints.Endpoints {
|
||||
cfg.serviceEndpoints[endpoint.ID] = &url.URL{
|
||||
Scheme: apiEndpointURL.Scheme,
|
||||
Host: endpoint.Address,
|
||||
}
|
||||
req, err := http.NewRequest("GET", apiURL, nil)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: cannot create new request for yandex cloud api url %s: %s", apiURL, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
req.Header.Set("Authorization", "Bearer "+creds.Token)
|
||||
resp, err := cfg.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query yandex cloud api url %s: %w", apiURL, err)
|
||||
}
|
||||
return readResponseBody(resp, apiURL)
|
||||
}
|
||||
|
||||
// readResponseBody reads body from http.Response.
|
||||
|
|
@ -243,30 +241,8 @@ func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
|
|||
return nil, fmt.Errorf("cannot read response from %q: %w", apiURL, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
|
||||
return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %s",
|
||||
apiURL, resp.StatusCode, http.StatusOK, data)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// getAPIResponse calls Yandex Cloud apiURL and returns response body.
|
||||
func getAPIResponse(apiURL string, cfg *apiConfig) ([]byte, error) {
|
||||
creds, err := cfg.getFreshAPICredentials()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create new request for yandex cloud api url %s: %w", apiURL, err)
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+creds.Token)
|
||||
resp, err := cfg.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query yandex cloud api url %s: %w", apiURL, err)
|
||||
}
|
||||
|
||||
return readResponseBody(resp, apiURL)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,11 +27,10 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instances = append(instances, inst...)
|
||||
}
|
||||
|
||||
logger.Infof("Collected %d instances", len(instances))
|
||||
logger.Infof("yandexcloud_sd: collected %d instances", len(instances))
|
||||
|
||||
return addInstanceLabels(instances), nil
|
||||
}
|
||||
|
|
@ -55,20 +54,20 @@ func addInstanceLabels(instances []instance) []map[string]string {
|
|||
m["__meta_yandexcloud_instance_label_"+discoveryutils.SanitizeLabelName(k)] = v
|
||||
}
|
||||
|
||||
for _, netInterface := range server.NetworkInterfaces {
|
||||
privateIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_ip_%s", netInterface.Index)
|
||||
m[privateIPLabel] = netInterface.PrimaryV4Address.Address
|
||||
if len(netInterface.PrimaryV4Address.OneToOneNat.Address) > 0 {
|
||||
publicIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_ip_%s", netInterface.Index)
|
||||
m[publicIPLabel] = netInterface.PrimaryV4Address.OneToOneNat.Address
|
||||
for _, ni := range server.NetworkInterfaces {
|
||||
privateIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_ip_%s", ni.Index)
|
||||
m[privateIPLabel] = ni.PrimaryV4Address.Address
|
||||
if len(ni.PrimaryV4Address.OneToOneNat.Address) > 0 {
|
||||
publicIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_ip_%s", ni.Index)
|
||||
m[publicIPLabel] = ni.PrimaryV4Address.OneToOneNat.Address
|
||||
}
|
||||
|
||||
for j, dnsRecord := range netInterface.PrimaryV4Address.DNSRecords {
|
||||
for j, dnsRecord := range ni.PrimaryV4Address.DNSRecords {
|
||||
dnsRecordLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_dns_%d", j)
|
||||
m[dnsRecordLabel] = dnsRecord.FQDN
|
||||
}
|
||||
|
||||
for j, dnsRecord := range netInterface.PrimaryV4Address.OneToOneNat.DNSRecords {
|
||||
for j, dnsRecord := range ni.PrimaryV4Address.OneToOneNat.DNSRecords {
|
||||
dnsRecordLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_dns_%d", j)
|
||||
m[dnsRecordLabel] = dnsRecord.FQDN
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,34 +2,10 @@ package yandexcloud
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type endpoint struct {
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
type endpoints struct {
|
||||
Endpoints []endpoint `json:"endpoints"`
|
||||
}
|
||||
|
||||
// See https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints
|
||||
func parseEndpoints(data []byte) (*endpoints, error) {
|
||||
var endpointsResponse endpoints
|
||||
if err := json.Unmarshal(data, &endpointsResponse); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse endpoints list: %w", err)
|
||||
}
|
||||
|
||||
if endpointsResponse.Endpoints == nil {
|
||||
return nil, errors.New("yandex cloud API endpoints list is empty")
|
||||
}
|
||||
|
||||
return &endpointsResponse, nil
|
||||
}
|
||||
|
||||
type organization struct {
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package yandexcloud
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"path"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
|
|
@ -36,142 +36,103 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
|
|||
}
|
||||
|
||||
func (cfg *apiConfig) getInstances(folderID string) ([]instance, error) {
|
||||
computeURL := *cfg.serviceEndpoints["compute"]
|
||||
computeURL.Path = path.Join(computeURL.Path, "compute", defaultAPIVersion, "instances")
|
||||
q := computeURL.Query()
|
||||
q.Set("folderId", folderID)
|
||||
computeURL.RawQuery = q.Encode()
|
||||
nextLink := computeURL.String()
|
||||
instancesURL := cfg.serviceEndpoints["compute"] + "/compute/v1/instances"
|
||||
instancesURL += "?folderId=" + url.QueryEscape(folderID)
|
||||
|
||||
instances := make([]instance, 0)
|
||||
var instances []instance
|
||||
nextLink := instancesURL
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
data, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot get instances: %w", err)
|
||||
}
|
||||
instancesPage, err := parseInstancesPage(resp)
|
||||
ip, err := parseInstancesPage(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot parse instances response from %q: %w; response body: %s", nextLink, err, data)
|
||||
}
|
||||
instances = append(instances, instancesPage.Instances...)
|
||||
if len(instancesPage.NextPageToken) == 0 {
|
||||
instances = append(instances, ip.Instances...)
|
||||
if len(ip.NextPageToken) == 0 {
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
q.Set("pageToken", instancesPage.NextPageToken)
|
||||
computeURL.RawQuery = q.Encode()
|
||||
nextLink = computeURL.String()
|
||||
nextLink = instancesURL + "&pageToken=" + url.QueryEscape(ip.NextPageToken)
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getFolders(clouds []cloud) ([]folder, error) {
|
||||
rmURL := *cfg.serviceEndpoints["resource-manager"]
|
||||
rmURL.Path = path.Join(rmURL.Path, "resource-manager", defaultAPIVersion, "folders")
|
||||
q := rmURL.Query()
|
||||
|
||||
folders := make([]folder, 0)
|
||||
foldersURL := cfg.serviceEndpoints["resource-manager"] + "/resource-manager/v1/folders"
|
||||
var folders []folder
|
||||
for _, cl := range clouds {
|
||||
q.Set("cloudId", cl.ID)
|
||||
rmURL.RawQuery = q.Encode()
|
||||
|
||||
nextLink := rmURL.String()
|
||||
cloudURL := foldersURL + "?cloudId=" + url.QueryEscape(cl.ID)
|
||||
nextLink := cloudURL
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
data, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot get folders: %w", err)
|
||||
}
|
||||
|
||||
foldersPage, err := parseFoldersPage(resp)
|
||||
fp, err := parseFoldersPage(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot parse folders response from %q: %w; response body: %s", nextLink, err, data)
|
||||
}
|
||||
|
||||
folders = append(folders, foldersPage.Folders...)
|
||||
|
||||
if len(foldersPage.NextPageToken) == 0 {
|
||||
folders = append(folders, fp.Folders...)
|
||||
if len(fp.NextPageToken) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
q.Set("pageToken", foldersPage.NextPageToken)
|
||||
rmURL.RawQuery = q.Encode()
|
||||
nextLink = rmURL.String()
|
||||
nextLink = cloudURL + "&pageToken=" + url.QueryEscape(fp.NextPageToken)
|
||||
}
|
||||
}
|
||||
|
||||
return folders, nil
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getClouds(organizations []organization) ([]cloud, error) {
|
||||
rmURL := *cfg.serviceEndpoints["resource-manager"]
|
||||
rmURL.Path = path.Join(rmURL.Path, "resource-manager", defaultAPIVersion, "clouds")
|
||||
q := rmURL.Query()
|
||||
|
||||
if len(organizations) == 0 {
|
||||
organizations = append(organizations, organization{
|
||||
func (cfg *apiConfig) getClouds(orgs []organization) ([]cloud, error) {
|
||||
cloudsURL := cfg.serviceEndpoints["resource-manager"] + "/resource-manager/v1/clouds"
|
||||
if len(orgs) == 0 {
|
||||
orgs = append(orgs, organization{
|
||||
ID: "",
|
||||
})
|
||||
}
|
||||
|
||||
clouds := make([]cloud, 0)
|
||||
for _, org := range organizations {
|
||||
var clouds []cloud
|
||||
for _, org := range orgs {
|
||||
orgURL := cloudsURL
|
||||
if org.ID != "" {
|
||||
q.Set("organizationId", org.ID)
|
||||
rmURL.RawQuery = q.Encode()
|
||||
orgURL += "?organizationId=" + url.QueryEscape(org.ID)
|
||||
}
|
||||
|
||||
nextLink := rmURL.String()
|
||||
nextLink := orgURL
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
data, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot get clouds: %w", err)
|
||||
}
|
||||
|
||||
cloudsPage, err := parseCloudsPage(resp)
|
||||
cp, err := parseCloudsPage(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot parse clouds response from %q: %w; response body: %s", nextLink, err, data)
|
||||
}
|
||||
|
||||
clouds = append(clouds, cloudsPage.Clouds...)
|
||||
|
||||
if len(cloudsPage.NextPageToken) == 0 {
|
||||
clouds = append(clouds, cp.Clouds...)
|
||||
if len(cp.NextPageToken) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
q.Set("pageToken", cloudsPage.NextPageToken)
|
||||
rmURL.RawQuery = q.Encode()
|
||||
nextLink = rmURL.String()
|
||||
nextLink = orgURL + "&pageToken=" + url.QueryEscape(cp.NextPageToken)
|
||||
}
|
||||
}
|
||||
|
||||
return clouds, nil
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getOrganizations() ([]organization, error) {
|
||||
omURL := *cfg.serviceEndpoints["organization-manager"]
|
||||
omURL.Path = path.Join(omURL.Path, "organization-manager", defaultAPIVersion, "organizations")
|
||||
q := omURL.Query()
|
||||
nextLink := omURL.String()
|
||||
|
||||
organizations := make([]organization, 0)
|
||||
orgsURL := cfg.serviceEndpoints["organization-manager"] + "/organization-manager/v1/organizations"
|
||||
var orgs []organization
|
||||
nextLink := orgsURL
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
data, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot get organizations: %w", err)
|
||||
}
|
||||
|
||||
organizationsPage, err := parseOrganizationsPage(resp)
|
||||
op, err := parseOrganizationsPage(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("cannot parse organizations response from %q: %w; response body: %s", nextLink, err, data)
|
||||
}
|
||||
|
||||
organizations = append(organizations, organizationsPage.Organizations...)
|
||||
|
||||
if len(organizationsPage.NextPageToken) == 0 {
|
||||
return organizations, nil
|
||||
orgs = append(orgs, op.Organizations...)
|
||||
if len(op.NextPageToken) == 0 {
|
||||
return orgs, nil
|
||||
}
|
||||
|
||||
q.Set("pageToken", organizationsPage.NextPageToken)
|
||||
omURL.RawQuery = q.Encode()
|
||||
nextLink = omURL.String()
|
||||
nextLink = orgsURL + "&pageToken=" + url.QueryEscape(op.NextPageToken)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue