diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5cb1cc45f..6e5e309bf 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -88,6 +88,7 @@ The previous behavior can be restored in the following ways: * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly calculate evaluation time for rules. Before, there was a low probability for discrepancy between actual time and rules evaluation time if evaluation interval was lower than the execution time for rules within the group. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): reset evaluation timestamp after modifying group interval. Before, there could have latency on rule evaluation time. * BUGFIX: vmselect: fix timestamp alignment for Prometheus querying API if time argument is less than 10m from the beginning of Unix epoch. +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): close HTTP connections to [service discovery](https://docs.victoriametrics.com/sd_configs.html) servers when they are no longer needed. This should prevent from possible connection exhasution in some cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not show [relabel debug](https://docs.victoriametrics.com/vmagent.html#relabel-debug) links at the `/targets` page when `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag, since it has no the original labels needed for relabel debug. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4597). * BUGFIX: vminsert: fixed decoding of label values with slash when accepting data via [pushgateway protocol](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format). This fixes Prometheus golang client compatibility. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4692). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly parse binary operations with reserved words on the right side such as `foo + (on{bar="baz"})`. Previously such queries could lead to panic. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4422). diff --git a/lib/promscrape/discovery/azure/azure.go b/lib/promscrape/discovery/azure/azure.go index b1fe35fe9..f9ec794e2 100644 --- a/lib/promscrape/discovery/azure/azure.go +++ b/lib/promscrape/discovery/azure/azure.go @@ -58,7 +58,11 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.c.Stop() + } } func appendMachineLabels(vms []virtualMachine, port int, sdc *SDConfig) []*promutils.Labels { diff --git a/lib/promscrape/discovery/azure/machine_test.go b/lib/promscrape/discovery/azure/machine_test.go index e323c9202..bb13a2f5a 100644 --- a/lib/promscrape/discovery/azure/machine_test.go +++ b/lib/promscrape/discovery/azure/machine_test.go @@ -71,6 +71,7 @@ func TestGetVirtualMachinesSuccess(t *testing.T) { if err != nil { t.Fatalf("unexpected error at client create: %s", err) } + defer c.Stop() ac := &apiConfig{ c: c, subscriptionID: "some-id", diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go index 630a31d27..8d815a39f 100644 --- a/lib/promscrape/discovery/consul/api.go +++ b/lib/promscrape/discovery/consul/api.go @@ -90,6 +90,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } dc, err := getDatacenter(client, sdc.Datacenter) if err != nil { + client.Stop() return nil, fmt.Errorf("cannot obtain consul datacenter: %w", err) } diff --git a/lib/promscrape/discovery/consulagent/api.go b/lib/promscrape/discovery/consulagent/api.go index a859bcf93..ba773ac18 100644 --- a/lib/promscrape/discovery/consulagent/api.go +++ b/lib/promscrape/discovery/consulagent/api.go @@ -84,6 +84,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } agent, err := consul.GetAgentInfo(client) if err != nil { + client.Stop() return nil, fmt.Errorf("cannot obtain consul datacenter: %w", err) } dc := sdc.Datacenter diff --git a/lib/promscrape/discovery/digitalocean/digitalocean.go b/lib/promscrape/discovery/digitalocean/digitalocean.go index cf25f2da3..aa4534c58 100644 --- a/lib/promscrape/discovery/digitalocean/digitalocean.go +++ b/lib/promscrape/discovery/digitalocean/digitalocean.go @@ -155,5 +155,9 @@ func addDropletLabels(droplets []droplet, defaultPort int) []*promutils.Labels { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.Stop() + } } diff --git a/lib/promscrape/discovery/docker/docker.go b/lib/promscrape/discovery/docker/docker.go index 6be908654..ade73572f 100644 --- a/lib/promscrape/discovery/docker/docker.go +++ b/lib/promscrape/discovery/docker/docker.go @@ -47,5 +47,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.Stop() + } } diff --git a/lib/promscrape/discovery/dockerswarm/dockerswarm.go b/lib/promscrape/discovery/dockerswarm/dockerswarm.go index bfe7a0697..f527a51fc 100644 --- a/lib/promscrape/discovery/dockerswarm/dockerswarm.go +++ b/lib/promscrape/discovery/dockerswarm/dockerswarm.go @@ -56,5 +56,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.Stop() + } } diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go index 4b73cf81c..e677e1ef2 100644 --- a/lib/promscrape/discovery/eureka/eureka.go +++ b/lib/promscrape/discovery/eureka/eureka.go @@ -101,7 +101,11 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.Stop() + } } func addInstanceLabels(apps *applications) []*promutils.Labels { diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go index 8fb9d76ec..c2953ebfc 100644 --- a/lib/promscrape/discovery/gce/api.go +++ b/lib/promscrape/discovery/gce/api.go @@ -42,6 +42,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { if len(project) == 0 { proj, err := getCurrentProject() if err != nil { + client.CloseIdleConnections() return nil, fmt.Errorf("cannot determine the current project; make sure `vmagent` runs inside GCE; error: %w", err) } project = proj @@ -52,6 +53,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { // Autodetect the current zone. zone, err := getCurrentZone() if err != nil { + client.CloseIdleConnections() return nil, fmt.Errorf("cannot determine the current zone; make sure `vmagent` runs inside GCE; error: %w", err) } zones = append(zones, zone) @@ -62,6 +64,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3202 zs, err := getZonesForProject(client, project) if err != nil { + client.CloseIdleConnections() return nil, fmt.Errorf("cannot obtain zones for project %q: %w", project, err) } zones = zs diff --git a/lib/promscrape/discovery/gce/gce.go b/lib/promscrape/discovery/gce/gce.go index 70ece8a04..a52e35962 100644 --- a/lib/promscrape/discovery/gce/gce.go +++ b/lib/promscrape/discovery/gce/gce.go @@ -73,5 +73,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.CloseIdleConnections() + } } diff --git a/lib/promscrape/discovery/http/http.go b/lib/promscrape/discovery/http/http.go index 23f4a5c01..479895131 100644 --- a/lib/promscrape/discovery/http/http.go +++ b/lib/promscrape/discovery/http/http.go @@ -57,5 +57,9 @@ func addHTTPTargetLabels(src []httpGroupTarget, sourceURL string) []*promutils.L // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.Stop() + } } diff --git a/lib/promscrape/discovery/kuma/api.go b/lib/promscrape/discovery/kuma/api.go index ce93572d0..26b62f081 100644 --- a/lib/promscrape/discovery/kuma/api.go +++ b/lib/promscrape/discovery/kuma/api.go @@ -80,6 +80,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { // The synchronous targets' update is needed for returning non-empty list of targets // just after the initialization. if err := cfg.updateTargetsLabels(ctx); err != nil { + client.Stop() return nil, fmt.Errorf("cannot discover Kuma targets: %w", err) } cfg.wg.Add(1) diff --git a/lib/promscrape/discovery/openstack/api.go b/lib/promscrape/discovery/openstack/api.go index c946a0803..fa04aadbd 100644 --- a/lib/promscrape/discovery/openstack/api.go +++ b/lib/promscrape/discovery/openstack/api.go @@ -91,6 +91,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } ac, err := opts.NewConfig() if err != nil { + cfg.client.CloseIdleConnections() return nil, err } cfg.client.Transport = &http.Transport{ @@ -111,6 +112,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { sdcAuth = readCredentialsFromEnv() } if strings.HasSuffix(sdcAuth.IdentityEndpoint, "v2.0") { + cfg.client.CloseIdleConnections() return nil, errors.New("identity_endpoint v2.0 is not supported") } // trim .0 from v3.0 for prometheus cfg compatibility @@ -118,11 +120,13 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { parsedURL, err := url.Parse(sdcAuth.IdentityEndpoint) if err != nil { + cfg.client.CloseIdleConnections() return nil, fmt.Errorf("cannot parse identity_endpoint: %s as url, err: %w", sdcAuth.IdentityEndpoint, err) } cfg.endpoint = parsedURL tokenReq, err := buildAuthRequestBody(&sdcAuth) if err != nil { + cfg.client.CloseIdleConnections() return nil, err } cfg.authTokenReq = tokenReq diff --git a/lib/promscrape/discovery/openstack/openstack.go b/lib/promscrape/discovery/openstack/openstack.go index df26b92b1..e380f66f8 100644 --- a/lib/promscrape/discovery/openstack/openstack.go +++ b/lib/promscrape/discovery/openstack/openstack.go @@ -57,5 +57,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { // MustStop stops further usage for sdc. func (sdc *SDConfig) MustStop() { - configMap.Delete(sdc) + v := configMap.Delete(sdc) + if v != nil { + cfg := v.(*apiConfig) + cfg.client.CloseIdleConnections() + } } diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go index ad54f3beb..161c207af 100644 --- a/lib/promscrape/discoveryutils/client.go +++ b/lib/promscrape/discoveryutils/client.go @@ -81,6 +81,12 @@ type HTTPClient struct { ReadTimeout time.Duration } +func (hc *HTTPClient) stop() { + // Close idle connections to server in order to free up resources. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724 + hc.client.CloseIdleConnections() +} + var defaultDialer = &net.Dialer{} // NewClient returns new Client for the given args. @@ -276,6 +282,8 @@ func (c *Client) APIServer() string { // Stop cancels all in-flight requests func (c *Client) Stop() { c.clientCancel() + c.client.stop() + c.blockingClient.stop() } func doRequestWithPossibleRetry(hc *HTTPClient, req *http.Request) (*http.Response, error) {