diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5cb1cc45f0..6e5e309bfc 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -88,6 +88,7 @@ The previous behavior can be restored in the following ways:
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly calculate evaluation time for rules. Before, there was a low probability for discrepancy between actual time and rules evaluation time if evaluation interval was lower than the execution time for rules within the group.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): reset evaluation timestamp after modifying group interval. Before, there could have latency on rule evaluation time.
* BUGFIX: vmselect: fix timestamp alignment for Prometheus querying API if time argument is less than 10m from the beginning of Unix epoch.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): close HTTP connections to [service discovery](https://docs.victoriametrics.com/sd_configs.html) servers when they are no longer needed. This should prevent from possible connection exhasution in some cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not show [relabel debug](https://docs.victoriametrics.com/vmagent.html#relabel-debug) links at the `/targets` page when `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag, since it has no the original labels needed for relabel debug. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4597).
* BUGFIX: vminsert: fixed decoding of label values with slash when accepting data via [pushgateway protocol](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format). This fixes Prometheus golang client compatibility. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4692).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly parse binary operations with reserved words on the right side such as `foo + (on{bar="baz"})`. Previously such queries could lead to panic. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4422).
diff --git a/lib/promscrape/discovery/azure/azure.go b/lib/promscrape/discovery/azure/azure.go
index b1fe35fe96..f9ec794e2d 100644
--- a/lib/promscrape/discovery/azure/azure.go
+++ b/lib/promscrape/discovery/azure/azure.go
@@ -58,7 +58,11 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.c.Stop()
+ }
}
func appendMachineLabels(vms []virtualMachine, port int, sdc *SDConfig) []*promutils.Labels {
diff --git a/lib/promscrape/discovery/azure/machine_test.go b/lib/promscrape/discovery/azure/machine_test.go
index e323c9202c..bb13a2f5af 100644
--- a/lib/promscrape/discovery/azure/machine_test.go
+++ b/lib/promscrape/discovery/azure/machine_test.go
@@ -71,6 +71,7 @@ func TestGetVirtualMachinesSuccess(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error at client create: %s", err)
}
+ defer c.Stop()
ac := &apiConfig{
c: c,
subscriptionID: "some-id",
diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go
index 630a31d277..8d815a39f9 100644
--- a/lib/promscrape/discovery/consul/api.go
+++ b/lib/promscrape/discovery/consul/api.go
@@ -90,6 +90,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
}
dc, err := getDatacenter(client, sdc.Datacenter)
if err != nil {
+ client.Stop()
return nil, fmt.Errorf("cannot obtain consul datacenter: %w", err)
}
diff --git a/lib/promscrape/discovery/consulagent/api.go b/lib/promscrape/discovery/consulagent/api.go
index a859bcf931..ba773ac182 100644
--- a/lib/promscrape/discovery/consulagent/api.go
+++ b/lib/promscrape/discovery/consulagent/api.go
@@ -84,6 +84,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
}
agent, err := consul.GetAgentInfo(client)
if err != nil {
+ client.Stop()
return nil, fmt.Errorf("cannot obtain consul datacenter: %w", err)
}
dc := sdc.Datacenter
diff --git a/lib/promscrape/discovery/digitalocean/digitalocean.go b/lib/promscrape/discovery/digitalocean/digitalocean.go
index cf25f2da3c..aa4534c587 100644
--- a/lib/promscrape/discovery/digitalocean/digitalocean.go
+++ b/lib/promscrape/discovery/digitalocean/digitalocean.go
@@ -155,5 +155,9 @@ func addDropletLabels(droplets []droplet, defaultPort int) []*promutils.Labels {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.Stop()
+ }
}
diff --git a/lib/promscrape/discovery/docker/docker.go b/lib/promscrape/discovery/docker/docker.go
index 6be9086541..ade73572f9 100644
--- a/lib/promscrape/discovery/docker/docker.go
+++ b/lib/promscrape/discovery/docker/docker.go
@@ -47,5 +47,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.Stop()
+ }
}
diff --git a/lib/promscrape/discovery/dockerswarm/dockerswarm.go b/lib/promscrape/discovery/dockerswarm/dockerswarm.go
index bfe7a06971..f527a51fc1 100644
--- a/lib/promscrape/discovery/dockerswarm/dockerswarm.go
+++ b/lib/promscrape/discovery/dockerswarm/dockerswarm.go
@@ -56,5 +56,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.Stop()
+ }
}
diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go
index 4b73cf81c4..e677e1ef23 100644
--- a/lib/promscrape/discovery/eureka/eureka.go
+++ b/lib/promscrape/discovery/eureka/eureka.go
@@ -101,7 +101,11 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.Stop()
+ }
}
func addInstanceLabels(apps *applications) []*promutils.Labels {
diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go
index 8fb9d76ecc..c2953ebfcf 100644
--- a/lib/promscrape/discovery/gce/api.go
+++ b/lib/promscrape/discovery/gce/api.go
@@ -42,6 +42,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
if len(project) == 0 {
proj, err := getCurrentProject()
if err != nil {
+ client.CloseIdleConnections()
return nil, fmt.Errorf("cannot determine the current project; make sure `vmagent` runs inside GCE; error: %w", err)
}
project = proj
@@ -52,6 +53,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
// Autodetect the current zone.
zone, err := getCurrentZone()
if err != nil {
+ client.CloseIdleConnections()
return nil, fmt.Errorf("cannot determine the current zone; make sure `vmagent` runs inside GCE; error: %w", err)
}
zones = append(zones, zone)
@@ -62,6 +64,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3202
zs, err := getZonesForProject(client, project)
if err != nil {
+ client.CloseIdleConnections()
return nil, fmt.Errorf("cannot obtain zones for project %q: %w", project, err)
}
zones = zs
diff --git a/lib/promscrape/discovery/gce/gce.go b/lib/promscrape/discovery/gce/gce.go
index 70ece8a04f..a52e359626 100644
--- a/lib/promscrape/discovery/gce/gce.go
+++ b/lib/promscrape/discovery/gce/gce.go
@@ -73,5 +73,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.CloseIdleConnections()
+ }
}
diff --git a/lib/promscrape/discovery/http/http.go b/lib/promscrape/discovery/http/http.go
index 23f4a5c013..479895131b 100644
--- a/lib/promscrape/discovery/http/http.go
+++ b/lib/promscrape/discovery/http/http.go
@@ -57,5 +57,9 @@ func addHTTPTargetLabels(src []httpGroupTarget, sourceURL string) []*promutils.L
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.Stop()
+ }
}
diff --git a/lib/promscrape/discovery/kuma/api.go b/lib/promscrape/discovery/kuma/api.go
index ce93572d07..26b62f0812 100644
--- a/lib/promscrape/discovery/kuma/api.go
+++ b/lib/promscrape/discovery/kuma/api.go
@@ -80,6 +80,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
// The synchronous targets' update is needed for returning non-empty list of targets
// just after the initialization.
if err := cfg.updateTargetsLabels(ctx); err != nil {
+ client.Stop()
return nil, fmt.Errorf("cannot discover Kuma targets: %w", err)
}
cfg.wg.Add(1)
diff --git a/lib/promscrape/discovery/openstack/api.go b/lib/promscrape/discovery/openstack/api.go
index c946a08030..fa04aadbda 100644
--- a/lib/promscrape/discovery/openstack/api.go
+++ b/lib/promscrape/discovery/openstack/api.go
@@ -91,6 +91,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
}
ac, err := opts.NewConfig()
if err != nil {
+ cfg.client.CloseIdleConnections()
return nil, err
}
cfg.client.Transport = &http.Transport{
@@ -111,6 +112,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
sdcAuth = readCredentialsFromEnv()
}
if strings.HasSuffix(sdcAuth.IdentityEndpoint, "v2.0") {
+ cfg.client.CloseIdleConnections()
return nil, errors.New("identity_endpoint v2.0 is not supported")
}
// trim .0 from v3.0 for prometheus cfg compatibility
@@ -118,11 +120,13 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
parsedURL, err := url.Parse(sdcAuth.IdentityEndpoint)
if err != nil {
+ cfg.client.CloseIdleConnections()
return nil, fmt.Errorf("cannot parse identity_endpoint: %s as url, err: %w", sdcAuth.IdentityEndpoint, err)
}
cfg.endpoint = parsedURL
tokenReq, err := buildAuthRequestBody(&sdcAuth)
if err != nil {
+ cfg.client.CloseIdleConnections()
return nil, err
}
cfg.authTokenReq = tokenReq
diff --git a/lib/promscrape/discovery/openstack/openstack.go b/lib/promscrape/discovery/openstack/openstack.go
index df26b92b1c..e380f66f8d 100644
--- a/lib/promscrape/discovery/openstack/openstack.go
+++ b/lib/promscrape/discovery/openstack/openstack.go
@@ -57,5 +57,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
// MustStop stops further usage for sdc.
func (sdc *SDConfig) MustStop() {
- configMap.Delete(sdc)
+ v := configMap.Delete(sdc)
+ if v != nil {
+ cfg := v.(*apiConfig)
+ cfg.client.CloseIdleConnections()
+ }
}
diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go
index ad54f3bebc..161c207af1 100644
--- a/lib/promscrape/discoveryutils/client.go
+++ b/lib/promscrape/discoveryutils/client.go
@@ -81,6 +81,12 @@ type HTTPClient struct {
ReadTimeout time.Duration
}
+func (hc *HTTPClient) stop() {
+ // Close idle connections to server in order to free up resources.
+ // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4724
+ hc.client.CloseIdleConnections()
+}
+
var defaultDialer = &net.Dialer{}
// NewClient returns new Client for the given args.
@@ -276,6 +282,8 @@ func (c *Client) APIServer() string {
// Stop cancels all in-flight requests
func (c *Client) Stop() {
c.clientCancel()
+ c.client.stop()
+ c.blockingClient.stop()
}
func doRequestWithPossibleRetry(hc *HTTPClient, req *http.Request) (*http.Response, error) {