From 202e5704e64ac965bf28cc396cadae2ac92b1581 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 15 Jul 2024 10:39:05 +0200 Subject: [PATCH] vendor: update github.com/VictoriaMetrics/metrics from v1.34.1 to v1.35.0 Fix potential memory leaks across VictoriaMetrics codebase after metrics.UnregisterSet(s) call because of missing s.UnregisterAllMetrics() call. This is a follow-up for 6a6e34ab8ec752ae9dfd03e0101625ffb242b9b5 . It is OK if some vmauth metrics aren't visible for a few microseconds when the previous metrics are unregistered and new metrics weren't registered yet. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4690 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6252 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5805 --- app/vlstorage/main.go | 2 +- app/vmauth/auth_config.go | 15 ++++----------- app/vmstorage/main.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- lib/streamaggr/deduplicator.go | 2 +- lib/streamaggr/streamaggr.go | 2 +- .../github.com/VictoriaMetrics/metrics/metrics.go | 9 +++++++-- vendor/modules.txt | 2 +- 9 files changed, 19 insertions(+), 21 deletions(-) diff --git a/app/vlstorage/main.go b/app/vlstorage/main.go index 3ea3db668..628919ce5 100644 --- a/app/vlstorage/main.go +++ b/app/vlstorage/main.go @@ -80,7 +80,7 @@ func Init() { // Stop stops vlstorage. func Stop() { - metrics.UnregisterSet(storageMetrics) + metrics.UnregisterSet(storageMetrics, true) storageMetrics = nil strg.MustClose() diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 18788c5c6..ce16a1558 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -696,22 +696,15 @@ func loadAuthConfig() (bool, error) { } logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath) - prevAc := authConfig.Load() - if prevAc != nil { - metrics.UnregisterSet(prevAc.ms) + acPrev := authConfig.Load() + if acPrev != nil { + metrics.UnregisterSet(acPrev.ms, true) } metrics.RegisterSet(ac.ms) + authConfig.Store(ac) authConfigData.Store(&data) authUsers.Store(&m) - if prevAc != nil { - // explicilty unregister metrics, since all summary type metrics - // are registered at global state of metrics package - // and must be removed from it to release memory. - // Metrics must be unregistered only after atomic.Value.Store calls above - // Otherwise it may lead to metric gaps, since UnregisterAllMetrics is slow operation - prevAc.ms.UnregisterAllMetrics() - } return true, nil } diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index c40ab47bd..e37c321e7 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -240,7 +240,7 @@ func GetSeriesCount(deadline uint64) (uint64, error) { // Stop stops the vmstorage func Stop() { // deregister storage metrics - metrics.UnregisterSet(storageMetrics) + metrics.UnregisterSet(storageMetrics, true) storageMetrics = nil logger.Infof("gracefully closing the storage at %s", *DataPath) diff --git a/go.mod b/go.mod index eeaa83abc..406f24406 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/VictoriaMetrics/easyproto v0.1.4 github.com/VictoriaMetrics/fastcache v1.12.2 - github.com/VictoriaMetrics/metrics v1.34.1 + github.com/VictoriaMetrics/metrics v1.35.0 github.com/VictoriaMetrics/metricsql v0.76.0 github.com/aws/aws-sdk-go-v2 v1.30.1 github.com/aws/aws-sdk-go-v2/config v1.27.24 diff --git a/go.sum b/go.sum index cc4b286ed..172b127d7 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/VictoriaMetrics/metrics v1.34.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= -github.com/VictoriaMetrics/metrics v1.34.1 h1:7EUEObv45ekfyY6PWat0K/ytluZ4q6aujzXN3g41g/A= -github.com/VictoriaMetrics/metrics v1.34.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= +github.com/VictoriaMetrics/metrics v1.35.0 h1:xWImz8UTwyhGZAzueTHFgYe4bnKbXCaLWBq2JBj7EzI= +github.com/VictoriaMetrics/metrics v1.35.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= github.com/VictoriaMetrics/metricsql v0.76.0 h1:hl7vqJqyH2d8zKImzalkFrkFiD5q4ACF8gl3s86DqKA= github.com/VictoriaMetrics/metricsql v0.76.0/go.mod h1:1g4hdCwlbJZ851PU9VN65xy9Rdlzupo6fx3SNZ8Z64U= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= diff --git a/lib/streamaggr/deduplicator.go b/lib/streamaggr/deduplicator.go index c49580dcc..ff7d3ddaa 100644 --- a/lib/streamaggr/deduplicator.go +++ b/lib/streamaggr/deduplicator.go @@ -71,7 +71,7 @@ func NewDeduplicator(pushFunc PushFunc, dedupInterval time.Duration, dropLabels // MustStop stops d. func (d *Deduplicator) MustStop() { - metrics.UnregisterSet(d.ms) + metrics.UnregisterSet(d.ms, true) d.ms = nil close(d.stopCh) diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index efdcb5baf..b28914b3d 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -323,7 +323,7 @@ func (a *Aggregators) MustStop() { return } - metrics.UnregisterSet(a.ms) + metrics.UnregisterSet(a.ms, true) a.ms = nil for _, aggr := range a.as { diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go index 51f236e32..74e97352c 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -55,11 +55,16 @@ func RegisterSet(s *Set) { // UnregisterSet stops exporting metrics for the given s via global WritePrometheus() call. // -// Call s.UnregisterAllMetrics() after unregistering s if it is no longer used. -func UnregisterSet(s *Set) { +// If destroySet is set to true, then s.UnregisterAllMetrics() is called on s after unregistering it, +// so s becomes destroyed. Otherwise the s can be registered again in the set by passing it to RegisterSet(). +func UnregisterSet(s *Set, destroySet bool) { registeredSetsLock.Lock() delete(registeredSets, s) registeredSetsLock.Unlock() + + if destroySet { + s.UnregisterAllMetrics() + } } // RegisterMetricsWriter registers writeMetrics callback for including metrics in the output generated by WritePrometheus. diff --git a/vendor/modules.txt b/vendor/modules.txt index 1950fd32b..2cf84791e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -115,7 +115,7 @@ github.com/VictoriaMetrics/easyproto # github.com/VictoriaMetrics/fastcache v1.12.2 ## explicit; go 1.13 github.com/VictoriaMetrics/fastcache -# github.com/VictoriaMetrics/metrics v1.34.1 +# github.com/VictoriaMetrics/metrics v1.35.0 ## explicit; go 1.17 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.76.0