diff --git a/app/victoria-logs/main.go b/app/victoria-logs/main.go
index 5cf3add9a8..62d952c07a 100644
--- a/app/victoria-logs/main.go
+++ b/app/victoria-logs/main.go
@@ -59,6 +59,8 @@ func main() {
}
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+ pushmetrics.Stop()
+
vlinsert.Stop()
vlselect.Stop()
vlstorage.Stop()
diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go
index 6a752919aa..b61ee7f24a 100644
--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -89,6 +89,7 @@ func main() {
if err := httpserver.Stop(*httpListenAddr); err != nil {
logger.Fatalf("cannot stop the webservice: %s", err)
}
+ pushmetrics.Stop()
vminsert.Stop()
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index 32c11614bd..151e957c4e 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -158,6 +158,7 @@ func main() {
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
}
+ pushmetrics.Stop()
promscrape.Stop()
if len(*influxListenAddr) > 0 {
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index ea02ad4061..bbf22dc842 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -195,6 +195,7 @@ func main() {
if err := httpserver.Stop(*httpListenAddr); err != nil {
logger.Fatalf("cannot stop the webservice: %s", err)
}
+ pushmetrics.Stop()
cancel()
manager.close()
}
diff --git a/app/vmauth/main.go b/app/vmauth/main.go
index 36a5301dd3..17b1baf27b 100644
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -67,6 +67,7 @@ func main() {
if err := httpserver.Stop(*httpListenAddr); err != nil {
logger.Fatalf("cannot stop the webservice: %s", err)
}
+ pushmetrics.Stop()
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
stopAuthConfig()
logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go
index f42dfa73fc..9b3f79996f 100644
--- a/app/vmbackup/main.go
+++ b/app/vmbackup/main.go
@@ -109,6 +109,7 @@ func main() {
if err := httpserver.Stop(*httpListenAddr); err != nil {
logger.Fatalf("cannot stop http server for metrics: %s", err)
}
+ pushmetrics.Stop()
logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
}
diff --git a/app/vmrestore/main.go b/app/vmrestore/main.go
index cb2efcd79d..799d56ba7e 100644
--- a/app/vmrestore/main.go
+++ b/app/vmrestore/main.go
@@ -65,6 +65,7 @@ func main() {
if err := httpserver.Stop(*httpListenAddr); err != nil {
logger.Fatalf("cannot stop http server for metrics: %s", err)
}
+ pushmetrics.Stop()
logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
}
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e9016da33b..8f1e1bdfe5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -18,6 +18,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly return results from [bottomk](https://docs.victoriametrics.com/MetricsQL.html#bottomk) and `bottomk_*()` functions when some of these results contain NaN values. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5506). Thanks to @xiaozongyang for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5509).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle queries, which wrap [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) with multiple arguments without explicitly specified lookbehind window in square brackets into [aggregate functions](https://docs.victoriametrics.com/MetricsQL.html#aggregate-functions). For example, `sum(quantile_over_time(0.5, process_resident_memory_bytes))` was resulting to `expecting at least 2 args to ...; got 1 args` error. Thanks to @atykhyy for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5414).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details.
+* BUGFIX: all: fix potential panic during components shutdown when `-pushmetrics.url` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): check for Error field in response from influx client during migration. Before, only network errors were checked. Thanks to @wozz for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5446).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried.
diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go
index 324362b367..5528d80d36 100644
--- a/lib/pushmetrics/pushmetrics.go
+++ b/lib/pushmetrics/pushmetrics.go
@@ -25,8 +25,7 @@ func init() {
}
var (
- // create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown.
- pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background())
+ pushCtx, cancelPushCtx = context.WithCancel(context.Background())
)
// Init must be called after logger.Init
@@ -36,12 +35,17 @@ func Init() {
opts := &metrics.PushOptions{
ExtraLabels: extraLabels,
}
- if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
+ if err := metrics.InitPushExtWithOptions(pushCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
logger.Fatalf("cannot initialize pushmetrics: %s", err)
}
}
}
-func StopPushMetrics() {
- cancelPushMetric()
+// Stop stops the periodic push of metrics.
+// It is important to stop the push of metrics before disposing resources
+// these metrics attached to. See related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548
+//
+// Stop must be called after Init.
+func Stop() {
+ cancelPushCtx()
}