From 91ccea236f62e07ead2ea71affc661c350cb5f65 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 9 Jan 2024 13:17:09 +0100 Subject: [PATCH] app/all: follow-up after https://github.com/VictoriaMetrics/VictoriaMetrics/commit/84d710beab98067e88badbe4af4e0b523e2f6076 https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548 Signed-off-by: hagen1778 --- app/victoria-logs/main.go | 2 ++ app/victoria-metrics/main.go | 1 + app/vmagent/main.go | 1 + app/vmalert/main.go | 1 + app/vmauth/main.go | 1 + app/vmbackup/main.go | 1 + app/vmrestore/main.go | 1 + docs/CHANGELOG.md | 1 + lib/pushmetrics/pushmetrics.go | 14 +++++++++----- 9 files changed, 18 insertions(+), 5 deletions(-) diff --git a/app/victoria-logs/main.go b/app/victoria-logs/main.go index 5cf3add9a..62d952c07 100644 --- a/app/victoria-logs/main.go +++ b/app/victoria-logs/main.go @@ -59,6 +59,8 @@ func main() { } logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) + pushmetrics.Stop() + vlinsert.Stop() vlselect.Stop() vlstorage.Stop() diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go index 149d13501..0639fbd2f 100644 --- a/app/victoria-metrics/main.go +++ b/app/victoria-metrics/main.go @@ -89,6 +89,7 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } + pushmetrics.Stop() vminsert.Stop() logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 9f66d54e8..c681a84fa 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -159,6 +159,7 @@ func main() { logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) } + pushmetrics.Stop() promscrape.Stop() if len(*influxListenAddr) > 0 { diff --git a/app/vmalert/main.go b/app/vmalert/main.go index a87897204..be8f0c1bd 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -187,6 +187,7 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } + pushmetrics.Stop() cancel() manager.close() } diff --git a/app/vmauth/main.go b/app/vmauth/main.go index b20661977..df9eeda00 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -80,6 +80,7 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } + pushmetrics.Stop() logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) stopAuthConfig() logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds()) diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go index 40c5c8bf4..2ce83d476 100644 --- a/app/vmbackup/main.go +++ b/app/vmbackup/main.go @@ -107,6 +107,7 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop http server for metrics: %s", err) } + pushmetrics.Stop() logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds()) } diff --git a/app/vmrestore/main.go b/app/vmrestore/main.go index cb2efcd79..799d56ba7 100644 --- a/app/vmrestore/main.go +++ b/app/vmrestore/main.go @@ -65,6 +65,7 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop http server for metrics: %s", err) } + pushmetrics.Stop() logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds()) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b23aa6d82..7fd23788d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -51,6 +51,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a link for the statistic inaccuracy explanation in the cardinality explorer tool. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5460). +* BUGFIX: all: fix potential panic during components shutdown when `-pushmetrics.url` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549). ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0) diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index 46d2be18e..0e5ddff3f 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -29,8 +29,7 @@ func init() { } var ( - // create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown. - pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background()) + pushCtx, cancelPushCtx = context.WithCancel(context.Background()) ) // Init must be called after logger.Init @@ -42,12 +41,17 @@ func Init() { Headers: *pushHeader, DisableCompression: *disableCompression, } - if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { + if err := metrics.InitPushExtWithOptions(pushCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) } } } -func StopPushMetrics() { - cancelPushMetric() +// Stop stops the periodic push of metrics. +// It is important to stop the push of metrics before disposing resources +// these metrics attached to. See related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548 +// +// Stop must be called after Init. +func Stop() { + cancelPushCtx() }