From fe2d9f6646a49c347b1ee03a0285971eaf681ed6 Mon Sep 17 00:00:00 2001 From: zhdd99 Date: Tue, 9 Jan 2024 13:01:03 +0100 Subject: [PATCH] lib/pushmetrics: fix a panic caused by pushing metrics during the graceful shutdown process of vmstorage nodes. (#5549) Co-authored-by: zhangdongdong Co-authored-by: Roman Khavronenko Signed-off-by: hagen1778 --- lib/pushmetrics/pushmetrics.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index ebb17bea4..46d2be18e 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -28,6 +28,11 @@ func init() { flagutil.RegisterSecretFlag("pushmetrics.url") } +var ( + // create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown. + pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background()) +) + // Init must be called after logger.Init func Init() { extraLabels := strings.Join(*pushExtraLabel, ",") @@ -37,8 +42,12 @@ func Init() { Headers: *pushHeader, DisableCompression: *disableCompression, } - if err := metrics.InitPushExtWithOptions(context.Background(), pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { + if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) } } } + +func StopPushMetrics() { + cancelPushMetric() +}