From 66bafee6518361c70caf56c72111f9552cef877f Mon Sep 17 00:00:00 2001 From: zhdd99 Date: Tue, 9 Jan 2024 13:01:03 +0100 Subject: [PATCH] lib/pushmetrics: fix a panic caused by pushing metrics during the graceful shutdown process of vmstorage nodes. (#5549) Co-authored-by: zhangdongdong Co-authored-by: Roman Khavronenko Signed-off-by: hagen1778 --- lib/pushmetrics/pushmetrics.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index a6d2dd0b5..324362b36 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -24,12 +24,24 @@ func init() { flagutil.RegisterSecretFlag("pushmetrics.url") } +var ( + // create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown. + pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background()) +) + // Init must be called after logger.Init func Init() { extraLabels := strings.Join(*pushExtraLabel, ",") for _, pu := range *pushURL { - if err := metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics); err != nil { + opts := &metrics.PushOptions{ + ExtraLabels: extraLabels, + } + if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) } } } + +func StopPushMetrics() { + cancelPushMetric() +}