From 7883248bf3da6e6d60b2b42b3210423990c2d5ef Mon Sep 17 00:00:00 2001 From: zhdd99 Date: Tue, 9 Jan 2024 20:01:03 +0800 Subject: [PATCH] lib/pushmetrics: fix a panic caused by pushing metrics during the graceful shutdown process of vmstorage nodes. (#5549) Co-authored-by: zhangdongdong Co-authored-by: Roman Khavronenko --- app/vmstorage/main.go | 4 ++++ lib/pushmetrics/pushmetrics.go | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index b38042936..82c0c843d 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -144,6 +144,10 @@ func main() { } logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds()) + // close the metric reporting goroutine + pushmetrics.StopPushMetrics() + logger.Infof("successfully stop push metric in %.3f seconds", time.Since(startTime).Seconds()) + logger.Infof("gracefully shutting down the service") startTime = time.Now() stopStaleSnapshotsRemover() diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index a6d2dd0b5..324362b36 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -24,12 +24,24 @@ func init() { flagutil.RegisterSecretFlag("pushmetrics.url") } +var ( + // create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown. + pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background()) +) + // Init must be called after logger.Init func Init() { extraLabels := strings.Join(*pushExtraLabel, ",") for _, pu := range *pushURL { - if err := metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics); err != nil { + opts := &metrics.PushOptions{ + ExtraLabels: extraLabels, + } + if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) } } } + +func StopPushMetrics() { + cancelPushMetric() +}