lib/pushmetrics: fix a panic caused by pushing metrics during the graceful shutdown process of vmstorage nodes. (#5549)

Co-authored-by: zhangdongdong <zhangdongdong@kuaishou.com>
Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>

app/vmstorage/main.go

@@ -144,6 +144,10 @@ func main() {
 	}
 	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
 
+	// close the metric reporting goroutine
+	pushmetrics.StopPushMetrics()
+	logger.Infof("successfully stop push metric in %.3f seconds", time.Since(startTime).Seconds())
+
 	logger.Infof("gracefully shutting down the service")
 	startTime = time.Now()
 	stopStaleSnapshotsRemover()

lib/pushmetrics/pushmetrics.go

@@ -24,12 +24,24 @@ func init() {
 	flagutil.RegisterSecretFlag("pushmetrics.url")
 }
 
+var (
+	// create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown.
+	pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background())
+)
+
 // Init must be called after logger.Init
 func Init() {
 	extraLabels := strings.Join(*pushExtraLabel, ",")
 	for _, pu := range *pushURL {
-		if err := metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics); err != nil {
+		opts := &metrics.PushOptions{
+			ExtraLabels: extraLabels,
+		}
+		if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
 			logger.Fatalf("cannot initialize pushmetrics: %s", err)
 		}
 	}
 }
+
+func StopPushMetrics() {
+	cancelPushMetric()
+}