app/all: follow-up after 84d710beab

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548
Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/victoria-logs/main.go

@@ -59,6 +59,8 @@ func main() {
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Stop()
+
 	vlinsert.Stop()
 	vlselect.Stop()
 	vlstorage.Stop()

app/victoria-metrics/main.go

@@ -89,6 +89,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	pushmetrics.Stop()
 	vminsert.Stop()
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 

app/vmagent/main.go

@@ -159,6 +159,7 @@ func main() {
 		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
 
+	pushmetrics.Stop()
 	promscrape.Stop()
 
 	if len(*influxListenAddr) > 0 {

app/vmalert/main.go

@@ -187,6 +187,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	pushmetrics.Stop()
 	cancel()
 	manager.close()
 }

app/vmauth/main.go

@@ -80,6 +80,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	stopAuthConfig()
 	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())

app/vmbackup/main.go

@@ -107,6 +107,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 

app/vmrestore/main.go

@@ -65,6 +65,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 

docs/CHANGELOG.md

@@ -51,6 +51,7 @@ The sandbox cluster installation is running under the constant load generated by
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a link for the statistic inaccuracy explanation in the cardinality explorer tool. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5460).
+* BUGFIX: all: fix potential panic during components shutdown when `-pushmetrics.url` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549).
 
 ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)
 

lib/pushmetrics/pushmetrics.go

@@ -29,8 +29,7 @@ func init() {
 }
 
 var (
-	// create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown.
+	pushCtx, cancelPushCtx = context.WithCancel(context.Background())
-	pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background())
 )
 
 // Init must be called after logger.Init
@@ -42,12 +41,17 @@ func Init() {
 			Headers:            *pushHeader,
 			DisableCompression: *disableCompression,
 		}
-		if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
+		if err := metrics.InitPushExtWithOptions(pushCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
 			logger.Fatalf("cannot initialize pushmetrics: %s", err)
 		}
 	}
 }
 
-func StopPushMetrics() {
+// Stop stops the periodic push of metrics.
-	cancelPushMetric()
+// It is important to stop the push of metrics before disposing resources
+// these metrics attached to. See related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548
+//
+// Stop must be called after Init.
+func Stop() {
+	cancelPushCtx()
 }