app/all: follow-up after 84d710beab

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548
Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/victoria-logs/main.go

@@ -59,6 +59,8 @@ func main() {
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Stop()
+
 	vlinsert.Stop()
 	vlselect.Stop()
 	vlstorage.Stop()

app/vmagent/main.go

@@ -158,6 +158,7 @@ func main() {
 		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
 
+	pushmetrics.Stop()
 	promscrape.Stop()
 
 	if len(*influxListenAddr) > 0 {

app/vmalert/main.go

@@ -195,6 +195,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	pushmetrics.Stop()
 	cancel()
 	manager.close()
 }

app/vmauth/main.go

@@ -67,6 +67,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	stopAuthConfig()
 	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())

app/vmbackup/main.go

@@ -109,6 +109,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 

app/vminsert/main.go

@@ -158,6 +158,8 @@ func main() {
 	}
 	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Stop()
+
 	if len(*clusternativeListenAddr) > 0 {
 		clusternativeServer.MustStop()
 	}

app/vmrestore/main.go

@@ -65,6 +65,7 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
+	pushmetrics.Stop()
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 

app/vmselect/main.go

@@ -145,6 +145,8 @@ func main() {
 	}
 	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Stop()
+
 	if vmselectapiServer != nil {
 		logger.Infof("stopping vmselectapi server...")
 		vmselectapiServer.MustStop()

app/vmstorage/main.go

@@ -144,9 +144,7 @@ func main() {
 	}
 	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
 
-	// close the metric reporting goroutine
-	pushmetrics.StopPushMetrics()
-	logger.Infof("successfully stop push metric in %.3f seconds", time.Since(startTime).Seconds())
+	pushmetrics.Stop()
 
 	logger.Infof("gracefully shutting down the service")
 	startTime = time.Now()

docs/CHANGELOG.md

@@ -18,6 +18,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly return results from [bottomk](https://docs.victoriametrics.com/MetricsQL.html#bottomk) and `bottomk_*()` functions when some of these results contain NaN values. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5506). Thanks to @xiaozongyang for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5509).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle queries, which wrap [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) with multiple arguments without explicitly specified lookbehind window in square brackets into [aggregate functions](https://docs.victoriametrics.com/MetricsQL.html#aggregate-functions). For example, `sum(quantile_over_time(0.5, process_resident_memory_bytes))` was resulting to `expecting at least 2 args to ...; got 1 args` error. Thanks to @atykhyy for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5414).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details.
+* BUGFIX: all: fix potential panic during components shutdown when `-pushmetrics.url` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): check for Error field in response from influx client during migration. Before, only network errors were checked. Thanks to @wozz for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5446).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried.
 

lib/pushmetrics/pushmetrics.go

@@ -25,8 +25,7 @@ func init() {
 }
 
 var (
-	// create a custom context for the pushmetrics module to close the metric reporting goroutine when the vmstorage process is shutdown.
-	pushMetricsCtx, cancelPushMetric = context.WithCancel(context.Background())
+	pushCtx, cancelPushCtx = context.WithCancel(context.Background())
 )
 
 // Init must be called after logger.Init
@@ -36,12 +35,17 @@ func Init() {
 		opts := &metrics.PushOptions{
 			ExtraLabels: extraLabels,
 		}
-		if err := metrics.InitPushExtWithOptions(pushMetricsCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
+		if err := metrics.InitPushExtWithOptions(pushCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil {
 			logger.Fatalf("cannot initialize pushmetrics: %s", err)
 		}
 	}
 }
 
-func StopPushMetrics() {
-	cancelPushMetric()
+// Stop stops the periodic push of metrics.
+// It is important to stop the push of metrics before disposing resources
+// these metrics attached to. See related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548
+//
+// Stop must be called after Init.
+func Stop() {
+	cancelPushCtx()
 }

lib/uint64set/uint64set_test.go

@@ -690,6 +690,40 @@ func testSetSparseItems(t *testing.T, itemsCount int) {
 	}
 }
 
+func FuzzAddMulti(f *testing.F) {
+	rand.Seed(time.Now().UnixNano())
+
+	f.Add(uint64(10))
+	f.Fuzz(func(t *testing.T, v uint64) {
+		var s1, s2 Set
+
+		v %= 1e3
+		vs := make([]uint64, v)
+		for i := range vs {
+			vs[i] = rand.Uint64()
+		}
+
+		s1.AddMulti(vs)
+		for _, v := range vs {
+			s2.Add(v)
+		}
+		if s1.Len() != s2.Len() {
+			t.Fatalf("unexpected number of items in the set; got %d; want %d\nset:\n%d", s1.Len(), s2.Len(), s1.AppendTo(nil))
+		}
+		for _, x := range vs {
+			if !s1.Has(x) {
+				t.Fatalf("missing item %d in the set", x)
+			}
+		}
+
+		a1 := s1.AppendTo(nil)
+		a2 := s2.AppendTo(nil)
+		if !reflect.DeepEqual(a1, a2) {
+			t.Fatalf("unexpected items in the set;\ngot\n%d\nwant\n%d", a1, a2)
+		}
+	})
+}
+
 func TestAddMulti(t *testing.T) {
 	f := func(a []uint64) {
 		t.Helper()