diff --git a/app/victoria-logs/main.go b/app/victoria-logs/main.go index 62d952c07..1a3aea2a9 100644 --- a/app/victoria-logs/main.go +++ b/app/victoria-logs/main.go @@ -37,7 +37,6 @@ func main() { cgroup.SetGOGC(*gogc) buildinfo.Init() logger.Init() - pushmetrics.Init() logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr) startTime := time.Now() @@ -49,8 +48,10 @@ func main() { go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler) logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds()) + pushmetrics.Init() sig := procutil.WaitForSigterm() logger.Infof("received signal %s", sig) + pushmetrics.Stop() logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr) startTime = time.Now() @@ -59,8 +60,6 @@ func main() { } logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) - pushmetrics.Stop() - vlinsert.Stop() vlselect.Stop() vlstorage.Stop() diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go index 0639fbd2f..f0c5f572b 100644 --- a/app/victoria-metrics/main.go +++ b/app/victoria-metrics/main.go @@ -48,7 +48,6 @@ func main() { envflag.Parse() buildinfo.Init() logger.Init() - pushmetrics.Init() if promscrape.IsDryRun() { *dryRun = true @@ -74,13 +73,16 @@ func main() { vmstorage.Init(promql.ResetRollupResultCacheIfNeeded) vmselect.Init() vminsert.Init() + startSelfScraper() go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler) logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds()) + pushmetrics.Init() sig := procutil.WaitForSigterm() logger.Infof("received signal %s", sig) + pushmetrics.Stop() stopSelfScraper() @@ -89,9 +91,8 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } - pushmetrics.Stop() - vminsert.Stop() logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) + vminsert.Stop() vmstorage.Stop() vmselect.Stop() diff --git a/app/vmagent/main.go b/app/vmagent/main.go index c681a84fa..913f832b8 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -96,7 +96,6 @@ func main() { remotewrite.InitSecretFlags() buildinfo.Init() logger.Init() - pushmetrics.Init() if promscrape.IsDryRun() { if err := promscrape.CheckConfig(); err != nil { @@ -147,8 +146,10 @@ func main() { } logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds()) + pushmetrics.Init() sig := procutil.WaitForSigterm() logger.Infof("received signal %s", sig) + pushmetrics.Stop() startTime = time.Now() if len(*httpListenAddr) > 0 { @@ -159,7 +160,6 @@ func main() { logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) } - pushmetrics.Stop() promscrape.Stop() if len(*influxListenAddr) > 0 { diff --git a/app/vmalert/main.go b/app/vmalert/main.go index be8f0c1bd..c2e43c280 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -96,7 +96,6 @@ func main() { notifier.InitSecretFlags() buildinfo.Init() logger.Init() - pushmetrics.Init() if !*remoteReadIgnoreRestoreErrors { logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.") @@ -182,12 +181,14 @@ func main() { rh := &requestHandler{m: manager} go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler) + pushmetrics.Init() sig := procutil.WaitForSigterm() logger.Infof("service received signal %s", sig) + pushmetrics.Stop() + if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } - pushmetrics.Stop() cancel() manager.close() } diff --git a/app/vmauth/main.go b/app/vmauth/main.go index df9eeda00..5fa311c6b 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -64,7 +64,6 @@ func main() { envflag.Parse() buildinfo.Init() logger.Init() - pushmetrics.Init() logger.Infof("starting vmauth at %q...", *httpListenAddr) startTime := time.Now() @@ -72,15 +71,16 @@ func main() { go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler) logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds()) + pushmetrics.Init() sig := procutil.WaitForSigterm() logger.Infof("received signal %s", sig) + pushmetrics.Stop() startTime = time.Now() logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr) if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop the webservice: %s", err) } - pushmetrics.Stop() logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds()) stopAuthConfig() logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds()) diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go index 2ce83d476..2be998aab 100644 --- a/app/vmbackup/main.go +++ b/app/vmbackup/main.go @@ -47,7 +47,6 @@ func main() { envflag.Parse() buildinfo.Init() logger.Init() - pushmetrics.Init() // Storing snapshot delete function to be able to call it in case // of error since logger.Fatal will exit the program without @@ -96,18 +95,19 @@ func main() { go httpserver.Serve(*httpListenAddr, false, nil) + pushmetrics.Init() err := makeBackup() deleteSnapshot() if err != nil { logger.Fatalf("cannot create backup: %s", err) } + pushmetrics.Stop() startTime := time.Now() logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr) if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop http server for metrics: %s", err) } - pushmetrics.Stop() logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds()) } diff --git a/app/vmrestore/main.go b/app/vmrestore/main.go index 799d56ba7..c0389bc43 100644 --- a/app/vmrestore/main.go +++ b/app/vmrestore/main.go @@ -36,7 +36,6 @@ func main() { envflag.Parse() buildinfo.Init() logger.Init() - pushmetrics.Init() go httpserver.Serve(*httpListenAddr, false, nil) @@ -54,9 +53,11 @@ func main() { Dst: dstFS, SkipBackupCompleteCheck: *skipBackupCompleteCheck, } + pushmetrics.Init() if err := a.Run(); err != nil { logger.Fatalf("cannot restore from backup: %s", err) } + pushmetrics.Stop() srcFS.MustStop() dstFS.MustStop() @@ -65,7 +66,6 @@ func main() { if err := httpserver.Stop(*httpListenAddr); err != nil { logger.Fatalf("cannot stop http server for metrics: %s", err) } - pushmetrics.Stop() logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds()) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c24cb6b78..9eeca32e0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -54,7 +54,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a link for the statistic inaccuracy explanation in the cardinality explorer tool. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5460). -* BUGFIX: all: fix potential panic during components shutdown when `-pushmetrics.url` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549). +* BUGFIX: all: fix potential panic during components shutdown when [metrics push](https://docs.victoriametrics.com/#push-metrics) is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549). ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0) diff --git a/go.mod b/go.mod index 911133e49..e9a4c9cf0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 - github.com/VictoriaMetrics/metrics v1.29.1 + github.com/VictoriaMetrics/metrics v1.30.0 github.com/VictoriaMetrics/metricsql v0.70.0 github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2/config v1.26.1 diff --git a/go.sum b/go.sum index 0cc9bfb78..ad4784542 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2blTJwfyU9I= github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metrics v1.29.1 h1:yTORfGeO1T0C6P/tEeT4Mf7rBU5TUu3kjmHvmlaoeO8= -github.com/VictoriaMetrics/metrics v1.29.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= +github.com/VictoriaMetrics/metrics v1.30.0 h1:m8o1sEDTpvFGwvliAmcaxxCDrIYS16rJPmOhwQNgavo= +github.com/VictoriaMetrics/metrics v1.30.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= github.com/VictoriaMetrics/metricsql v0.70.0 h1:G0k/m1yAF6pmk0dM3VT9/XI5PZ8dL7EbcLhREf4bgeI= github.com/VictoriaMetrics/metricsql v0.70.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go index 0e5ddff3f..c03a19a25 100644 --- a/lib/pushmetrics/pushmetrics.go +++ b/lib/pushmetrics/pushmetrics.go @@ -4,6 +4,7 @@ import ( "context" "flag" "strings" + "sync" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" @@ -30,6 +31,7 @@ func init() { var ( pushCtx, cancelPushCtx = context.WithCancel(context.Background()) + wgDone sync.WaitGroup ) // Init must be called after logger.Init @@ -40,6 +42,7 @@ func Init() { ExtraLabels: extraLabels, Headers: *pushHeader, DisableCompression: *disableCompression, + WaitGroup: &wgDone, } if err := metrics.InitPushExtWithOptions(pushCtx, pu, *pushInterval, appmetrics.WritePrometheusMetrics, opts); err != nil { logger.Fatalf("cannot initialize pushmetrics: %s", err) @@ -54,4 +57,5 @@ func Init() { // Stop must be called after Init. func Stop() { cancelPushCtx() + wgDone.Wait() } diff --git a/vendor/github.com/VictoriaMetrics/metrics/push.go b/vendor/github.com/VictoriaMetrics/metrics/push.go index c62c82379..15b105874 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/push.go +++ b/vendor/github.com/VictoriaMetrics/metrics/push.go @@ -31,6 +31,9 @@ type PushOptions struct { // // By default the compression is enabled. DisableCompression bool + + // Optional WaitGroup for waiting until all the push workers created with this WaitGroup are stopped. + WaitGroup *sync.WaitGroup } // InitPushWithOptions sets up periodic push for globally registered metrics to the given pushURL with the given interval. @@ -207,6 +210,13 @@ func InitPushExtWithOptions(ctx context.Context, pushURL string, interval time.D } pushMetricsSet.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pc.pushURLRedacted)).Set(interval.Seconds()) + var wg *sync.WaitGroup + if opts != nil { + wg = opts.WaitGroup + if wg != nil { + wg.Add(1) + } + } go func() { ticker := time.NewTicker(interval) defer ticker.Stop() @@ -221,6 +231,9 @@ func InitPushExtWithOptions(ctx context.Context, pushURL string, interval time.D log.Printf("ERROR: metrics.push: %s", err) } case <-stopCh: + if wg != nil { + wg.Done() + } return } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 7cd0ba25c..ee7e34987 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -100,7 +100,7 @@ github.com/VictoriaMetrics/fastcache github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless -# github.com/VictoriaMetrics/metrics v1.29.1 +# github.com/VictoriaMetrics/metrics v1.30.0 ## explicit; go 1.17 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.70.0