From 0a796f7c3a81852e9e7610ff34d365f1c0c284d9 Mon Sep 17 00:00:00 2001 From: Zongyang <8848479+xiaozongyang@users.noreply.github.com> Date: Wed, 16 Jun 2021 17:16:44 +0800 Subject: [PATCH] Change default value of '-remoteWrite.queues' to cgroup.AvailableCPUs * 2 (#1385) * Change default value of '-remoteWrite.queues' to cgroup.AvailableCPUS() * 2 to reduce scrape interval Default value of vmagent option '-remotewrite.queues' is 4 and default size of vmagent ScheudleUnmarshalWorkers is number of CPUs, when available CPUs is much greater than 4, e.g 32, worker are competing push queues which will increase scrape interval and may cause scrape timeout. * Update README and flag description Co-authored-by: xiaozy --- app/vmagent/README.md | 2 +- app/vmagent/remotewrite/remotewrite.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 901309994..641a7ab9a 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -719,7 +719,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234 Supports an array of values separated by comma or specified via multiple flags. -remoteWrite.queues int - The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage (default 4) + The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage (default 2 * numberOfAvailableCPUs) -remoteWrite.rateLimit array Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage Supports array of values separated by comma or specified via multiple flags. diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 1b5b9b3f5..c187515b1 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -28,8 +28,8 @@ var ( "Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems") tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+ "See also -remoteWrite.maxDiskUsagePerURL") - queues = flag.Int("remoteWrite.queues", 4, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+ - "isn't enough for sending high volume of collected data to remote storage") + queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs() * 2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+ + "isn't enough for sending high volume of collected data to remote storage. Default value if 2 * numberOfAvailableCPUs") showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+ "It is hidden by default, since it can contain sensitive info such as auth key") maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+