From bc17f4828c8f9d0fa073033a0ff7863921a60f43 Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Wed, 26 Apr 2023 13:23:01 +0300 Subject: [PATCH] app/vmagent,lib/persistentqueue: show warning message if `--remoteWrite.maxDiskUsagePerURL` flag lower than 500MB (#4196) * app/vmagent,lib/persistentqueue: show warning message if `--remoteWrite.maxDiskUsagePerURL` flag lower than 500MB * app/vmagent,lib/persistentqueue: linter fix * app/vmagent,lib/persistentqueue: fix comment --- app/vmagent/README.md | 2 +- app/vmagent/remotewrite/remotewrite.go | 5 ++++- docs/vmagent.md | 2 +- lib/persistentqueue/persistentqueue.go | 5 +++-- lib/persistentqueue/persistentqueue_test.go | 8 ++++---- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 9ed20561b9..8e321950b3 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -1485,7 +1485,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.maxDailySeries int The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter -remoteWrite.maxDiskUsagePerURL array - The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0 + The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. Disk usage is unlimited if the value is set to 0 Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB. Supports array of values separated by comma or specified via multiple flags. -remoteWrite.maxHourlySeries int diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index 0a9258c73e..79b1ab03fb 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -47,7 +47,7 @@ var ( "It is hidden by default, since it can contain sensitive info such as auth key") maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+ "for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+ - "Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+ + "Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+ "Disk usage is unlimited if the value is set to 0") significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+ "to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+ @@ -523,6 +523,9 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI h := xxhash.Sum64([]byte(pqURL.String())) queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h)) maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0) + if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize { + logger.Warnf("specified flag `--remoteWrite.maxDiskUsagePerURL` is lower than buffered data chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB.") + } fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes) _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 { return float64(fq.GetPendingBytes()) diff --git a/docs/vmagent.md b/docs/vmagent.md index f497c47d0f..a33222d348 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1489,7 +1489,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.maxDailySeries int The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter -remoteWrite.maxDiskUsagePerURL array - The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0 + The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. Disk usage is unlimited if the value is set to 0 Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB. Supports array of values separated by comma or specified via multiple flags. -remoteWrite.maxHourlySeries int diff --git a/lib/persistentqueue/persistentqueue.go b/lib/persistentqueue/persistentqueue.go index c86973008f..1bc582d36e 100644 --- a/lib/persistentqueue/persistentqueue.go +++ b/lib/persistentqueue/persistentqueue.go @@ -22,7 +22,8 @@ import ( // MaxBlockSize is the maximum size of the block persistent queue can work with. const MaxBlockSize = 32 * 1024 * 1024 -const defaultChunkFileSize = (MaxBlockSize + 8) * 16 +// DefaultChunkFileSize represents default chunk file size +const DefaultChunkFileSize = (MaxBlockSize + 8) * 16 var chunkFileNameRegex = regexp.MustCompile("^[0-9A-F]{16}$") @@ -122,7 +123,7 @@ func mustOpen(path, name string, maxPendingBytes int64) *queue { if maxPendingBytes < 0 { maxPendingBytes = 0 } - return mustOpenInternal(path, name, defaultChunkFileSize, MaxBlockSize, uint64(maxPendingBytes)) + return mustOpenInternal(path, name, DefaultChunkFileSize, MaxBlockSize, uint64(maxPendingBytes)) } func mustOpenInternal(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) *queue { diff --git a/lib/persistentqueue/persistentqueue_test.go b/lib/persistentqueue/persistentqueue_test.go index 19a3c9c1b2..2099d9e751 100644 --- a/lib/persistentqueue/persistentqueue_test.go +++ b/lib/persistentqueue/persistentqueue_test.go @@ -53,7 +53,7 @@ func TestQueueOpen(t *testing.T) { path := "queue-open-too-new-chunk" mustCreateDir(path) mustCreateEmptyMetainfo(path, "foobar") - mustCreateFile(filepath.Join(path, fmt.Sprintf("%016X", 100*uint64(defaultChunkFileSize))), "asdf") + mustCreateFile(filepath.Join(path, fmt.Sprintf("%016X", 100*uint64(DefaultChunkFileSize))), "asdf") q := mustOpen(path, "foobar", 0) q.MustClose() mustDeleteDir(path) @@ -63,8 +63,8 @@ func TestQueueOpen(t *testing.T) { mustCreateDir(path) mi := &metainfo{ Name: "foobar", - ReaderOffset: defaultChunkFileSize, - WriterOffset: defaultChunkFileSize, + ReaderOffset: DefaultChunkFileSize, + WriterOffset: DefaultChunkFileSize, } if err := mi.WriteToFile(filepath.Join(path, metainfoFilename)); err != nil { t.Fatalf("unexpected error: %s", err) @@ -79,7 +79,7 @@ func TestQueueOpen(t *testing.T) { mustCreateDir(path) mi := &metainfo{ Name: "foobar", - ReaderOffset: defaultChunkFileSize + 123, + ReaderOffset: DefaultChunkFileSize + 123, } if err := mi.WriteToFile(filepath.Join(path, metainfoFilename)); err != nil { t.Fatalf("unexpected error: %s", err)