From d23334a427f324c2a4b4d109d23cd23253cafb0e Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 30 Oct 2023 10:52:57 +0100 Subject: [PATCH] lib/storage: log warning about RO mode only on state change (#5191) Before, vmstorage would log the same message each second producing excessive amount of logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5159 Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 1 + lib/storage/storage.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1e1bcf61d3..cc635cf88b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly drop samples if `-streamAggr.dropInput` command-line flag is set and `-remoteWrite.streamAggr.config` contains an empty file. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5207). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not print redundant error logs when failed to scrape consul or nomad target. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5239). * BUGFIX: [vmstorage](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): prevent deleted series to be searchable via `/api/v1/series` API if they were re-ingested with staleness markers. This situation could happen if user deletes the series from the target and from VM, and then vmagent sends stale markers for absent series. Thanks to @ilyatrefilov for the [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069) and [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5174). +* BUGFIX: [vmstorage](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): log warning about switching to ReadOnly mode only on state change. Before, vmstorage would log this warning every 1s. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5159) for details. ## [v1.93.6](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.6) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index f7b39e87c9..ed95f750ad 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -665,9 +665,12 @@ func (s *Storage) startFreeDiskSpaceWatcher() { f := func() { freeSpaceBytes := fs.MustGetFreeSpace(s.path) if freeSpaceBytes < freeDiskSpaceLimitBytes { + if atomic.LoadUint32(&s.isReadOnly) == 0 { + // log notification only on state change + logger.Warnf("switching the storage at %s to read-only mode, since it has less than -storage.minFreeDiskSpaceBytes=%d of free space: %d bytes left", + s.path, freeDiskSpaceLimitBytes, freeSpaceBytes) + } // Switch the storage to readonly mode if there is no enough free space left at s.path - logger.Warnf("switching the storage at %s to read-only mode, since it has less than -storage.minFreeDiskSpaceBytes=%d of free space: %d bytes left", - s.path, freeDiskSpaceLimitBytes, freeSpaceBytes) atomic.StoreUint32(&s.isReadOnly, 1) return }