From b04f5b7313e367deaf52b398dabe096778987bf6 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Sat, 1 Apr 2023 08:50:27 +0200 Subject: [PATCH] lib/storage: check for free disk space before opening tables (#4035) * lib/storage: check for free disk space before opening tables We check for free disk space before call to `openTable`, so `Storage` can be set to ReadOnly before mergeWorkers start. Before the change, there was a chance that merges will start even if Storage has to start in ReadOnly mode because of `-storage.minFreeDiskSpaceBytes` limit. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4023 Signed-off-by: hagen1778 * lib/storage: chore Signed-off-by: hagen1778 * Update lib/storage/storage.go --------- Signed-off-by: hagen1778 Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 1 + lib/storage/storage.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 01661df060..8cf17d4183 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * SECURITY: upgrade base docker image (alpine) from 3.17.2 to 3.17.3. See [alpine 3.17.3 release notes](https://alpinelinux.org/posts/Alpine-3.17.3-released.html). +* BUGFIX: prevent unexpected merges on start-up when `-storage.minFreeDiskSpaceBytes` is set. See [the issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4023). * BUGFIX: verify response code when fetching configuration files via HTTP. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4034). ## [v1.79.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.11) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 3e9a2753c4..2043d7166f 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -259,6 +259,10 @@ func OpenStorage(path string, retentionMsecs int64, maxHourlySeries, maxDailySer s.setDeletedMetricIDs(dmisCurr) s.updateDeletedMetricIDs(dmisPrev) + // check for free disk space before opening the table + // to prevent unexpected part merges. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4023 + s.startFreeDiskSpaceWatcher() + // Load data tablePath := path + "/data" tb, err := openTable(tablePath, s.getDeletedMetricIDs, retentionMsecs, &s.isReadOnly) @@ -271,7 +275,6 @@ func OpenStorage(path string, retentionMsecs int64, maxHourlySeries, maxDailySer s.startCurrHourMetricIDsUpdater() s.startNextDayMetricIDsUpdater() s.startRetentionWatcher() - s.startFreeDiskSpaceWatcher() return s, nil }