From 5eb5df96e26bae3fbf771afbe9eee313c45ad25d Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 15 Jun 2023 11:19:22 +0200 Subject: [PATCH] lib/storage: creates parts.json on start-up if it not exists. (#4450) * lib/storage: creates parts.json on start-up if it not exists. It fixes migrations from versions below v1.90.0. Previously parts.json was created only after successful merge. But if merge was interruped for some reason (OOM or shutdown), parts.json wasn't created and partitions left after interruped merge weren't properly deleted. Since VM cannot check if it must be removed or not. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336 * Apply suggestions from code review Co-authored-by: Roman Khavronenko * Update lib/storage/partition.go Co-authored-by: Roman Khavronenko --------- Co-authored-by: Roman Khavronenko --- docs/CHANGELOG.md | 1 + lib/mergeset/table.go | 5 +++++ lib/storage/partition.go | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f2ea6ed6e..dc61c396a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix panic on vmagent shutdown which could lead to loosing aggregation results which were not flushed to remote yet. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4407) for details. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fixed service name detection for [consulagent service discovery](https://docs.victoriametrics.com/sd_configs.html?highlight=consulagent#consulagent_sd_configs) in case of a difference in service name and service id. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4390) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix an issue with `vmbackupmanager` not being able to restore data from a backup stored in GCS. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4420) for details. +* BUGFIX: [storage](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): Properly creates `parts.json` after migration from versions below `v1.90.0. It must fix errors on start-up after unclean shutdown. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336) for details. ## [v1.91.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.2) diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index f436307cf..86bb0c56f 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -1386,6 +1386,11 @@ func mustOpenParts(path string) []*partWrapper { } pws = append(pws, pw) } + partNamesPath := filepath.Join(path, partsFilename) + if !fs.IsPathExist(partNamesPath) { + // create parts.json file on migration from previous versions before v1.90.0 + mustWritePartNames(pws, path) + } return pws } diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 37f071286..779c2cb21 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -267,6 +267,12 @@ func mustOpenPartition(smallPartsPath, bigPartsPath string, s *Storage) *partiti smallParts := mustOpenParts(smallPartsPath, partNamesSmall) bigParts := mustOpenParts(bigPartsPath, partNamesBig) + partNamesPath := filepath.Join(smallPartsPath, partsFilename) + if !fs.IsPathExist(partNamesPath) { + // create parts.json file on migration from previous versions before v1.90.0 + mustWritePartNames(smallParts, bigParts, smallPartsPath) + } + pt := newPartition(name, smallPartsPath, bigPartsPath, s) pt.smallParts = smallParts pt.bigParts = bigParts