From 85d09e5a2de0a83b1a0212aeda62d131bf406160 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 16 Apr 2024 19:07:36 +0200 Subject: [PATCH] lib/{mergeset,storage}: log deleting directories inside partitions if they are missing in parts.json This should improve debuggability of unexpected deletion of directories inside partitions. While at it, log the proper path to parts.json when the directory for big part is missing in the partition. parts.json is located inside directory with small parts, and there is no parts.json file inside directory with big parts. --- lib/mergeset/table.go | 23 +++++++++++------------ lib/storage/partition.go | 31 +++++++++++++++---------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index b9202f4e5..52cc74ea8 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -1471,7 +1471,8 @@ func mustOpenParts(path string) []*partWrapper { fs.MustRemoveAll(filepath.Join(path, "txn")) fs.MustRemoveAll(filepath.Join(path, "tmp")) - partNames := mustReadPartNames(path) + partsFile := filepath.Join(path, partsFilename) + partNames := mustReadPartNames(partsFile, path) // Remove dirs missing in partNames. These dirs may be left after unclean shutdown // or after the update from versions prior to v1.90.0. @@ -1484,7 +1485,6 @@ func mustOpenParts(path string) []*partWrapper { // including unclean shutdown. partPath := filepath.Join(path, partName) if !fs.IsPathExist(partPath) { - partsFile := filepath.Join(path, partsFilename) logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", partPath, partsFile, partsFile, partsFile) @@ -1500,6 +1500,7 @@ func mustOpenParts(path string) []*partWrapper { fn := de.Name() if _, ok := m[fn]; !ok { deletePath := filepath.Join(path, fn) + logger.Infof("deleting %q because it isn't listed in %q; this is the expected case after unclean shutdown", deletePath, partsFile) fs.MustRemoveAll(deletePath) } } @@ -1516,8 +1517,7 @@ func mustOpenParts(path string) []*partWrapper { pw.incRef() pws = append(pws, pw) } - partNamesPath := filepath.Join(path, partsFilename) - if !fs.IsPathExist(partNamesPath) { + if !fs.IsPathExist(partsFile) { // Create parts.json file if it doesn't exist yet. // This should protect from possible carshloops just after the migration from versions below v1.90.0 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336 @@ -1595,20 +1595,19 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) { if err != nil { logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err) } - partNamesPath := filepath.Join(dstDir, partsFilename) - fs.MustWriteAtomic(partNamesPath, data, true) + partsFile := filepath.Join(dstDir, partsFilename) + fs.MustWriteAtomic(partsFile, data, true) } -func mustReadPartNames(srcDir string) []string { - partNamesPath := filepath.Join(srcDir, partsFilename) - if fs.IsPathExist(partNamesPath) { - data, err := os.ReadFile(partNamesPath) +func mustReadPartNames(partsFile, srcDir string) []string { + if fs.IsPathExist(partsFile) { + data, err := os.ReadFile(partsFile) if err != nil { - logger.Panicf("FATAL: cannot read %s file: %s", partsFilename, err) + logger.Panicf("FATAL: cannot read %q: %s", partsFile, err) } var partNames []string if err := json.Unmarshal(data, &partNames); err != nil { - logger.Panicf("FATAL: cannot parse %s: %s", partNamesPath, err) + logger.Panicf("FATAL: cannot parse %q: %s", partsFile, err) } return partNames } diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 704fa8e93..2756fc943 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -246,13 +246,13 @@ func mustOpenPartition(smallPartsPath, bigPartsPath string, s *Storage) *partiti logger.Panicf("FATAL: partition name in bigPartsPath %q doesn't match smallPartsPath %q; want %q", bigPartsPath, smallPartsPath, name) } - partNamesSmall, partNamesBig := mustReadPartNames(smallPartsPath, bigPartsPath) + partsFile := filepath.Join(smallPartsPath, partsFilename) + partNamesSmall, partNamesBig := mustReadPartNames(partsFile, smallPartsPath, bigPartsPath) - smallParts := mustOpenParts(smallPartsPath, partNamesSmall) - bigParts := mustOpenParts(bigPartsPath, partNamesBig) + smallParts := mustOpenParts(partsFile, smallPartsPath, partNamesSmall) + bigParts := mustOpenParts(partsFile, bigPartsPath, partNamesBig) - partNamesPath := filepath.Join(smallPartsPath, partsFilename) - if !fs.IsPathExist(partNamesPath) { + if !fs.IsPathExist(partsFile) { // Create parts.json file if it doesn't exist yet. // This should protect from possible carshloops just after the migration from versions below v1.90.0 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336 @@ -1905,7 +1905,7 @@ func getPartsSize(pws []*partWrapper) uint64 { return n } -func mustOpenParts(path string, partNames []string) []*partWrapper { +func mustOpenParts(partsFile, path string, partNames []string) []*partWrapper { // The path can be missing after restoring from backup, so create it if needed. fs.MustMkdirIfNotExist(path) fs.MustRemoveTemporaryDirs(path) @@ -1926,7 +1926,6 @@ func mustOpenParts(path string, partNames []string) []*partWrapper { // including unclean shutdown. partPath := filepath.Join(path, partName) if !fs.IsPathExist(partPath) { - partsFile := filepath.Join(path, partsFilename) logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", partPath, partsFile, partsFile, partsFile) @@ -1942,6 +1941,7 @@ func mustOpenParts(path string, partNames []string) []*partWrapper { fn := de.Name() if _, ok := m[fn]; !ok { deletePath := filepath.Join(path, fn) + logger.Infof("deleting %q because it isn't listed in %q; this is the expected case after unclean shutdown", deletePath, partsFile) fs.MustRemoveAll(deletePath) } } @@ -2037,8 +2037,8 @@ func mustWritePartNames(pwsSmall, pwsBig []*partWrapper, dstDir string) { if err != nil { logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err) } - partNamesPath := filepath.Join(dstDir, partsFilename) - fs.MustWriteAtomic(partNamesPath, data, true) + partsFile := filepath.Join(dstDir, partsFilename) + fs.MustWriteAtomic(partsFile, data, true) } func getPartNames(pws []*partWrapper) []string { @@ -2055,20 +2055,19 @@ func getPartNames(pws []*partWrapper) []string { return partNames } -func mustReadPartNames(smallPartsPath, bigPartsPath string) ([]string, []string) { - partNamesPath := filepath.Join(smallPartsPath, partsFilename) - if fs.IsPathExist(partNamesPath) { - data, err := os.ReadFile(partNamesPath) +func mustReadPartNames(partsFile, smallPartsPath, bigPartsPath string) ([]string, []string) { + if fs.IsPathExist(partsFile) { + data, err := os.ReadFile(partsFile) if err != nil { - logger.Panicf("FATAL: cannot read %s file: %s", partsFilename, err) + logger.Panicf("FATAL: cannot read %q: %s", partsFile, err) } var partNames partNamesJSON if err := json.Unmarshal(data, &partNames); err != nil { - logger.Panicf("FATAL: cannot parse %s: %s", partNamesPath, err) + logger.Panicf("FATAL: cannot parse %q: %s", partsFile, err) } return partNames.Small, partNames.Big } - // The partsFilename is missing. This is the upgrade from versions previous to v1.90.0. + // The partsFile is missing. This is the upgrade from versions previous to v1.90.0. // Read part names from smallPartsPath and bigPartsPath directories partNamesSmall := mustReadPartNamesFromDir(smallPartsPath) partNamesBig := mustReadPartNamesFromDir(bigPartsPath)