From bea3431ed1814dffab2e4035a62232f740a9bad5 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 19 Sep 2023 13:17:41 +0400 Subject: [PATCH] lib/storage/partition: add check to ensure parts exist on disk (#5017) * lib/storage/partition: add check to ensure parts exist on disk If part exists in parts.json but is missing on disk there will be a misleading error similar to "unexpected number of substrings in the part name". This change forces verification of part existence and throws a correct error in case it is missing on disk. Such issue can be result of https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5005 or disk corruption. Signed-off-by: Zakhar Bessarab * lib/storage/partition: use filepath.Join instead of string concatenation Signed-off-by: Zakhar Bessarab * lib/storage/partition: add action points for error message Signed-off-by: Zakhar Bessarab * all: add a check for missing part in lib/mergeset and lib/logstorage --------- Signed-off-by: Zakhar Bessarab Co-authored-by: Aliaksandr Valialkin --- lib/logstorage/datadb.go | 11 +++++++++++ lib/mergeset/table.go | 12 ++++++++++++ lib/storage/partition.go | 12 ++++++++++++ 3 files changed, 35 insertions(+) diff --git a/lib/logstorage/datadb.go b/lib/logstorage/datadb.go index d23c05eed..41f9c31a6 100644 --- a/lib/logstorage/datadb.go +++ b/lib/logstorage/datadb.go @@ -149,7 +149,18 @@ func mustOpenDatadb(pt *partition, path string, flushInterval time.Duration) *da pws := make([]*partWrapper, len(partNames)) for i, partName := range partNames { + // Make sure the partName exists on disk. + // If it is missing, then manual action from the user is needed, + // since this is unexpected state, which cannot occur under normal operation, + // including unclean shutdown. partPath := filepath.Join(path, partName) + if !fs.IsPathExist(partPath) { + partsFile := filepath.Join(path, partsFilename) + logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ + "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", + partPath, partsFile, partsFile, partsFile) + } + p := mustOpenFilePart(pt, partPath) pws[i] = newPartWrapper(p, nil, time.Time{}) } diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index 8b916770e..bf7274a8e 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -1349,6 +1349,18 @@ func mustOpenParts(path string) []*partWrapper { des := fs.MustReadDir(path) m := make(map[string]struct{}, len(partNames)) for _, partName := range partNames { + // Make sure the partName exists on disk. + // If it is missing, then manual action from the user is needed, + // since this is unexpected state, which cannot occur under normal operation, + // including unclean shutdown. + partPath := filepath.Join(path, partName) + if !fs.IsPathExist(partPath) { + partsFile := filepath.Join(path, partsFilename) + logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ + "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", + partPath, partsFile, partsFile, partsFile) + } + m[partName] = struct{}{} } for _, de := range des { diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 695496c46..ddad2b67d 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -1779,6 +1779,18 @@ func mustOpenParts(path string, partNames []string) []*partWrapper { des := fs.MustReadDir(path) m := make(map[string]struct{}, len(partNames)) for _, partName := range partNames { + // Make sure the partName exists on disk. + // If it is missing, then manual action from the user is needed, + // since this is unexpected state, which cannot occur under normal operation, + // including unclean shutdown. + partPath := filepath.Join(path, partName) + if !fs.IsPathExist(partPath) { + partsFile := filepath.Join(path, partsFilename) + logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ + "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", + partPath, partsFile, partsFile, partsFile) + } + m[partName] = struct{}{} } for _, de := range des {