github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/{mergeset,storage}: log deleting directories inside partitions if they are missing in parts.json

Browse source 
This should improve debuggability of unexpected deletion of directories inside partitions.

While at it, log the proper path to parts.json when the directory for big part is missing in the partition.
parts.json is located inside directory with small parts, and there is no parts.json file inside directory with big parts.
This commit is contained in:
Aliaksandr Valialkin 2024-04-16 19:07:36 +02:00
parent 6bcc6c938b
commit 85d09e5a2d
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
2 changed files with 26 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
lib/mergeset/table.go
View file

@ -1471,7 +1471,8 @@ func mustOpenParts(path string) []*partWrapper {
fs.MustRemoveAll(filepath.Join(path, "txn")) fs.MustRemoveAll(filepath.Join(path, "txn"))
fs.MustRemoveAll(filepath.Join(path, "tmp")) fs.MustRemoveAll(filepath.Join(path, "tmp"))
partNames := mustReadPartNames(path) partsFile := filepath.Join(path, partsFilename)
partNames := mustReadPartNames(partsFile, path)
// Remove dirs missing in partNames. These dirs may be left after unclean shutdown // Remove dirs missing in partNames. These dirs may be left after unclean shutdown
// or after the update from versions prior to v1.90.0. // or after the update from versions prior to v1.90.0.
@ -1484,7 +1485,6 @@ func mustOpenParts(path string) []*partWrapper {
// including unclean shutdown. // including unclean shutdown.
partPath := filepath.Join(path, partName) partPath := filepath.Join(path, partName)
if !fs.IsPathExist(partPath) { if !fs.IsPathExist(partPath) {
partsFile := filepath.Join(path, partsFilename)
logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+
"ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts",
partPath, partsFile, partsFile, partsFile) partPath, partsFile, partsFile, partsFile)
@ -1500,6 +1500,7 @@ func mustOpenParts(path string) []*partWrapper {
fn := de.Name() fn := de.Name()
if _, ok := m[fn]; !ok { if _, ok := m[fn]; !ok {
deletePath := filepath.Join(path, fn) deletePath := filepath.Join(path, fn)
logger.Infof("deleting %q because it isn't listed in %q; this is the expected case after unclean shutdown", deletePath, partsFile)
fs.MustRemoveAll(deletePath) fs.MustRemoveAll(deletePath)
} }
} }
@ -1516,8 +1517,7 @@ func mustOpenParts(path string) []*partWrapper {
pw.incRef() pw.incRef()
pws = append(pws, pw) pws = append(pws, pw)
} }
partNamesPath := filepath.Join(path, partsFilename) if !fs.IsPathExist(partsFile) {
if !fs.IsPathExist(partNamesPath) {
// Create parts.json file if it doesn't exist yet. // Create parts.json file if it doesn't exist yet.
// This should protect from possible carshloops just after the migration from versions below v1.90.0 // This should protect from possible carshloops just after the migration from versions below v1.90.0
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336
@ -1595,20 +1595,19 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) {
if err != nil { if err != nil {
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err) logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
} }
partNamesPath := filepath.Join(dstDir, partsFilename) partsFile := filepath.Join(dstDir, partsFilename)
fs.MustWriteAtomic(partNamesPath, data, true) fs.MustWriteAtomic(partsFile, data, true)
} }
func mustReadPartNames(srcDir string) []string { func mustReadPartNames(partsFile, srcDir string) []string {
partNamesPath := filepath.Join(srcDir, partsFilename) if fs.IsPathExist(partsFile) {
if fs.IsPathExist(partNamesPath) { data, err := os.ReadFile(partsFile)
data, err := os.ReadFile(partNamesPath)
if err != nil { if err != nil {
logger.Panicf("FATAL: cannot read %s file: %s", partsFilename, err) logger.Panicf("FATAL: cannot read %q: %s", partsFile, err)
} }
var partNames []string var partNames []string
if err := json.Unmarshal(data, &partNames); err != nil { if err := json.Unmarshal(data, &partNames); err != nil {
logger.Panicf("FATAL: cannot parse %s: %s", partNamesPath, err) logger.Panicf("FATAL: cannot parse %q: %s", partsFile, err)
} }
return partNames return partNames
} }

31
lib/storage/partition.go
View file

@ -246,13 +246,13 @@ func mustOpenPartition(smallPartsPath, bigPartsPath string, s *Storage) *partiti
logger.Panicf("FATAL: partition name in bigPartsPath %q doesn't match smallPartsPath %q; want %q", bigPartsPath, smallPartsPath, name) logger.Panicf("FATAL: partition name in bigPartsPath %q doesn't match smallPartsPath %q; want %q", bigPartsPath, smallPartsPath, name)
} }
partNamesSmall, partNamesBig := mustReadPartNames(smallPartsPath, bigPartsPath) partsFile := filepath.Join(smallPartsPath, partsFilename)
partNamesSmall, partNamesBig := mustReadPartNames(partsFile, smallPartsPath, bigPartsPath)
smallParts := mustOpenParts(smallPartsPath, partNamesSmall) smallParts := mustOpenParts(partsFile, smallPartsPath, partNamesSmall)
bigParts := mustOpenParts(bigPartsPath, partNamesBig) bigParts := mustOpenParts(partsFile, bigPartsPath, partNamesBig)
partNamesPath := filepath.Join(smallPartsPath, partsFilename) if !fs.IsPathExist(partsFile) {
if !fs.IsPathExist(partNamesPath) {
// Create parts.json file if it doesn't exist yet. // Create parts.json file if it doesn't exist yet.
// This should protect from possible carshloops just after the migration from versions below v1.90.0 // This should protect from possible carshloops just after the migration from versions below v1.90.0
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4336
@ -1905,7 +1905,7 @@ func getPartsSize(pws []*partWrapper) uint64 {
return n return n
} }
func mustOpenParts(path string, partNames []string) []*partWrapper { func mustOpenParts(partsFile, path string, partNames []string) []*partWrapper {
// The path can be missing after restoring from backup, so create it if needed. // The path can be missing after restoring from backup, so create it if needed.
fs.MustMkdirIfNotExist(path) fs.MustMkdirIfNotExist(path)
fs.MustRemoveTemporaryDirs(path) fs.MustRemoveTemporaryDirs(path)
@ -1926,7 +1926,6 @@ func mustOpenParts(path string, partNames []string) []*partWrapper {
// including unclean shutdown. // including unclean shutdown.
partPath := filepath.Join(path, partName) partPath := filepath.Join(path, partName)
if !fs.IsPathExist(partPath) { if !fs.IsPathExist(partPath) {
partsFile := filepath.Join(path, partsFilename)
logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+ logger.Panicf("FATAL: part %q is listed in %q, but is missing on disk; "+
"ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts", "ensure %q contents is not corrupted; remove %q to rebuild its' content from the list of existing parts",
partPath, partsFile, partsFile, partsFile) partPath, partsFile, partsFile, partsFile)
@ -1942,6 +1941,7 @@ func mustOpenParts(path string, partNames []string) []*partWrapper {
fn := de.Name() fn := de.Name()
if _, ok := m[fn]; !ok { if _, ok := m[fn]; !ok {
deletePath := filepath.Join(path, fn) deletePath := filepath.Join(path, fn)
logger.Infof("deleting %q because it isn't listed in %q; this is the expected case after unclean shutdown", deletePath, partsFile)
fs.MustRemoveAll(deletePath) fs.MustRemoveAll(deletePath)
} }
} }
@ -2037,8 +2037,8 @@ func mustWritePartNames(pwsSmall, pwsBig []*partWrapper, dstDir string) {
if err != nil { if err != nil {
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err) logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
} }
partNamesPath := filepath.Join(dstDir, partsFilename) partsFile := filepath.Join(dstDir, partsFilename)
fs.MustWriteAtomic(partNamesPath, data, true) fs.MustWriteAtomic(partsFile, data, true)
} }
func getPartNames(pws []*partWrapper) []string { func getPartNames(pws []*partWrapper) []string {
@ -2055,20 +2055,19 @@ func getPartNames(pws []*partWrapper) []string {
return partNames return partNames
} }
func mustReadPartNames(smallPartsPath, bigPartsPath string) ([]string, []string) { func mustReadPartNames(partsFile, smallPartsPath, bigPartsPath string) ([]string, []string) {
partNamesPath := filepath.Join(smallPartsPath, partsFilename) if fs.IsPathExist(partsFile) {
if fs.IsPathExist(partNamesPath) { data, err := os.ReadFile(partsFile)
data, err := os.ReadFile(partNamesPath)
if err != nil { if err != nil {
logger.Panicf("FATAL: cannot read %s file: %s", partsFilename, err) logger.Panicf("FATAL: cannot read %q: %s", partsFile, err)
} }
var partNames partNamesJSON var partNames partNamesJSON
if err := json.Unmarshal(data, &partNames); err != nil { if err := json.Unmarshal(data, &partNames); err != nil {
logger.Panicf("FATAL: cannot parse %s: %s", partNamesPath, err) logger.Panicf("FATAL: cannot parse %q: %s", partsFile, err)
} }
return partNames.Small, partNames.Big return partNames.Small, partNames.Big
} }
// The partsFilename is missing. This is the upgrade from versions previous to v1.90.0. // The partsFile is missing. This is the upgrade from versions previous to v1.90.0.
// Read part names from smallPartsPath and bigPartsPath directories // Read part names from smallPartsPath and bigPartsPath directories
partNamesSmall := mustReadPartNamesFromDir(smallPartsPath) partNamesSmall := mustReadPartNamesFromDir(smallPartsPath)
partNamesBig := mustReadPartNamesFromDir(bigPartsPath) partNamesBig := mustReadPartNamesFromDir(bigPartsPath)