lib/mergeset: consistently use OS-independent separator in file paths

This is needed for Windows support, which uses `\` instead of `/` as file separator

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
This commit is contained in:
Aliaksandr Valialkin 2023-03-25 13:39:38 -07:00
parent b14d96618c
commit 36bbdd7d4b
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
7 changed files with 39 additions and 27 deletions

View file

@ -147,7 +147,7 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
return fmt.Errorf("cannot read metadata from %q: %w", path, err) return fmt.Errorf("cannot read metadata from %q: %w", path, err)
} }
metaindexPath := path + "/metaindex.bin" metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Open(metaindexPath, true) metaindexFile, err := filestream.Open(metaindexPath, true)
if err != nil { if err != nil {
return fmt.Errorf("cannot open metaindex file in stream mode: %w", err) return fmt.Errorf("cannot open metaindex file in stream mode: %w", err)
@ -158,20 +158,20 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err) return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err)
} }
indexPath := path + "/index.bin" indexPath := filepath.Join(path, indexFilename)
indexFile, err := filestream.Open(indexPath, true) indexFile, err := filestream.Open(indexPath, true)
if err != nil { if err != nil {
return fmt.Errorf("cannot open index file in stream mode: %w", err) return fmt.Errorf("cannot open index file in stream mode: %w", err)
} }
itemsPath := path + "/items.bin" itemsPath := filepath.Join(path, itemsFilename)
itemsFile, err := filestream.Open(itemsPath, true) itemsFile, err := filestream.Open(itemsPath, true)
if err != nil { if err != nil {
indexFile.MustClose() indexFile.MustClose()
return fmt.Errorf("cannot open items file in stream mode: %w", err) return fmt.Errorf("cannot open items file in stream mode: %w", err)
} }
lensPath := path + "/lens.bin" lensPath := filepath.Join(path, lensFilename)
lensFile, err := filestream.Open(lensPath, true) lensFile, err := filestream.Open(lensPath, true)
if err != nil { if err != nil {
indexFile.MustClose() indexFile.MustClose()

View file

@ -88,14 +88,14 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
// Always cache metaindex file in OS page cache, since it is immediately // Always cache metaindex file in OS page cache, since it is immediately
// read after the merge. // read after the merge.
metaindexPath := path + "/metaindex.bin" metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Create(metaindexPath, false) metaindexFile, err := filestream.Create(metaindexPath, false)
if err != nil { if err != nil {
fs.MustRemoveDirAtomic(path) fs.MustRemoveDirAtomic(path)
return fmt.Errorf("cannot create metaindex file: %w", err) return fmt.Errorf("cannot create metaindex file: %w", err)
} }
indexPath := path + "/index.bin" indexPath := filepath.Join(path, indexFilename)
indexFile, err := filestream.Create(indexPath, nocache) indexFile, err := filestream.Create(indexPath, nocache)
if err != nil { if err != nil {
metaindexFile.MustClose() metaindexFile.MustClose()
@ -103,7 +103,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
return fmt.Errorf("cannot create index file: %w", err) return fmt.Errorf("cannot create index file: %w", err)
} }
itemsPath := path + "/items.bin" itemsPath := filepath.Join(path, itemsFilename)
itemsFile, err := filestream.Create(itemsPath, nocache) itemsFile, err := filestream.Create(itemsPath, nocache)
if err != nil { if err != nil {
metaindexFile.MustClose() metaindexFile.MustClose()
@ -112,7 +112,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
return fmt.Errorf("cannot create items file: %w", err) return fmt.Errorf("cannot create items file: %w", err)
} }
lensPath := path + "/lens.bin" lensPath := filepath.Join(path, lensFilename)
lensFile, err := filestream.Create(lensPath, nocache) lensFile, err := filestream.Create(lensPath, nocache)
if err != nil { if err != nil {
metaindexFile.MustClose() metaindexFile.MustClose()

10
lib/mergeset/filenames.go Normal file
View file

@ -0,0 +1,10 @@
package mergeset
const (
metaindexFilename = "metaindex.bin"
indexFilename = "index.bin"
itemsFilename = "items.bin"
lensFilename = "lens.bin"
metadataFilename = "metadata.json"
partsFilename = "parts.json"
)

View file

@ -37,19 +37,19 @@ func (mp *inmemoryPart) StoreToDisk(path string) error {
if err := fs.MkdirAllIfNotExist(path); err != nil { if err := fs.MkdirAllIfNotExist(path); err != nil {
return fmt.Errorf("cannot create directory %q: %w", path, err) return fmt.Errorf("cannot create directory %q: %w", path, err)
} }
metaindexPath := path + "/metaindex.bin" metaindexPath := filepath.Join(path, metaindexFilename)
if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil { if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil {
return fmt.Errorf("cannot store metaindex: %w", err) return fmt.Errorf("cannot store metaindex: %w", err)
} }
indexPath := path + "/index.bin" indexPath := filepath.Join(path, indexFilename)
if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil { if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil {
return fmt.Errorf("cannot store index: %w", err) return fmt.Errorf("cannot store index: %w", err)
} }
itemsPath := path + "/items.bin" itemsPath := filepath.Join(path, itemsFilename)
if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil { if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil {
return fmt.Errorf("cannot store items: %w", err) return fmt.Errorf("cannot store items: %w", err)
} }
lensPath := path + "/lens.bin" lensPath := filepath.Join(path, lensFilename)
if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil { if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil {
return fmt.Errorf("cannot store lens: %w", err) return fmt.Errorf("cannot store lens: %w", err)
} }

View file

@ -2,6 +2,7 @@ package mergeset
import ( import (
"fmt" "fmt"
"path/filepath"
"sync" "sync"
"unsafe" "unsafe"
@ -72,22 +73,22 @@ func openFilePart(path string) (*part, error) {
return nil, fmt.Errorf("cannot read part metadata: %w", err) return nil, fmt.Errorf("cannot read part metadata: %w", err)
} }
metaindexPath := path + "/metaindex.bin" metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Open(metaindexPath, true) metaindexFile, err := filestream.Open(metaindexPath, true)
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err) return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
} }
metaindexSize := fs.MustFileSize(metaindexPath) metaindexSize := fs.MustFileSize(metaindexPath)
indexPath := path + "/index.bin" indexPath := filepath.Join(path, indexFilename)
indexFile := fs.MustOpenReaderAt(indexPath) indexFile := fs.MustOpenReaderAt(indexPath)
indexSize := fs.MustFileSize(indexPath) indexSize := fs.MustFileSize(indexPath)
itemsPath := path + "/items.bin" itemsPath := filepath.Join(path, itemsFilename)
itemsFile := fs.MustOpenReaderAt(itemsPath) itemsFile := fs.MustOpenReaderAt(itemsPath)
itemsSize := fs.MustFileSize(itemsPath) itemsSize := fs.MustFileSize(itemsPath)
lensPath := path + "/lens.bin" lensPath := filepath.Join(path, lensFilename)
lensFile := fs.MustOpenReaderAt(lensPath) lensFile := fs.MustOpenReaderAt(lensPath)
lensSize := fs.MustFileSize(lensPath) lensSize := fs.MustFileSize(lensPath)

View file

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"path/filepath"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@ -81,7 +82,7 @@ func (ph *partHeader) ReadMetadata(partPath string) error {
ph.Reset() ph.Reset()
// Read ph fields from metadata. // Read ph fields from metadata.
metadataPath := partPath + "/metadata.json" metadataPath := filepath.Join(partPath, metadataFilename)
metadata, err := os.ReadFile(metadataPath) metadata, err := os.ReadFile(metadataPath)
if err != nil { if err != nil {
return fmt.Errorf("cannot read %q: %w", metadataPath, err) return fmt.Errorf("cannot read %q: %w", metadataPath, err)
@ -123,7 +124,7 @@ func (ph *partHeader) WriteMetadata(partPath string) error {
if err != nil { if err != nil {
logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err) logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err)
} }
metadataPath := partPath + "/metadata.json" metadataPath := filepath.Join(partPath, metadataFilename)
if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil { if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil {
return fmt.Errorf("cannot create %q: %w", metadataPath, err) return fmt.Errorf("cannot create %q: %w", metadataPath, err)
} }

View file

@ -1085,7 +1085,7 @@ func (tb *Table) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFinal
mergeIdx := tb.nextMergeIdx() mergeIdx := tb.nextMergeIdx()
dstPartPath := "" dstPartPath := ""
if dstPartType == partFile { if dstPartType == partFile {
dstPartPath = fmt.Sprintf("%s/%016X", tb.path, mergeIdx) dstPartPath = filepath.Join(tb.path, fmt.Sprintf("%016X", mergeIdx))
} }
if isFinal && len(pws) == 1 && pws[0].mp != nil { if isFinal && len(pws) == 1 && pws[0].mp != nil {
@ -1379,8 +1379,8 @@ func openParts(path string) ([]*partWrapper, error) {
// Remove txn and tmp directories, which may be left after the upgrade // Remove txn and tmp directories, which may be left after the upgrade
// to v1.90.0 and newer versions. // to v1.90.0 and newer versions.
fs.MustRemoveAll(path + "/txn") fs.MustRemoveAll(filepath.Join(path, "txn"))
fs.MustRemoveAll(path + "/tmp") fs.MustRemoveAll(filepath.Join(path, "tmp"))
partNames := mustReadPartNames(path) partNames := mustReadPartNames(path)
@ -1401,7 +1401,7 @@ func openParts(path string) ([]*partWrapper, error) {
} }
fn := de.Name() fn := de.Name()
if _, ok := m[fn]; !ok { if _, ok := m[fn]; !ok {
deletePath := path + "/" + fn deletePath := filepath.Join(path, fn)
fs.MustRemoveAll(deletePath) fs.MustRemoveAll(deletePath)
} }
} }
@ -1410,7 +1410,7 @@ func openParts(path string) ([]*partWrapper, error) {
// Open parts // Open parts
var pws []*partWrapper var pws []*partWrapper
for _, partName := range partNames { for _, partName := range partNames {
partPath := path + "/" + partName partPath := filepath.Join(path, partName)
p, err := openFilePart(partPath) p, err := openFilePart(partPath)
if err != nil { if err != nil {
mustCloseParts(pws) mustCloseParts(pws)
@ -1456,7 +1456,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
if err != nil { if err != nil {
return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err) return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err)
} }
if strings.HasPrefix(dstDir, srcDir+"/") { if strings.HasPrefix(dstDir, srcDir+string(filepath.Separator)) {
return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir) return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir)
} }
@ -1483,7 +1483,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path) return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path)
} }
srcPartPath := pw.p.path srcPartPath := pw.p.path
dstPartPath := dstDir + "/" + filepath.Base(srcPartPath) dstPartPath := filepath.Join(dstDir, filepath.Base(srcPartPath))
if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil { if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil {
return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err) return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err)
} }
@ -1512,14 +1512,14 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) {
if err != nil { if err != nil {
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err) logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
} }
partNamesPath := dstDir + "/parts.json" partNamesPath := filepath.Join(dstDir, partsFilename)
if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil { if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil {
logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err) logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err)
} }
} }
func mustReadPartNames(srcDir string) []string { func mustReadPartNames(srcDir string) []string {
partNamesPath := srcDir + "/parts.json" partNamesPath := filepath.Join(srcDir, partsFilename)
data, err := os.ReadFile(partNamesPath) data, err := os.ReadFile(partNamesPath)
if err == nil { if err == nil {
var partNames []string var partNames []string