lib/mergeset: consistently use OS-independent separator in file paths

This is needed for Windows support, which uses `\` instead of `/` as file separator

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
This commit is contained in:
Aliaksandr Valialkin 2023-03-25 13:39:38 -07:00
parent b14d96618c
commit 36bbdd7d4b
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
7 changed files with 39 additions and 27 deletions

View file

@ -147,7 +147,7 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
return fmt.Errorf("cannot read metadata from %q: %w", path, err)
}
metaindexPath := path + "/metaindex.bin"
metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Open(metaindexPath, true)
if err != nil {
return fmt.Errorf("cannot open metaindex file in stream mode: %w", err)
@ -158,20 +158,20 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err)
}
indexPath := path + "/index.bin"
indexPath := filepath.Join(path, indexFilename)
indexFile, err := filestream.Open(indexPath, true)
if err != nil {
return fmt.Errorf("cannot open index file in stream mode: %w", err)
}
itemsPath := path + "/items.bin"
itemsPath := filepath.Join(path, itemsFilename)
itemsFile, err := filestream.Open(itemsPath, true)
if err != nil {
indexFile.MustClose()
return fmt.Errorf("cannot open items file in stream mode: %w", err)
}
lensPath := path + "/lens.bin"
lensPath := filepath.Join(path, lensFilename)
lensFile, err := filestream.Open(lensPath, true)
if err != nil {
indexFile.MustClose()

View file

@ -88,14 +88,14 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
// Always cache metaindex file in OS page cache, since it is immediately
// read after the merge.
metaindexPath := path + "/metaindex.bin"
metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Create(metaindexPath, false)
if err != nil {
fs.MustRemoveDirAtomic(path)
return fmt.Errorf("cannot create metaindex file: %w", err)
}
indexPath := path + "/index.bin"
indexPath := filepath.Join(path, indexFilename)
indexFile, err := filestream.Create(indexPath, nocache)
if err != nil {
metaindexFile.MustClose()
@ -103,7 +103,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
return fmt.Errorf("cannot create index file: %w", err)
}
itemsPath := path + "/items.bin"
itemsPath := filepath.Join(path, itemsFilename)
itemsFile, err := filestream.Create(itemsPath, nocache)
if err != nil {
metaindexFile.MustClose()
@ -112,7 +112,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
return fmt.Errorf("cannot create items file: %w", err)
}
lensPath := path + "/lens.bin"
lensPath := filepath.Join(path, lensFilename)
lensFile, err := filestream.Create(lensPath, nocache)
if err != nil {
metaindexFile.MustClose()

10
lib/mergeset/filenames.go Normal file
View file

@ -0,0 +1,10 @@
package mergeset
const (
metaindexFilename = "metaindex.bin"
indexFilename = "index.bin"
itemsFilename = "items.bin"
lensFilename = "lens.bin"
metadataFilename = "metadata.json"
partsFilename = "parts.json"
)

View file

@ -37,19 +37,19 @@ func (mp *inmemoryPart) StoreToDisk(path string) error {
if err := fs.MkdirAllIfNotExist(path); err != nil {
return fmt.Errorf("cannot create directory %q: %w", path, err)
}
metaindexPath := path + "/metaindex.bin"
metaindexPath := filepath.Join(path, metaindexFilename)
if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil {
return fmt.Errorf("cannot store metaindex: %w", err)
}
indexPath := path + "/index.bin"
indexPath := filepath.Join(path, indexFilename)
if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil {
return fmt.Errorf("cannot store index: %w", err)
}
itemsPath := path + "/items.bin"
itemsPath := filepath.Join(path, itemsFilename)
if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil {
return fmt.Errorf("cannot store items: %w", err)
}
lensPath := path + "/lens.bin"
lensPath := filepath.Join(path, lensFilename)
if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil {
return fmt.Errorf("cannot store lens: %w", err)
}

View file

@ -2,6 +2,7 @@ package mergeset
import (
"fmt"
"path/filepath"
"sync"
"unsafe"
@ -72,22 +73,22 @@ func openFilePart(path string) (*part, error) {
return nil, fmt.Errorf("cannot read part metadata: %w", err)
}
metaindexPath := path + "/metaindex.bin"
metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile, err := filestream.Open(metaindexPath, true)
if err != nil {
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
}
metaindexSize := fs.MustFileSize(metaindexPath)
indexPath := path + "/index.bin"
indexPath := filepath.Join(path, indexFilename)
indexFile := fs.MustOpenReaderAt(indexPath)
indexSize := fs.MustFileSize(indexPath)
itemsPath := path + "/items.bin"
itemsPath := filepath.Join(path, itemsFilename)
itemsFile := fs.MustOpenReaderAt(itemsPath)
itemsSize := fs.MustFileSize(itemsPath)
lensPath := path + "/lens.bin"
lensPath := filepath.Join(path, lensFilename)
lensFile := fs.MustOpenReaderAt(lensPath)
lensSize := fs.MustFileSize(lensPath)

View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@ -81,7 +82,7 @@ func (ph *partHeader) ReadMetadata(partPath string) error {
ph.Reset()
// Read ph fields from metadata.
metadataPath := partPath + "/metadata.json"
metadataPath := filepath.Join(partPath, metadataFilename)
metadata, err := os.ReadFile(metadataPath)
if err != nil {
return fmt.Errorf("cannot read %q: %w", metadataPath, err)
@ -123,7 +124,7 @@ func (ph *partHeader) WriteMetadata(partPath string) error {
if err != nil {
logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err)
}
metadataPath := partPath + "/metadata.json"
metadataPath := filepath.Join(partPath, metadataFilename)
if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil {
return fmt.Errorf("cannot create %q: %w", metadataPath, err)
}

View file

@ -1085,7 +1085,7 @@ func (tb *Table) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFinal
mergeIdx := tb.nextMergeIdx()
dstPartPath := ""
if dstPartType == partFile {
dstPartPath = fmt.Sprintf("%s/%016X", tb.path, mergeIdx)
dstPartPath = filepath.Join(tb.path, fmt.Sprintf("%016X", mergeIdx))
}
if isFinal && len(pws) == 1 && pws[0].mp != nil {
@ -1379,8 +1379,8 @@ func openParts(path string) ([]*partWrapper, error) {
// Remove txn and tmp directories, which may be left after the upgrade
// to v1.90.0 and newer versions.
fs.MustRemoveAll(path + "/txn")
fs.MustRemoveAll(path + "/tmp")
fs.MustRemoveAll(filepath.Join(path, "txn"))
fs.MustRemoveAll(filepath.Join(path, "tmp"))
partNames := mustReadPartNames(path)
@ -1401,7 +1401,7 @@ func openParts(path string) ([]*partWrapper, error) {
}
fn := de.Name()
if _, ok := m[fn]; !ok {
deletePath := path + "/" + fn
deletePath := filepath.Join(path, fn)
fs.MustRemoveAll(deletePath)
}
}
@ -1410,7 +1410,7 @@ func openParts(path string) ([]*partWrapper, error) {
// Open parts
var pws []*partWrapper
for _, partName := range partNames {
partPath := path + "/" + partName
partPath := filepath.Join(path, partName)
p, err := openFilePart(partPath)
if err != nil {
mustCloseParts(pws)
@ -1456,7 +1456,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
if err != nil {
return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err)
}
if strings.HasPrefix(dstDir, srcDir+"/") {
if strings.HasPrefix(dstDir, srcDir+string(filepath.Separator)) {
return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir)
}
@ -1483,7 +1483,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path)
}
srcPartPath := pw.p.path
dstPartPath := dstDir + "/" + filepath.Base(srcPartPath)
dstPartPath := filepath.Join(dstDir, filepath.Base(srcPartPath))
if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil {
return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err)
}
@ -1512,14 +1512,14 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) {
if err != nil {
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
}
partNamesPath := dstDir + "/parts.json"
partNamesPath := filepath.Join(dstDir, partsFilename)
if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil {
logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err)
}
}
func mustReadPartNames(srcDir string) []string {
partNamesPath := srcDir + "/parts.json"
partNamesPath := filepath.Join(srcDir, partsFilename)
data, err := os.ReadFile(partNamesPath)
if err == nil {
var partNames []string