mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/mergeset: consistently use OS-independent separator in file paths
This is needed for Windows support, which uses `\` instead of `/` as file separator Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
This commit is contained in:
parent
b14d96618c
commit
36bbdd7d4b
7 changed files with 39 additions and 27 deletions
|
@ -147,7 +147,7 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
|
|||
return fmt.Errorf("cannot read metadata from %q: %w", path, err)
|
||||
}
|
||||
|
||||
metaindexPath := path + "/metaindex.bin"
|
||||
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||
metaindexFile, err := filestream.Open(metaindexPath, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open metaindex file in stream mode: %w", err)
|
||||
|
@ -158,20 +158,20 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
|
|||
return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err)
|
||||
}
|
||||
|
||||
indexPath := path + "/index.bin"
|
||||
indexPath := filepath.Join(path, indexFilename)
|
||||
indexFile, err := filestream.Open(indexPath, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open index file in stream mode: %w", err)
|
||||
}
|
||||
|
||||
itemsPath := path + "/items.bin"
|
||||
itemsPath := filepath.Join(path, itemsFilename)
|
||||
itemsFile, err := filestream.Open(itemsPath, true)
|
||||
if err != nil {
|
||||
indexFile.MustClose()
|
||||
return fmt.Errorf("cannot open items file in stream mode: %w", err)
|
||||
}
|
||||
|
||||
lensPath := path + "/lens.bin"
|
||||
lensPath := filepath.Join(path, lensFilename)
|
||||
lensFile, err := filestream.Open(lensPath, true)
|
||||
if err != nil {
|
||||
indexFile.MustClose()
|
||||
|
|
|
@ -88,14 +88,14 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
|||
|
||||
// Always cache metaindex file in OS page cache, since it is immediately
|
||||
// read after the merge.
|
||||
metaindexPath := path + "/metaindex.bin"
|
||||
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||
metaindexFile, err := filestream.Create(metaindexPath, false)
|
||||
if err != nil {
|
||||
fs.MustRemoveDirAtomic(path)
|
||||
return fmt.Errorf("cannot create metaindex file: %w", err)
|
||||
}
|
||||
|
||||
indexPath := path + "/index.bin"
|
||||
indexPath := filepath.Join(path, indexFilename)
|
||||
indexFile, err := filestream.Create(indexPath, nocache)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
|
@ -103,7 +103,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
|||
return fmt.Errorf("cannot create index file: %w", err)
|
||||
}
|
||||
|
||||
itemsPath := path + "/items.bin"
|
||||
itemsPath := filepath.Join(path, itemsFilename)
|
||||
itemsFile, err := filestream.Create(itemsPath, nocache)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
|
@ -112,7 +112,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
|||
return fmt.Errorf("cannot create items file: %w", err)
|
||||
}
|
||||
|
||||
lensPath := path + "/lens.bin"
|
||||
lensPath := filepath.Join(path, lensFilename)
|
||||
lensFile, err := filestream.Create(lensPath, nocache)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
|
|
10
lib/mergeset/filenames.go
Normal file
10
lib/mergeset/filenames.go
Normal file
|
@ -0,0 +1,10 @@
|
|||
package mergeset
|
||||
|
||||
const (
|
||||
metaindexFilename = "metaindex.bin"
|
||||
indexFilename = "index.bin"
|
||||
itemsFilename = "items.bin"
|
||||
lensFilename = "lens.bin"
|
||||
metadataFilename = "metadata.json"
|
||||
partsFilename = "parts.json"
|
||||
)
|
|
@ -37,19 +37,19 @@ func (mp *inmemoryPart) StoreToDisk(path string) error {
|
|||
if err := fs.MkdirAllIfNotExist(path); err != nil {
|
||||
return fmt.Errorf("cannot create directory %q: %w", path, err)
|
||||
}
|
||||
metaindexPath := path + "/metaindex.bin"
|
||||
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||
if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil {
|
||||
return fmt.Errorf("cannot store metaindex: %w", err)
|
||||
}
|
||||
indexPath := path + "/index.bin"
|
||||
indexPath := filepath.Join(path, indexFilename)
|
||||
if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil {
|
||||
return fmt.Errorf("cannot store index: %w", err)
|
||||
}
|
||||
itemsPath := path + "/items.bin"
|
||||
itemsPath := filepath.Join(path, itemsFilename)
|
||||
if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil {
|
||||
return fmt.Errorf("cannot store items: %w", err)
|
||||
}
|
||||
lensPath := path + "/lens.bin"
|
||||
lensPath := filepath.Join(path, lensFilename)
|
||||
if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil {
|
||||
return fmt.Errorf("cannot store lens: %w", err)
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package mergeset
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
|
@ -72,22 +73,22 @@ func openFilePart(path string) (*part, error) {
|
|||
return nil, fmt.Errorf("cannot read part metadata: %w", err)
|
||||
}
|
||||
|
||||
metaindexPath := path + "/metaindex.bin"
|
||||
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||
metaindexFile, err := filestream.Open(metaindexPath, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
|
||||
}
|
||||
metaindexSize := fs.MustFileSize(metaindexPath)
|
||||
|
||||
indexPath := path + "/index.bin"
|
||||
indexPath := filepath.Join(path, indexFilename)
|
||||
indexFile := fs.MustOpenReaderAt(indexPath)
|
||||
indexSize := fs.MustFileSize(indexPath)
|
||||
|
||||
itemsPath := path + "/items.bin"
|
||||
itemsPath := filepath.Join(path, itemsFilename)
|
||||
itemsFile := fs.MustOpenReaderAt(itemsPath)
|
||||
itemsSize := fs.MustFileSize(itemsPath)
|
||||
|
||||
lensPath := path + "/lens.bin"
|
||||
lensPath := filepath.Join(path, lensFilename)
|
||||
lensFile := fs.MustOpenReaderAt(lensPath)
|
||||
lensSize := fs.MustFileSize(lensPath)
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
|
@ -81,7 +82,7 @@ func (ph *partHeader) ReadMetadata(partPath string) error {
|
|||
ph.Reset()
|
||||
|
||||
// Read ph fields from metadata.
|
||||
metadataPath := partPath + "/metadata.json"
|
||||
metadataPath := filepath.Join(partPath, metadataFilename)
|
||||
metadata, err := os.ReadFile(metadataPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %q: %w", metadataPath, err)
|
||||
|
@ -123,7 +124,7 @@ func (ph *partHeader) WriteMetadata(partPath string) error {
|
|||
if err != nil {
|
||||
logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err)
|
||||
}
|
||||
metadataPath := partPath + "/metadata.json"
|
||||
metadataPath := filepath.Join(partPath, metadataFilename)
|
||||
if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil {
|
||||
return fmt.Errorf("cannot create %q: %w", metadataPath, err)
|
||||
}
|
||||
|
|
|
@ -1085,7 +1085,7 @@ func (tb *Table) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFinal
|
|||
mergeIdx := tb.nextMergeIdx()
|
||||
dstPartPath := ""
|
||||
if dstPartType == partFile {
|
||||
dstPartPath = fmt.Sprintf("%s/%016X", tb.path, mergeIdx)
|
||||
dstPartPath = filepath.Join(tb.path, fmt.Sprintf("%016X", mergeIdx))
|
||||
}
|
||||
|
||||
if isFinal && len(pws) == 1 && pws[0].mp != nil {
|
||||
|
@ -1379,8 +1379,8 @@ func openParts(path string) ([]*partWrapper, error) {
|
|||
|
||||
// Remove txn and tmp directories, which may be left after the upgrade
|
||||
// to v1.90.0 and newer versions.
|
||||
fs.MustRemoveAll(path + "/txn")
|
||||
fs.MustRemoveAll(path + "/tmp")
|
||||
fs.MustRemoveAll(filepath.Join(path, "txn"))
|
||||
fs.MustRemoveAll(filepath.Join(path, "tmp"))
|
||||
|
||||
partNames := mustReadPartNames(path)
|
||||
|
||||
|
@ -1401,7 +1401,7 @@ func openParts(path string) ([]*partWrapper, error) {
|
|||
}
|
||||
fn := de.Name()
|
||||
if _, ok := m[fn]; !ok {
|
||||
deletePath := path + "/" + fn
|
||||
deletePath := filepath.Join(path, fn)
|
||||
fs.MustRemoveAll(deletePath)
|
||||
}
|
||||
}
|
||||
|
@ -1410,7 +1410,7 @@ func openParts(path string) ([]*partWrapper, error) {
|
|||
// Open parts
|
||||
var pws []*partWrapper
|
||||
for _, partName := range partNames {
|
||||
partPath := path + "/" + partName
|
||||
partPath := filepath.Join(path, partName)
|
||||
p, err := openFilePart(partPath)
|
||||
if err != nil {
|
||||
mustCloseParts(pws)
|
||||
|
@ -1456,7 +1456,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err)
|
||||
}
|
||||
if strings.HasPrefix(dstDir, srcDir+"/") {
|
||||
if strings.HasPrefix(dstDir, srcDir+string(filepath.Separator)) {
|
||||
return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir)
|
||||
}
|
||||
|
||||
|
@ -1483,7 +1483,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
|
|||
return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path)
|
||||
}
|
||||
srcPartPath := pw.p.path
|
||||
dstPartPath := dstDir + "/" + filepath.Base(srcPartPath)
|
||||
dstPartPath := filepath.Join(dstDir, filepath.Base(srcPartPath))
|
||||
if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil {
|
||||
return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err)
|
||||
}
|
||||
|
@ -1512,14 +1512,14 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) {
|
|||
if err != nil {
|
||||
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
|
||||
}
|
||||
partNamesPath := dstDir + "/parts.json"
|
||||
partNamesPath := filepath.Join(dstDir, partsFilename)
|
||||
if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil {
|
||||
logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadPartNames(srcDir string) []string {
|
||||
partNamesPath := srcDir + "/parts.json"
|
||||
partNamesPath := filepath.Join(srcDir, partsFilename)
|
||||
data, err := os.ReadFile(partNamesPath)
|
||||
if err == nil {
|
||||
var partNames []string
|
||||
|
|
Loading…
Reference in a new issue