mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/mergeset: consistently use OS-independent separator in file paths
This is needed for Windows support, which uses `\` instead of `/` as file separator Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
This commit is contained in:
parent
b14d96618c
commit
36bbdd7d4b
7 changed files with 39 additions and 27 deletions
|
@ -147,7 +147,7 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
|
||||||
return fmt.Errorf("cannot read metadata from %q: %w", path, err)
|
return fmt.Errorf("cannot read metadata from %q: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
metaindexPath := path + "/metaindex.bin"
|
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||||
metaindexFile, err := filestream.Open(metaindexPath, true)
|
metaindexFile, err := filestream.Open(metaindexPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot open metaindex file in stream mode: %w", err)
|
return fmt.Errorf("cannot open metaindex file in stream mode: %w", err)
|
||||||
|
@ -158,20 +158,20 @@ func (bsr *blockStreamReader) InitFromFilePart(path string) error {
|
||||||
return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err)
|
return fmt.Errorf("cannot unmarshal metaindex rows from file %q: %w", metaindexPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
indexPath := path + "/index.bin"
|
indexPath := filepath.Join(path, indexFilename)
|
||||||
indexFile, err := filestream.Open(indexPath, true)
|
indexFile, err := filestream.Open(indexPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot open index file in stream mode: %w", err)
|
return fmt.Errorf("cannot open index file in stream mode: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
itemsPath := path + "/items.bin"
|
itemsPath := filepath.Join(path, itemsFilename)
|
||||||
itemsFile, err := filestream.Open(itemsPath, true)
|
itemsFile, err := filestream.Open(itemsPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
indexFile.MustClose()
|
indexFile.MustClose()
|
||||||
return fmt.Errorf("cannot open items file in stream mode: %w", err)
|
return fmt.Errorf("cannot open items file in stream mode: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
lensPath := path + "/lens.bin"
|
lensPath := filepath.Join(path, lensFilename)
|
||||||
lensFile, err := filestream.Open(lensPath, true)
|
lensFile, err := filestream.Open(lensPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
indexFile.MustClose()
|
indexFile.MustClose()
|
||||||
|
|
|
@ -88,14 +88,14 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
||||||
|
|
||||||
// Always cache metaindex file in OS page cache, since it is immediately
|
// Always cache metaindex file in OS page cache, since it is immediately
|
||||||
// read after the merge.
|
// read after the merge.
|
||||||
metaindexPath := path + "/metaindex.bin"
|
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||||
metaindexFile, err := filestream.Create(metaindexPath, false)
|
metaindexFile, err := filestream.Create(metaindexPath, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fs.MustRemoveDirAtomic(path)
|
fs.MustRemoveDirAtomic(path)
|
||||||
return fmt.Errorf("cannot create metaindex file: %w", err)
|
return fmt.Errorf("cannot create metaindex file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
indexPath := path + "/index.bin"
|
indexPath := filepath.Join(path, indexFilename)
|
||||||
indexFile, err := filestream.Create(indexPath, nocache)
|
indexFile, err := filestream.Create(indexPath, nocache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metaindexFile.MustClose()
|
metaindexFile.MustClose()
|
||||||
|
@ -103,7 +103,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
||||||
return fmt.Errorf("cannot create index file: %w", err)
|
return fmt.Errorf("cannot create index file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
itemsPath := path + "/items.bin"
|
itemsPath := filepath.Join(path, itemsFilename)
|
||||||
itemsFile, err := filestream.Create(itemsPath, nocache)
|
itemsFile, err := filestream.Create(itemsPath, nocache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metaindexFile.MustClose()
|
metaindexFile.MustClose()
|
||||||
|
@ -112,7 +112,7 @@ func (bsw *blockStreamWriter) InitFromFilePart(path string, nocache bool, compre
|
||||||
return fmt.Errorf("cannot create items file: %w", err)
|
return fmt.Errorf("cannot create items file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
lensPath := path + "/lens.bin"
|
lensPath := filepath.Join(path, lensFilename)
|
||||||
lensFile, err := filestream.Create(lensPath, nocache)
|
lensFile, err := filestream.Create(lensPath, nocache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metaindexFile.MustClose()
|
metaindexFile.MustClose()
|
||||||
|
|
10
lib/mergeset/filenames.go
Normal file
10
lib/mergeset/filenames.go
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
package mergeset
|
||||||
|
|
||||||
|
const (
|
||||||
|
metaindexFilename = "metaindex.bin"
|
||||||
|
indexFilename = "index.bin"
|
||||||
|
itemsFilename = "items.bin"
|
||||||
|
lensFilename = "lens.bin"
|
||||||
|
metadataFilename = "metadata.json"
|
||||||
|
partsFilename = "parts.json"
|
||||||
|
)
|
|
@ -37,19 +37,19 @@ func (mp *inmemoryPart) StoreToDisk(path string) error {
|
||||||
if err := fs.MkdirAllIfNotExist(path); err != nil {
|
if err := fs.MkdirAllIfNotExist(path); err != nil {
|
||||||
return fmt.Errorf("cannot create directory %q: %w", path, err)
|
return fmt.Errorf("cannot create directory %q: %w", path, err)
|
||||||
}
|
}
|
||||||
metaindexPath := path + "/metaindex.bin"
|
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||||
if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil {
|
if err := fs.WriteFileAndSync(metaindexPath, mp.metaindexData.B); err != nil {
|
||||||
return fmt.Errorf("cannot store metaindex: %w", err)
|
return fmt.Errorf("cannot store metaindex: %w", err)
|
||||||
}
|
}
|
||||||
indexPath := path + "/index.bin"
|
indexPath := filepath.Join(path, indexFilename)
|
||||||
if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil {
|
if err := fs.WriteFileAndSync(indexPath, mp.indexData.B); err != nil {
|
||||||
return fmt.Errorf("cannot store index: %w", err)
|
return fmt.Errorf("cannot store index: %w", err)
|
||||||
}
|
}
|
||||||
itemsPath := path + "/items.bin"
|
itemsPath := filepath.Join(path, itemsFilename)
|
||||||
if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil {
|
if err := fs.WriteFileAndSync(itemsPath, mp.itemsData.B); err != nil {
|
||||||
return fmt.Errorf("cannot store items: %w", err)
|
return fmt.Errorf("cannot store items: %w", err)
|
||||||
}
|
}
|
||||||
lensPath := path + "/lens.bin"
|
lensPath := filepath.Join(path, lensFilename)
|
||||||
if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil {
|
if err := fs.WriteFileAndSync(lensPath, mp.lensData.B); err != nil {
|
||||||
return fmt.Errorf("cannot store lens: %w", err)
|
return fmt.Errorf("cannot store lens: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package mergeset
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
|
@ -72,22 +73,22 @@ func openFilePart(path string) (*part, error) {
|
||||||
return nil, fmt.Errorf("cannot read part metadata: %w", err)
|
return nil, fmt.Errorf("cannot read part metadata: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
metaindexPath := path + "/metaindex.bin"
|
metaindexPath := filepath.Join(path, metaindexFilename)
|
||||||
metaindexFile, err := filestream.Open(metaindexPath, true)
|
metaindexFile, err := filestream.Open(metaindexPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
|
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
|
||||||
}
|
}
|
||||||
metaindexSize := fs.MustFileSize(metaindexPath)
|
metaindexSize := fs.MustFileSize(metaindexPath)
|
||||||
|
|
||||||
indexPath := path + "/index.bin"
|
indexPath := filepath.Join(path, indexFilename)
|
||||||
indexFile := fs.MustOpenReaderAt(indexPath)
|
indexFile := fs.MustOpenReaderAt(indexPath)
|
||||||
indexSize := fs.MustFileSize(indexPath)
|
indexSize := fs.MustFileSize(indexPath)
|
||||||
|
|
||||||
itemsPath := path + "/items.bin"
|
itemsPath := filepath.Join(path, itemsFilename)
|
||||||
itemsFile := fs.MustOpenReaderAt(itemsPath)
|
itemsFile := fs.MustOpenReaderAt(itemsPath)
|
||||||
itemsSize := fs.MustFileSize(itemsPath)
|
itemsSize := fs.MustFileSize(itemsPath)
|
||||||
|
|
||||||
lensPath := path + "/lens.bin"
|
lensPath := filepath.Join(path, lensFilename)
|
||||||
lensFile := fs.MustOpenReaderAt(lensPath)
|
lensFile := fs.MustOpenReaderAt(lensPath)
|
||||||
lensSize := fs.MustFileSize(lensPath)
|
lensSize := fs.MustFileSize(lensPath)
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
|
@ -81,7 +82,7 @@ func (ph *partHeader) ReadMetadata(partPath string) error {
|
||||||
ph.Reset()
|
ph.Reset()
|
||||||
|
|
||||||
// Read ph fields from metadata.
|
// Read ph fields from metadata.
|
||||||
metadataPath := partPath + "/metadata.json"
|
metadataPath := filepath.Join(partPath, metadataFilename)
|
||||||
metadata, err := os.ReadFile(metadataPath)
|
metadata, err := os.ReadFile(metadataPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot read %q: %w", metadataPath, err)
|
return fmt.Errorf("cannot read %q: %w", metadataPath, err)
|
||||||
|
@ -123,7 +124,7 @@ func (ph *partHeader) WriteMetadata(partPath string) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err)
|
logger.Panicf("BUG: cannot marshal partHeader metadata: %s", err)
|
||||||
}
|
}
|
||||||
metadataPath := partPath + "/metadata.json"
|
metadataPath := filepath.Join(partPath, metadataFilename)
|
||||||
if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil {
|
if err := fs.WriteFileAtomically(metadataPath, metadata, false); err != nil {
|
||||||
return fmt.Errorf("cannot create %q: %w", metadataPath, err)
|
return fmt.Errorf("cannot create %q: %w", metadataPath, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1085,7 +1085,7 @@ func (tb *Table) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFinal
|
||||||
mergeIdx := tb.nextMergeIdx()
|
mergeIdx := tb.nextMergeIdx()
|
||||||
dstPartPath := ""
|
dstPartPath := ""
|
||||||
if dstPartType == partFile {
|
if dstPartType == partFile {
|
||||||
dstPartPath = fmt.Sprintf("%s/%016X", tb.path, mergeIdx)
|
dstPartPath = filepath.Join(tb.path, fmt.Sprintf("%016X", mergeIdx))
|
||||||
}
|
}
|
||||||
|
|
||||||
if isFinal && len(pws) == 1 && pws[0].mp != nil {
|
if isFinal && len(pws) == 1 && pws[0].mp != nil {
|
||||||
|
@ -1379,8 +1379,8 @@ func openParts(path string) ([]*partWrapper, error) {
|
||||||
|
|
||||||
// Remove txn and tmp directories, which may be left after the upgrade
|
// Remove txn and tmp directories, which may be left after the upgrade
|
||||||
// to v1.90.0 and newer versions.
|
// to v1.90.0 and newer versions.
|
||||||
fs.MustRemoveAll(path + "/txn")
|
fs.MustRemoveAll(filepath.Join(path, "txn"))
|
||||||
fs.MustRemoveAll(path + "/tmp")
|
fs.MustRemoveAll(filepath.Join(path, "tmp"))
|
||||||
|
|
||||||
partNames := mustReadPartNames(path)
|
partNames := mustReadPartNames(path)
|
||||||
|
|
||||||
|
@ -1401,7 +1401,7 @@ func openParts(path string) ([]*partWrapper, error) {
|
||||||
}
|
}
|
||||||
fn := de.Name()
|
fn := de.Name()
|
||||||
if _, ok := m[fn]; !ok {
|
if _, ok := m[fn]; !ok {
|
||||||
deletePath := path + "/" + fn
|
deletePath := filepath.Join(path, fn)
|
||||||
fs.MustRemoveAll(deletePath)
|
fs.MustRemoveAll(deletePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1410,7 +1410,7 @@ func openParts(path string) ([]*partWrapper, error) {
|
||||||
// Open parts
|
// Open parts
|
||||||
var pws []*partWrapper
|
var pws []*partWrapper
|
||||||
for _, partName := range partNames {
|
for _, partName := range partNames {
|
||||||
partPath := path + "/" + partName
|
partPath := filepath.Join(path, partName)
|
||||||
p, err := openFilePart(partPath)
|
p, err := openFilePart(partPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
mustCloseParts(pws)
|
mustCloseParts(pws)
|
||||||
|
@ -1456,7 +1456,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err)
|
return fmt.Errorf("cannot obtain absolute dir for %q: %w", dstDir, err)
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(dstDir, srcDir+"/") {
|
if strings.HasPrefix(dstDir, srcDir+string(filepath.Separator)) {
|
||||||
return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir)
|
return fmt.Errorf("cannot create snapshot %q inside the data dir %q", dstDir, srcDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1483,7 +1483,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string, deadline uint64) error {
|
||||||
return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path)
|
return fmt.Errorf("cannot create snapshot for %q: timeout exceeded", tb.path)
|
||||||
}
|
}
|
||||||
srcPartPath := pw.p.path
|
srcPartPath := pw.p.path
|
||||||
dstPartPath := dstDir + "/" + filepath.Base(srcPartPath)
|
dstPartPath := filepath.Join(dstDir, filepath.Base(srcPartPath))
|
||||||
if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil {
|
if err := fs.HardLinkFiles(srcPartPath, dstPartPath); err != nil {
|
||||||
return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err)
|
return fmt.Errorf("cannot create hard links from %q to %q: %w", srcPartPath, dstPartPath, err)
|
||||||
}
|
}
|
||||||
|
@ -1512,14 +1512,14 @@ func mustWritePartNames(pws []*partWrapper, dstDir string) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
|
logger.Panicf("BUG: cannot marshal partNames to JSON: %s", err)
|
||||||
}
|
}
|
||||||
partNamesPath := dstDir + "/parts.json"
|
partNamesPath := filepath.Join(dstDir, partsFilename)
|
||||||
if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil {
|
if err := fs.WriteFileAtomically(partNamesPath, data, true); err != nil {
|
||||||
logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err)
|
logger.Panicf("FATAL: cannot update %s: %s", partNamesPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mustReadPartNames(srcDir string) []string {
|
func mustReadPartNames(srcDir string) []string {
|
||||||
partNamesPath := srcDir + "/parts.json"
|
partNamesPath := filepath.Join(srcDir, partsFilename)
|
||||||
data, err := os.ReadFile(partNamesPath)
|
data, err := os.ReadFile(partNamesPath)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
var partNames []string
|
var partNames []string
|
||||||
|
|
Loading…
Reference in a new issue