2019-05-22 21:16:55 +00:00
|
|
|
package mergeset
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"path/filepath"
|
|
|
|
"sync"
|
2021-02-08 22:34:18 +00:00
|
|
|
"unsafe"
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/blockcache"
|
2019-05-22 21:16:55 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
|
|
|
|
)
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
var idxbCache = blockcache.NewCache(getMaxIndexBlocksCacheSize)
|
|
|
|
var ibCache = blockcache.NewCache(getMaxInmemoryBlocksCacheSize)
|
|
|
|
|
2022-02-21 11:50:34 +00:00
|
|
|
// SetIndexBlocksCacheSize overrides the default size of indexdb/indexBlock cache
|
|
|
|
func SetIndexBlocksCacheSize(size int) {
|
|
|
|
maxIndexBlockCacheSize = size
|
|
|
|
}
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
func getMaxIndexBlocksCacheSize() int {
|
|
|
|
maxIndexBlockCacheSizeOnce.Do(func() {
|
2022-02-21 11:50:34 +00:00
|
|
|
if maxIndexBlockCacheSize <= 0 {
|
|
|
|
maxIndexBlockCacheSize = int(0.10 * float64(memory.Allowed()))
|
|
|
|
}
|
2019-05-22 21:16:55 +00:00
|
|
|
})
|
2022-01-20 16:34:59 +00:00
|
|
|
return maxIndexBlockCacheSize
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2022-01-20 16:34:59 +00:00
|
|
|
maxIndexBlockCacheSize int
|
|
|
|
maxIndexBlockCacheSizeOnce sync.Once
|
2019-05-22 21:16:55 +00:00
|
|
|
)
|
|
|
|
|
2022-02-21 11:50:34 +00:00
|
|
|
// SetDataBlocksCacheSize overrides the default size of indexdb/dataBlocks cache
|
|
|
|
func SetDataBlocksCacheSize(size int) {
|
|
|
|
maxInmemoryBlockCacheSize = size
|
|
|
|
}
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
func getMaxInmemoryBlocksCacheSize() int {
|
|
|
|
maxInmemoryBlockCacheSizeOnce.Do(func() {
|
2022-02-21 11:50:34 +00:00
|
|
|
if maxInmemoryBlockCacheSize <= 0 {
|
2022-02-21 15:58:22 +00:00
|
|
|
maxInmemoryBlockCacheSize = int(0.25 * float64(memory.Allowed()))
|
2022-02-21 11:50:34 +00:00
|
|
|
}
|
2019-05-22 21:16:55 +00:00
|
|
|
})
|
2022-01-20 16:34:59 +00:00
|
|
|
return maxInmemoryBlockCacheSize
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2022-01-20 16:34:59 +00:00
|
|
|
maxInmemoryBlockCacheSize int
|
|
|
|
maxInmemoryBlockCacheSizeOnce sync.Once
|
2019-05-22 21:16:55 +00:00
|
|
|
)
|
|
|
|
|
2020-01-14 20:47:04 +00:00
|
|
|
type part struct {
|
2019-05-22 21:16:55 +00:00
|
|
|
ph partHeader
|
|
|
|
|
|
|
|
path string
|
|
|
|
|
2019-07-04 16:09:40 +00:00
|
|
|
size uint64
|
|
|
|
|
2019-05-22 21:16:55 +00:00
|
|
|
mrs []metaindexRow
|
|
|
|
|
2020-01-30 11:22:15 +00:00
|
|
|
indexFile fs.MustReadAtCloser
|
|
|
|
itemsFile fs.MustReadAtCloser
|
|
|
|
lensFile fs.MustReadAtCloser
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func openFilePart(path string) (*part, error) {
|
|
|
|
path = filepath.Clean(path)
|
|
|
|
|
|
|
|
var ph partHeader
|
|
|
|
if err := ph.ParseFromPath(path); err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot parse path to part: %w", err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
metaindexPath := path + "/metaindex.bin"
|
|
|
|
metaindexFile, err := filestream.Open(metaindexPath, true)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
return nil, fmt.Errorf("cannot open %q: %w", metaindexPath, err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2019-07-04 16:09:40 +00:00
|
|
|
metaindexSize := fs.MustFileSize(metaindexPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
|
|
|
indexPath := path + "/index.bin"
|
2020-11-23 07:55:38 +00:00
|
|
|
indexFile := fs.MustOpenReaderAt(indexPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
indexSize := fs.MustFileSize(indexPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
|
|
|
itemsPath := path + "/items.bin"
|
2020-11-23 07:55:38 +00:00
|
|
|
itemsFile := fs.MustOpenReaderAt(itemsPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
itemsSize := fs.MustFileSize(itemsPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
|
|
|
lensPath := path + "/lens.bin"
|
2020-11-23 07:55:38 +00:00
|
|
|
lensFile := fs.MustOpenReaderAt(lensPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
lensSize := fs.MustFileSize(lensPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2019-07-04 16:09:40 +00:00
|
|
|
size := metaindexSize + indexSize + itemsSize + lensSize
|
|
|
|
return newPart(&ph, path, size, metaindexFile, indexFile, itemsFile, lensFile)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2020-01-30 11:22:15 +00:00
|
|
|
func newPart(ph *partHeader, path string, size uint64, metaindexReader filestream.ReadCloser, indexFile, itemsFile, lensFile fs.MustReadAtCloser) (*part, error) {
|
2019-05-22 21:16:55 +00:00
|
|
|
var errors []error
|
|
|
|
mrs, err := unmarshalMetaindexRows(nil, metaindexReader)
|
|
|
|
if err != nil {
|
2020-06-30 19:58:18 +00:00
|
|
|
errors = append(errors, fmt.Errorf("cannot unmarshal metaindexRows: %w", err))
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
metaindexReader.MustClose()
|
|
|
|
|
2019-10-17 15:22:56 +00:00
|
|
|
var p part
|
|
|
|
p.path = path
|
|
|
|
p.size = size
|
|
|
|
p.mrs = mrs
|
|
|
|
|
|
|
|
p.indexFile = indexFile
|
|
|
|
p.itemsFile = itemsFile
|
|
|
|
p.lensFile = lensFile
|
2019-05-22 21:16:55 +00:00
|
|
|
|
|
|
|
p.ph.CopyFrom(ph)
|
|
|
|
if len(errors) > 0 {
|
|
|
|
// Return only the first error, since it has no sense in returning all errors.
|
2020-06-30 19:58:18 +00:00
|
|
|
err := fmt.Errorf("error opening part %s: %w", p.path, errors[0])
|
2019-05-22 21:16:55 +00:00
|
|
|
p.MustClose()
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-10-17 15:22:56 +00:00
|
|
|
return &p, nil
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *part) MustClose() {
|
|
|
|
p.indexFile.MustClose()
|
|
|
|
p.itemsFile.MustClose()
|
|
|
|
p.lensFile.MustClose()
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
idxbCache.RemoveBlocksForPart(p)
|
|
|
|
ibCache.RemoveBlocksForPart(p)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type indexBlock struct {
|
|
|
|
bhs []blockHeader
|
2022-11-16 10:06:14 +00:00
|
|
|
|
|
|
|
// The buffer for holding the data referrred by bhs
|
|
|
|
buf []byte
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
2021-02-08 22:34:18 +00:00
|
|
|
func (idxb *indexBlock) SizeBytes() int {
|
2021-02-21 19:18:49 +00:00
|
|
|
bhs := idxb.bhs[:cap(idxb.bhs)]
|
|
|
|
n := int(unsafe.Sizeof(*idxb))
|
|
|
|
for i := range bhs {
|
|
|
|
n += bhs[i].SizeBytes()
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
2021-02-21 19:18:49 +00:00
|
|
|
return n
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|