2019-05-22 21:16:55 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"path/filepath"
|
|
|
|
"sync"
|
2021-02-08 22:34:18 +00:00
|
|
|
"unsafe"
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/blockcache"
|
2019-05-22 21:16:55 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
2023-04-14 22:46:09 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
2019-05-22 21:16:55 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
|
|
|
|
)
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize)
|
|
|
|
|
|
|
|
func getMaxIndexBlocksCacheSize() int {
|
|
|
|
maxIndexBlockCacheSizeOnce.Do(func() {
|
2022-01-21 10:45:29 +00:00
|
|
|
maxIndexBlockCacheSize = int(0.1 * float64(memory.Allowed()))
|
2019-05-22 21:16:55 +00:00
|
|
|
})
|
2022-01-20 16:34:59 +00:00
|
|
|
return maxIndexBlockCacheSize
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2022-01-20 16:34:59 +00:00
|
|
|
maxIndexBlockCacheSize int
|
|
|
|
maxIndexBlockCacheSizeOnce sync.Once
|
2019-05-22 21:16:55 +00:00
|
|
|
)
|
|
|
|
|
2020-01-14 20:47:04 +00:00
|
|
|
// part represents a searchable part containing time series data.
|
|
|
|
type part struct {
|
2019-05-22 21:16:55 +00:00
|
|
|
ph partHeader
|
|
|
|
|
|
|
|
// Filesystem path to the part.
|
|
|
|
//
|
|
|
|
// Empty for in-memory part.
|
|
|
|
path string
|
|
|
|
|
2019-07-04 16:09:40 +00:00
|
|
|
// Total size in bytes of part data.
|
|
|
|
size uint64
|
|
|
|
|
2020-01-30 11:22:15 +00:00
|
|
|
timestampsFile fs.MustReadAtCloser
|
|
|
|
valuesFile fs.MustReadAtCloser
|
|
|
|
indexFile fs.MustReadAtCloser
|
2019-05-22 21:16:55 +00:00
|
|
|
|
|
|
|
metaindex []metaindexRow
|
|
|
|
}
|
|
|
|
|
2023-04-14 22:46:09 +00:00
|
|
|
// mustOpenFilePart opens file-based part from the given path.
|
|
|
|
func mustOpenFilePart(path string) *part {
|
2019-05-22 21:16:55 +00:00
|
|
|
path = filepath.Clean(path)
|
|
|
|
|
|
|
|
var ph partHeader
|
2023-04-14 22:46:09 +00:00
|
|
|
ph.MustReadMetadata(path)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-03-25 21:33:54 +00:00
|
|
|
timestampsPath := filepath.Join(path, timestampsFilename)
|
2020-11-23 07:55:38 +00:00
|
|
|
timestampsFile := fs.MustOpenReaderAt(timestampsPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
timestampsSize := fs.MustFileSize(timestampsPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-03-25 21:33:54 +00:00
|
|
|
valuesPath := filepath.Join(path, valuesFilename)
|
2020-11-23 07:55:38 +00:00
|
|
|
valuesFile := fs.MustOpenReaderAt(valuesPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
valuesSize := fs.MustFileSize(valuesPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-03-25 21:33:54 +00:00
|
|
|
indexPath := filepath.Join(path, indexFilename)
|
2020-11-23 07:55:38 +00:00
|
|
|
indexFile := fs.MustOpenReaderAt(indexPath)
|
2019-07-04 16:09:40 +00:00
|
|
|
indexSize := fs.MustFileSize(indexPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-03-25 21:33:54 +00:00
|
|
|
metaindexPath := filepath.Join(path, metaindexFilename)
|
2023-04-14 22:03:39 +00:00
|
|
|
metaindexFile := filestream.MustOpen(metaindexPath, true)
|
2019-07-04 16:09:40 +00:00
|
|
|
metaindexSize := fs.MustFileSize(metaindexPath)
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2019-07-04 16:09:40 +00:00
|
|
|
size := timestampsSize + valuesSize + indexSize + metaindexSize
|
|
|
|
return newPart(&ph, path, size, metaindexFile, timestampsFile, valuesFile, indexFile)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// newPart returns new part initialized with the given arguments.
|
|
|
|
//
|
|
|
|
// The returned part calls MustClose on all the files passed to newPart
|
|
|
|
// when calling part.MustClose.
|
2023-04-14 22:46:09 +00:00
|
|
|
func newPart(ph *partHeader, path string, size uint64, metaindexReader filestream.ReadCloser, timestampsFile, valuesFile, indexFile fs.MustReadAtCloser) *part {
|
2019-05-22 21:16:55 +00:00
|
|
|
metaindex, err := unmarshalMetaindexRows(nil, metaindexReader)
|
|
|
|
if err != nil {
|
2023-04-14 22:46:09 +00:00
|
|
|
logger.Panicf("FATAL: cannot unmarshal metaindex data from %q: %s", path, err)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
metaindexReader.MustClose()
|
|
|
|
|
2019-10-17 15:22:56 +00:00
|
|
|
var p part
|
|
|
|
p.ph = *ph
|
|
|
|
p.path = path
|
|
|
|
p.size = size
|
|
|
|
p.timestampsFile = timestampsFile
|
|
|
|
p.valuesFile = valuesFile
|
|
|
|
p.indexFile = indexFile
|
|
|
|
p.metaindex = metaindex
|
2019-05-22 21:16:55 +00:00
|
|
|
|
2023-04-14 22:46:09 +00:00
|
|
|
return &p
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// String returns human-readable representation of p.
|
|
|
|
func (p *part) String() string {
|
|
|
|
if len(p.path) > 0 {
|
|
|
|
return p.path
|
|
|
|
}
|
|
|
|
return p.ph.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
// MustClose closes all the part files.
|
|
|
|
func (p *part) MustClose() {
|
|
|
|
p.timestampsFile.MustClose()
|
|
|
|
p.valuesFile.MustClose()
|
|
|
|
p.indexFile.MustClose()
|
|
|
|
|
2022-01-20 16:34:59 +00:00
|
|
|
ibCache.RemoveBlocksForPart(p)
|
2019-05-22 21:16:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type indexBlock struct {
|
|
|
|
bhs []blockHeader
|
|
|
|
}
|
|
|
|
|
2021-02-08 22:34:18 +00:00
|
|
|
func (idxb *indexBlock) SizeBytes() int {
|
|
|
|
return cap(idxb.bhs) * int(unsafe.Sizeof(blockHeader{}))
|
|
|
|
}
|