VictoriaMetrics/lib/storage/part.go

122 lines
3.1 KiB
Go
Raw Permalink Normal View History

2019-05-22 21:16:55 +00:00
package storage
import (
"path/filepath"
"sync"
"unsafe"
2019-05-22 21:16:55 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/blockcache"
2019-05-22 21:16:55 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
2019-05-22 21:16:55 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
)
var ibCache = blockcache.NewCache(getMaxIndexBlocksCacheSize)
func getMaxIndexBlocksCacheSize() int {
maxIndexBlockCacheSizeOnce.Do(func() {
maxIndexBlockCacheSize = int(0.1 * float64(memory.Allowed()))
2019-05-22 21:16:55 +00:00
})
return maxIndexBlockCacheSize
2019-05-22 21:16:55 +00:00
}
var (
maxIndexBlockCacheSize int
maxIndexBlockCacheSizeOnce sync.Once
2019-05-22 21:16:55 +00:00
)
// part represents a searchable part containing time series data.
type part struct {
2019-05-22 21:16:55 +00:00
ph partHeader
// Filesystem path to the part.
//
// Empty for in-memory part.
path string
// Total size in bytes of part data.
size uint64
timestampsFile fs.MustReadAtCloser
valuesFile fs.MustReadAtCloser
indexFile fs.MustReadAtCloser
2019-05-22 21:16:55 +00:00
metaindex []metaindexRow
}
// mustOpenFilePart opens file-based part from the given path.
func mustOpenFilePart(path string) *part {
2019-05-22 21:16:55 +00:00
path = filepath.Clean(path)
var ph partHeader
ph.MustReadMetadata(path)
2019-05-22 21:16:55 +00:00
timestampsPath := filepath.Join(path, timestampsFilename)
timestampsFile := fs.MustOpenReaderAt(timestampsPath)
timestampsSize := fs.MustFileSize(timestampsPath)
2019-05-22 21:16:55 +00:00
valuesPath := filepath.Join(path, valuesFilename)
valuesFile := fs.MustOpenReaderAt(valuesPath)
valuesSize := fs.MustFileSize(valuesPath)
2019-05-22 21:16:55 +00:00
indexPath := filepath.Join(path, indexFilename)
indexFile := fs.MustOpenReaderAt(indexPath)
indexSize := fs.MustFileSize(indexPath)
2019-05-22 21:16:55 +00:00
metaindexPath := filepath.Join(path, metaindexFilename)
metaindexFile := filestream.MustOpen(metaindexPath, true)
metaindexSize := fs.MustFileSize(metaindexPath)
2019-05-22 21:16:55 +00:00
size := timestampsSize + valuesSize + indexSize + metaindexSize
return newPart(&ph, path, size, metaindexFile, timestampsFile, valuesFile, indexFile)
2019-05-22 21:16:55 +00:00
}
// newPart returns new part initialized with the given arguments.
//
// The returned part calls MustClose on all the files passed to newPart
// when calling part.MustClose.
func newPart(ph *partHeader, path string, size uint64, metaindexReader filestream.ReadCloser, timestampsFile, valuesFile, indexFile fs.MustReadAtCloser) *part {
2019-05-22 21:16:55 +00:00
metaindex, err := unmarshalMetaindexRows(nil, metaindexReader)
if err != nil {
logger.Panicf("FATAL: cannot unmarshal metaindex data from %q: %s", path, err)
2019-05-22 21:16:55 +00:00
}
metaindexReader.MustClose()
var p part
p.ph = *ph
p.path = path
p.size = size
p.timestampsFile = timestampsFile
p.valuesFile = valuesFile
p.indexFile = indexFile
p.metaindex = metaindex
2019-05-22 21:16:55 +00:00
return &p
2019-05-22 21:16:55 +00:00
}
// String returns human-readable representation of p.
func (p *part) String() string {
if len(p.path) > 0 {
return p.path
}
return p.ph.String()
}
// MustClose closes all the part files.
func (p *part) MustClose() {
p.timestampsFile.MustClose()
p.valuesFile.MustClose()
p.indexFile.MustClose()
ibCache.RemoveBlocksForPart(p)
2019-05-22 21:16:55 +00:00
}
type indexBlock struct {
bhs []blockHeader
}
func (idxb *indexBlock) SizeBytes() int {
return cap(idxb.bhs) * int(unsafe.Sizeof(blockHeader{}))
}