github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-21 15:45:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: use unversioned keys for tag cache in extDB

Browse source 
Data in ExtDB cannot be changed, so it is OK to use unversioned keys for tag cache.
This should improve performance for index lookups over big amount of time series.
This commit is contained in:
Aliaksandr Valialkin 2019-06-25 13:08:56 +03:00
parent 51e2f3b48f
commit 362e187011
1 changed files with 24 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
lib/storage/index_db.go
View file

@ -340,22 +340,25 @@ func (db *indexDB) putMetricNameToCache(metricID uint64, metricName []byte) {
db.metricNameCache.Set(key[:], metricName) db.metricNameCache.Set(key[:], metricName)
} }
func (db *indexDB) marshalTagFiltersKeyVersioned(dst []byte, tfss []*TagFilters) []byte { func (db *indexDB) marshalTagFiltersKey(dst []byte, tfss []*TagFilters, versioned bool) []byte {
if len(tfss) == 0 { if len(tfss) == 0 {
return nil return nil
} }
k := accountProjectKey{ prefix := ^uint64(0)
AccountID: tfss[0].accountID, if versioned {
ProjectID: tfss[0].projectID, k := accountProjectKey{
} AccountID: tfss[0].accountID,
db.tagCachePrefixesLock.RLock() ProjectID: tfss[0].projectID,
prefix := db.tagCachePrefixes[k] }
db.tagCachePrefixesLock.RUnlock() db.tagCachePrefixesLock.RLock()
if prefix == 0 { prefix = db.tagCachePrefixes[k]
// Create missing prefix. db.tagCachePrefixesLock.RUnlock()
// It is OK if multiple concurrent goroutines call invalidateTagCache if prefix == 0 {
// for the same (accountID, projectID). // Create missing prefix.
prefix = db.invalidateTagCache(k.AccountID, k.ProjectID) // It is OK if multiple concurrent goroutines call invalidateTagCache
// for the same (accountID, projectID).
prefix = db.invalidateTagCache(k.AccountID, k.ProjectID)
}
} }
dst = encoding.MarshalUint64(dst, prefix) dst = encoding.MarshalUint64(dst, prefix)
for _, tfs := range tfss { for _, tfs := range tfss {
@ -967,7 +970,7 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int)
tfKeyBuf := tagFiltersKeyBufPool.Get() tfKeyBuf := tagFiltersKeyBufPool.Get()
defer tagFiltersKeyBufPool.Put(tfKeyBuf) defer tagFiltersKeyBufPool.Put(tfKeyBuf)
tfKeyBuf.B = db.marshalTagFiltersKeyVersioned(tfKeyBuf.B[:0], tfss) tfKeyBuf.B = db.marshalTagFiltersKey(tfKeyBuf.B[:0], tfss, true)
tsids, ok := db.getFromTagCache(tfKeyBuf.B) tsids, ok := db.getFromTagCache(tfKeyBuf.B)
if ok { if ok {
// Fast path - tsids found in the cache. // Fast path - tsids found in the cache.
@ -984,7 +987,12 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int)
var extTSIDs []TSID var extTSIDs []TSID
if db.doExtDB(func(extDB *indexDB) { if db.doExtDB(func(extDB *indexDB) {
tsids, ok := extDB.getFromTagCache(tfKeyBuf.B) tfKeyExtBuf := tagFiltersKeyBufPool.Get()
defer tagFiltersKeyBufPool.Put(tfKeyExtBuf)
// Data in extDB cannot be changed, so use unversioned keys for tag cache.
tfKeyExtBuf.B = extDB.marshalTagFiltersKey(tfKeyExtBuf.B[:0], tfss, false)
tsids, ok := extDB.getFromTagCache(tfKeyExtBuf.B)
if ok { if ok {
extTSIDs = tsids extTSIDs = tsids
return return
@ -993,8 +1001,7 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int)
extTSIDs, err = is.searchTSIDs(tfss, tr, maxMetrics) extTSIDs, err = is.searchTSIDs(tfss, tr, maxMetrics)
extDB.putIndexSearch(is) extDB.putIndexSearch(is)
// Do not store found tsids into extDB.tagCache, db.putToTagCache(tsids, tfKeyExtBuf.B)
// since they will be stored into outer cache instead.
}) { }) {
if err != nil { if err != nil {
return nil, err return nil, err