github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: follow-up for 76f00cea6b

Browse source 
Store the deadline when the metricID entries must be deleted from indexdb
if metricID->metricName entry isn't found after the deadline. This should
make the code more clear comparing the the previous version, where the timestamp
of the first metricID->metricName lookup miss was stored in missingMetricIDs.

Remove the misleading comment about the importance of the order for creating entries
in the inverted index when registering new time series. The order doesn't matter,
since any subset of the created entries can become visible for search
before any other subset after registering in indexdb.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5948
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5959
This commit is contained in:
Aliaksandr Valialkin 2024-03-27 10:51:03 +02:00
parent 20d0183195
commit 4a359d5f67
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
2 changed files with 9 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
lib/storage/index_db.go
View file

@ -506,9 +506,6 @@ func generateTSID(dst *TSID, mn *MetricName) {
} }
func (is *indexSearch) createGlobalIndexes(tsid *TSID, mn *MetricName) { func (is *indexSearch) createGlobalIndexes(tsid *TSID, mn *MetricName) {
// The order of index items is important.
// It guarantees index consistency.
ii := getIndexItems() ii := getIndexItems()
defer putIndexItems(ii) defer putIndexItems(ii)
@ -1504,17 +1501,17 @@ func (db *indexDB) searchMetricNameWithCache(dst []byte, metricID uint64) ([]byt
db.s.missingMetricIDs = nil db.s.missingMetricIDs = nil
db.s.missingMetricIDsResetDeadline = ct + 2*60 db.s.missingMetricIDsResetDeadline = ct + 2*60
} }
timestamp, ok := db.s.missingMetricIDs[metricID] deleteDeadline, ok := db.s.missingMetricIDs[metricID]
if !ok { if !ok {
if db.s.missingMetricIDs == nil { if db.s.missingMetricIDs == nil {
db.s.missingMetricIDs = make(map[uint64]uint64) db.s.missingMetricIDs = make(map[uint64]uint64)
} }
db.s.missingMetricIDs[metricID] = ct deleteDeadline = ct + 60
timestamp = ct db.s.missingMetricIDs[metricID] = deleteDeadline
} }
db.s.missingMetricIDsLock.Unlock() db.s.missingMetricIDsLock.Unlock()
if ct > timestamp+60 { if ct > deleteDeadline {
// Cannot find the MetricName for the given metricID for the last 60 seconds. // Cannot find the MetricName for the given metricID for the last 60 seconds.
// It is likely the indexDB contains incomplete set of metricID -> metricName entries // It is likely the indexDB contains incomplete set of metricID -> metricName entries
// after unclean shutdown or after restoring from a snapshot. // after unclean shutdown or after restoring from a snapshot.

8
lib/storage/storage.go
View file

@ -147,9 +147,11 @@ type Storage struct {
deletedMetricIDs atomic.Pointer[uint64set.Set] deletedMetricIDs atomic.Pointer[uint64set.Set]
deletedMetricIDsUpdateLock sync.Mutex deletedMetricIDsUpdateLock sync.Mutex
// missingMetricIDs maps metricID to the timestamp of first unsuccessful lookup // missingMetricIDs maps metricID to the deadline in unix timestamp seconds
// of metricName by the given metricID. // after which all the indexdb entries for the given metricID
// must be deleted if metricName isn't found by the given metricID.
// This is used inside searchMetricNameWithCache() for detecting permanently missing metricID->metricName entries. // This is used inside searchMetricNameWithCache() for detecting permanently missing metricID->metricName entries.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5959
missingMetricIDsLock sync.Mutex missingMetricIDsLock sync.Mutex
missingMetricIDs map[uint64]uint64 missingMetricIDs map[uint64]uint64
missingMetricIDsResetDeadline uint64 missingMetricIDsResetDeadline uint64
@ -1151,7 +1153,7 @@ func (s *Storage) SearchMetricNames(qt *querytracer.Tracer, tfss []*TagFilters,
metricName, ok = idb.searchMetricNameWithCache(metricName[:0], metricID) metricName, ok = idb.searchMetricNameWithCache(metricName[:0], metricID)
if !ok { if !ok {
// Skip missing metricName for metricID. // Skip missing metricName for metricID.
// It should be automatically fixed. See indexDB.searchMetricName for details. // It should be automatically fixed. See indexDB.searchMetricNameWithCache for details.
continue continue
} }
if _, ok := metricNamesSeen[string(metricName)]; ok { if _, ok := metricNamesSeen[string(metricName)]; ok {