github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: do not drop (date, metricID) entries for the date older than 2 days if samples are ingested at this date

Browse source 
Previously the (date, metricID) entries for dates older than the last 2 days were removed.
This could lead to slow check for the (date, metricID) entry in the indexdb during ingesting historical data (aka backfilling).

The issue has been introduced in 431aa16c8d
This commit is contained in:
Aliaksandr Valialkin 2024-02-23 04:04:03 +02:00
parent f46eaf92eb
commit f79944532b
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
1 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/storage/storage.go
View file

@ -2342,7 +2342,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
byDateMutable := dmc.byDateMutable
byDateMutable.hotEntry.Store(&byDateMetricIDEntry{})
keepDatesMap := make(map[uint64]struct{}, len(byDateMutable.m))
for k, e := range byDateMutable.m {
keepDatesMap[k.date] = struct{}{}
v := byDate.get(k.generation, k.date)
if v == nil {
// Nothing to merge
@ -2358,7 +2360,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
}
// Copy entries from byDate, which are missing in byDateMutable
allDatesMap := make(map[uint64]struct{}, len(byDate.m))
for k, e := range byDate.m {
allDatesMap[k.date] = struct{}{}
v := byDateMutable.get(k.generation, k.date)
if v != nil {
continue
@ -2367,18 +2371,22 @@ func (dmc *dateMetricIDCache) syncLocked() {
}
if len(byDateMutable.m) > 2 {
// Keep only entries for the last two dates - these are usually
// the current date and the next date.
dates := make([]uint64, 0, len(byDateMutable.m))
for k := range byDateMutable.m {
dates = append(dates, k.date)
// Keep only entries for the last two dates from allDatesMap plus all the entries for byDateMutable.
dates := make([]uint64, 0, len(allDatesMap))
for date := range allDatesMap {
dates = append(dates, date)
}
sort.Slice(dates, func(i, j int) bool {
return dates[i] < dates[j]
})
maxDate := dates[len(dates)-2]
if len(dates) > 2 {
dates = dates[len(dates)-2:]
}
for _, date := range dates {
keepDatesMap[date] = struct{}{}
}
for k := range byDateMutable.m {
if k.date < maxDate {
if _, ok := keepDatesMap[k.date]; !ok {
delete(byDateMutable.m, k)
}
}