github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: do not drop (date, metricID) entries for the date older than 2 days if samples are ingested at this date

Browse source 
Previously the (date, metricID) entries for dates older than the last 2 days were removed.
This could lead to slow check for the (date, metricID) entry in the indexdb during ingesting historical data (aka backfilling).

The issue has been introduced in 431aa16c8d
This commit is contained in:
Aliaksandr Valialkin 2024-02-23 04:04:03 +02:00
parent 83217b7473
commit 42437e05c7
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
1 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/storage/storage.go
View file

@ -2461,7 +2461,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
byDateMutable := dmc.byDateMutable byDateMutable := dmc.byDateMutable
byDateMutable.hotEntry.Store(&byDateMetricIDEntry{}) byDateMutable.hotEntry.Store(&byDateMetricIDEntry{})
keepDatesMap := make(map[uint64]struct{}, len(byDateMutable.m))
for k, e := range byDateMutable.m { for k, e := range byDateMutable.m {
keepDatesMap[k.date] = struct{}{}
v := byDate.get(k.generation, k.date) v := byDate.get(k.generation, k.date)
if v == nil { if v == nil {
// Nothing to merge // Nothing to merge
@ -2477,7 +2479,9 @@ func (dmc *dateMetricIDCache) syncLocked() {
} }
// Copy entries from byDate, which are missing in byDateMutable // Copy entries from byDate, which are missing in byDateMutable
allDatesMap := make(map[uint64]struct{}, len(byDate.m))
for k, e := range byDate.m { for k, e := range byDate.m {
allDatesMap[k.date] = struct{}{}
v := byDateMutable.get(k.generation, k.date) v := byDateMutable.get(k.generation, k.date)
if v != nil { if v != nil {
continue continue
@ -2486,18 +2490,22 @@ func (dmc *dateMetricIDCache) syncLocked() {
} }
if len(byDateMutable.m) > 2 { if len(byDateMutable.m) > 2 {
// Keep only entries for the last two dates - these are usually // Keep only entries for the last two dates from allDatesMap plus all the entries for byDateMutable.
// the current date and the next date. dates := make([]uint64, 0, len(allDatesMap))
dates := make([]uint64, 0, len(byDateMutable.m)) for date := range allDatesMap {
for k := range byDateMutable.m { dates = append(dates, date)
dates = append(dates, k.date)
} }
sort.Slice(dates, func(i, j int) bool { sort.Slice(dates, func(i, j int) bool {
return dates[i] < dates[j] return dates[i] < dates[j]
}) })
maxDate := dates[len(dates)-2] if len(dates) > 2 {
dates = dates[len(dates)-2:]
}
for _, date := range dates {
keepDatesMap[date] = struct{}{}
}
for k := range byDateMutable.m { for k := range byDateMutable.m {
if k.date < maxDate { if _, ok := keepDatesMap[k.date]; !ok {
delete(byDateMutable.m, k) delete(byDateMutable.m, k)
} }
} }