github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: keep (date, metricID) entries only for the last two dates

Browse source 
Entries for the previous dates is usually not used, so there is little sense in keeping them in memory.

This should reduce the size of storage/date_metricID cache, which can be monitored
via vm_cache_entries{type="storage/date_metricID"} metric.
This commit is contained in:
Aliaksandr Valialkin 2024-01-29 18:42:45 +01:00
parent e7844f2efd
commit 431aa16c8d
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
2 changed files with 24 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
lib/storage/storage.go
View file

@ -2352,6 +2352,8 @@ func (dmc *dateMetricIDCache) syncLocked() {
// Merge data from byDate into byDateMutable and then atomically replace byDate with the merged data.
byDate := dmc.byDate.Load()
byDateMutable := dmc.byDateMutable
byDateMutable.hotEntry.Store(&byDateMetricIDEntry{})
for k, e := range byDateMutable.m {
v := byDate.get(k.generation, k.date)
if v == nil {
@ -2365,11 +2367,8 @@ func (dmc *dateMetricIDCache) syncLocked() {
v: *v,
}
byDateMutable.m[k] = dme
he := byDateMutable.hotEntry.Load()
if he.k == k {
byDateMutable.hotEntry.Store(dme)
}
}
// Copy entries from byDate, which are missing in byDateMutable
for k, e := range byDate.m {
v := byDateMutable.get(k.generation, k.date)
@ -2379,6 +2378,24 @@ func (dmc *dateMetricIDCache) syncLocked() {
byDateMutable.m[k] = e
}
if len(byDateMutable.m) > 2 {
// Keep only entries for the last two dates - these are usually
// the current date and the next date.
dates := make([]uint64, 0, len(byDateMutable.m))
for k := range byDateMutable.m {
dates = append(dates, k.date)
}
sort.Slice(dates, func(i, j int) bool {
return dates[i] < dates[j]
})
maxDate := dates[len(dates)-2]
for k := range byDateMutable.m {
if k.date < maxDate {
delete(byDateMutable.m, k)
}
}
}
// Atomically replace byDate with byDateMutable
dmc.byDate.Store(dmc.byDateMutable)
dmc.byDateMutable = newByDateMetricIDMap()

6
lib/storage/storage_test.go
View file

@ -98,7 +98,7 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
m := make(map[dmk]bool)
for i := 0; i < 1e5; i++ {
generation := uint64(i) % 2
date := uint64(i) % 3
date := uint64(i) % 2
metricID := uint64(i) % 1237
if !concurrent && c.Has(generation, date, metricID) {
if !m[dmk{generation, date, metricID}] {
@ -127,7 +127,7 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
// Verify fast path after sync.
for i := 0; i < 1e5; i++ {
generation := uint64(i) % 2
date := uint64(i) % 3
date := uint64(i) % 2
metricID := uint64(i) % 123
c.Set(generation, date, metricID)
}
@ -136,7 +136,7 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
c.mu.Unlock()
for i := 0; i < 1e5; i++ {
generation := uint64(i) % 2
date := uint64(i) % 3
date := uint64(i) % 2
metricID := uint64(i) % 123
if !concurrent && !c.Has(generation, date, metricID) {
return fmt.Errorf("c.Has(%d, %d, %d) must return true after sync", generation, date, metricID)