github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/blockcache: increase the lifetime for rarely accessed blocks from 2 minutes to 5 minutes

Browse source 
This should improve data ingestion speed if time series samples are ingested with interval bigger than 2 minutes.
The actual interval could exceed 2 minutes if the original interval between samples doesn't exceed 2 minutes
in the case of slow inserts. Slow inserts may appear in the following cases:

* Big number of new time series are pushed to VictoriaMetrics, so they couldn't be registered in 2 minutes.
* MetricName->tsid cache reset on indexdb rotation or due to unclean shutdown.
  In this case VictoriaMetrics needs to load MetricName->tsid entries for all the incoming series from IndexDB.
  IndexDB uses the block cache for increasing lookup performance. If the cache has no the needed block,
  then IndexDB reads and unpacks the block from disk. This requires an extra disk read IO and CPU.
  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007

This also should increase performance for periodically executed queries with intervals from 2 minutes to 5 minutes.
See the previous similar commit - 43103be011

It is possible that the timeout can be increased further. Let's collect production numbers for this change
so the timeout could be adjusted further.
This commit is contained in:
Aliaksandr Valialkin 2022-02-08 00:11:19 +02:00
parent b7cefff7b0
commit 10476738a8
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/blockcache/blockcache.go
View file

@ -92,7 +92,7 @@ func (c *Cache) updateSizeBytes(n int) {
// cleaner periodically cleans least recently used entries in c. // cleaner periodically cleans least recently used entries in c.
func (c *Cache) cleaner() { func (c *Cache) cleaner() {
ticker := time.NewTicker(30 * time.Second) ticker := time.NewTicker(57 * time.Second)
defer ticker.Stop() defer ticker.Stop()
perKeyMissesTicker := time.NewTicker(2 * time.Minute) perKeyMissesTicker := time.NewTicker(2 * time.Minute)
defer perKeyMissesTicker.Stop() defer perKeyMissesTicker.Stop()
@ -113,9 +113,9 @@ func (c *Cache) cleanByTimeout() {
c.mu.Lock() c.mu.Lock()
for _, pes := range c.m { for _, pes := range c.m {
for offset, e := range pes { for offset, e := range pes {
// Delete items accessed more than two minutes ago. // Delete items accessed more than five minutes ago.
// This time should be enough for repeated queries. // This time should be enough for repeated queries.
if currentTime-atomic.LoadUint64(&e.lastAccessTime) > 2*60 { if currentTime-atomic.LoadUint64(&e.lastAccessTime) > 5*60 {
c.updateSizeBytes(-e.block.SizeBytes()) c.updateSizeBytes(-e.block.SizeBytes())
delete(pes, offset) delete(pes, offset)
// do not delete the entry from c.perKeyMisses, since it is removed by Cache.cleaner later. // do not delete the entry from c.perKeyMisses, since it is removed by Cache.cleaner later.