github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/mergeset: cache indexBlock items only on the second request

Browse source 
This should reduce the indexdb/indexBlocks cache size, since it won't contain one-time-wonders items.
This commit is contained in:
Aliaksandr Valialkin 2021-07-07 15:21:24 +03:00
parent 9c19719ad6
commit 41754e12f8
2 changed files with 22 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/CHANGELOG.md
View file

@ -7,7 +7,7 @@ sort: 15
## tip ## tip
* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420). * FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
* FEATURE: reduce memory usage when performing heavy queries over big number of time series. * FEATURE: reduce memory usage by up to 30% on production workloads.
* FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors. * FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
* FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution for the served requests. * FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution for the served requests.

21
lib/mergeset/part.go
View file

@ -156,6 +156,9 @@ type indexBlockCache struct {
m map[uint64]*indexBlockCacheEntry m map[uint64]*indexBlockCacheEntry
mu sync.RWMutex mu sync.RWMutex
perKeyMisses map[uint64]int
perKeyMissesLock sync.Mutex
cleanerStopCh chan struct{} cleanerStopCh chan struct{}
cleanerWG sync.WaitGroup cleanerWG sync.WaitGroup
} }
@ -172,6 +175,7 @@ type indexBlockCacheEntry struct {
func newIndexBlockCache() *indexBlockCache { func newIndexBlockCache() *indexBlockCache {
var idxbc indexBlockCache var idxbc indexBlockCache
idxbc.m = make(map[uint64]*indexBlockCacheEntry) idxbc.m = make(map[uint64]*indexBlockCacheEntry)
idxbc.perKeyMisses = make(map[uint64]int)
idxbc.cleanerStopCh = make(chan struct{}) idxbc.cleanerStopCh = make(chan struct{})
idxbc.cleanerWG.Add(1) idxbc.cleanerWG.Add(1)
go func() { go func() {
@ -185,6 +189,7 @@ func (idxbc *indexBlockCache) MustClose() {
close(idxbc.cleanerStopCh) close(idxbc.cleanerStopCh)
idxbc.cleanerWG.Wait() idxbc.cleanerWG.Wait()
idxbc.m = nil idxbc.m = nil
idxbc.perKeyMisses = nil
} }
// cleaner periodically cleans least recently used items. // cleaner periodically cleans least recently used items.
@ -212,6 +217,10 @@ func (idxbc *indexBlockCache) cleanByTimeout() {
} }
} }
idxbc.mu.Unlock() idxbc.mu.Unlock()
idxbc.perKeyMissesLock.Lock()
idxbc.perKeyMisses = make(map[uint64]int, len(idxbc.perKeyMisses))
idxbc.perKeyMissesLock.Unlock()
} }
func (idxbc *indexBlockCache) Get(k uint64) *indexBlock { func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
@ -227,12 +236,24 @@ func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
} }
return idxbe.idxb return idxbe.idxb
} }
idxbc.perKeyMissesLock.Lock()
idxbc.perKeyMisses[k]++
idxbc.perKeyMissesLock.Unlock()
atomic.AddUint64(&idxbc.misses, 1) atomic.AddUint64(&idxbc.misses, 1)
return nil return nil
} }
// Put puts idxb under the key k into idxbc. // Put puts idxb under the key k into idxbc.
func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) { func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) {
idxbc.perKeyMissesLock.Lock()
doNotCache := idxbc.perKeyMisses[k] == 1
idxbc.perKeyMissesLock.Unlock()
if doNotCache {
// Do not cache ib if it has been requested only once (aka one-time-wonders items).
// This should reduce memory usage for the ibc cache.
return
}
idxbc.mu.Lock() idxbc.mu.Lock()
// Remove superfluous entries. // Remove superfluous entries.
if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 { if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {