mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
78121642df
The main change is getting rid of interning of sample key. It was discovered that for cases with many unique time series aggregated by vmagent interned keys could grow up to hundreds of millions of objects. This has negative impact on the following aspects: 1. It slows down garbage collection cycles, as GC has to scan all inuse objects periodically. The higher is the number of inuse objects, the longer it takes/the more CPU it takes. 2. It slows down the hot path of samples aggregation where each key needs to be looked up in the map first. The change makes code more fragile, but suppose to provide performance optimization for heavy-loaded vmagents with stream aggregation enabled. --------- Signed-off-by: hagen1778 <roman@victoriametrics.com> Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
116 lines
2.7 KiB
Go
116 lines
2.7 KiB
Go
package streamaggr
|
|
|
|
import (
|
|
"math"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
|
"github.com/VictoriaMetrics/metrics"
|
|
)
|
|
|
|
// histogramBucketAggrState calculates output=histogram_bucket, e.g. VictoriaMetrics histogram over input samples.
|
|
type histogramBucketAggrState struct {
|
|
m sync.Map
|
|
|
|
stalenessSecs uint64
|
|
}
|
|
|
|
type histogramBucketStateValue struct {
|
|
mu sync.Mutex
|
|
h metrics.Histogram
|
|
deleteDeadline uint64
|
|
deleted bool
|
|
}
|
|
|
|
func newHistogramBucketAggrState(stalenessInterval time.Duration) *histogramBucketAggrState {
|
|
stalenessSecs := roundDurationToSecs(stalenessInterval)
|
|
return &histogramBucketAggrState{
|
|
stalenessSecs: stalenessSecs,
|
|
}
|
|
}
|
|
|
|
func (as *histogramBucketAggrState) pushSamples(samples []pushSample) {
|
|
currentTime := fasttime.UnixTimestamp()
|
|
deleteDeadline := currentTime + as.stalenessSecs
|
|
for i := range samples {
|
|
s := &samples[i]
|
|
outputKey := getOutputKey(s.key)
|
|
|
|
again:
|
|
v, ok := as.m.Load(outputKey)
|
|
if !ok {
|
|
// The entry is missing in the map. Try creating it.
|
|
v = &histogramBucketStateValue{}
|
|
vNew, loaded := as.m.LoadOrStore(strings.Clone(outputKey), v)
|
|
if loaded {
|
|
// Use the entry created by a concurrent goroutine.
|
|
v = vNew
|
|
}
|
|
}
|
|
sv := v.(*histogramBucketStateValue)
|
|
sv.mu.Lock()
|
|
deleted := sv.deleted
|
|
if !deleted {
|
|
sv.h.Update(s.value)
|
|
sv.deleteDeadline = deleteDeadline
|
|
}
|
|
sv.mu.Unlock()
|
|
if deleted {
|
|
// The entry has been deleted by the concurrent call to flushState
|
|
// Try obtaining and updating the entry again.
|
|
goto again
|
|
}
|
|
}
|
|
}
|
|
|
|
func (as *histogramBucketAggrState) removeOldEntries(currentTime uint64) {
|
|
m := &as.m
|
|
m.Range(func(k, v interface{}) bool {
|
|
sv := v.(*histogramBucketStateValue)
|
|
|
|
sv.mu.Lock()
|
|
deleted := currentTime > sv.deleteDeadline
|
|
if deleted {
|
|
// Mark the current entry as deleted
|
|
sv.deleted = deleted
|
|
}
|
|
sv.mu.Unlock()
|
|
|
|
if deleted {
|
|
m.Delete(k)
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
func (as *histogramBucketAggrState) flushState(ctx *flushCtx, resetState bool) {
|
|
_ = resetState // it isn't used here
|
|
currentTime := fasttime.UnixTimestamp()
|
|
currentTimeMsec := int64(currentTime) * 1000
|
|
|
|
as.removeOldEntries(currentTime)
|
|
|
|
m := &as.m
|
|
m.Range(func(k, v interface{}) bool {
|
|
sv := v.(*histogramBucketStateValue)
|
|
sv.mu.Lock()
|
|
if !sv.deleted {
|
|
key := k.(string)
|
|
sv.h.VisitNonZeroBuckets(func(vmrange string, count uint64) {
|
|
ctx.appendSeriesWithExtraLabel(key, "histogram_bucket", currentTimeMsec, float64(count), "vmrange", vmrange)
|
|
})
|
|
}
|
|
sv.mu.Unlock()
|
|
return true
|
|
})
|
|
}
|
|
|
|
func roundDurationToSecs(d time.Duration) uint64 {
|
|
if d < 0 {
|
|
return 0
|
|
}
|
|
secs := d.Seconds()
|
|
return uint64(math.Ceil(secs))
|
|
}
|