VictoriaMetrics/lib/tenantmetrics/counter_map.go
Will Jordan 209a5024ce
lib/tenantmetrics: improves CounterMap performance with large numbers of tenants
Previously, map for storing tenant metrics was re-created to each newly ingested tenant. It has significant performance impact for systems with large number of tenants. 

 This commit addresses this issue by changing algorithm of creating tenant metric records at map. Instead of map re-creation, it uses `sync.Map` primitive. 


Benchmark results:

```
goos: linux
goarch: amd64
pkg: github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics
cpu: AMD Ryzen 9 5900X 12-Core Processor            
                                            │ lib/tenantmetrics/orig.bench │     lib/tenantmetrics/new.bench     │
                                            │            sec/op            │    sec/op     vs base               │
CounterMapGrowth/n=100,nProcs=GOMAXPROCS-24                  1943.2µ ±  5%   248.0µ ± 11%  -87.24% (p=0.001 n=7)
CounterMapGrowth/n=100-24                                    434.63µ ±  5%   98.82µ ± 16%  -77.26% (p=0.001 n=7)
CounterMapGrowth/n=1000-24                                   32.719m ± 20%   1.425m ±  5%  -95.65% (p=0.001 n=7)
CounterMapGrowth/n=10000-24                                 3653.60m ±  5%   18.00m ±  2%  -99.51% (p=0.001 n=7)
geomean                                                       17.83m         890.4µ        -95.00%
```

Related issue:
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7482

---
Co-authored-by: Artem Fetishev <rtm@victoriametrics.com>
2024-11-20 18:36:06 +01:00

101 lines
2.5 KiB
Go

package tenantmetrics
import (
"fmt"
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// TenantID defines metric tenant.
type TenantID struct {
AccountID uint32
ProjectID uint32
}
// CounterMap is a map of counters keyed by tenant.
type CounterMap struct {
metric string
m sync.Map
// mt holds value for multi-tenant metrics.
mt atomic.Value
}
// NewCounterMap creates new CounterMap for the given metric.
func NewCounterMap(metric string) *CounterMap {
return &CounterMap{
metric: metric,
}
}
// Get returns counter for the given at
func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter {
if at == nil {
return cm.GetByTenant(nil)
}
key := TenantID{
AccountID: at.AccountID,
ProjectID: at.ProjectID,
}
return cm.GetByTenant(&key)
}
// MultiAdd adds multiple values grouped by auth.Token
func (cm *CounterMap) MultiAdd(perTenantValues map[auth.Token]int) {
for token, value := range perTenantValues {
cm.Get(&token).Add(value)
}
}
// GetByTenant returns counter for the given key.
func (cm *CounterMap) GetByTenant(key *TenantID) *metrics.Counter {
if key == nil {
mtm := cm.mt.Load()
if mtm == nil {
mtc := metrics.GetOrCreateCounter(createMetricNameMultitenant(cm.metric))
cm.mt.Store(mtc)
return mtc
}
return mtm.(*metrics.Counter)
}
if counter, ok := cm.m.Load(*key); ok {
return counter.(*metrics.Counter)
}
// Slow path - create missing counter for k.
metricName := createMetricName(cm.metric, *key)
c := metrics.GetOrCreateCounter(metricName)
cm.m.Store(*key, c)
return c
}
func createMetricName(metric string, key TenantID) string {
if len(metric) == 0 {
logger.Panicf("BUG: metric cannot be empty")
}
if metric[len(metric)-1] != '}' {
// Metric without labels.
return fmt.Sprintf(`%s{accountID="%d",projectID="%d"}`, metric, key.AccountID, key.ProjectID)
}
// Metric with labels.
return fmt.Sprintf(`%s,accountID="%d",projectID="%d"}`, metric[:len(metric)-1], key.AccountID, key.ProjectID)
}
func createMetricNameMultitenant(metric string) string {
if len(metric) == 0 {
logger.Panicf("BUG: metric cannot be empty")
}
if metric[len(metric)-1] != '}' {
// Metric without labels.
return fmt.Sprintf(`%s{accountID="multitenant",projectID="multitenant"}`, metric)
}
// Metric with labels.
return fmt.Sprintf(`%s,accountID="multitenant",projectID="multitenant"}`, metric[:len(metric)-1])
}