VictoriaMetrics/lib/streamaggr/count_series.go

package streamaggr

import (
	"sync"

	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
	"github.com/cespare/xxhash/v2"
)

// countSeriesAggrState calculates output=count_series, e.g. the number of unique series.
type countSeriesAggrState struct {
	m sync.Map
}

type countSeriesStateValue struct {
	mu             sync.Mutex
	m              map[uint64]struct{}
	deleted        bool
	deleteDeadline int64
}

func newCountSeriesAggrState() *countSeriesAggrState {
	return &countSeriesAggrState{}
}

func (as *countSeriesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, _ bool) {
	for i := range samples {
		s := &samples[i]
		inputKey, outputKey := getInputOutputKey(s.key)

		// Count unique hashes over the inputKeys instead of unique inputKey values.
		// This reduces memory usage at the cost of possible hash collisions for distinct inputKey values.
		h := xxhash.Sum64(bytesutil.ToUnsafeBytes(inputKey))

	again:
		v, ok := as.m.Load(outputKey)
		if !ok {
			// The entry is missing in the map. Try creating it.
			v = &countSeriesStateValue{
				m: make(map[uint64]struct{}),
			}
			outputKey = bytesutil.InternString(outputKey)
			vNew, loaded := as.m.LoadOrStore(outputKey, v)
			if loaded {
				// Update the entry created by a concurrent goroutine.
				v = vNew
			}
		}
		sv := v.(*countSeriesStateValue)
		sv.mu.Lock()
		deleted := sv.deleted
		if !deleted {
			if _, ok := sv.m[h]; !ok {
				sv.m[h] = struct{}{}
			}
			sv.deleteDeadline = deleteDeadline
		}
		sv.mu.Unlock()
		if deleted {
			// The entry has been deleted by the concurrent call to flushState
			// Try obtaining and updating the entry again.
			goto again
		}
	}
}

func (as *countSeriesAggrState) flushState(ctx *flushCtx) {
	m := &as.m
	m.Range(func(k, v any) bool {
		// Atomically delete the entry from the map, so new entry is created for the next flush.
		m.Delete(k)

		sv := v.(*countSeriesStateValue)
		sv.mu.Lock()
		if ctx.flushTimestamp > sv.deleteDeadline {
			sv.deleted = true
			sv.mu.Unlock()
			key := k.(string)
			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
			m.Delete(k)
			return true
		}
		n := len(sv.m)
		if n == 0 {
			sv.mu.Unlock()
			return true
		}

		sv.m = make(map[uint64]struct{})
		sv.mu.Unlock()

		key := k.(string)
		ctx.appendSeries(key, "count_series", float64(n))
		return true
	})
}