2023-01-04 06:19:18 +00:00
|
|
|
package streamaggr
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sync"
|
|
|
|
|
2024-06-07 14:24:09 +00:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
2023-01-04 06:19:18 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// avgAggrState calculates output=avg, e.g. the average value over input samples.
|
|
|
|
type avgAggrState struct {
|
|
|
|
m sync.Map
|
|
|
|
}
|
|
|
|
|
2024-07-03 10:42:45 +00:00
|
|
|
type avgState struct {
|
|
|
|
sum float64
|
|
|
|
count float64
|
|
|
|
}
|
|
|
|
|
2023-01-04 06:19:18 +00:00
|
|
|
type avgStateValue struct {
|
2024-07-03 10:42:45 +00:00
|
|
|
mu sync.Mutex
|
|
|
|
state [aggrStateSize]avgState
|
|
|
|
deleted bool
|
|
|
|
deleteDeadline int64
|
2023-01-04 06:19:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func newAvgAggrState() *avgAggrState {
|
|
|
|
return &avgAggrState{}
|
|
|
|
}
|
|
|
|
|
2024-07-03 10:42:45 +00:00
|
|
|
func (as *avgAggrState) pushSamples(samples []pushSample, deleteDeadline int64, idx int) {
|
lib/streamaggr: huge pile of changes
- Reduce memory usage by up to 5x when de-duplicating samples across big number of time series.
- Reduce memory usage by up to 5x when aggregating across big number of output time series.
- Add lib/promutils.LabelsCompressor, which is going to be used by other VictoriaMetrics components
for reducing memory usage for marshaled []prompbmarshal.Label.
- Add `dedup_interval` option at aggregation config, which allows setting individual
deduplication intervals per each aggregation.
- Add `keep_metric_names` option at aggregation config, which allows keeping the original
metric names in the output samples.
- Add `unique_samples` output, which counts the number of unique sample values.
- Add `increase_prometheus` and `total_prometheus` outputs, which ignore the first sample
per each newly encountered time series.
- Use 64-bit hashes instead of marshaled labels as map keys when calculating `count_series` output.
This makes obsolete https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5579
- Expose various metrics, which may help debugging stream aggregation:
- vm_streamaggr_dedup_state_size_bytes - the size of data structures responsible for deduplication
- vm_streamaggr_dedup_state_items_count - the number of items in the deduplication data structures
- vm_streamaggr_labels_compressor_size_bytes - the size of labels compressor data structures
- vm_streamaggr_labels_compressor_items_count - the number of entries in the labels compressor
- vm_streamaggr_flush_duration_seconds - a histogram, which shows the duration of stream aggregation flushes
- vm_streamaggr_dedup_flush_duration_seconds - a histogram, which shows the duration of deduplication flushes
- vm_streamaggr_flush_timeouts_total - counter for timed out stream aggregation flushes,
which took longer than the configured interval
- vm_streamaggr_dedup_flush_timeouts_total - counter for timed out deduplication flushes,
which took longer than the configured dedup_interval
- Actualize docs/stream-aggregation.md
The memory usage reduction increases CPU usage during stream aggregation by up to 30%.
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5898
2024-03-02 00:42:26 +00:00
|
|
|
for i := range samples {
|
|
|
|
s := &samples[i]
|
|
|
|
outputKey := getOutputKey(s.key)
|
|
|
|
|
|
|
|
again:
|
|
|
|
v, ok := as.m.Load(outputKey)
|
|
|
|
if !ok {
|
|
|
|
// The entry is missing in the map. Try creating it.
|
2024-07-03 10:42:45 +00:00
|
|
|
v = &avgStateValue{}
|
2024-06-07 14:24:09 +00:00
|
|
|
outputKey = bytesutil.InternString(outputKey)
|
|
|
|
vNew, loaded := as.m.LoadOrStore(outputKey, v)
|
2024-07-03 10:42:45 +00:00
|
|
|
if loaded {
|
|
|
|
// Use the entry created by a concurrent goroutine.
|
|
|
|
v = vNew
|
lib/streamaggr: huge pile of changes
- Reduce memory usage by up to 5x when de-duplicating samples across big number of time series.
- Reduce memory usage by up to 5x when aggregating across big number of output time series.
- Add lib/promutils.LabelsCompressor, which is going to be used by other VictoriaMetrics components
for reducing memory usage for marshaled []prompbmarshal.Label.
- Add `dedup_interval` option at aggregation config, which allows setting individual
deduplication intervals per each aggregation.
- Add `keep_metric_names` option at aggregation config, which allows keeping the original
metric names in the output samples.
- Add `unique_samples` output, which counts the number of unique sample values.
- Add `increase_prometheus` and `total_prometheus` outputs, which ignore the first sample
per each newly encountered time series.
- Use 64-bit hashes instead of marshaled labels as map keys when calculating `count_series` output.
This makes obsolete https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5579
- Expose various metrics, which may help debugging stream aggregation:
- vm_streamaggr_dedup_state_size_bytes - the size of data structures responsible for deduplication
- vm_streamaggr_dedup_state_items_count - the number of items in the deduplication data structures
- vm_streamaggr_labels_compressor_size_bytes - the size of labels compressor data structures
- vm_streamaggr_labels_compressor_items_count - the number of entries in the labels compressor
- vm_streamaggr_flush_duration_seconds - a histogram, which shows the duration of stream aggregation flushes
- vm_streamaggr_dedup_flush_duration_seconds - a histogram, which shows the duration of deduplication flushes
- vm_streamaggr_flush_timeouts_total - counter for timed out stream aggregation flushes,
which took longer than the configured interval
- vm_streamaggr_dedup_flush_timeouts_total - counter for timed out deduplication flushes,
which took longer than the configured dedup_interval
- Actualize docs/stream-aggregation.md
The memory usage reduction increases CPU usage during stream aggregation by up to 30%.
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5898
2024-03-02 00:42:26 +00:00
|
|
|
}
|
2023-01-04 06:19:18 +00:00
|
|
|
}
|
lib/streamaggr: huge pile of changes
- Reduce memory usage by up to 5x when de-duplicating samples across big number of time series.
- Reduce memory usage by up to 5x when aggregating across big number of output time series.
- Add lib/promutils.LabelsCompressor, which is going to be used by other VictoriaMetrics components
for reducing memory usage for marshaled []prompbmarshal.Label.
- Add `dedup_interval` option at aggregation config, which allows setting individual
deduplication intervals per each aggregation.
- Add `keep_metric_names` option at aggregation config, which allows keeping the original
metric names in the output samples.
- Add `unique_samples` output, which counts the number of unique sample values.
- Add `increase_prometheus` and `total_prometheus` outputs, which ignore the first sample
per each newly encountered time series.
- Use 64-bit hashes instead of marshaled labels as map keys when calculating `count_series` output.
This makes obsolete https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5579
- Expose various metrics, which may help debugging stream aggregation:
- vm_streamaggr_dedup_state_size_bytes - the size of data structures responsible for deduplication
- vm_streamaggr_dedup_state_items_count - the number of items in the deduplication data structures
- vm_streamaggr_labels_compressor_size_bytes - the size of labels compressor data structures
- vm_streamaggr_labels_compressor_items_count - the number of entries in the labels compressor
- vm_streamaggr_flush_duration_seconds - a histogram, which shows the duration of stream aggregation flushes
- vm_streamaggr_dedup_flush_duration_seconds - a histogram, which shows the duration of deduplication flushes
- vm_streamaggr_flush_timeouts_total - counter for timed out stream aggregation flushes,
which took longer than the configured interval
- vm_streamaggr_dedup_flush_timeouts_total - counter for timed out deduplication flushes,
which took longer than the configured dedup_interval
- Actualize docs/stream-aggregation.md
The memory usage reduction increases CPU usage during stream aggregation by up to 30%.
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5898
2024-03-02 00:42:26 +00:00
|
|
|
sv := v.(*avgStateValue)
|
|
|
|
sv.mu.Lock()
|
|
|
|
deleted := sv.deleted
|
|
|
|
if !deleted {
|
2024-07-03 10:42:45 +00:00
|
|
|
sv.state[idx].sum += s.value
|
|
|
|
sv.state[idx].count++
|
|
|
|
sv.deleteDeadline = deleteDeadline
|
lib/streamaggr: huge pile of changes
- Reduce memory usage by up to 5x when de-duplicating samples across big number of time series.
- Reduce memory usage by up to 5x when aggregating across big number of output time series.
- Add lib/promutils.LabelsCompressor, which is going to be used by other VictoriaMetrics components
for reducing memory usage for marshaled []prompbmarshal.Label.
- Add `dedup_interval` option at aggregation config, which allows setting individual
deduplication intervals per each aggregation.
- Add `keep_metric_names` option at aggregation config, which allows keeping the original
metric names in the output samples.
- Add `unique_samples` output, which counts the number of unique sample values.
- Add `increase_prometheus` and `total_prometheus` outputs, which ignore the first sample
per each newly encountered time series.
- Use 64-bit hashes instead of marshaled labels as map keys when calculating `count_series` output.
This makes obsolete https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5579
- Expose various metrics, which may help debugging stream aggregation:
- vm_streamaggr_dedup_state_size_bytes - the size of data structures responsible for deduplication
- vm_streamaggr_dedup_state_items_count - the number of items in the deduplication data structures
- vm_streamaggr_labels_compressor_size_bytes - the size of labels compressor data structures
- vm_streamaggr_labels_compressor_items_count - the number of entries in the labels compressor
- vm_streamaggr_flush_duration_seconds - a histogram, which shows the duration of stream aggregation flushes
- vm_streamaggr_dedup_flush_duration_seconds - a histogram, which shows the duration of deduplication flushes
- vm_streamaggr_flush_timeouts_total - counter for timed out stream aggregation flushes,
which took longer than the configured interval
- vm_streamaggr_dedup_flush_timeouts_total - counter for timed out deduplication flushes,
which took longer than the configured dedup_interval
- Actualize docs/stream-aggregation.md
The memory usage reduction increases CPU usage during stream aggregation by up to 30%.
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5898
2024-03-02 00:42:26 +00:00
|
|
|
}
|
|
|
|
sv.mu.Unlock()
|
|
|
|
if deleted {
|
2024-03-03 23:21:39 +00:00
|
|
|
// The entry has been deleted by the concurrent call to flushState
|
lib/streamaggr: huge pile of changes
- Reduce memory usage by up to 5x when de-duplicating samples across big number of time series.
- Reduce memory usage by up to 5x when aggregating across big number of output time series.
- Add lib/promutils.LabelsCompressor, which is going to be used by other VictoriaMetrics components
for reducing memory usage for marshaled []prompbmarshal.Label.
- Add `dedup_interval` option at aggregation config, which allows setting individual
deduplication intervals per each aggregation.
- Add `keep_metric_names` option at aggregation config, which allows keeping the original
metric names in the output samples.
- Add `unique_samples` output, which counts the number of unique sample values.
- Add `increase_prometheus` and `total_prometheus` outputs, which ignore the first sample
per each newly encountered time series.
- Use 64-bit hashes instead of marshaled labels as map keys when calculating `count_series` output.
This makes obsolete https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5579
- Expose various metrics, which may help debugging stream aggregation:
- vm_streamaggr_dedup_state_size_bytes - the size of data structures responsible for deduplication
- vm_streamaggr_dedup_state_items_count - the number of items in the deduplication data structures
- vm_streamaggr_labels_compressor_size_bytes - the size of labels compressor data structures
- vm_streamaggr_labels_compressor_items_count - the number of entries in the labels compressor
- vm_streamaggr_flush_duration_seconds - a histogram, which shows the duration of stream aggregation flushes
- vm_streamaggr_dedup_flush_duration_seconds - a histogram, which shows the duration of deduplication flushes
- vm_streamaggr_flush_timeouts_total - counter for timed out stream aggregation flushes,
which took longer than the configured interval
- vm_streamaggr_dedup_flush_timeouts_total - counter for timed out deduplication flushes,
which took longer than the configured dedup_interval
- Actualize docs/stream-aggregation.md
The memory usage reduction increases CPU usage during stream aggregation by up to 30%.
This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5850
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5898
2024-03-02 00:42:26 +00:00
|
|
|
// Try obtaining and updating the entry again.
|
|
|
|
goto again
|
2023-01-04 06:19:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-08-07 09:34:18 +00:00
|
|
|
func (as *avgAggrState) flushState(ctx *flushCtx) {
|
2023-01-04 06:19:18 +00:00
|
|
|
m := &as.m
|
2024-07-09 22:14:15 +00:00
|
|
|
m.Range(func(k, v any) bool {
|
2024-08-07 09:34:18 +00:00
|
|
|
// Atomically delete the entry from the map, so new entry is created for the next flush.
|
2023-01-04 06:19:18 +00:00
|
|
|
sv := v.(*avgStateValue)
|
|
|
|
sv.mu.Lock()
|
2024-03-04 17:12:06 +00:00
|
|
|
|
2024-07-03 10:42:45 +00:00
|
|
|
// check for stale entries
|
|
|
|
deleted := ctx.flushTimestamp > sv.deleteDeadline
|
|
|
|
if deleted {
|
|
|
|
// Mark the current entry as deleted
|
|
|
|
sv.deleted = deleted
|
|
|
|
sv.mu.Unlock()
|
|
|
|
m.Delete(k)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
state := sv.state[ctx.idx]
|
|
|
|
sv.state[ctx.idx] = avgState{}
|
|
|
|
sv.mu.Unlock()
|
|
|
|
if state.count > 0 {
|
|
|
|
key := k.(string)
|
|
|
|
avg := state.sum/state.count
|
|
|
|
ctx.appendSeries(key, "avg", avg)
|
|
|
|
}
|
2023-01-04 06:19:18 +00:00
|
|
|
return true
|
|
|
|
})
|
|
|
|
}
|