github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-19 15:30:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmagent/remotewrite: reduce memory usage when samples with big number of labels are sent to remote storage

Browse source
This commit is contained in:
Aliaksandr Valialkin 2021-03-31 00:44:31 +03:00
parent 108bf68a91
commit b873b965af
2 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/vmagent/remotewrite/pendingseries.go
View file

@ -27,6 +27,9 @@ var (
// the maximum number of rows to send per each block. // the maximum number of rows to send per each block.
const maxRowsPerBlock = 10000 const maxRowsPerBlock = 10000
// the maximum number of labels to send per each block.
const maxLabelsPerBlock = 40000
type pendingSeries struct { type pendingSeries struct {
mu sync.Mutex mu sync.Mutex
wr writeRequest wr writeRequest
@ -153,7 +156,7 @@ func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
for i := range src { for i := range src {
tssDst = append(tssDst, prompbmarshal.TimeSeries{}) tssDst = append(tssDst, prompbmarshal.TimeSeries{})
wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i]) wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
if len(wr.samples) >= maxRowsPerBlock { if len(wr.samples) >= maxRowsPerBlock || len(wr.labels) >= maxLabelsPerBlock {
wr.tss = tssDst wr.tss = tssDst
wr.flush() wr.flush()
tssDst = wr.tss tssDst = wr.tss

4
app/vmagent/remotewrite/remotewrite.go
View file

@ -151,11 +151,13 @@ func Push(wr *prompbmarshal.WriteRequest) {
for len(tss) > 0 { for len(tss) > 0 {
// Process big tss in smaller blocks in order to reduce the maximum memory usage // Process big tss in smaller blocks in order to reduce the maximum memory usage
samplesCount := 0 samplesCount := 0
labelsCount := 0
i := 0 i := 0
for i < len(tss) { for i < len(tss) {
samplesCount += len(tss[i].Samples) samplesCount += len(tss[i].Samples)
labelsCount += len(tss[i].Labels)
i++ i++
if samplesCount > maxRowsPerBlock { if samplesCount >= maxRowsPerBlock || labelsCount >= maxLabelsPerBlock {
break break
} }
} }