From 5dd37ad836ef1381ce7d31e9efdd68d7aae984b5 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 22 Jan 2024 23:56:25 +0200
Subject: [PATCH] app/vmselect/netstorage: use []blockRef from blockRefPool in
order to reduce memory allocations
---
app/vmselect/netstorage/netstorage.go | 39 ++++++++++++++++++++-------
1 file changed, 30 insertions(+), 9 deletions(-)
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index da4468f5ad..f60ad3f67b 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -5,10 +5,12 @@ import (
"errors"
"flag"
"fmt"
+ "reflect"
"sort"
"sync"
"sync/atomic"
"time"
+ "unsafe"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -1169,8 +1171,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
indexSearchDuration.UpdateDuration(startTime)
type blockRefs struct {
- brsPrealloc [4]blockRef
- brs []blockRef
+ brs []blockRef
}
blocksRead := 0
@@ -1178,7 +1179,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
tbf := getTmpBlocksFile()
var buf []byte
- // metricNamesBuf is used for holding all the loaded unique metric names.
+ // metricNamesBuf is used for holding all the loaded unique metric names at m and orderedMetricNames.
// It should reduce pressure on Go GC by reducing the number of string allocations
// when constructing metricName string from byte slice.
var metricNamesBuf []byte
@@ -1187,6 +1188,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
// It should reduce pressure on Go GC by reducing the number of blockRefs allocations.
brssPool := make([]blockRefs, 0, maxSeriesCount)
+ // brsPool is used for holding the most of blockRefs.brs slices across all the loaded time series.
+ // It should reduce pressure on Go GC by reducing the number of allocations for blockRefs.brs slices.
+ brsPool := make([]blockRef, 0, maxSeriesCount)
+
// m maps from metricName to the index of blockRefs inside brssPool
m := make(map[string]int, maxSeriesCount)
@@ -1224,14 +1229,24 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
brssPool = append(brssPool, blockRefs{})
}
brsIdx = len(brssPool) - 1
- brs := &brssPool[brsIdx]
- brs.brs = brs.brsPrealloc[:0]
}
brs := &brssPool[brsIdx]
- brs.brs = append(brs.brs, blockRef{
- partRef: br.PartRef(),
- addr: addr,
- })
+ partRef := br.PartRef()
+ if brs.brs == nil || haveSameBlockRefTails(brs.brs, brsPool) {
+ // It is safe appending blockRef to brsPool, since there are no other items added there yet.
+ brsPool = append(brsPool, blockRef{
+ partRef: partRef,
+ addr: addr,
+ })
+ brs.brs = brsPool[len(brsPool)-len(brs.brs)-1 : len(brsPool) : len(brsPool)]
+ } else {
+ // It is unsafe appending blockRef to brsPool, since there are other items added there.
+ // So just append it to brs.brs.
+ brs.brs = append(brs.brs, blockRef{
+ partRef: partRef,
+ addr: addr,
+ })
+ }
if len(brs.brs) == 1 {
metricNamesBufLen := len(metricNamesBuf)
metricNamesBuf = append(metricNamesBuf, metricName...)
@@ -1279,6 +1294,12 @@ type blockRef struct {
addr tmpBlockAddr
}
+func haveSameBlockRefTails(a, b []blockRef) bool {
+ sha := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+ shb := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+ return sha.Data+uintptr(sha.Len)*unsafe.Sizeof(blockRef{}) == shb.Data+uintptr(shb.Len)*unsafe.Sizeof(blockRef{})
+}
+
func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]storage.TagFilter, maxMetrics int, deadline searchutils.Deadline) ([]*storage.TagFilters, error) {
tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
for _, tagFilters := range tagFilterss {