github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vminsert: reduce the max packet size, which vminsert can send to vmstorage

Browse source 
This reduces the max memory usage for vminsert and vmstorage under heavy ingestion rate
by up to 50% on production workload
This commit is contained in:
Aliaksandr Valialkin 2022-04-05 15:35:08 +03:00
parent a8337c7170
commit 8752cce157
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
4 changed files with 14 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/vminsert/netstorage/netstorage.go
View file

@ -512,8 +512,8 @@ func InitStorageNodes(addrs []string, hashSeed uint64) {
}
maxBufSizePerStorageNode = memory.Allowed() / 8 / len(storageNodes)
if maxBufSizePerStorageNode > consts.MaxInsertPacketSize {
maxBufSizePerStorageNode = consts.MaxInsertPacketSize
if maxBufSizePerStorageNode > consts.MaxInsertPacketSizeForVMInsert {
maxBufSizePerStorageNode = consts.MaxInsertPacketSizeForVMInsert
}
for idx, sn := range storageNodes {

1
docs/CHANGELOG.md
View file

@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add pre-defined dasbhoards for per-job CPU usage, memory usage and disk IO usage. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2243) for details.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): improve compatibility with [Prometheus Alert Generator specification](https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2340).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `-datasource.disableKeepAlive` command-line flag, which can be used for disabling [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) to datasources. This option can be useful for distributing load among multiple datasources behind TCP proxy such as [HAProxy](http://www.haproxy.org/).
* FEATURE: [Cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce memory usage by up to 50% for `vminsert` and `vmstorage` under high ingestion rate.
* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway.html): Allow to read `-ratelimit.config` file from URL. Also add `-atelimit.configCheckInterval` command-line option. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2241).
* FEATURE: add the following command-line flags, which can be used for fine-grained limiting of CPU and memory usage during various API calls:

10
lib/consts/consts.go
View file

@ -1,4 +1,10 @@
package consts
// MaxInsertPacketSize is the maximum packet size in bytes vminsert may send to vmstorage.
const MaxInsertPacketSize = 100 * 1024 * 1024
// MaxInsertPacketSizeForVMStorage is the maximum packet size in bytes vmstorage can accept from vmstorage.
// It cannot be reduced due to backwards compatibility :(
const MaxInsertPacketSizeForVMStorage = 100 * 1024 * 1024
// MaxInsertPacketSizeForVMInsert is the maximum packet size in bytes vminsert may send to vmstorage.
// It is smaller than MaxInsertPacketSizeForVMStorage in order to reduce
// max memory usage occupied by buffers at vminsert and vmstorage.
const MaxInsertPacketSizeForVMInsert = 30 * 1024 * 1024

6
lib/protoparser/clusternative/streamparser.go
View file

@ -31,7 +31,7 @@ func ParseStream(bc *handshake.BufferedConn, callback func(rows []storage.Metric
)
for {
// Do not use unmarshalWork pool, since every unmarshalWork structure usually occupies
// big amounts of memory (more than consts.MaxInsertPacketSize bytes).
// big amounts of memory (more than consts.MaxInsertPacketSizeForVMStorage bytes).
// The pool would result in increased memory usage.
uw := &unmarshalWork{}
uw.callback = func(rows []storage.MetricRow) {
@ -74,9 +74,9 @@ func readBlock(dst []byte, bc *handshake.BufferedConn, isReadOnly func() bool) (
return dst, err
}
packetSize := encoding.UnmarshalUint64(sizeBuf.B)
if packetSize > consts.MaxInsertPacketSize {
if packetSize > consts.MaxInsertPacketSizeForVMStorage {
parseErrors.Inc()
return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSize)
return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSizeForVMStorage)
}
dstLen := len(dst)
dst = bytesutil.ResizeWithCopyMayOverallocate(dst, dstLen+int(packetSize))