diff --git a/app/vminsert/netstorage/netstorage.go b/app/vminsert/netstorage/netstorage.go index 2c65dea37..ecfbce4c3 100644 --- a/app/vminsert/netstorage/netstorage.go +++ b/app/vminsert/netstorage/netstorage.go @@ -512,8 +512,8 @@ func InitStorageNodes(addrs []string, hashSeed uint64) { } maxBufSizePerStorageNode = memory.Allowed() / 8 / len(storageNodes) - if maxBufSizePerStorageNode > consts.MaxInsertPacketSize { - maxBufSizePerStorageNode = consts.MaxInsertPacketSize + if maxBufSizePerStorageNode > consts.MaxInsertPacketSizeForVMInsert { + maxBufSizePerStorageNode = consts.MaxInsertPacketSizeForVMInsert } for idx, sn := range storageNodes { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b2a0472e5..c502dbec0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add pre-defined dasbhoards for per-job CPU usage, memory usage and disk IO usage. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2243) for details. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): improve compatibility with [Prometheus Alert Generator specification](https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2340). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `-datasource.disableKeepAlive` command-line flag, which can be used for disabling [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) to datasources. This option can be useful for distributing load among multiple datasources behind TCP proxy such as [HAProxy](http://www.haproxy.org/). +* FEATURE: [Cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce memory usage by up to 50% for `vminsert` and `vmstorage` under high ingestion rate. * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway.html): Allow to read `-ratelimit.config` file from URL. Also add `-atelimit.configCheckInterval` command-line option. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2241). * FEATURE: add the following command-line flags, which can be used for fine-grained limiting of CPU and memory usage during various API calls: diff --git a/lib/consts/consts.go b/lib/consts/consts.go index 3b16e56f9..7232b2a6a 100644 --- a/lib/consts/consts.go +++ b/lib/consts/consts.go @@ -1,4 +1,10 @@ package consts -// MaxInsertPacketSize is the maximum packet size in bytes vminsert may send to vmstorage. -const MaxInsertPacketSize = 100 * 1024 * 1024 +// MaxInsertPacketSizeForVMStorage is the maximum packet size in bytes vmstorage can accept from vmstorage. +// It cannot be reduced due to backwards compatibility :( +const MaxInsertPacketSizeForVMStorage = 100 * 1024 * 1024 + +// MaxInsertPacketSizeForVMInsert is the maximum packet size in bytes vminsert may send to vmstorage. +// It is smaller than MaxInsertPacketSizeForVMStorage in order to reduce +// max memory usage occupied by buffers at vminsert and vmstorage. +const MaxInsertPacketSizeForVMInsert = 30 * 1024 * 1024 diff --git a/lib/protoparser/clusternative/streamparser.go b/lib/protoparser/clusternative/streamparser.go index 623172cd9..51ffda38b 100644 --- a/lib/protoparser/clusternative/streamparser.go +++ b/lib/protoparser/clusternative/streamparser.go @@ -31,7 +31,7 @@ func ParseStream(bc *handshake.BufferedConn, callback func(rows []storage.Metric ) for { // Do not use unmarshalWork pool, since every unmarshalWork structure usually occupies - // big amounts of memory (more than consts.MaxInsertPacketSize bytes). + // big amounts of memory (more than consts.MaxInsertPacketSizeForVMStorage bytes). // The pool would result in increased memory usage. uw := &unmarshalWork{} uw.callback = func(rows []storage.MetricRow) { @@ -74,9 +74,9 @@ func readBlock(dst []byte, bc *handshake.BufferedConn, isReadOnly func() bool) ( return dst, err } packetSize := encoding.UnmarshalUint64(sizeBuf.B) - if packetSize > consts.MaxInsertPacketSize { + if packetSize > consts.MaxInsertPacketSizeForVMStorage { parseErrors.Inc() - return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSize) + return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSizeForVMStorage) } dstLen := len(dst) dst = bytesutil.ResizeWithCopyMayOverallocate(dst, dstLen+int(packetSize))