app/vminsert: reduce the max packet size, which vminsert can send to vmstorage

This reduces the max memory usage for vminsert and vmstorage under heavy ingestion rate
by up to 50% on production workload

app/vminsert/netstorage/netstorage.go

@@ -512,8 +512,8 @@ func InitStorageNodes(addrs []string, hashSeed uint64) {
 	}
 
 	maxBufSizePerStorageNode = memory.Allowed() / 8 / len(storageNodes)
-	if maxBufSizePerStorageNode > consts.MaxInsertPacketSize {
+	if maxBufSizePerStorageNode > consts.MaxInsertPacketSizeForVMInsert {
-		maxBufSizePerStorageNode = consts.MaxInsertPacketSize
+		maxBufSizePerStorageNode = consts.MaxInsertPacketSizeForVMInsert
 	}
 
 	for idx, sn := range storageNodes {

docs/CHANGELOG.md

@@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add pre-defined dasbhoards for per-job CPU usage, memory usage and disk IO usage. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2243) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): improve compatibility with [Prometheus Alert Generator specification](https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2340).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `-datasource.disableKeepAlive` command-line flag, which can be used for disabling [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) to datasources. This option can be useful for distributing load among multiple datasources behind TCP proxy such as [HAProxy](http://www.haproxy.org/).
+* FEATURE: [Cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce memory usage by up to 50% for `vminsert` and `vmstorage` under high ingestion rate.
 * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway.html): Allow to read `-ratelimit.config` file from URL. Also add `-atelimit.configCheckInterval` command-line option. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2241).
 * FEATURE: add the following command-line flags, which can be used for fine-grained limiting of CPU and memory usage during various API calls:
 

lib/consts/consts.go

@@ -1,4 +1,10 @@
 package consts
 
-// MaxInsertPacketSize is the maximum packet size in bytes vminsert may send to vmstorage.
+// MaxInsertPacketSizeForVMStorage is the maximum packet size in bytes vmstorage can accept from vmstorage.
-const MaxInsertPacketSize = 100 * 1024 * 1024
+// It cannot be reduced due to backwards compatibility :(
+const MaxInsertPacketSizeForVMStorage = 100 * 1024 * 1024
+
+// MaxInsertPacketSizeForVMInsert is the maximum packet size in bytes vminsert may send to vmstorage.
+// It is smaller than MaxInsertPacketSizeForVMStorage in order to reduce
+// max memory usage occupied by buffers at vminsert and vmstorage.
+const MaxInsertPacketSizeForVMInsert = 30 * 1024 * 1024

lib/protoparser/clusternative/streamparser.go

@@ -31,7 +31,7 @@ func ParseStream(bc *handshake.BufferedConn, callback func(rows []storage.Metric
 	)
 	for {
 		// Do not use unmarshalWork pool, since every unmarshalWork structure usually occupies
-		// big amounts of memory (more than consts.MaxInsertPacketSize bytes).
+		// big amounts of memory (more than consts.MaxInsertPacketSizeForVMStorage bytes).
 		// The pool would result in increased memory usage.
 		uw := &unmarshalWork{}
 		uw.callback = func(rows []storage.MetricRow) {
@@ -74,9 +74,9 @@ func readBlock(dst []byte, bc *handshake.BufferedConn, isReadOnly func() bool) (
 		return dst, err
 	}
 	packetSize := encoding.UnmarshalUint64(sizeBuf.B)
-	if packetSize > consts.MaxInsertPacketSize {
+	if packetSize > consts.MaxInsertPacketSizeForVMStorage {
 		parseErrors.Inc()
-		return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSize)
+		return dst, fmt.Errorf("too big packet size: %d; shouldn't exceed %d", packetSize, consts.MaxInsertPacketSizeForVMStorage)
 	}
 	dstLen := len(dst)
 	dst = bytesutil.ResizeWithCopyMayOverallocate(dst, dstLen+int(packetSize))