github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-21 15:45:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vminsert/netstorage: reduce the maximum buffer size for rerouted rows, so it occupies less RAM

Browse source
This commit is contained in:
Aliaksandr Valialkin 2019-09-11 14:25:53 +03:00
parent b101064f8b
commit ed50b8792b
1 changed files with 25 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/vminsert/netstorage/netstorage.go
View file

@ -107,14 +107,14 @@ func (sn *storageNode) flushBufLocked() error {
} }
func (sn *storageNode) sendBufLocked(buf []byte) error { func (sn *storageNode) sendBufLocked(buf []byte) error {
if len(buf) == 0 {
return nil
}
if sn.bc == nil { if sn.bc == nil {
if err := sn.dial(); err != nil { if err := sn.dial(); err != nil {
return fmt.Errorf("cannot dial %q: %s", sn.dialer.Addr(), err) return fmt.Errorf("cannot dial %q: %s", sn.dialer.Addr(), err)
} }
} }
if len(buf) == 0 {
return nil
}
timeoutSeconds := len(buf) / 1e6 timeoutSeconds := len(buf) / 1e6
if timeoutSeconds < 10 { if timeoutSeconds < 10 {
timeoutSeconds = 10 timeoutSeconds = 10
@ -317,7 +317,7 @@ func InitStorageNodes(addrs []string) {
}(addr) }(addr)
} }
reroutedBufMaxSize = memory.Allowed() / 8 reroutedBufMaxSize = memory.Allowed() / 16
rerouteWorkerWG.Add(1) rerouteWorkerWG.Add(1)
go func() { go func() {
rerouteWorker(rerouteWorkerStopCh) rerouteWorker(rerouteWorkerStopCh)
@ -396,7 +396,9 @@ func spreadReroutedBufToStorageNodes(swapBuf []byte) ([]byte, error) {
idx = 0 idx = 0
} }
attempts++ attempts++
if attempts == len(healthyStorageNodes) { if attempts < len(healthyStorageNodes) {
continue
}
// There are no healthy nodes. // There are no healthy nodes.
// Try returning the remaining data to reroutedBuf if it has enough free space. // Try returning the remaining data to reroutedBuf if it has enough free space.
rowsRemaining := rows - rowsProcessed rowsRemaining := rows - rowsProcessed
@ -417,7 +419,6 @@ func spreadReroutedBufToStorageNodes(swapBuf []byte) ([]byte, error) {
rowsLostTotal.Add(rowsRemaining) rowsLostTotal.Add(rowsRemaining)
return swapBuf, fmt.Errorf("all the %d vmstorage nodes are unavailable; lost %d rows; last error: %s", len(storageNodes), rowsRemaining, err) return swapBuf, fmt.Errorf("all the %d vmstorage nodes are unavailable; lost %d rows; last error: %s", len(storageNodes), rowsRemaining, err)
} }
}
rowsProcessed++ rowsProcessed++
} }
if rowsProcessed != rows { if rowsProcessed != rows {