github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vminsert/netstorage: log network errors when sending data to vmstorage nodes

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/175
This commit is contained in:
Aliaksandr Valialkin 2019-09-13 22:25:15 +03:00
parent 41ef6b060e
commit 550a12415a
1 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
app/vminsert/netstorage/netstorage.go
View file

@ -85,7 +85,8 @@ func (sn *storageNode) sendReroutedRow(buf []byte) error {
var errBrokenStorageNode = fmt.Errorf("the vmstorage node is temporarily broken") var errBrokenStorageNode = fmt.Errorf("the vmstorage node is temporarily broken")
func (sn *storageNode) flushBufLocked() error { func (sn *storageNode) flushBufLocked() error {
if err := sn.sendBufLocked(sn.buf); err == nil { err := sn.sendBufLocked(sn.buf)
if err == nil {
// Successful flush. Remove broken flag. // Successful flush. Remove broken flag.
sn.broken = false sn.broken = false
sn.rowsSent.Add(sn.rows) sn.rowsSent.Add(sn.rows)
@ -96,8 +97,9 @@ func (sn *storageNode) flushBufLocked() error {
// Couldn't flush sn.buf to vmstorage. Mark sn as broken // Couldn't flush sn.buf to vmstorage. Mark sn as broken
// and try re-routing sn.buf to healthy vmstorage nodes. // and try re-routing sn.buf to healthy vmstorage nodes.
logger.Errorf("cannot send data to vmstorage %s: %s; re-routing data to healthy vmstorage nodes", sn.dialer.Addr(), err)
sn.broken = true sn.broken = true
err := addToReroutedBuf(sn.buf, sn.rows) err = addToReroutedBuf(sn.buf, sn.rows)
if err != nil { if err != nil {
rowsLostTotal.Add(sn.rows) rowsLostTotal.Add(sn.rows)
} }
@ -116,8 +118,8 @@ func (sn *storageNode) sendBufLocked(buf []byte) error {
} }
} }
timeoutSeconds := len(buf) / 1e6 timeoutSeconds := len(buf) / 1e6
if timeoutSeconds < 10 { if timeoutSeconds < 60 {
timeoutSeconds = 10 timeoutSeconds = 60
} }
timeout := time.Duration(timeoutSeconds) * time.Second timeout := time.Duration(timeoutSeconds) * time.Second
deadline := time.Now().Add(timeout) deadline := time.Now().Add(timeout)