diff --git a/app/vminsert/netstorage/netstorage.go b/app/vminsert/netstorage/netstorage.go index 9f32e4eea6..e2600f8be6 100644 --- a/app/vminsert/netstorage/netstorage.go +++ b/app/vminsert/netstorage/netstorage.go @@ -808,9 +808,22 @@ func (sn *storageNode) checkReadOnlyMode() { atomic.StoreUint32(&sn.isReadOnly, 0) return } - if !errors.Is(err, errStorageReadOnly) { - logger.Errorf("cannot check storage readonly mode for -storageNode=%q: %s", sn.dialer.Addr(), err) + if errors.Is(err, errStorageReadOnly) { + // The storage remains in read-only mode + return } + + // There was an error when sending nil buf to the storage. + logger.Errorf("cannot check storage readonly mode for -storageNode=%q: %s", sn.dialer.Addr(), err) + + // Mark the connection to the storage as broken. + if err = sn.bc.Close(); err != nil { + cannotCloseStorageNodeConnLogger.Warnf("cannot close connection to storageNode %q: %s", sn.dialer.Addr(), err) + } + sn.bc = nil + atomic.StoreUint32(&sn.broken, 1) + sn.brCond.Broadcast() + sn.connectionErrors.Inc() } var ( diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7c4568b423..bccb5bc868 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## v1.87.x long-time support release (LTS) +* BUGFIX: [vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly close broken vmstorage connection during [read-only state](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode) checks at `vmstorage`. Previously it wasn't properly closed, which prevents restoring `vmstorage` node from read-only mode. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4870). + ## [v1.87.8](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.8) Released at 2023-09-01