From 5420989018fc98e7e40763ac01b9545c731b26cf Mon Sep 17 00:00:00 2001
From: Zakhar Bessarab <z.bessarab@victoriametrics.com>
Date: Tue, 9 Jul 2024 17:15:48 +0400
Subject: [PATCH] app/vminsert/netstorage: reinitialize snb on vmstorage
 connection restore

It is needed to rebuild snb in order to ensure that list of storage nodes and consistent hash are in sync.
Updating just consistent hash ring is not safe because it can cause misalignment of indexes of alive nodes in snb.sns and hash slots.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
---
 app/vminsert/netstorage/insert_ctx.go |  3 ++-
 app/vminsert/netstorage/netstorage.go | 18 ++++++------------
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/app/vminsert/netstorage/insert_ctx.go b/app/vminsert/netstorage/insert_ctx.go
index efb77a6630..26d9526f0a 100644
--- a/app/vminsert/netstorage/insert_ctx.go
+++ b/app/vminsert/netstorage/insert_ctx.go
@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/cespare/xxhash/v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -12,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/cespare/xxhash/v2"
 )
 
 // InsertCtx is a generic context for inserting data.
diff --git a/app/vminsert/netstorage/netstorage.go b/app/vminsert/netstorage/netstorage.go
index dc0ef01f8f..0258233f76 100644
--- a/app/vminsert/netstorage/netstorage.go
+++ b/app/vminsert/netstorage/netstorage.go
@@ -671,20 +671,14 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 
 				if sn.isReady() {
 				again:
-					oldSnb := getStorageNodesBucket()
-					snbNew := storageNodesBucket{
-						ms:     oldSnb.ms,
-						wg:     oldSnb.wg,
-						stopCh: oldSnb.stopCh,
-						sns:    oldSnb.sns,
-					}
-
-					newNodeIDs := append(oldSnb.nodesHash.nodeHashes, sn.getID())
-					snbNew.nodesHash = newConsistentHash(newNodeIDs, hashSeed)
-
-					if !storageNodes.CompareAndSwap(oldSnb, &snbNew) {
+					currentSnb := getStorageNodesBucket()
+					newSnb := initStorageNodes(addrs, hashSeed)
+					if !storageNodes.CompareAndSwap(currentSnb, newSnb) {
+						mustStopStorageNodes(newSnb)
 						goto again
 					}
+
+					mustStopStorageNodes(currentSnb)
 					break
 				}
 			}