app/vminsert/netstorage: reinitialize snb on vmstorage connection restore

It is needed to rebuild snb in order to ensure that list of storage nodes and consistent hash are in sync.
Updating just consistent hash ring is not safe because it can cause misalignment of indexes of alive nodes in snb.sns and hash slots.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

app/vminsert/netstorage/insert_ctx.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/cespare/xxhash/v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -12,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/cespare/xxhash/v2"
 )
 
 // InsertCtx is a generic context for inserting data.

app/vminsert/netstorage/netstorage.go

@@ -671,20 +671,14 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 
 				if sn.isReady() {
 				again:
-					oldSnb := getStorageNodesBucket()
-					snbNew := storageNodesBucket{
-						ms:     oldSnb.ms,
-						wg:     oldSnb.wg,
-						stopCh: oldSnb.stopCh,
-						sns:    oldSnb.sns,
-					}
-
-					newNodeIDs := append(oldSnb.nodesHash.nodeHashes, sn.getID())
-					snbNew.nodesHash = newConsistentHash(newNodeIDs, hashSeed)
-
-					if !storageNodes.CompareAndSwap(oldSnb, &snbNew) {
+					currentSnb := getStorageNodesBucket()
+					newSnb := initStorageNodes(addrs, hashSeed)
+					if !storageNodes.CompareAndSwap(currentSnb, newSnb) {
+						mustStopStorageNodes(newSnb)
 						goto again
 					}
+
+					mustStopStorageNodes(currentSnb)
 					break
 				}
 			}