app/vmselect: fixes multitenant cache init

Previously multitenant cache was inited before flag.Parse call. It
didn't allow to change cache expiration value and default value was
always used.

 This commit adds cache init at the first time cache was called.

 Also this commit adds small cache improvements:
* chore for cleanup cache, it now uses common pattern for in-place items
filtering
* fail cache request fast if item is already expired


---------
Signed-off-by: f41gh7 <nik@victoriametrics.com>
Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>

app/vmselect/netstorage/tenant_cache.go

@@ -24,12 +24,15 @@ var (
 func TenantsCached(qt *querytracer.Tracer, tr storage.TimeRange, deadline searchutils.Deadline) ([]storage.TenantToken, error) {
 	qt.Printf("fetching tenants on timeRange=%s", tr.String())
 
+	initTenantsCacheVOnce.Do(func() {
+		tenantsCacheV = newTenantsCache(*tenantsCacheDuration)
+	})
+
 	cached := tenantsCacheV.get(tr)
 	qt.Printf("fetched %d tenants from cache", len(cached))
 	if len(cached) > 0 {
 		return cached, nil
 	}
-
 	tenants, err := Tenants(qt, tr, deadline)
 	if err != nil {
 		return nil, fmt.Errorf("cannot obtain tenants: %w", err)
@@ -53,10 +56,10 @@ func TenantsCached(qt *querytracer.Tracer, tr storage.TimeRange, deadline search
 	return tt, nil
 }
 
-var tenantsCacheV = func() *tenantsCache {
-	tc := newTenantsCache(*tenantsCacheDuration)
-	return tc
-}()
+var (
+	initTenantsCacheVOnce sync.Once
+	tenantsCacheV         *tenantsCache
+)
 
 type tenantsCacheItem struct {
 	tenants []storage.TenantToken
@@ -97,11 +100,14 @@ func newTenantsCache(expiration time.Duration) *tenantsCache {
 
 func (tc *tenantsCache) cleanupLocked() {
 	expires := time.Now().Add(tc.itemExpiration)
-	for i := len(tc.items) - 1; i >= 0; i-- {
-		if tc.items[i].expires.Before(expires) {
-			tc.items = append(tc.items[:i], tc.items[i+1:]...)
+	itemsTmp := tc.items[:0]
+	for _, item := range tc.items {
+		if item.expires.After(expires) {
+			itemsTmp = append(itemsTmp, item)
 		}
 	}
+
+	tc.items = itemsTmp
 }
 
 func (tc *tenantsCache) put(tr storage.TimeRange, tenants []storage.TenantToken) {
@@ -147,13 +153,15 @@ func (tc *tenantsCache) getInternal(tr storage.TimeRange) []storage.TenantToken
 	if len(tc.items) == 0 {
 		return nil
 	}
+	ct := time.Now()
 
 	result := make(map[storage.TenantToken]struct{})
 	cleanupNeeded := false
 	for idx := range tc.items {
 		ci := tc.items[idx]
-		if ci.expires.Before(time.Now()) {
+		if ci.expires.Before(ct) {
 			cleanupNeeded = true
+			continue
 		}
 
 		if hasIntersection(tr, ci.tr) {

docs/changelog/CHANGELOG.md

@@ -29,6 +29,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix rendering of isolated data points on the graph that are not connected to other points.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): improve the correctness of alert [state restoration](https://docs.victoriametrics.com/vmalert/#alerts-state-on-restarts). Previously, it could result in false-positive alerts if alert was resolved shortly before vmalert restart.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Properly handle [multitenant](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels) query request errors and correctly perform search for available tenants. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7549) for details. This is follow-up after [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1) release changes.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly set `multitenant` cache expiration duration with `search.tenantCacheExpireDuration`. Previously flag always used default value.
 
 ## [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)