lib/storage: make indexdb/tagFilters cache size configurable (#2667)

The default size of `indexdb/tagFilters` now can be overridden via
`storage.cacheSizeIndexDBTagFilters` flag.
Please, be careful with changing default size since it may
lead to inefficient work of the vmstorage or OOM exceptions.

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2663
Signed-off-by: hagen1778 <roman@victoriametrics.com>

Co-authored-by: Nikolay <nik@victoriametrics.com>

app/vmstorage/main.go

@@ -57,6 +57,7 @@ var (
 	cacheSizeStorageTSID        = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBDataBlocks  = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
+	cacheSizeIndexDBTagFilters  = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFilters cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 )
 
 func main() {
@@ -75,6 +76,7 @@ func main() {
 	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.N)
+	storage.SetTagFilterCacheSize(cacheSizeIndexDBTagFilters.N)
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.N)
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.N)
 

docs/CHANGELOG.md

@@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 ## tip
 
+* FEATURE: allow overriding default limits for in-memory cache `indexdb/tagFilters` via flag `-storage.cacheSizeIndexDBTagFilters`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2663).
 * FEATURE: add support of `lowercase` and `uppercase` relabeling actions for compatibility reasons. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2664).
 * FEATURE: support query tracing, which allows determining bottlenecks during query processing. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#query-tracing) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1403).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): remove dependency on Internet access in `http://vmagent:8429/targets` page. Previously the page layout was broken without Internet access. See [shis issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2594).

docs/README.md

@@ -1537,7 +1537,8 @@ The panel `Cache usage %` in `Troubleshooting` section shows the percentage of u
 from the allowed size by type. If the percentage is below 100%, then no further tuning needed.
 
 Please note, default cache sizes were carefully adjusted accordingly to the most
-practical scenarios and workloads. Change the defaults only if you understand the implications.
+practical scenarios and workloads. Change the defaults only if you understand the implications
+and vmstorage has enough free memory to accommodate the new size. 
 
 To override the default values see command-line flags with `-storage.cacheSize` prefix.
 See the full description of flags [here](#list-of-command-line-flags).

docs/Single-server-VictoriaMetrics.md

@@ -1541,7 +1541,8 @@ The panel `Cache usage %` in `Troubleshooting` section shows the percentage of u
 from the allowed size by type. If the percentage is below 100%, then no further tuning needed.
 
 Please note, default cache sizes were carefully adjusted accordingly to the most
-practical scenarios and workloads. Change the defaults only if you understand the implications.
+practical scenarios and workloads. Change the defaults only if you understand the implications
+and vmstorage has enough free memory to accommodate the new size. 
 
 To override the default values see command-line flags with `-storage.cacheSize` prefix.
 See the full description of flags [here](#list-of-command-line-flags).

lib/storage/index_db.go

@@ -110,6 +110,20 @@ type indexDB struct {
 	indexSearchPool sync.Pool
 }
 
+var maxTagFilterCacheSize int
+
+// SetTagFilterCacheSize overrides the default size of indexdb/tagFilters cache
+func SetTagFilterCacheSize(size int) {
+	maxTagFilterCacheSize = size
+}
+
+func getTagFilterCacheSize() int {
+	if maxTagFilterCacheSize <= 0 {
+		return int(float64(memory.Allowed()) / 32)
+	}
+	return maxTagFilterCacheSize
+}
+
 // openIndexDB opens index db from the given path.
 //
 // The last segment of the path should contain unique hex value which
@@ -143,7 +157,7 @@ func openIndexDB(path string, s *Storage, rotationTimestamp uint64, isReadOnly *
 		tb:                tb,
 		name:              name,
 
-		tagFiltersCache:            workingsetcache.New(mem / 32),
+		tagFiltersCache:            workingsetcache.New(getTagFilterCacheSize()),
 		s:                          s,
 		loopsPerDateTagFilterCache: workingsetcache.New(mem / 128),
 	}

lib/storage/storage.go

@@ -291,7 +291,7 @@ func (s *Storage) RetentionMsecs() int64 {
 
 var maxTSIDCacheSize int
 
-// SetTSIDCacheSize overrides the default size of storage/tsid cahce
+// SetTSIDCacheSize overrides the default size of storage/tsid cache
 func SetTSIDCacheSize(size int) {
 	maxTSIDCacheSize = size
 }