From 7f34158db5f5543f624a095c005701fea6a1ae06 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Sun, 12 Mar 2023 00:42:17 -0800
Subject: [PATCH] app/vmselect/netstorage: do not intern string representation
of MetricName for time series received from vmstorage
It has been appeared that this interning may lead to increased memory usage and increased CPU usage
when vmselect performs queries, which select big number of time series.
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3863
---
app/vmselect/netstorage/netstorage.go | 11 +++++++----
docs/CHANGELOG.md | 3 ++-
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index 4380cfd8df..2949a8ca48 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -1365,9 +1365,11 @@ func (tbfw *tmpBlocksFileWrapper) RegisterAndWriteBlock(mb *storage.MetricBlock,
if err != nil {
return err
}
- metricName := bytesutil.InternBytes(mb.MetricName)
+ // Do not intern mb.MetricName, since it leads to increased memory usage.
+ // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
+ metricName := mb.MetricName
m := tbfw.ms[workerID]
- addrs := m[metricName]
+ addrs := m[string(metricName)]
if addrs == nil {
addrs = newBlockAddrs()
}
@@ -1376,8 +1378,9 @@ func (tbfw *tmpBlocksFileWrapper) RegisterAndWriteBlock(mb *storage.MetricBlock,
// An optimization for big number of time series with long names: store only a single copy of metricNameStr
// in both tbfw.orderedMetricNamess and tbfw.ms.
orderedMetricNames := tbfw.orderedMetricNamess[workerID]
- orderedMetricNames = append(orderedMetricNames, metricName)
- m[metricName] = addrs
+ metricNameStr := string(metricName)
+ orderedMetricNames = append(orderedMetricNames, metricNameStr)
+ m[metricNameStr] = addrs
tbfw.orderedMetricNamess[workerID] = orderedMetricNames
}
return nil
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index feec47873e..8d6387484d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,7 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
## v1.87.x long-time support release (LTS)
-* BUGFIX: fix a bug, which could lead to incomplete or empty results for heavy queries selecting tens of thousands of time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3946).
+* BUGFIX: vmstorage: fix a bug, which could lead to incomplete or empty results for heavy queries selecting tens of thousands of time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3946).
+* BUGFIX: vmselect: reduce memory usage and CPU usage when performing heavy queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692).
* BUGFIX: prevent from possible `invalid memory address or nil pointer dereference` panic during [background merge](https://docs.victoriametrics.com/#storage). The issue has been introduced at [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3897).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not register `vm_promscrape_config_*` metrics if `-promscrape.config` flag is not used. Previously those metrics were registered and never updated, which was confusing and could trigger false-positive alerts.
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): skip measurements with no fields when migrating data from influxdb. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3837).