diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5993e681ed..3b7ec6b27a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduce memory usage when sending stale markers for targets, which expose big number of metrics. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3668) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3675) issues. +* FEATURE: add `-internStringMaxLen` command-line flag, which can be used for fine-tuning RAM vs CPU usage in certain workloads. For example, if the stored time series contain long labels, then it may be useful reducing the `-internStringMaxLen` in order to reduce memory usage at the cost of increased CPU usage. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692). * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): propagate all the timeout-related errors from `vmstorage` to `vmselect` when `vmstorage`. Previously some timeout errors weren't returned from `vmselect` to `vmstorage`. Instead, `vmstorage` could log the error and close the connection to `vmselect`, so `vmselect` was logging cryptic errors such as `cannot execute funcName="..." on vmstorage "...": EOF`. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): add support for time zone selection for older versions of browsers. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3680). diff --git a/lib/bytesutil/internstring.go b/lib/bytesutil/internstring.go index f14a8527f1..cfb652e5ee 100644 --- a/lib/bytesutil/internstring.go +++ b/lib/bytesutil/internstring.go @@ -1,6 +1,7 @@ package bytesutil import ( + "flag" "strings" "sync" "sync/atomic" @@ -8,6 +9,9 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" ) +var internStringMaxLen = flag.Int("internStringMaxLen", 300, "The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. "+ + "See https://en.wikipedia.org/wiki/String_interning") + // InternBytes interns b as a string func InternBytes(b []byte) string { s := ToUnsafeString(b) @@ -30,6 +34,12 @@ func InternString(s string) string { } // Make a new copy for s in order to remove references from possible bigger string s refers to. sCopy := strings.Clone(s) + if len(sCopy) > *internStringMaxLen { + // Do not intern long strings, since this may result in high memory usage + // like in https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692 + return sCopy + } + e := &ismEntry{ lastAccessTime: ct, s: sCopy,