mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
84 lines
2.8 KiB
Go
84 lines
2.8 KiB
Go
package bytesutil
|
|
|
|
import (
|
|
"flag"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
|
)
|
|
|
|
var (
|
|
internStringMaxLen = flag.Int("internStringMaxLen", 500, "The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. "+
|
|
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration")
|
|
disableCache = flag.Bool("internStringDisableCache", false, "Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. "+
|
|
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen")
|
|
cacheExpireDuration = flag.Duration("internStringCacheExpireDuration", 6*time.Minute, "The expiry duration for caches for interned strings. "+
|
|
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache")
|
|
)
|
|
|
|
func isSkipCache(s string) bool {
|
|
return *disableCache || len(s) > *internStringMaxLen
|
|
}
|
|
|
|
// InternBytes interns b as a string
|
|
func InternBytes(b []byte) string {
|
|
s := ToUnsafeString(b)
|
|
return InternString(s)
|
|
}
|
|
|
|
// InternString returns interned s.
|
|
//
|
|
// This may be needed for reducing the amounts of allocated memory.
|
|
func InternString(s string) string {
|
|
if isSkipCache(s) {
|
|
// Make a new copy for s in order to remove references from possible bigger string s refers to.
|
|
// This also protects from cases when s points to unsafe string - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227
|
|
return strings.Clone(s)
|
|
}
|
|
|
|
ct := fasttime.UnixTimestamp()
|
|
if v, ok := internStringsMap.Load(s); ok {
|
|
e := v.(*ismEntry)
|
|
if atomic.LoadUint64(&e.lastAccessTime)+10 < ct {
|
|
// Reduce the frequency of e.lastAccessTime update to once per 10 seconds
|
|
// in order to improve the fast path speed on systems with many CPU cores.
|
|
atomic.StoreUint64(&e.lastAccessTime, ct)
|
|
}
|
|
return e.s
|
|
}
|
|
// Make a new copy for s in order to remove references from possible bigger string s refers to.
|
|
sCopy := strings.Clone(s)
|
|
e := &ismEntry{
|
|
lastAccessTime: ct,
|
|
s: sCopy,
|
|
}
|
|
internStringsMap.Store(sCopy, e)
|
|
|
|
if needCleanup(&internStringsMapLastCleanupTime, ct) {
|
|
// Perform a global cleanup for internStringsMap by removing items, which weren't accessed during the last 5 minutes.
|
|
m := &internStringsMap
|
|
deadline := ct - uint64(cacheExpireDuration.Seconds())
|
|
m.Range(func(k, v interface{}) bool {
|
|
e := v.(*ismEntry)
|
|
if atomic.LoadUint64(&e.lastAccessTime) < deadline {
|
|
m.Delete(k)
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
return sCopy
|
|
}
|
|
|
|
type ismEntry struct {
|
|
lastAccessTime uint64
|
|
s string
|
|
}
|
|
|
|
var (
|
|
internStringsMap sync.Map
|
|
internStringsMapLastCleanupTime uint64
|
|
)
|