github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-05-09 20:25:20 +02:00
parent 28db8795ee
commit 6da2f28d36
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
1 changed files with 8 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
lib/logstorage/tokenizer.go
View file

@ -10,16 +10,12 @@ import (
// the order of returned tokens is unspecified.
func tokenizeStrings(dst, a []string) []string {
t := getTokenizer()
m := t.m
for i, s := range a {
if i > 0 && s == a[i-1] {
// This string has been already tokenized
continue
}
tokenizeString(m, s)
}
for k := range t.m {
dst = append(dst, k)
dst = t.tokenizeString(dst, s)
}
putTokenizer(t)
@ -34,7 +30,8 @@ func (t *tokenizer) reset() {
clear(t.m)
}
func tokenizeString(dst map[string]struct{}, s string) {
func (t *tokenizer) tokenizeString(dst []string, s string) []string {
m := t.m
for len(s) > 0 {
// Search for the next token.
nextIdx := len(s)
@ -55,10 +52,14 @@ func tokenizeString(dst map[string]struct{}, s string) {
}
token := s[:nextIdx]
if len(token) > 0 {
dst[token] = struct{}{}
if _, ok := m[token]; ok {
m[token] = struct{}{}
dst = append(dst, token)
}
}
s = s[nextIdx:]
}
return dst
}
func isTokenRune(c rune) bool {