github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/promrelabel: optimize relabeling performance for common cases

Browse source
This commit is contained in:
Aliaksandr Valialkin 2021-02-22 00:50:57 +02:00
parent 8b87398333
commit 63c16c3fdf
4 changed files with 446 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -11,6 +11,7 @@
* `process_io_storage_read_bytes_total` - the number of bytes read from storage layer
* `process_io_storage_written_bytes_total` - the number of bytes written to storage layer
* FEATURE: vmagent: export `vm_promscrape_target_relabel_duration_seconds` metric, which can be used for monitoring the time spend on relabeling for discovered targets.
* FEATURE: vmagent: optimize [relabeling](https://victoriametrics.github.io/vmagent.html#relabeling) performance for common cases.
* BUGFIX: vmagent: properly perform graceful shutdown on `SIGINT` and `SIGTERM` signals. The graceful shutdown has been broken in `v1.54.0`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1065

88
lib/promrelabel/relabel.go
View file

@ -115,12 +115,23 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "replace":
bb := relabelBufPool.Get()
bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
if len(bb.B) == 0 && prc.Regex == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInReplacement && !prc.hasCaptureGroupInTargetLabel {
// Fast path for the following rule that just sets label value:
// - target_label: foobar
// replacement: something-here
relabelBufPool.Put(bb)
return setLabelValue(labels, labelsOffset, prc.TargetLabel, prc.Replacement)
if prc.Regex == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInTargetLabel {
if prc.Replacement == "$1" {
// Fast path for the rule that copies source label values to destination:
// - source_labels: [...]
// target_label: foobar
valueStr := string(bb.B)
relabelBufPool.Put(bb)
return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
}
if !prc.hasCaptureGroupInReplacement {
// Fast path for the rule that sets label value:
// - target_label: foobar
// replacement: something-here
relabelBufPool.Put(bb)
labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, prc.Replacement)
return labels
}
}
match := prc.Regex.FindSubmatchIndex(bb.B)
if match == nil {
@ -139,6 +150,13 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "replace_all":
bb := relabelBufPool.Get()
bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
// Fast path - string replacement without regexp.
sourceStr := string(bb.B)
relabelBufPool.Put(bb)
valueStr := strings.ReplaceAll(sourceStr, prefix, prc.Replacement)
return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
}
if !prc.Regex.Match(bb.B) {
// Fast path - nothing to replace.
relabelBufPool.Put(bb)
@ -175,7 +193,14 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "keep":
bb := relabelBufPool.Get()
bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
keep := prc.Regex.Match(bb.B)
keep := false
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
// Fast path - simple string match
keep = prefix == string(bb.B)
} else {
// Slow path - match by regexp
keep = prc.Regex.Match(bb.B)
}
relabelBufPool.Put(bb)
if !keep {
return labels[:labelsOffset]
@ -184,7 +209,14 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "drop":
bb := relabelBufPool.Get()
bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
drop := prc.Regex.Match(bb.B)
drop := false
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
// Fast path - simple string match
drop = prefix == string(bb.B)
} else {
// Slow path - match by regexp
drop = prc.Regex.Match(bb.B)
}
relabelBufPool.Put(bb)
if drop {
return labels[:labelsOffset]
@ -214,16 +246,28 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "labelmap_all":
for i := range src {
label := &src[i]
if !prc.Regex.MatchString(label.Name) {
continue
if prefix, complete := prc.Regex.LiteralPrefix(); complete && !prc.hasCaptureGroupInReplacement {
// Fast path - replace without regexp
label.Name = strings.ReplaceAll(label.Name, prefix, prc.Replacement)
} else {
// Slow path - replace with regexp.
if !prc.Regex.MatchString(label.Name) {
continue
}
label.Name = prc.Regex.ReplaceAllString(label.Name, prc.Replacement)
}
label.Name = prc.Regex.ReplaceAllString(label.Name, prc.Replacement)
}
return labels
case "labeldrop":
keepSrc := true
for i := range src {
if prc.Regex.MatchString(src[i].Name) {
label := &src[i]
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
if prefix == label.Name {
keepSrc = false
break
}
} else if prc.Regex.MatchString(label.Name) {
keepSrc = false
break
}
@ -234,7 +278,11 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
dst := labels[:labelsOffset]
for i := range src {
label := &src[i]
if !prc.Regex.MatchString(label.Name) {
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
if prefix != label.Name {
dst = append(dst, *label)
}
} else if !prc.Regex.MatchString(label.Name) {
dst = append(dst, *label)
}
}
@ -242,7 +290,13 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
case "labelkeep":
keepSrc := true
for i := range src {
if !prc.Regex.MatchString(src[i].Name) {
label := &src[i]
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
if prefix != label.Name {
keepSrc = false
break
}
} else if !prc.Regex.MatchString(src[i].Name) {
keepSrc = false
break
}
@ -253,7 +307,11 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
dst := labels[:labelsOffset]
for i := range src {
label := &src[i]
if prc.Regex.MatchString(label.Name) {
if prefix, complete := prc.Regex.LiteralPrefix(); complete {
if prefix == label.Name {
dst = append(dst, *label)
}
} else if prc.Regex.MatchString(label.Name) {
dst = append(dst, *label)
}
}

181
lib/promrelabel/relabel_test.go
View file

@ -232,6 +232,28 @@ func TestApplyRelabelConfigs(t *testing.T) {
})
})
t.Run("replace_all-hit", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "replace_all",
SourceLabels: []string{"xxx"},
Separator: ";",
TargetLabel: "xxx",
Regex: regexp.MustCompile("-"),
Replacement: ".",
},
}, []prompbmarshal.Label{
{
Name: "xxx",
Value: "a-b-c",
},
}, false, []prompbmarshal.Label{
{
Name: "xxx",
Value: "a.b.c",
},
})
})
t.Run("replace_all-regex-hit", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "replace_all",
@ -265,11 +287,12 @@ func TestApplyRelabelConfigs(t *testing.T) {
hasCaptureGroupInReplacement: true,
},
{
Action: "replace",
SourceLabels: []string{"bar"},
TargetLabel: "zar",
Regex: defaultRegexForRelabelConfig,
Replacement: "b-$1",
Action: "replace",
SourceLabels: []string{"bar"},
TargetLabel: "zar",
Regex: defaultRegexForRelabelConfig,
Replacement: "b-$1",
hasCaptureGroupInReplacement: true,
},
}, []prompbmarshal.Label{
{
@ -444,6 +467,25 @@ func TestApplyRelabelConfigs(t *testing.T) {
}, true, []prompbmarshal.Label{})
})
t.Run("keep-hit", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "keep",
SourceLabels: []string{"foo"},
Regex: regexp.MustCompile("yyy"),
},
}, []prompbmarshal.Label{
{
Name: "foo",
Value: "yyy",
},
}, false, []prompbmarshal.Label{
{
Name: "foo",
Value: "yyy",
},
})
})
t.Run("keep-hit-regexp", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "keep",
@ -489,6 +531,20 @@ func TestApplyRelabelConfigs(t *testing.T) {
})
})
t.Run("drop-hit", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "drop",
SourceLabels: []string{"foo"},
Regex: regexp.MustCompile("yyy"),
},
}, []prompbmarshal.Label{
{
Name: "foo",
Value: "yyy",
},
}, true, []prompbmarshal.Label{})
})
t.Run("drop-hit-regexp", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "drop",
@ -608,7 +664,80 @@ func TestApplyRelabelConfigs(t *testing.T) {
},
})
})
t.Run("labelmap_all-regexp", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "labelmap_all",
Regex: regexp.MustCompile(`ba(.)`),
Replacement: "${1}ss",
},
}, []prompbmarshal.Label{
{
Name: "foo.bar.baz",
Value: "yyy",
},
{
Name: "foozar",
Value: "aaa",
},
}, true, []prompbmarshal.Label{
{
Name: "foo.rss.zss",
Value: "yyy",
},
{
Name: "foozar",
Value: "aaa",
},
})
})
t.Run("labeldrop", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "labeldrop",
Regex: regexp.MustCompile("dropme"),
},
}, []prompbmarshal.Label{
{
Name: "aaa",
Value: "bbb",
},
}, true, []prompbmarshal.Label{
{
Name: "aaa",
Value: "bbb",
},
})
f([]ParsedRelabelConfig{
{
Action: "labeldrop",
Regex: regexp.MustCompile("dropme"),
},
}, []prompbmarshal.Label{
{
Name: "xxx",
Value: "yyy",
},
{
Name: "dropme",
Value: "aaa",
},
{
Name: "foo",
Value: "bar",
},
}, false, []prompbmarshal.Label{
{
Name: "foo",
Value: "bar",
},
{
Name: "xxx",
Value: "yyy",
},
})
})
t.Run("labeldrop-regexp", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "labeldrop",
@ -655,6 +784,48 @@ func TestApplyRelabelConfigs(t *testing.T) {
})
})
t.Run("labelkeep", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "labelkeep",
Regex: regexp.MustCompile("keepme"),
},
}, []prompbmarshal.Label{
{
Name: "keepme",
Value: "aaa",
},
}, true, []prompbmarshal.Label{
{
Name: "keepme",
Value: "aaa",
},
})
f([]ParsedRelabelConfig{
{
Action: "labelkeep",
Regex: regexp.MustCompile("keepme"),
},
}, []prompbmarshal.Label{
{
Name: "keepme",
Value: "aaa",
},
{
Name: "aaaa",
Value: "awef",
},
{
Name: "keepme-aaa",
Value: "234",
},
}, false, []prompbmarshal.Label{
{
Name: "keepme",
Value: "aaa",
},
})
})
t.Run("labelkeep-regexp", func(t *testing.T) {
f([]ParsedRelabelConfig{
{
Action: "labelkeep",

197
lib/promrelabel/relabel_timing_test.go
View file

@ -183,7 +183,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
}
})
})
b.Run("replace-match", func(b *testing.B) {
b.Run("replace-match-regex", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "replace",
@ -272,6 +272,37 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
})
})
b.Run("drop-match", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "drop",
SourceLabels: []string{"id"},
Regex: regexp.MustCompile("yes"),
},
}
labelsOrig := []prompbmarshal.Label{
{
Name: "__name__",
Value: "metric",
},
{
Name: "id",
Value: "yes",
},
}
b.ReportAllocs()
b.SetBytes(1)
b.RunParallel(func(pb *testing.PB) {
var labels []prompbmarshal.Label
for pb.Next() {
labels = append(labels[:0], labelsOrig...)
labels = ApplyRelabelConfigs(labels, 0, prcs, true)
if len(labels) != 0 {
panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 0, labels))
}
}
})
})
b.Run("drop-match-regexp", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "drop",
@ -334,6 +365,49 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
})
})
b.Run("keep-match", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "keep",
SourceLabels: []string{"id"},
Regex: regexp.MustCompile("yes"),
},
}
labelsOrig := []prompbmarshal.Label{
{
Name: "__name__",
Value: "metric",
},
{
Name: "id",
Value: "yes",
},
}
b.ReportAllocs()
b.SetBytes(1)
b.RunParallel(func(pb *testing.PB) {
var labels []prompbmarshal.Label
for pb.Next() {
labels = append(labels[:0], labelsOrig...)
labels = ApplyRelabelConfigs(labels, 0, prcs, true)
if len(labels) != len(labelsOrig) {
panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
}
if labels[0].Name != "__name__" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
}
if labels[0].Value != "metric" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "metric"))
}
if labels[1].Name != "id" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[1].Name, "id"))
}
if labels[1].Value != "yes" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "yes"))
}
}
})
})
b.Run("keep-match-regexp", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "keep",
@ -454,6 +528,42 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
}
})
})
b.Run("labeldrop-match-regexp", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "labeldrop",
Regex: regexp.MustCompile("id.*"),
},
}
labelsOrig := []prompbmarshal.Label{
{
Name: "__name__",
Value: "metric",
},
{
Name: "id",
Value: "foobar-random-string-here",
},
}
b.ReportAllocs()
b.SetBytes(1)
b.RunParallel(func(pb *testing.PB) {
var labels []prompbmarshal.Label
for pb.Next() {
labels = append(labels[:0], labelsOrig...)
labels = ApplyRelabelConfigs(labels, 0, prcs, true)
if len(labels) != 1 {
panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
}
if labels[0].Name != "__name__" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
}
if labels[0].Value != "metric" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "metric"))
}
}
})
})
b.Run("labelkeep-mismatch", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
@ -520,6 +630,91 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
}
})
})
b.Run("labelkeep-match-regexp", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "labelkeep",
Regex: regexp.MustCompile("id.*"),
},
}
labelsOrig := []prompbmarshal.Label{
{
Name: "__name__",
Value: "metric",
},
{
Name: "id",
Value: "foobar-random-string-here",
},
}
b.ReportAllocs()
b.SetBytes(1)
b.RunParallel(func(pb *testing.PB) {
var labels []prompbmarshal.Label
for pb.Next() {
labels = append(labels[:0], labelsOrig...)
labels = ApplyRelabelConfigs(labels, 0, prcs, true)
if len(labels) != 1 {
panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 1, labels))
}
if labels[0].Name != "id" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "id"))
}
if labels[0].Value != "foobar-random-string-here" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
}
}
})
})
b.Run("labelmap", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{
Action: "labelmap",
Regex: regexp.MustCompile("a(.*)"),
Replacement: "$1",
},
}
labelsOrig := []prompbmarshal.Label{
{
Name: "aabc",
Value: "foobar-random-string-here",
},
{
Name: "foo",
Value: "bar",
},
}
b.ReportAllocs()
b.SetBytes(1)
b.RunParallel(func(pb *testing.PB) {
var labels []prompbmarshal.Label
for pb.Next() {
labels = append(labels[:0], labelsOrig...)
labels = ApplyRelabelConfigs(labels, 0, prcs, true)
if len(labels) != 3 {
panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), 3, labels))
}
if labels[0].Name != "aabc" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "aabc"))
}
if labels[0].Value != "foobar-random-string-here" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[0].Value, "foobar-random-string-here"))
}
if labels[1].Name != "abc" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[1].Name, "abc"))
}
if labels[1].Value != "foobar-random-string-here" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[1].Value, "foobar-random-string-here"))
}
if labels[2].Name != "foo" {
panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[2].Name, "foo"))
}
if labels[2].Value != "bar" {
panic(fmt.Errorf("unexpected label value; got %q; want %q", labels[2].Value, "bar"))
}
}
})
})
b.Run("hashmod", func(b *testing.B) {
prcs := []ParsedRelabelConfig{
{