From 9fc2817f41ce6fbbf5b1aab238d01c5b88f7b5ff Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 30 Sep 2022 12:25:02 +0300 Subject: [PATCH] lib/promrelabel: optimize `action: replace` for non-trivial regex values Cache `action: replace` results for non-trivial regexs and return them next time instead of performing CPU-intensive regex replacement. Optimize also `action: labelmap_all` and `action: replace_all` in the same way. --- docs/CHANGELOG.md | 1 + lib/promrelabel/config.go | 1 + lib/promrelabel/config_test.go | 1 + lib/promrelabel/relabel.go | 50 ++++++++++++++++++++-------------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b5eabc002..f9080cee1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,7 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +* FEATURE: improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x if non-trivial `regex` values are used. * FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105). * FEATURE: add `-usePromCompatibleNaming` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent.html), to single-node VictoriaMetrics and to `vminsert` component of VictoriaMetrics cluster. This flag can be used for normalizing the ingested metric names and label names to [Prometheus-compatible form](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). If this flag is set, then all the chars unsupported by Prometheus are replaced with `_` chars in metric names and labels of the ingested samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113). * FEATURE: accept whitespace in metric names and tags ingested via [Graphite plaintext protocol](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) according to [the specs](https://graphite.readthedocs.io/en/latest/tags.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3102). diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 65c383dd3..eee9f002c 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -368,6 +368,7 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { hasLabelReferenceInReplacement: strings.Contains(replacement, "{{"), } prc.stringReplacer = bytesutil.NewFastStringTransformer(prc.replaceFullStringSlow) + prc.submatchReplacer = bytesutil.NewFastStringTransformer(prc.replaceStringSubmatchesSlow) return prc, nil } diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index db0919a05..dcc689b27 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -164,6 +164,7 @@ func TestParseRelabelConfigsSuccess(t *testing.T) { if pcs != nil { for _, prc := range pcs.prcs { prc.stringReplacer = nil + prc.submatchReplacer = nil } } if !reflect.DeepEqual(pcs, pcsExpected) { diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 32e65e513..fd78f4f75 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -37,6 +37,7 @@ type parsedRelabelConfig struct { hasLabelReferenceInReplacement bool stringReplacer *bytesutil.FastStringTransformer + submatchReplacer *bytesutil.FastStringTransformer } // String returns human-readable representation for prc. @@ -211,17 +212,23 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset relabelBufPool.Put(bb) return labels } - match := prc.RegexAnchored.FindSubmatchIndex(bb.B) - if match == nil { - // Fast path - nothing to replace. - relabelBufPool.Put(bb) - return labels + var valueStr string + if replacement == prc.Replacement { + // Fast path - the replacement wasn't modified, so it is safe calling stringReplacer.Transform. + valueStr = prc.stringReplacer.Transform(sourceStr) + } else { + // Slow path - the replacement has been modified, so the valueStr must be calculated + // from scratch based on the new replacement value. + match := prc.RegexAnchored.FindSubmatchIndex(bb.B) + valueStr = prc.expandCaptureGroups(replacement, sourceStr, match) } nameStr := prc.TargetLabel if prc.hasCaptureGroupInTargetLabel { + // Slow path - target_label contains regex capture groups, so the target_label + // must be calculated from the regex match. + match := prc.RegexAnchored.FindSubmatchIndex(bb.B) nameStr = prc.expandCaptureGroups(nameStr, sourceStr, match) } - valueStr := prc.expandCaptureGroups(replacement, sourceStr, match) relabelBufPool.Put(bb) return setLabelValue(labels, labelsOffset, nameStr, valueStr) case "replace_all": @@ -231,8 +238,8 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator) sourceStr := string(bb.B) relabelBufPool.Put(bb) - valueStr, ok := prc.replaceStringSubmatches(sourceStr, prc.Replacement, prc.hasCaptureGroupInReplacement) - if ok { + valueStr := prc.replaceStringSubmatchesFast(sourceStr) + if valueStr != sourceStr { labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr) } return labels @@ -317,7 +324,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset // Replace all the occurrences of `regex` at label names with `replacement` for i := range src { label := &src[i] - label.Name, _ = prc.replaceStringSubmatches(label.Name, prc.Replacement, prc.hasCaptureGroupInReplacement) + label.Name = prc.replaceStringSubmatchesFast(label.Name) } return labels case "labeldrop": @@ -421,19 +428,20 @@ func (prc *parsedRelabelConfig) replaceFullStringSlow(s string) string { return prc.expandCaptureGroups(prc.Replacement, s, match) } -func (prc *parsedRelabelConfig) replaceStringSubmatches(s, replacement string, hasCaptureGroupInReplacement bool) (string, bool) { - re := prc.regexOriginal - prefix, complete := re.LiteralPrefix() - if complete && !hasCaptureGroupInReplacement { - if !strings.Contains(s, prefix) { - return s, false - } - return strings.ReplaceAll(s, prefix, replacement), true +// replaceStringSubmatchesFast replaces all the regex matches with the replacement in s. +func (prc *parsedRelabelConfig) replaceStringSubmatchesFast(s string) string { + prefix, complete := prc.regexOriginal.LiteralPrefix() + if complete && !prc.hasCaptureGroupInReplacement && !strings.Contains(s, prefix) { + // Fast path - zero regex matches in s. + return s } - if !re.MatchString(s) { - return s, false - } - return re.ReplaceAllString(s, replacement), true + // Slow path - replace all the regex matches in s with the replacement. + return prc.submatchReplacer.Transform(s) +} + +// replaceStringSubmatchesSlow replaces all the regex matches with the replacement in s. +func (prc *parsedRelabelConfig) replaceStringSubmatchesSlow(s string) string { + return prc.regexOriginal.ReplaceAllString(s, prc.Replacement) } func (prc *parsedRelabelConfig) expandCaptureGroups(template, source string, match []int) string {