diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4d8d9f8fa..f262ef197 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -21,6 +21,7 @@ created by v1.90.0 or newer versions. The solution is to upgrade to v1.90.0 or n * FEATURE: publish VictoriaMetrics binaries for Windows. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3236), [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3821) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70) issues. * FEATURE: log metrics with truncated labels if the length of label value in the ingested metric exceeds `-maxLabelValueLen`. This should simplify debugging for this case. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): show target URL when debugging [target relabeling](https://docs.victoriametrics.com/vmagent.html#relabel-debug). This should simplify target relabel debugging a bit. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3882). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent.html#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `--kafka.consumer.topic.concurrency` command-line flag. It controls the number of Kafka consumer workers to use by `vmagent`. It should eliminate the need to start multiple `vmagent` instances to improve data transfer rate. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1957). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [Kafka producer and consumer](https://docs.victoriametrics.com/vmagent.html#kafka-integration) on `arm64` machines. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2271). diff --git a/lib/promrelabel/debug.go b/lib/promrelabel/debug.go index 360792d13..d4de9e3e5 100644 --- a/lib/promrelabel/debug.go +++ b/lib/promrelabel/debug.go @@ -21,30 +21,32 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela if metric == "" { metric = "{}" } + targetURL := "" if err != nil { - WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err) + WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err) return } labels, err := promutils.NewLabelsFromString(metric) if err != nil { err = fmt.Errorf("cannot parse metric: %s", err) - WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err) + WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err) return } pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs)) if err != nil { err = fmt.Errorf("cannot parse relabel configs: %s", err) - WriteRelabelDebugSteps(w, isTargetRelabel, targetID, nil, metric, relabelConfigs, err) + WriteRelabelDebugSteps(w, targetURL, targetID, nil, metric, relabelConfigs, err) return } - dss := newDebugRelabelSteps(pcs, labels, isTargetRelabel) - WriteRelabelDebugSteps(w, isTargetRelabel, targetID, dss, metric, relabelConfigs, nil) + dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel) + WriteRelabelDebugSteps(w, targetURL, targetID, dss, metric, relabelConfigs, nil) } -func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTargetRelabel bool) []DebugStep { - // The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork if isTragetRelabeling=true +func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTargetRelabel bool) ([]DebugStep, string) { + // The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork if isTargetRelabel=true // and with the code at scrapeWork.addRowToTimeseries when isTargetRelabeling=false + targetURL := "" // Prevent from modifying the original labels labels = labels.Clone() @@ -70,6 +72,9 @@ func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTarget } } + // Generate targetURL + targetURL, _ = GetScrapeURL(labels, nil) + // Remove labels with __ prefix inStr := outStr labels.RemoveLabelsWithDoubleUnderscorePrefix() @@ -96,7 +101,7 @@ func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutils.Labels, isTarget } // There is no need in labels' sorting, since LabelsToString() automatically sorts labels. - return dss + return dss, targetURL } func getChangedLabelNames(in, out *promutils.Labels) map[string]struct{} { diff --git a/lib/promrelabel/debug.qtpl b/lib/promrelabel/debug.qtpl index 2f8244d25..1d894c0ae 100644 --- a/lib/promrelabel/debug.qtpl +++ b/lib/promrelabel/debug.qtpl @@ -5,7 +5,7 @@ {% stripspace %} -{% func RelabelDebugSteps(isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %} +{% func RelabelDebugSteps(targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %} @@ -27,7 +27,7 @@ function submitRelabelDebugForm(e) {
Relabeling docs{% space %} - {% if isTargetRelabel %} + {% if targetURL != "" %} Metric relabel debug {% else %} Target relabel debug @@ -53,7 +53,7 @@ function submitRelabelDebugForm(e) {
- {%= relabelDebugSteps(dss) %} + {%= relabelDebugSteps(dss, targetURL, targetID) %}
@@ -73,10 +73,10 @@ function submitRelabelDebugForm(e) { {% endfunc %} -{% func relabelDebugSteps(dss []DebugStep) %} +{% func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) %} {% if len(dss) > 0 %}
- Original labels: {%= mustFormatLabels(dss[0].In) %} + Original labels: {%= mustFormatLabels(dss[0].In) %}
{% endif %} @@ -114,7 +114,16 @@ function submitRelabelDebugForm(e) {
{% if len(dss) > 0 %}
- Resulting labels: {%= mustFormatLabels(dss[len(dss)-1].Out) %} + Resulting labels: {%= mustFormatLabels(dss[len(dss)-1].Out) %} + {% if targetURL != "" %} +
+ Target URL:{% space %}{%s targetURL %} + {% if targetID != "" %} + {% space %} + (response) + {% endif %} +
+ {% endif %}
{% endif %} {% endfunc %} diff --git a/lib/promrelabel/debug.qtpl.go b/lib/promrelabel/debug.qtpl.go index 4e78569b9..e2d680eff 100644 --- a/lib/promrelabel/debug.qtpl.go +++ b/lib/promrelabel/debug.qtpl.go @@ -24,7 +24,7 @@ var ( ) //line lib/promrelabel/debug.qtpl:8 -func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) { +func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) { //line lib/promrelabel/debug.qtpl:8 qw422016.N().S(``) //line lib/promrelabel/debug.qtpl:12 @@ -38,7 +38,7 @@ func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, isTargetRelabel bool, ta //line lib/promrelabel/debug.qtpl:28 qw422016.N().S(` `) //line lib/promrelabel/debug.qtpl:30 - if isTargetRelabel { + if targetURL != "" { //line lib/promrelabel/debug.qtpl:30 qw422016.N().S(`
`) //line lib/promrelabel/debug.qtpl:56 - streamrelabelDebugSteps(qw422016, dss) + streamrelabelDebugSteps(qw422016, dss, targetURL, targetID) //line lib/promrelabel/debug.qtpl:56 qw422016.N().S(`
`) //line lib/promrelabel/debug.qtpl:62 } //line lib/promrelabel/debug.qtpl:62 -func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) { +func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) { //line lib/promrelabel/debug.qtpl:62 qw422016 := qt422016.AcquireWriter(qq422016) //line lib/promrelabel/debug.qtpl:62 - StreamRelabelDebugSteps(qw422016, isTargetRelabel, targetID, dss, metric, relabelConfigs, err) + StreamRelabelDebugSteps(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err) //line lib/promrelabel/debug.qtpl:62 qt422016.ReleaseWriter(qw422016) //line lib/promrelabel/debug.qtpl:62 } //line lib/promrelabel/debug.qtpl:62 -func RelabelDebugSteps(isTargetRelabel bool, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string { +func RelabelDebugSteps(targetURL string, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string { //line lib/promrelabel/debug.qtpl:62 qb422016 := qt422016.AcquireByteBuffer() //line lib/promrelabel/debug.qtpl:62 - WriteRelabelDebugSteps(qb422016, isTargetRelabel, targetID, dss, metric, relabelConfigs, err) + WriteRelabelDebugSteps(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err) //line lib/promrelabel/debug.qtpl:62 qs422016 := string(qb422016.B) //line lib/promrelabel/debug.qtpl:62 @@ -178,7 +178,7 @@ func relabelDebugFormInputs(metric, relabelConfigs string) string { } //line lib/promrelabel/debug.qtpl:76 -func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep) { +func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) { //line lib/promrelabel/debug.qtpl:77 if len(dss) > 0 { //line lib/promrelabel/debug.qtpl:77 @@ -227,41 +227,75 @@ func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep) { //line lib/promrelabel/debug.qtpl:117 streammustFormatLabels(qw422016, dss[len(dss)-1].Out) //line lib/promrelabel/debug.qtpl:117 - qw422016.N().S(``) -//line lib/promrelabel/debug.qtpl:119 - } + qw422016.N().S(``) +//line lib/promrelabel/debug.qtpl:118 + if targetURL != "" { +//line lib/promrelabel/debug.qtpl:118 + qw422016.N().S(`
Target URL:`) //line lib/promrelabel/debug.qtpl:120 -} - + qw422016.N().S(` `) //line lib/promrelabel/debug.qtpl:120 -func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep) { + qw422016.N().S(``) //line lib/promrelabel/debug.qtpl:120 - qt422016.ReleaseWriter(qw422016) + qw422016.E().S(targetURL) //line lib/promrelabel/debug.qtpl:120 -} - -//line lib/promrelabel/debug.qtpl:120 -func relabelDebugSteps(dss []DebugStep) string { -//line lib/promrelabel/debug.qtpl:120 - qb422016 := qt422016.AcquireByteBuffer() -//line lib/promrelabel/debug.qtpl:120 - writerelabelDebugSteps(qb422016, dss) -//line lib/promrelabel/debug.qtpl:120 - qs422016 := string(qb422016.B) -//line lib/promrelabel/debug.qtpl:120 - qt422016.ReleaseByteBuffer(qb422016) -//line lib/promrelabel/debug.qtpl:120 - return qs422016 -//line lib/promrelabel/debug.qtpl:120 -} - + qw422016.N().S(``) +//line lib/promrelabel/debug.qtpl:121 + if targetID != "" { //line lib/promrelabel/debug.qtpl:122 -func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { + qw422016.N().S(` `) +//line lib/promrelabel/debug.qtpl:122 + qw422016.N().S(`(response)`) //line lib/promrelabel/debug.qtpl:124 + } +//line lib/promrelabel/debug.qtpl:124 + qw422016.N().S(`
`) +//line lib/promrelabel/debug.qtpl:126 + } +//line lib/promrelabel/debug.qtpl:126 + qw422016.N().S(``) +//line lib/promrelabel/debug.qtpl:128 + } +//line lib/promrelabel/debug.qtpl:129 +} + +//line lib/promrelabel/debug.qtpl:129 +func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) { +//line lib/promrelabel/debug.qtpl:129 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promrelabel/debug.qtpl:129 + streamrelabelDebugSteps(qw422016, dss, targetURL, targetID) +//line lib/promrelabel/debug.qtpl:129 + qt422016.ReleaseWriter(qw422016) +//line lib/promrelabel/debug.qtpl:129 +} + +//line lib/promrelabel/debug.qtpl:129 +func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string { +//line lib/promrelabel/debug.qtpl:129 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promrelabel/debug.qtpl:129 + writerelabelDebugSteps(qb422016, dss, targetURL, targetID) +//line lib/promrelabel/debug.qtpl:129 + qs422016 := string(qb422016.B) +//line lib/promrelabel/debug.qtpl:129 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promrelabel/debug.qtpl:129 + return qs422016 +//line lib/promrelabel/debug.qtpl:129 +} + +//line lib/promrelabel/debug.qtpl:131 +func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { +//line lib/promrelabel/debug.qtpl:133 labelsList := labels.GetLabels() metricName := "" for i, label := range labelsList { @@ -272,40 +306,10 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labe } } -//line lib/promrelabel/debug.qtpl:134 - if metricName != "" { -//line lib/promrelabel/debug.qtpl:135 - if _, ok := highlight["__name__"]; ok { -//line lib/promrelabel/debug.qtpl:135 - qw422016.N().S(``) -//line lib/promrelabel/debug.qtpl:136 - qw422016.E().S(metricName) -//line lib/promrelabel/debug.qtpl:136 - qw422016.N().S(``) -//line lib/promrelabel/debug.qtpl:137 - } else { -//line lib/promrelabel/debug.qtpl:138 - qw422016.E().S(metricName) -//line lib/promrelabel/debug.qtpl:139 - } -//line lib/promrelabel/debug.qtpl:140 - if len(labelsList) == 0 { -//line lib/promrelabel/debug.qtpl:140 - return -//line lib/promrelabel/debug.qtpl:140 - } -//line lib/promrelabel/debug.qtpl:141 - } -//line lib/promrelabel/debug.qtpl:141 - qw422016.N().S(`{`) //line lib/promrelabel/debug.qtpl:143 - for i, label := range labelsList { + if metricName != "" { //line lib/promrelabel/debug.qtpl:144 - if _, ok := highlight[label.Name]; ok { + if _, ok := highlight["__name__"]; ok { //line lib/promrelabel/debug.qtpl:144 qw422016.N().S(``) //line lib/promrelabel/debug.qtpl:145 - qw422016.E().S(label.Name) -//line lib/promrelabel/debug.qtpl:145 - qw422016.N().S(`=`) -//line lib/promrelabel/debug.qtpl:145 - qw422016.E().Q(label.Value) + qw422016.E().S(metricName) //line lib/promrelabel/debug.qtpl:145 qw422016.N().S(``) //line lib/promrelabel/debug.qtpl:146 } else { //line lib/promrelabel/debug.qtpl:147 - qw422016.E().S(label.Name) -//line lib/promrelabel/debug.qtpl:147 - qw422016.N().S(`=`) -//line lib/promrelabel/debug.qtpl:147 - qw422016.E().Q(label.Value) + qw422016.E().S(metricName) //line lib/promrelabel/debug.qtpl:148 } //line lib/promrelabel/debug.qtpl:149 - if i < len(labelsList)-1 { + if len(labelsList) == 0 { //line lib/promrelabel/debug.qtpl:149 - qw422016.N().S(`,`) -//line lib/promrelabel/debug.qtpl:149 - qw422016.N().S(` `) + return //line lib/promrelabel/debug.qtpl:149 } //line lib/promrelabel/debug.qtpl:150 } //line lib/promrelabel/debug.qtpl:150 - qw422016.N().S(`}`) + qw422016.N().S(`{`) //line lib/promrelabel/debug.qtpl:152 -} - -//line lib/promrelabel/debug.qtpl:152 -func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { -//line lib/promrelabel/debug.qtpl:152 - qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promrelabel/debug.qtpl:152 - streamlabelsWithHighlight(qw422016, labels, highlight, color) -//line lib/promrelabel/debug.qtpl:152 - qt422016.ReleaseWriter(qw422016) -//line lib/promrelabel/debug.qtpl:152 -} - -//line lib/promrelabel/debug.qtpl:152 -func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string { -//line lib/promrelabel/debug.qtpl:152 - qb422016 := qt422016.AcquireByteBuffer() -//line lib/promrelabel/debug.qtpl:152 - writelabelsWithHighlight(qb422016, labels, highlight, color) -//line lib/promrelabel/debug.qtpl:152 - qs422016 := string(qb422016.B) -//line lib/promrelabel/debug.qtpl:152 - qt422016.ReleaseByteBuffer(qb422016) -//line lib/promrelabel/debug.qtpl:152 - return qs422016 -//line lib/promrelabel/debug.qtpl:152 -} - + for i, label := range labelsList { +//line lib/promrelabel/debug.qtpl:153 + if _, ok := highlight[label.Name]; ok { +//line lib/promrelabel/debug.qtpl:153 + qw422016.N().S(``) +//line lib/promrelabel/debug.qtpl:154 + qw422016.E().S(label.Name) +//line lib/promrelabel/debug.qtpl:154 + qw422016.N().S(`=`) +//line lib/promrelabel/debug.qtpl:154 + qw422016.E().Q(label.Value) +//line lib/promrelabel/debug.qtpl:154 + qw422016.N().S(``) //line lib/promrelabel/debug.qtpl:155 + } else { +//line lib/promrelabel/debug.qtpl:156 + qw422016.E().S(label.Name) +//line lib/promrelabel/debug.qtpl:156 + qw422016.N().S(`=`) +//line lib/promrelabel/debug.qtpl:156 + qw422016.E().Q(label.Value) +//line lib/promrelabel/debug.qtpl:157 + } +//line lib/promrelabel/debug.qtpl:158 + if i < len(labelsList)-1 { +//line lib/promrelabel/debug.qtpl:158 + qw422016.N().S(`,`) +//line lib/promrelabel/debug.qtpl:158 + qw422016.N().S(` `) +//line lib/promrelabel/debug.qtpl:158 + } +//line lib/promrelabel/debug.qtpl:159 + } +//line lib/promrelabel/debug.qtpl:159 + qw422016.N().S(`}`) +//line lib/promrelabel/debug.qtpl:161 +} + +//line lib/promrelabel/debug.qtpl:161 +func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { +//line lib/promrelabel/debug.qtpl:161 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promrelabel/debug.qtpl:161 + streamlabelsWithHighlight(qw422016, labels, highlight, color) +//line lib/promrelabel/debug.qtpl:161 + qt422016.ReleaseWriter(qw422016) +//line lib/promrelabel/debug.qtpl:161 +} + +//line lib/promrelabel/debug.qtpl:161 +func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string { +//line lib/promrelabel/debug.qtpl:161 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promrelabel/debug.qtpl:161 + writelabelsWithHighlight(qb422016, labels, highlight, color) +//line lib/promrelabel/debug.qtpl:161 + qs422016 := string(qb422016.B) +//line lib/promrelabel/debug.qtpl:161 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promrelabel/debug.qtpl:161 + return qs422016 +//line lib/promrelabel/debug.qtpl:161 +} + +//line lib/promrelabel/debug.qtpl:163 +func streammustFormatLabels(qw422016 *qt422016.Writer, s string) { +//line lib/promrelabel/debug.qtpl:164 labels := promutils.MustNewLabelsFromString(s) -//line lib/promrelabel/debug.qtpl:156 +//line lib/promrelabel/debug.qtpl:165 streamlabelsWithHighlight(qw422016, labels, nil, "") -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 } -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 func writemustFormatLabels(qq422016 qtio422016.Writer, s string) { -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 streammustFormatLabels(qw422016, s) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 qt422016.ReleaseWriter(qw422016) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 } -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 func mustFormatLabels(s string) string { -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 writemustFormatLabels(qb422016, s) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 qs422016 := string(qb422016.B) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 return qs422016 -//line lib/promrelabel/debug.qtpl:157 +//line lib/promrelabel/debug.qtpl:166 } diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 9c55d76a3..35a47e74d 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -49,7 +49,7 @@ type DebugStep struct { // Rule contains string representation of the rule step Rule string - // In contains the input labels before the exeuction of the rule step + // In contains the input labels before the execution of the rule step In string // Out contains the output labels after the execution of the rule step diff --git a/lib/promrelabel/scrape_url.go b/lib/promrelabel/scrape_url.go new file mode 100644 index 000000000..55d7d5d5d --- /dev/null +++ b/lib/promrelabel/scrape_url.go @@ -0,0 +1,114 @@ +package promrelabel + +import ( + "net/url" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// GetScrapeURL makes scrape url and __address_ labels for the given labels and extraParams. +func GetScrapeURL(labels *promutils.Labels, extraParams map[string][]string) (string, string) { + // See https://www.robustperception.io/life-of-a-label + scheme := labels.Get("__scheme__") + if len(scheme) == 0 { + scheme = "http" + } + metricsPath := labels.Get("__metrics_path__") + if len(metricsPath) == 0 { + metricsPath = "/metrics" + } + address := labels.Get("__address__") + if len(address) == 0 { + return "", "" + } + // Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__. + // Prometheus silently drops targets with __address__ containing scheme or metricsPath + // according to https://www.robustperception.io/life-of-a-label/ . + if strings.HasPrefix(address, "http://") { + scheme = "http" + address = address[len("http://"):] + } else if strings.HasPrefix(address, "https://") { + scheme = "https" + address = address[len("https://"):] + } + if n := strings.IndexByte(address, '/'); n >= 0 { + metricsPath = address[n:] + address = address[:n] + } + address = addMissingPort(address, scheme == "https") + + if !strings.HasPrefix(metricsPath, "/") { + metricsPath = "/" + metricsPath + } + params := getParamsFromLabels(labels, extraParams) + optionalQuestion := "" + if len(params) > 0 { + optionalQuestion = "?" + if strings.Contains(metricsPath, "?") { + optionalQuestion = "&" + } + } + paramsStr := url.Values(params).Encode() + scrapeURL := buildScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr) + return scrapeURL, address +} + +func getParamsFromLabels(labels *promutils.Labels, extraParams map[string][]string) map[string][]string { + // See https://www.robustperception.io/life-of-a-label + var m map[string][]string + for _, label := range labels.GetLabels() { + if !strings.HasPrefix(label.Name, "__param_") { + continue + } + name := label.Name[len("__param_"):] + values := []string{label.Value} + if p := extraParams[name]; len(p) > 1 { + values = append(values, p[1:]...) + } + if m == nil { + m = make(map[string][]string) + } + m[name] = values + } + return m +} + +func buildScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr string) string { + bb := bbPool.Get() + b := bb.B[:0] + b = append(b, scheme...) + b = append(b, "://"...) + b = append(b, address...) + b = append(b, metricsPath...) + b = append(b, optionalQuestion...) + b = append(b, paramsStr...) + s := bytesutil.InternBytes(b) + bb.B = b + bbPool.Put(bb) + return s +} + +func addMissingPort(addr string, isTLS bool) string { + if strings.Contains(addr, ":") { + return addr + } + if isTLS { + return concatTwoStrings(addr, ":443") + } + return concatTwoStrings(addr, ":80") +} + +func concatTwoStrings(x, y string) string { + bb := bbPool.Get() + b := bb.B[:0] + b = append(b, x...) + b = append(b, y...) + s := bytesutil.InternBytes(b) + bb.B = b + bbPool.Put(bb) + return s +} + +var bbPool bytesutil.ByteBufferPool diff --git a/lib/promrelabel/scrape_url_test.go b/lib/promrelabel/scrape_url_test.go new file mode 100644 index 000000000..de8da3a32 --- /dev/null +++ b/lib/promrelabel/scrape_url_test.go @@ -0,0 +1,54 @@ +package promrelabel + +import ( + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +func TestGetScrapeURL(t *testing.T) { + f := func(labelsStr, expectedScrapeURL, expectedAddress string) { + t.Helper() + labels := promutils.MustNewLabelsFromString(labelsStr) + scrapeURL, address := GetScrapeURL(labels, nil) + if scrapeURL != expectedScrapeURL { + t.Fatalf("unexpected scrapeURL; got %q; want %q", scrapeURL, expectedScrapeURL) + } + if address != expectedAddress { + t.Fatalf("unexpected address; got %q; want %q", address, expectedAddress) + } + } + + // Missing __address__ + f("{}", "", "") + f(`{foo="bar"}`, "", "") + + // __address__ without port + f(`{__address__="foo"}`, "http://foo:80/metrics", "foo:80") + + // __address__ with explicit port + f(`{__address__="foo:1234"}`, "http://foo:1234/metrics", "foo:1234") + + // explicit __scheme__ + f(`{__address__="foo",__scheme__="https"}`, "https://foo:443/metrics", "foo:443") + f(`{__address__="foo:1234",__scheme__="https"}`, "https://foo:1234/metrics", "foo:1234") + + // explicit __metrics_path__ + f(`{__address__="foo",__metrics_path__="abc"}`, "http://foo:80/abc", "foo:80") + f(`{__address__="foo",__metrics_path__="/abc"}`, "http://foo:80/abc", "foo:80") + f(`{__address__="foo",__metrics_path__="/ab/c?d=ef&aa=bb"}`, "http://foo:80/ab/c?d=ef&aa=bb", "foo:80") + + // explitit __param_* + f(`{__address__="foo",__param_x="y"}`, "http://foo:80/metrics?x=y", "foo:80") + f(`{__address__="foo",__param_x="y",__param_y="aa"}`, "http://foo:80/metrics?x=y&y=aa", "foo:80") + f(`{__address__="foo",__param_x="y",__metrics_path__="?abc=de"}`, "http://foo:80/?abc=de&x=y", "foo:80") + f(`{__address__="foo",__param_abc="y",__metrics_path__="?abc=de"}`, "http://foo:80/?abc=de&abc=y", "foo:80") + + // __address__ with metrics path and/or scheme + f(`{__address__="foo/bar/baz?abc=de"}`, "http://foo:80/bar/baz?abc=de", "foo:80") + f(`{__address__="foo:784/bar/baz?abc=de"}`, "http://foo:784/bar/baz?abc=de", "foo:784") + f(`{__address__="foo:784/bar/baz?abc=de",__param_xx="yy"}`, "http://foo:784/bar/baz?abc=de&xx=yy", "foo:784") + f(`{__address__="foo:784/bar/baz?abc=de",__param_xx="yy",__scheme__="https"}`, "https://foo:784/bar/baz?abc=de&xx=yy", "foo:784") + f(`{__address__="http://foo/bar/baz?abc=de",__param_xx="yy"}`, "http://foo:80/bar/baz?abc=de&xx=yy", "foo:80") + f(`{__address__="https://foo/bar/baz?abc=de",__param_xx="yy"}`, "https://foo:443/bar/baz?abc=de&xx=yy", "foo:443") +} diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 4ae3369e9..3914c5f71 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -1258,36 +1258,15 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel droppedTargetsMap.Register(originalLabels, swc.relabelConfigs) return nil, nil } - // See https://www.robustperception.io/life-of-a-label - scheme := labels.Get("__scheme__") - if len(scheme) == 0 { - scheme = "http" - } - metricsPath := labels.Get("__metrics_path__") - if len(metricsPath) == 0 { - metricsPath = "/metrics" - } - address := labels.Get("__address__") - if len(address) == 0 { - // Drop target without scrape address. + scrapeURL, address := promrelabel.GetScrapeURL(labels, swc.params) + if scrapeURL == "" { + // Drop target without URL. droppedTargetsMap.Register(originalLabels, swc.relabelConfigs) return nil, nil } - // Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__. - // Prometheus silently drops targets with __address__ containing scheme or metricsPath - // according to https://www.robustperception.io/life-of-a-label/ . - if strings.HasPrefix(address, "http://") { - scheme = "http" - address = address[len("http://"):] - } else if strings.HasPrefix(address, "https://") { - scheme = "https" - address = address[len("https://"):] + if _, err := url.Parse(scrapeURL); err != nil { + return nil, fmt.Errorf("invalid target url=%q for job=%q: %w", scrapeURL, swc.jobName, err) } - if n := strings.IndexByte(address, '/'); n >= 0 { - metricsPath = address[n:] - address = address[:n] - } - address = addMissingPort(address, scheme == "https") var at *auth.Token tenantID := labels.Get("__tenant_id__") @@ -1299,23 +1278,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel at = newToken } - if !strings.HasPrefix(metricsPath, "/") { - metricsPath = "/" + metricsPath - } - params := getParamsFromLabels(labels, swc.params) - optionalQuestion := "" - if len(params) > 0 { - optionalQuestion = "?" - if strings.Contains(metricsPath, "?") { - optionalQuestion = "&" - } - } - paramsStr := url.Values(params).Encode() - scrapeURL := getScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr) - if _, err := url.Parse(scrapeURL); err != nil { - return nil, fmt.Errorf("invalid url %q for scheme=%q, target=%q, address=%q, metrics_path=%q for job=%q: %w", - scrapeURL, scheme, target, address, metricsPath, swc.jobName, err) - } // Read __scrape_interval__ and __scrape_timeout__ from labels. scrapeInterval := swc.scrapeInterval if s := labels.Get("__scrape_interval__"); len(s) > 0 { @@ -1398,41 +1360,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel return sw, nil } -func getScrapeURL(scheme, address, metricsPath, optionalQuestion, paramsStr string) string { - bb := bbPool.Get() - b := bb.B[:0] - b = append(b, scheme...) - b = append(b, "://"...) - b = append(b, address...) - b = append(b, metricsPath...) - b = append(b, optionalQuestion...) - b = append(b, paramsStr...) - s := bytesutil.InternBytes(b) - bb.B = b - bbPool.Put(bb) - return s -} - -func getParamsFromLabels(labels *promutils.Labels, paramsOrig map[string][]string) map[string][]string { - // See https://www.robustperception.io/life-of-a-label - var m map[string][]string - for _, label := range labels.GetLabels() { - if !strings.HasPrefix(label.Name, "__param_") { - continue - } - name := label.Name[len("__param_"):] - values := []string{label.Value} - if p := paramsOrig[name]; len(p) > 1 { - values = append(values, p[1:]...) - } - if m == nil { - m = make(map[string][]string) - } - m[name] = values - } - return m -} - func mergeLabels(dst *promutils.Labels, swc *scrapeWorkConfig, target string, extraLabels, metaLabels *promutils.Labels) { if n := dst.Len(); n > 0 { logger.Panicf("BUG: len(dst.Labels) must be 0; got %d", n)