From d1289383eb0ea2a2ca505b3bca4056f5c7920927 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 2 Dec 2021 13:43:49 +0200 Subject: [PATCH] lib/protoparser/graphite: allow multiple separators between metric name, value and timestamp --- docs/CHANGELOG.md | 2 +- lib/protoparser/graphite/parser.go | 48 +++++++++++++++---------- lib/protoparser/graphite/parser_test.go | 8 +++++ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2729cb1092..8564be0274 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ sort: 15 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): store the display type in URL, so it isn't lost when copy-pasting the URL. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1804). * FEATURE: vmalert: make `-notifier.url` command-line flag optional. This flag can be omitted if `vmalert` is used solely for recording rules and doesn't evaluate alerting rules. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1870). * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html), [vmrestore](https://docs.victoriametrics.com/vmrestore.html): export internal metrics at `http://vmbackup:8420/metrics` and `http://vmrestore:8421/metrics` for better visibility of the backup/restore process. +* FEATURE: allow trailing whitespace after the timestamp when [parsing Graphite plaintext lines](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1865). * BUGFIX: vmagent: prevent from scraping duplicate targets if `-promscrape.dropOriginalLabels` command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1830). Thanks to @guidao for the fix. * BUGFIX: vmstorage [enterprise](https://victoriametrics.com/enterprise.html): added missing `vm_tenant_used_tenant_bytes` metric, which shows the approximate per-tenant disk usage. See [these docs](https://docs.victoriametrics.com/PerTenantStatistic.html) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1605). @@ -27,7 +28,6 @@ sort: 15 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): properly resume downloading for partially downloaded big files. Previously such files were re-downloaded from the beginning after the interrupt. Now only the remaining parts of the file are downloaded. This allows saving network bandwidth. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/487). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): do not store the last query across vmui page reloads. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1694). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `Cannot read properties of undefined` error at table view. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1797). -* BUGFIX: properly parse Graphite plaintext lines with whitespace after the timestamp. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1865). ## [v1.69.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.69.0) diff --git a/lib/protoparser/graphite/parser.go b/lib/protoparser/graphite/parser.go index e62bbd6a76..5be4cae48c 100644 --- a/lib/protoparser/graphite/parser.go +++ b/lib/protoparser/graphite/parser.go @@ -93,7 +93,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) { return tagsPool, fmt.Errorf("cannot find separator between metric and value in %q", s) } metricAndTags := s[:n] - tail := s[n+1:] + tail := stripLeadingWhitespace(s[n+1:]) tagsPool, err := r.UnmarshalMetricAndTags(metricAndTags, tagsPool) if err != nil { @@ -114,7 +114,8 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) { if err != nil { return tagsPool, fmt.Errorf("cannot unmarshal value from %q: %w", tail[:n], err) } - tail = stripTailSpace(tail[n+1:]) + tail = stripLeadingWhitespace(tail[n+1:]) + tail = stripTrailingWhitespace(tail) ts, err := fastfloat.Parse(tail) if err != nil { return tagsPool, fmt.Errorf("cannot unmarshal timestamp from %q: %w", tail, err) @@ -124,22 +125,6 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) { return tagsPool, nil } -func stripTailSpace(s string) string { - n := len(s) - for { - n-- - if n < 0 { - return "" - } - ch := s[n] - // graphite text line protocol may use white space or tab as separator - // See https://github.com/grobian/carbon-c-relay/commit/f3ffe6cc2b52b07d14acbda649ad3fd6babdd528 - if ch != ' ' && ch != '\t' { - return s[:n+1] - } - } -} - func unmarshalRows(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag) { for len(s) > 0 { n := strings.IndexByte(s, '\n') @@ -232,3 +217,30 @@ func (t *Tag) unmarshal(s string) { t.Value = s[n+1:] } } + +func stripTrailingWhitespace(s string) string { + n := len(s) + for { + n-- + if n < 0 { + return "" + } + ch := s[n] + // graphite text line protocol may use white space or tab as separator + // See https://github.com/grobian/carbon-c-relay/commit/f3ffe6cc2b52b07d14acbda649ad3fd6babdd528 + if ch != ' ' && ch != '\t' { + return s[:n+1] + } + } +} + +func stripLeadingWhitespace(s string) string { + for len(s) > 0 { + ch := s[0] + if ch != ' ' && ch != '\t' { + return s + } + s = s[1:] + } + return "" +} diff --git a/lib/protoparser/graphite/parser_test.go b/lib/protoparser/graphite/parser_test.go index 24c8ab9dab..e2922548ab 100644 --- a/lib/protoparser/graphite/parser_test.go +++ b/lib/protoparser/graphite/parser_test.go @@ -305,6 +305,14 @@ func TestRowsUnmarshalSuccess(t *testing.T) { }, }) + // Multiple whitespaces as separators + f("foo.baz \t125 1789 \t\n", &Rows{ + Rows: []Row{{ + Metric: "foo.baz", + Value: 125, + Timestamp: 1789, + }}, + }) } func Test_streamContext_Read(t *testing.T) {