From c22122212dd5931c8be9a6618293aa20e78c8e2c Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 29 Dec 2022 20:52:10 +0100 Subject: [PATCH] csvimport: support empty values (#3565) Before, if the imported line contained multiple metrics and one or more of them had an empty values - the whole line was ignored. Now, only metrics with empty values are ignored, and the rest of the metrics are accepted successfully. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540 Signed-off-by: hagen1778 Signed-off-by: hagen1778 Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 3 +- lib/protoparser/csvimport/parser.go | 4 +- lib/protoparser/csvimport/parser_test.go | 65 ++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 26cb25181..8cb249a47 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,7 +18,8 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386). * FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). * FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`. -* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556). +* FEATURE: [csvimport](https://docs.victoriametrics.com/#how-to-import-csv-data): support empty values for imported metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540). +* FEATURE: [vmalert](httpпоs://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513). * BUGFIX: properly parse floating-point numbers without integer or fractional parts such as `.123` and `20.` during [data import](https://docs.victoriametrics.com/#how-to-import-time-series-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3544). diff --git a/lib/protoparser/csvimport/parser.go b/lib/protoparser/csvimport/parser.go index 004d581b6..56e7560b5 100644 --- a/lib/protoparser/csvimport/parser.go +++ b/lib/protoparser/csvimport/parser.go @@ -68,7 +68,7 @@ type metric struct { Value float64 } -// Unmarshal unmarshal csv lines from s according to the given cds. +// Unmarshal unmarshals csv lines from s according to the given cds. func (rs *Rows) Unmarshal(s string, cds []ColumnDescriptor) { rs.sc.Init(s) rs.Rows, rs.tagsPool, rs.metricsPool = parseRows(&rs.sc, rs.Rows[:0], rs.tagsPool[:0], rs.metricsPool[:0], cds) @@ -105,7 +105,7 @@ func parseRows(sc *scanner, dst []Row, tags []Tag, metrics []metric, cds []Colum continue } metricName := cd.MetricName - if metricName == "" { + if metricName == "" || sc.Column == "" { // The given field is ignored. continue } diff --git a/lib/protoparser/csvimport/parser_test.go b/lib/protoparser/csvimport/parser_test.go index ea1fbcf2b..342a1d357 100644 --- a/lib/protoparser/csvimport/parser_test.go +++ b/lib/protoparser/csvimport/parser_test.go @@ -116,9 +116,9 @@ func TestRowsUnmarshalSuccess(t *testing.T) { }) f("2:label:symbol,3:time:custom:2006-01-02 15:04:05.999Z,4:metric:bid,5:metric:ask", ` -"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273 -"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276 -`, []Row{ + "aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273 + "aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276 + `, []Row{ { Metric: "bid", Tags: []Tag{ @@ -178,4 +178,63 @@ func TestRowsUnmarshalSuccess(t *testing.T) { Value: -45.6, }, }) + // skip metrics with empty values + f("1:metric:foo,2:metric:bar,3:metric:baz,4:metric:quux", `1,,,2`, []Row{ + { + Metric: "foo", + Value: 1, + }, + { + Metric: "quux", + Value: 2, + }, + }) + // see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540 + f("1:label:mytest,2:time:rfc3339,3:metric:M10,4:metric:M20,5:metric:M30,6:metric:M40,7:metric:M50,8:metric:M60", + `test,2022-12-25T16:57:12+01:00,10,20,30,,,60,70,80`, []Row{ + { + Metric: "M10", + Tags: []Tag{ + { + Key: "mytest", + Value: "test", + }, + }, + Timestamp: 1671983832000, + Value: 10, + }, + { + Metric: "M20", + Tags: []Tag{ + { + Key: "mytest", + Value: "test", + }, + }, + Timestamp: 1671983832000, + Value: 20, + }, + { + Metric: "M30", + Tags: []Tag{ + { + Key: "mytest", + Value: "test", + }, + }, + Timestamp: 1671983832000, + Value: 30, + }, + { + Metric: "M60", + Tags: []Tag{ + { + Key: "mytest", + Value: "test", + }, + }, + Timestamp: 1671983832000, + Value: 60, + }, + }) }