csvimport: support empty values (#3565)

Before, if the imported line contained multiple metrics and one
or more of them had an empty values - the whole line was ignored.

Now, only metrics with empty values are ignored, and the rest
of the metrics are accepted successfully.

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540

Signed-off-by: hagen1778 <roman@victoriametrics.com>

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

docs/CHANGELOG.md

@@ -18,7 +18,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386).
 * FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats).
 * FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`.
-* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556).
+* FEATURE: [csvimport](https://docs.victoriametrics.com/#how-to-import-csv-data): support empty values for imported metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540).
+* FEATURE: [vmalert](httpпоs://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556).
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513).
 * BUGFIX: properly parse floating-point numbers without integer or fractional parts such as `.123` and `20.` during [data import](https://docs.victoriametrics.com/#how-to-import-time-series-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3544).

lib/protoparser/csvimport/parser.go

@@ -68,7 +68,7 @@ type metric struct {
 	Value float64
 }
 
-// Unmarshal unmarshal csv lines from s according to the given cds.
+// Unmarshal unmarshals csv lines from s according to the given cds.
 func (rs *Rows) Unmarshal(s string, cds []ColumnDescriptor) {
 	rs.sc.Init(s)
 	rs.Rows, rs.tagsPool, rs.metricsPool = parseRows(&rs.sc, rs.Rows[:0], rs.tagsPool[:0], rs.metricsPool[:0], cds)
@@ -105,7 +105,7 @@ func parseRows(sc *scanner, dst []Row, tags []Tag, metrics []metric, cds []Colum
 				continue
 			}
 			metricName := cd.MetricName
-			if metricName == "" {
+			if metricName == "" || sc.Column == "" {
 				// The given field is ignored.
 				continue
 			}

lib/protoparser/csvimport/parser_test.go

@@ -116,9 +116,9 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		})
 	f("2:label:symbol,3:time:custom:2006-01-02 15:04:05.999Z,4:metric:bid,5:metric:ask",
 		`
-"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273
-"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276
-`, []Row{
+		"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273
+		"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276
+		`, []Row{
 			{
 				Metric: "bid",
 				Tags: []Tag{
@@ -178,4 +178,63 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			Value:  -45.6,
 		},
 	})
+	// skip metrics with empty values
+	f("1:metric:foo,2:metric:bar,3:metric:baz,4:metric:quux", `1,,,2`, []Row{
+		{
+			Metric: "foo",
+			Value:  1,
+		},
+		{
+			Metric: "quux",
+			Value:  2,
+		},
+	})
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540
+	f("1:label:mytest,2:time:rfc3339,3:metric:M10,4:metric:M20,5:metric:M30,6:metric:M40,7:metric:M50,8:metric:M60",
+		`test,2022-12-25T16:57:12+01:00,10,20,30,,,60,70,80`, []Row{
+			{
+				Metric: "M10",
+				Tags: []Tag{
+					{
+						Key:   "mytest",
+						Value: "test",
+					},
+				},
+				Timestamp: 1671983832000,
+				Value:     10,
+			},
+			{
+				Metric: "M20",
+				Tags: []Tag{
+					{
+						Key:   "mytest",
+						Value: "test",
+					},
+				},
+				Timestamp: 1671983832000,
+				Value:     20,
+			},
+			{
+				Metric: "M30",
+				Tags: []Tag{
+					{
+						Key:   "mytest",
+						Value: "test",
+					},
+				},
+				Timestamp: 1671983832000,
+				Value:     30,
+			},
+			{
+				Metric: "M60",
+				Tags: []Tag{
+					{
+						Key:   "mytest",
+						Value: "test",
+					},
+				},
+				Timestamp: 1671983832000,
+				Value:     60,
+			},
+		})
 }