mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
csvimport: support empty values (#3565)
Before, if the imported line contained multiple metrics and one or more of them had an empty values - the whole line was ignored. Now, only metrics with empty values are ignored, and the rest of the metrics are accepted successfully. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540 Signed-off-by: hagen1778 <roman@victoriametrics.com> Signed-off-by: hagen1778 <roman@victoriametrics.com> Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Roman Khavronenko
GitHub
parent
83870aeb8d
commit
2cedb3e883
3 changed files with 66 additions and 6 deletions
|
|
@ -18,7 +18,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386).
|
||||
* FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats).
|
||||
* FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`.
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556).
|
||||
* FEATURE: [csvimport](https://docs.victoriametrics.com/#how-to-import-csv-data): support empty values for imported metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540).
|
||||
* FEATURE: [vmalert](httpпоs://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556).
|
||||
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513).
|
||||
* BUGFIX: properly parse floating-point numbers without integer or fractional parts such as `.123` and `20.` during [data import](https://docs.victoriametrics.com/#how-to-import-time-series-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3544).
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ type metric struct {
|
|||
Value float64
|
||||
}
|
||||
|
||||
// Unmarshal unmarshal csv lines from s according to the given cds.
|
||||
// Unmarshal unmarshals csv lines from s according to the given cds.
|
||||
func (rs *Rows) Unmarshal(s string, cds []ColumnDescriptor) {
|
||||
rs.sc.Init(s)
|
||||
rs.Rows, rs.tagsPool, rs.metricsPool = parseRows(&rs.sc, rs.Rows[:0], rs.tagsPool[:0], rs.metricsPool[:0], cds)
|
||||
|
|
@ -105,7 +105,7 @@ func parseRows(sc *scanner, dst []Row, tags []Tag, metrics []metric, cds []Colum
|
|||
continue
|
||||
}
|
||||
metricName := cd.MetricName
|
||||
if metricName == "" {
|
||||
if metricName == "" || sc.Column == "" {
|
||||
// The given field is ignored.
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,9 +116,9 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
|||
})
|
||||
f("2:label:symbol,3:time:custom:2006-01-02 15:04:05.999Z,4:metric:bid,5:metric:ask",
|
||||
`
|
||||
"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273
|
||||
"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276
|
||||
`, []Row{
|
||||
"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273
|
||||
"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276
|
||||
`, []Row{
|
||||
{
|
||||
Metric: "bid",
|
||||
Tags: []Tag{
|
||||
|
|
@ -178,4 +178,63 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
|||
Value: -45.6,
|
||||
},
|
||||
})
|
||||
// skip metrics with empty values
|
||||
f("1:metric:foo,2:metric:bar,3:metric:baz,4:metric:quux", `1,,,2`, []Row{
|
||||
{
|
||||
Metric: "foo",
|
||||
Value: 1,
|
||||
},
|
||||
{
|
||||
Metric: "quux",
|
||||
Value: 2,
|
||||
},
|
||||
})
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540
|
||||
f("1:label:mytest,2:time:rfc3339,3:metric:M10,4:metric:M20,5:metric:M30,6:metric:M40,7:metric:M50,8:metric:M60",
|
||||
`test,2022-12-25T16:57:12+01:00,10,20,30,,,60,70,80`, []Row{
|
||||
{
|
||||
Metric: "M10",
|
||||
Tags: []Tag{
|
||||
{
|
||||
Key: "mytest",
|
||||
Value: "test",
|
||||
},
|
||||
},
|
||||
Timestamp: 1671983832000,
|
||||
Value: 10,
|
||||
},
|
||||
{
|
||||
Metric: "M20",
|
||||
Tags: []Tag{
|
||||
{
|
||||
Key: "mytest",
|
||||
Value: "test",
|
||||
},
|
||||
},
|
||||
Timestamp: 1671983832000,
|
||||
Value: 20,
|
||||
},
|
||||
{
|
||||
Metric: "M30",
|
||||
Tags: []Tag{
|
||||
{
|
||||
Key: "mytest",
|
||||
Value: "test",
|
||||
},
|
||||
},
|
||||
Timestamp: 1671983832000,
|
||||
Value: 30,
|
||||
},
|
||||
{
|
||||
Metric: "M60",
|
||||
Tags: []Tag{
|
||||
{
|
||||
Key: "mytest",
|
||||
Value: "test",
|
||||
},
|
||||
},
|
||||
Timestamp: 1671983832000,
|
||||
Value: 60,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue