mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-02-19 15:30:17 +00:00
app/vminsert/influx: allow escaping newline char
Though newline char isn't mentioned in escape rules at https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/ , there are reports that such chars occur in real life
This commit is contained in:
parent
44d7fc599d
commit
a18317adbc
2 changed files with 48 additions and 6 deletions
app/vminsert/influx
|
@ -62,9 +62,8 @@ func (r *Row) reset() {
|
||||||
r.Timestamp = 0
|
r.Timestamp = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Row) unmarshal(s string, tagsPool []Tag, fieldsPool []Field) ([]Tag, []Field, error) {
|
func (r *Row) unmarshal(s string, tagsPool []Tag, fieldsPool []Field, noEscapeChars bool) ([]Tag, []Field, error) {
|
||||||
r.reset()
|
r.reset()
|
||||||
noEscapeChars := strings.IndexByte(s, '\\') < 0
|
|
||||||
n := nextUnescapedChar(s, ' ', noEscapeChars)
|
n := nextUnescapedChar(s, ' ', noEscapeChars)
|
||||||
if n < 0 {
|
if n < 0 {
|
||||||
return tagsPool, fieldsPool, fmt.Errorf("cannot find Whitespace I in %q", s)
|
return tagsPool, fieldsPool, fmt.Errorf("cannot find Whitespace I in %q", s)
|
||||||
|
@ -175,8 +174,9 @@ func (f *Field) unmarshal(s string, noEscapeChars, hasQuotedFields bool) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]Row, []Tag, []Field, error) {
|
func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]Row, []Tag, []Field, error) {
|
||||||
|
noEscapeChars := strings.IndexByte(s, '\\') < 0
|
||||||
for len(s) > 0 {
|
for len(s) > 0 {
|
||||||
n := strings.IndexByte(s, '\n')
|
n := nextUnquotedChar(s, '\n', noEscapeChars, true)
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
// Skip empty line
|
// Skip empty line
|
||||||
s = s[1:]
|
s = s[1:]
|
||||||
|
@ -200,7 +200,7 @@ func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]R
|
||||||
if n < 0 {
|
if n < 0 {
|
||||||
// The last line.
|
// The last line.
|
||||||
var err error
|
var err error
|
||||||
tagsPool, fieldsPool, err = r.unmarshal(s, tagsPool, fieldsPool)
|
tagsPool, fieldsPool, err = r.unmarshal(s, tagsPool, fieldsPool, noEscapeChars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s, err)
|
err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s, err)
|
||||||
return dst, tagsPool, fieldsPool, err
|
return dst, tagsPool, fieldsPool, err
|
||||||
|
@ -208,7 +208,7 @@ func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]R
|
||||||
return dst, tagsPool, fieldsPool, nil
|
return dst, tagsPool, fieldsPool, nil
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
tagsPool, fieldsPool, err = r.unmarshal(s[:n], tagsPool, fieldsPool)
|
tagsPool, fieldsPool, err = r.unmarshal(s[:n], tagsPool, fieldsPool, noEscapeChars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s[:n], err)
|
err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s[:n], err)
|
||||||
return dst, tagsPool, fieldsPool, err
|
return dst, tagsPool, fieldsPool, err
|
||||||
|
@ -281,7 +281,7 @@ func unescapeTagValue(s string, noEscapeChars bool) string {
|
||||||
return string(append(dst, '\\'))
|
return string(append(dst, '\\'))
|
||||||
}
|
}
|
||||||
ch := s[0]
|
ch := s[0]
|
||||||
if ch != ' ' && ch != ',' && ch != '=' && ch != '\\' {
|
if ch != ' ' && ch != ',' && ch != '=' && ch != '\\' && ch != '\n' {
|
||||||
dst = append(dst, '\\')
|
dst = append(dst, '\\')
|
||||||
}
|
}
|
||||||
dst = append(dst, ch)
|
dst = append(dst, ch)
|
||||||
|
|
|
@ -337,6 +337,27 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
||||||
}},
|
}},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Escape newline
|
||||||
|
f("fo\\\nb\\,ar,x\\\ny=\\\n\\y a=\"foo\nbar\",b\\\nc\\==34\n", &Rows{
|
||||||
|
Rows: []Row{{
|
||||||
|
Measurement: "fo\nb,ar",
|
||||||
|
Tags: []Tag{{
|
||||||
|
Key: "x\ny",
|
||||||
|
Value: "\n\\y",
|
||||||
|
}},
|
||||||
|
Fields: []Field{
|
||||||
|
{
|
||||||
|
Key: "a",
|
||||||
|
Value: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Key: "b\nc=",
|
||||||
|
Value: 34,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
|
||||||
// Multiple lines
|
// Multiple lines
|
||||||
f("foo,tag=xyz field=1.23 48934\n"+
|
f("foo,tag=xyz field=1.23 48934\n"+
|
||||||
"bar x=-1i\n\n", &Rows{
|
"bar x=-1i\n\n", &Rows{
|
||||||
|
@ -362,6 +383,7 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
// No newline after the second line.
|
// No newline after the second line.
|
||||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/82
|
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/82
|
||||||
f("foo,tag=xyz field=1.23 48934\n"+
|
f("foo,tag=xyz field=1.23 48934\n"+
|
||||||
|
@ -388,4 +410,24 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
f("x,y=z,g=p:\\ \\ 5432\\,\\ gp\\ mon\\ [lol]\\ con10\\ cmd5\\ SELECT f=1", &Rows{
|
||||||
|
Rows: []Row{{
|
||||||
|
Measurement: "x",
|
||||||
|
Tags: []Tag{
|
||||||
|
{
|
||||||
|
Key: "y",
|
||||||
|
Value: "z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Key: "g",
|
||||||
|
Value: "p: 5432, gp mon [lol] con10 cmd5 SELECT",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Fields: []Field{{
|
||||||
|
Key: "f",
|
||||||
|
Value: 1,
|
||||||
|
}},
|
||||||
|
}},
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue