github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vlselect: do not show empty fields in query results

Browse source 
Empty fields are treated as non-existing fields by VictoriaLogs data model.
So there is no sense in returning empty fields in query results, since they may mislead and confuse users.
This commit is contained in:
Aliaksandr Valialkin 2024-10-14 23:39:29 +02:00
parent 343463fc0f
commit bac193e50b
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
13 changed files with 203 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
app/vlinsert/elasticsearch/elasticsearch_test.go
View file

@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
msgField := "message" msgField := "message"
rowsExpected := 4 rowsExpected := 4
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000} timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"} {"_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"} {"_msg":"xyz","x":"y"}
{"_msg":"qwe rty","@timestamp":""}` {"_msg":"qwe rty"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected) f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
} }

6
app/vlinsert/jsonline/jsonline_test.go
View file

@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
msgField := "message" msgField := "message"
rowsExpected := 3 rowsExpected := 3
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000} timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"} {"_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"}` {"_msg":"xyz","x":"y"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected) f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
} }

6
app/vlinsert/syslog/syslog_test.go
View file

@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
currentYear := 2023 currentYear := 2023
rowsExpected := 3 rowsExpected := 3
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000} timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."} resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"} {"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}` {"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected) f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
} }

35
app/vlselect/logsql/query_response.qtpl
View file

@ -6,15 +6,31 @@
// JSONRow creates JSON row from the given fields. // JSONRow creates JSON row from the given fields.
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %} {% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
{ {% code
{% code c := &columns[0] %} i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
%}
{% if len(columns) == 0 %}
{% return %}
{% endif %}
{
{% code c := &columns[0] %}
{%q= c.Name %}:{%q= c.Values[rowIdx] %} {%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% code columns = columns[1:] %} {% code columns = columns[1:] %}
{% for colIdx := range columns %} {% for colIdx := range columns %}
{% code c := &columns[colIdx] %} {% code
c := &columns[colIdx]
v := c.Values[rowIdx]
%}
{% if v == "" %}
{% continue %}
{% endif %}
,{%q= c.Name %}:{%q= c.Values[rowIdx] %} ,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% endfor %} {% endfor %}
}{% newline %} }{% newline %}
{% endfunc %} {% endfunc %}
// JSONRows prints formatted rows // JSONRows prints formatted rows
@ -23,7 +39,11 @@
{% return %} {% return %}
{% endif %} {% endif %}
{% for _, fields := range rows %} {% for _, fields := range rows %}
{ {% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
{% if len(fields) == 0 %}
{% continue %}
{% endif %}
{
{% if len(fields) > 0 %} {% if len(fields) > 0 %}
{% code {% code
f := fields[0] f := fields[0]
@ -31,10 +51,13 @@
%} %}
{%q= f.Name %}:{%q= f.Value %} {%q= f.Name %}:{%q= f.Value %}
{% for _, f := range fields %} {% for _, f := range fields %}
{% if f.Value == "" %}
{% continue %}
{% endif %}
,{%q= f.Name %}:{%q= f.Value %} ,{%q= f.Name %}:{%q= f.Value %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
}{% newline %} }{% newline %}
{% endfor %} {% endfor %}
{% endfunc %} {% endfunc %}

159
app/vlselect/logsql/query_response.qtpl.go
View file

@ -26,141 +26,176 @@ var (
//line app/vlselect/logsql/query_response.qtpl:8 //line app/vlselect/logsql/query_response.qtpl:8
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) { func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:8
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:10 //line app/vlselect/logsql/query_response.qtpl:10
i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
//line app/vlselect/logsql/query_response.qtpl:16
if len(columns) == 0 {
//line app/vlselect/logsql/query_response.qtpl:17
return
//line app/vlselect/logsql/query_response.qtpl:18
}
//line app/vlselect/logsql/query_response.qtpl:18
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:20
c := &columns[0] c := &columns[0]
//line app/vlselect/logsql/query_response.qtpl:11 //line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Name) qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:11 //line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().S(`:`) qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:11 //line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Values[rowIdx]) qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:12 //line app/vlselect/logsql/query_response.qtpl:22
columns = columns[1:] columns = columns[1:]
//line app/vlselect/logsql/query_response.qtpl:13 //line app/vlselect/logsql/query_response.qtpl:23
for colIdx := range columns { for colIdx := range columns {
//line app/vlselect/logsql/query_response.qtpl:14 //line app/vlselect/logsql/query_response.qtpl:25
c := &columns[colIdx] c := &columns[colIdx]
v := c.Values[rowIdx]
//line app/vlselect/logsql/query_response.qtpl:14 //line app/vlselect/logsql/query_response.qtpl:28
if v == "" {
//line app/vlselect/logsql/query_response.qtpl:29
continue
//line app/vlselect/logsql/query_response.qtpl:30
}
//line app/vlselect/logsql/query_response.qtpl:30
qw422016.N().S(`,`) qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:15 //line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Name) qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:15 //line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().S(`:`) qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:15 //line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Values[rowIdx]) qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:16 //line app/vlselect/logsql/query_response.qtpl:32
} }
//line app/vlselect/logsql/query_response.qtpl:16 //line app/vlselect/logsql/query_response.qtpl:32
qw422016.N().S(`}`) qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:17 //line app/vlselect/logsql/query_response.qtpl:33
qw422016.N().S(` qw422016.N().S(`
`) `)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
} }
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) { func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
qw422016 := qt422016.AcquireWriter(qq422016) qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
StreamJSONRow(qw422016, columns, rowIdx) StreamJSONRow(qw422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseWriter(qw422016) qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
} }
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string { func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
qb422016 := qt422016.AcquireByteBuffer() qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
WriteJSONRow(qb422016, columns, rowIdx) WriteJSONRow(qb422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
qs422016 := string(qb422016.B) qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseByteBuffer(qb422016) qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
return qs422016 return qs422016
//line app/vlselect/logsql/query_response.qtpl:18 //line app/vlselect/logsql/query_response.qtpl:34
} }
// JSONRows prints formatted rows // JSONRows prints formatted rows
//line app/vlselect/logsql/query_response.qtpl:21 //line app/vlselect/logsql/query_response.qtpl:37
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) { func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:22 //line app/vlselect/logsql/query_response.qtpl:38
if len(rows) == 0 { if len(rows) == 0 {
//line app/vlselect/logsql/query_response.qtpl:23 //line app/vlselect/logsql/query_response.qtpl:39
return return
//line app/vlselect/logsql/query_response.qtpl:24 //line app/vlselect/logsql/query_response.qtpl:40
} }
//line app/vlselect/logsql/query_response.qtpl:25 //line app/vlselect/logsql/query_response.qtpl:41
for _, fields := range rows { for _, fields := range rows {
//line app/vlselect/logsql/query_response.qtpl:25 //line app/vlselect/logsql/query_response.qtpl:42
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
//line app/vlselect/logsql/query_response.qtpl:43
if len(fields) == 0 {
//line app/vlselect/logsql/query_response.qtpl:44
continue
//line app/vlselect/logsql/query_response.qtpl:45
}
//line app/vlselect/logsql/query_response.qtpl:45
qw422016.N().S(`{`) qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:27 //line app/vlselect/logsql/query_response.qtpl:47
if len(fields) > 0 { if len(fields) > 0 {
//line app/vlselect/logsql/query_response.qtpl:29 //line app/vlselect/logsql/query_response.qtpl:49
f := fields[0] f := fields[0]
fields = fields[1:] fields = fields[1:]
//line app/vlselect/logsql/query_response.qtpl:32 //line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Name) qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:32 //line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().S(`:`) qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:32 //line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Value) qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:33 //line app/vlselect/logsql/query_response.qtpl:53
for _, f := range fields { for _, f := range fields {
//line app/vlselect/logsql/query_response.qtpl:33 //line app/vlselect/logsql/query_response.qtpl:54
if f.Value == "" {
//line app/vlselect/logsql/query_response.qtpl:55
continue
//line app/vlselect/logsql/query_response.qtpl:56
}
//line app/vlselect/logsql/query_response.qtpl:56
qw422016.N().S(`,`) qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:34 //line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Name) qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:34 //line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().S(`:`) qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:34 //line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Value) qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:35 //line app/vlselect/logsql/query_response.qtpl:58
} }
//line app/vlselect/logsql/query_response.qtpl:36 //line app/vlselect/logsql/query_response.qtpl:59
} }
//line app/vlselect/logsql/query_response.qtpl:36 //line app/vlselect/logsql/query_response.qtpl:59
qw422016.N().S(`}`) qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:37 //line app/vlselect/logsql/query_response.qtpl:60
qw422016.N().S(` qw422016.N().S(`
`) `)
//line app/vlselect/logsql/query_response.qtpl:38 //line app/vlselect/logsql/query_response.qtpl:61
} }
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
} }
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) { func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
qw422016 := qt422016.AcquireWriter(qq422016) qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
StreamJSONRows(qw422016, rows) StreamJSONRows(qw422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseWriter(qw422016) qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
} }
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
func JSONRows(rows [][]logstorage.Field) string { func JSONRows(rows [][]logstorage.Field) string {
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
qb422016 := qt422016.AcquireByteBuffer() qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
WriteJSONRows(qb422016, rows) WriteJSONRows(qb422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
qs422016 := string(qb422016.B) qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseByteBuffer(qb422016) qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
return qs422016 return qs422016
//line app/vlselect/logsql/query_response.qtpl:39 //line app/vlselect/logsql/query_response.qtpl:62
} }

1
docs/VictoriaLogs/CHANGELOG.md
View file

@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip ## tip
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge). * FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries. * BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163). * BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).

20
lib/logstorage/logfmt_parser_test.go
View file

@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
defer putLogfmtParser(p) defer putLogfmtParser(p)
p.parse(s) p.parse(s)
result := MarshalFieldsToJSON(nil, p.fields) result := MarshalFieldsToLogfmt(nil, p.fields)
if string(result) != resultExpected { if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected) t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
} }
} }
f(``, `{}`) f(``, ``)
f(`foo=bar`, `{"foo":"bar"}`) f(`foo=bar`, `foo=bar`)
f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`) f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
f(`foo=`, `{"foo":""}`) f(`foo=`, `foo=`)
f(`foo`, `{"foo":""}`) f(`foo`, `foo=`)
f(`foo bar`, `{"foo":"","bar":""}`) f(`foo bar`, `foo= bar=`)
f(`foo bar=baz`, `{"foo":"","bar":"baz"}`) f(`foo bar=baz`, `foo= bar=baz`)
f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`) f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`) f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
} }

4
lib/logstorage/pipe_pack_json_test.go
View file

@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
{"_msg", `x`}, {"_msg", `x`},
{"foo", `abc`}, {"foo", `abc`},
{"bar", `cde`}, {"bar", `cde`},
{"a", `{"foo":"abc","baz":""}`}, {"a", `{"foo":"abc"}`},
}, },
{ {
{"a", `{"foo":"","baz":""}`}, {"a", `{}`},
{"c", "d"}, {"c", "d"},
}, },
}) })

23
lib/logstorage/rows.go
View file

@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
} }
func (f *Field) marshalToLogfmt(dst []byte) []byte { func (f *Field) marshalToLogfmt(dst []byte) []byte {
dst = append(dst, f.Name...) name := f.Name
if name == "" {
name = "_msg"
}
dst = append(dst, name...)
dst = append(dst, '=') dst = append(dst, '=')
if needLogfmtQuoting(f.Value) { if needLogfmtQuoting(f.Value) {
dst = quicktemplate.AppendJSONString(dst, f.Value, true) dst = quicktemplate.AppendJSONString(dst, f.Value, true)
@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result. // MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte { func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
fields = SkipLeadingFieldsWithoutValues(fields)
dst = append(dst, '{') dst = append(dst, '{')
if len(fields) > 0 { if len(fields) > 0 {
dst = fields[0].marshalToJSON(dst) dst = fields[0].marshalToJSON(dst)
fields = fields[1:] fields = fields[1:]
for i := range fields { for i := range fields {
f := &fields[i]
if f.Value == "" {
// Skip fields without values
continue
}
dst = append(dst, ',') dst = append(dst, ',')
dst = fields[i].marshalToJSON(dst) dst = f.marshalToJSON(dst)
} }
} }
dst = append(dst, '}') dst = append(dst, '}')
@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
return dst return dst
} }
// SkipLeadingFieldsWithoutValues skips leading fields without values.
func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
i := 0
for i < len(fields) && fields[i].Value == "" {
i++
}
return fields[i:]
}
func appendFields(a *arena, dst, src []Field) []Field { func appendFields(a *arena, dst, src []Field) []Field {
for _, f := range src { for _, f := range src {
dst = append(dst, Field{ dst = append(dst, Field{

6
lib/logstorage/stats_row_any_test.go
View file

@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
}, },
}, [][]Field{ }, [][]Field{
{ {
{"x", `{"a":"2","x":"","b":"3"}`}, {"x", `{"a":"2","b":"3"}`},
}, },
}) })
@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
}, [][]Field{ }, [][]Field{
{ {
{"a", "1"}, {"a", "1"},
{"x", `{"c":""}`}, {"x", `{}`},
}, },
{ {
{"a", "3"}, {"a", "3"},
@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
{ {
{"a", "1"}, {"a", "1"},
{"b", "3"}, {"b", "3"},
{"x", `{"c":""}`}, {"x", `{}`},
}, },
{ {
{"a", "1"}, {"a", "1"},

4
lib/logstorage/stats_row_max_test.go
View file

@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
}, },
}, [][]Field{ }, [][]Field{
{ {
{"x", `{"a":"3","x":"","b":"54"}`}, {"x", `{"a":"3","b":"54"}`},
}, },
}) })
@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
}, [][]Field{ }, [][]Field{
{ {
{"a", "1"}, {"a", "1"},
{"x", `{"c":""}`}, {"x", `{}`},
}, },
{ {
{"a", "3"}, {"a", "3"},

4
lib/logstorage/stats_row_min_test.go
View file

@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
}, },
}, [][]Field{ }, [][]Field{
{ {
{"x", `{"a":"2","x":"","b":"3"}`}, {"x", `{"a":"2","b":"3"}`},
}, },
}) })
@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
}, [][]Field{ }, [][]Field{
{ {
{"a", "1"}, {"a", "1"},
{"x", `{"c":""}`}, {"x", `{}`},
}, },
{ {
{"a", "3"}, {"a", "3"},

52
lib/logstorage/syslog_parser_test.go
View file

@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
defer PutSyslogParser(p) defer PutSyslogParser(p)
p.Parse(s) p.Parse(s)
result := MarshalFieldsToJSON(nil, p.Fields) result := MarshalFieldsToLogfmt(nil, p.Fields)
if string(result) != resultExpected { if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected) t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
} }
@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
// RFC 3164 // RFC 3164
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC, f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`) `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC, f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`) `priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC, f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`) `format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC, f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`) `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC, f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`) `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
// RFC 5424 // RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC, f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`) `format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC, f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC, f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
// Incomplete RFC 3164 // Incomplete RFC 3164
f("", time.UTC, `{}`) f("", time.UTC, ``)
f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`) f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`) f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`) f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`) f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`) f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`) f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`) f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`) f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
// Incomplete RFC 5424 // Incomplete RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC, f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`) `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
f(`<165>1 `, time.UTC, f(`<165>1 `, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`) `priority=165 facility=20 severity=5 format=rfc5424`)
} }