mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-03-11 15:34:56 +00:00
app/vlselect: do not show empty fields in query results
Empty fields are treated as non-existing fields by VictoriaLogs data model.
So there is no sense in returning empty fields in query results, since they may mislead and confuse users.
(cherry picked from commit bac193e50b)
This commit is contained in:
Aliaksandr Valialkin
hagen1778
parent
ac382d8b79
commit
0881e5fd5c
13 changed files with 203 additions and 125 deletions
|
|
@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
|||
msgField := "message"
|
||||
rowsExpected := 4
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}
|
||||
{"_msg":"qwe rty","@timestamp":""}`
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}
|
||||
{"_msg":"qwe rty"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
|||
msgField := "message"
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
|||
currentYear := 2023
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
|
||||
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,15 +6,31 @@
|
|||
|
||||
// JSONRow creates JSON row from the given fields.
|
||||
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{% code
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
%}
|
||||
{% if len(columns) == 0 %}
|
||||
{% return %}
|
||||
{% endif %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% code columns = columns[1:] %}
|
||||
{% for colIdx := range columns %}
|
||||
{% code c := &columns[colIdx] %}
|
||||
{% code
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
%}
|
||||
{% if v == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% endfor %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfunc %}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
|
|
@ -23,7 +39,11 @@
|
|||
{% return %}
|
||||
{% endif %}
|
||||
{% for _, fields := range rows %}
|
||||
{
|
||||
{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
|
||||
{% if len(fields) == 0 %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
{
|
||||
{% if len(fields) > 0 %}
|
||||
{% code
|
||||
f := fields[0]
|
||||
|
|
@ -31,10 +51,13 @@
|
|||
%}
|
||||
{%q= f.Name %}:{%q= f.Value %}
|
||||
{% for _, f := range fields %}
|
||||
{% if f.Value == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
,{%q= f.Name %}:{%q= f.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfor %}
|
||||
{% endfunc %}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,141 +26,176 @@ var (
|
|||
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:10
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
if len(columns) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:20
|
||||
c := &columns[0]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:12
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
columns = columns[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:13
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
for colIdx := range columns {
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
//line app/vlselect/logsql/query_response.qtpl:28
|
||||
if v == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
StreamJSONRow(qw422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
WriteJSONRow(qb422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
if len(rows) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:24
|
||||
//line app/vlselect/logsql/query_response.qtpl:40
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
//line app/vlselect/logsql/query_response.qtpl:41
|
||||
for _, fields := range rows {
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
//line app/vlselect/logsql/query_response.qtpl:42
|
||||
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:43
|
||||
if len(fields) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:44
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:27
|
||||
//line app/vlselect/logsql/query_response.qtpl:47
|
||||
if len(fields) > 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
//line app/vlselect/logsql/query_response.qtpl:49
|
||||
f := fields[0]
|
||||
fields = fields[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
//line app/vlselect/logsql/query_response.qtpl:53
|
||||
for _, f := range fields {
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
//line app/vlselect/logsql/query_response.qtpl:54
|
||||
if f.Value == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:55
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:35
|
||||
//line app/vlselect/logsql/query_response.qtpl:58
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
//line app/vlselect/logsql/query_response.qtpl:60
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
//line app/vlselect/logsql/query_response.qtpl:61
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
StreamJSONRows(qw422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
func JSONRows(rows [][]logstorage.Field) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
WriteJSONRows(qb422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
|
|||
## tip
|
||||
|
||||
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
|
||||
* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||
|
||||
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
|
||||
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).
|
||||
|
|
|
|||
|
|
@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
|
|||
defer putLogfmtParser(p)
|
||||
|
||||
p.parse(s)
|
||||
result := MarshalFieldsToJSON(nil, p.fields)
|
||||
result := MarshalFieldsToLogfmt(nil, p.fields)
|
||||
if string(result) != resultExpected {
|
||||
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(``, `{}`)
|
||||
f(`foo=bar`, `{"foo":"bar"}`)
|
||||
f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`)
|
||||
f(`foo=`, `{"foo":""}`)
|
||||
f(`foo`, `{"foo":""}`)
|
||||
f(`foo bar`, `{"foo":"","bar":""}`)
|
||||
f(`foo bar=baz`, `{"foo":"","bar":"baz"}`)
|
||||
f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`)
|
||||
f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`)
|
||||
f(``, ``)
|
||||
f(`foo=bar`, `foo=bar`)
|
||||
f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
|
||||
f(`foo=`, `foo=`)
|
||||
f(`foo`, `foo=`)
|
||||
f(`foo bar`, `foo= bar=`)
|
||||
f(`foo bar=baz`, `foo= bar=baz`)
|
||||
f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
|
||||
f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
|
|||
{"_msg", `x`},
|
||||
{"foo", `abc`},
|
||||
{"bar", `cde`},
|
||||
{"a", `{"foo":"abc","baz":""}`},
|
||||
{"a", `{"foo":"abc"}`},
|
||||
},
|
||||
{
|
||||
{"a", `{"foo":"","baz":""}`},
|
||||
{"a", `{}`},
|
||||
{"c", "d"},
|
||||
},
|
||||
})
|
||||
|
|
|
|||
|
|
@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
|
|||
}
|
||||
|
||||
func (f *Field) marshalToLogfmt(dst []byte) []byte {
|
||||
dst = append(dst, f.Name...)
|
||||
name := f.Name
|
||||
if name == "" {
|
||||
name = "_msg"
|
||||
}
|
||||
dst = append(dst, name...)
|
||||
dst = append(dst, '=')
|
||||
if needLogfmtQuoting(f.Value) {
|
||||
dst = quicktemplate.AppendJSONString(dst, f.Value, true)
|
||||
|
|
@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
|
|||
|
||||
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
|
||||
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
|
||||
fields = SkipLeadingFieldsWithoutValues(fields)
|
||||
dst = append(dst, '{')
|
||||
if len(fields) > 0 {
|
||||
dst = fields[0].marshalToJSON(dst)
|
||||
fields = fields[1:]
|
||||
for i := range fields {
|
||||
f := &fields[i]
|
||||
if f.Value == "" {
|
||||
// Skip fields without values
|
||||
continue
|
||||
}
|
||||
dst = append(dst, ',')
|
||||
dst = fields[i].marshalToJSON(dst)
|
||||
dst = f.marshalToJSON(dst)
|
||||
}
|
||||
}
|
||||
dst = append(dst, '}')
|
||||
|
|
@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
|
|||
return dst
|
||||
}
|
||||
|
||||
// SkipLeadingFieldsWithoutValues skips leading fields without values.
|
||||
func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
|
||||
i := 0
|
||||
for i < len(fields) && fields[i].Value == "" {
|
||||
i++
|
||||
}
|
||||
return fields[i:]
|
||||
}
|
||||
|
||||
func appendFields(a *arena, dst, src []Field) []Field {
|
||||
for _, f := range src {
|
||||
dst = append(dst, Field{
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"2","x":"","b":"3"}`},
|
||||
{"x", `{"a":"2","b":"3"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
{
|
||||
{"a", "1"},
|
||||
{"b", "3"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "1"},
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"3","x":"","b":"54"}`},
|
||||
{"x", `{"a":"3","b":"54"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"2","x":"","b":"3"}`},
|
||||
{"x", `{"a":"2","b":"3"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
|
|||
defer PutSyslogParser(p)
|
||||
|
||||
p.Parse(s)
|
||||
result := MarshalFieldsToJSON(nil, p.Fields)
|
||||
result := MarshalFieldsToLogfmt(nil, p.Fields)
|
||||
if string(result) != resultExpected {
|
||||
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
|
||||
}
|
||||
|
|
@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
|
|||
|
||||
// RFC 3164
|
||||
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
|
||||
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
|
||||
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
|
||||
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
|
||||
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
|
||||
|
||||
// RFC 5424
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
|
||||
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
|
||||
`{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
|
||||
`format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
|
||||
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
|
||||
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
|
||||
|
||||
// Incomplete RFC 3164
|
||||
f("", time.UTC, `{}`)
|
||||
f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`)
|
||||
f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`)
|
||||
f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`)
|
||||
f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`)
|
||||
f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`)
|
||||
f("", time.UTC, ``)
|
||||
f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
|
||||
f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
|
||||
f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
|
||||
f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
|
||||
f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
|
||||
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
|
||||
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
|
||||
f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
|
||||
|
||||
// Incomplete RFC 5424
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
|
||||
f(`<165>1 `, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424`)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue