mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
app/vlselect: do not show empty fields in query results
Empty fields are treated as non-existing fields by VictoriaLogs data model. So there is no sense in returning empty fields in query results, since they may mislead and confuse users.
This commit is contained in:
Aliaksandr Valialkin
parent
343463fc0f
commit
bac193e50b
13 changed files with 203 additions and 125 deletions
|
|
@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
|||
msgField := "message"
|
||||
rowsExpected := 4
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}
|
||||
{"_msg":"qwe rty","@timestamp":""}`
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}
|
||||
{"_msg":"qwe rty"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
|||
msgField := "message"
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
|||
currentYear := 2023
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
|
||||
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,15 +6,31 @@
|
|||
|
||||
// JSONRow creates JSON row from the given fields.
|
||||
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{% code
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
%}
|
||||
{% if len(columns) == 0 %}
|
||||
{% return %}
|
||||
{% endif %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% code columns = columns[1:] %}
|
||||
{% for colIdx := range columns %}
|
||||
{% code c := &columns[colIdx] %}
|
||||
{% code
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
%}
|
||||
{% if v == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% endfor %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfunc %}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
|
|
@ -23,7 +39,11 @@
|
|||
{% return %}
|
||||
{% endif %}
|
||||
{% for _, fields := range rows %}
|
||||
{
|
||||
{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
|
||||
{% if len(fields) == 0 %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
{
|
||||
{% if len(fields) > 0 %}
|
||||
{% code
|
||||
f := fields[0]
|
||||
|
|
@ -31,10 +51,13 @@
|
|||
%}
|
||||
{%q= f.Name %}:{%q= f.Value %}
|
||||
{% for _, f := range fields %}
|
||||
{% if f.Value == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
,{%q= f.Name %}:{%q= f.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfor %}
|
||||
{% endfunc %}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,141 +26,176 @@ var (
|
|||
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:10
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
if len(columns) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:20
|
||||
c := &columns[0]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:12
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
columns = columns[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:13
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
for colIdx := range columns {
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
//line app/vlselect/logsql/query_response.qtpl:28
|
||||
if v == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
StreamJSONRow(qw422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
WriteJSONRow(qb422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
if len(rows) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:24
|
||||
//line app/vlselect/logsql/query_response.qtpl:40
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
//line app/vlselect/logsql/query_response.qtpl:41
|
||||
for _, fields := range rows {
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
//line app/vlselect/logsql/query_response.qtpl:42
|
||||
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:43
|
||||
if len(fields) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:44
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:27
|
||||
//line app/vlselect/logsql/query_response.qtpl:47
|
||||
if len(fields) > 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
//line app/vlselect/logsql/query_response.qtpl:49
|
||||
f := fields[0]
|
||||
fields = fields[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
//line app/vlselect/logsql/query_response.qtpl:53
|
||||
for _, f := range fields {
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
//line app/vlselect/logsql/query_response.qtpl:54
|
||||
if f.Value == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:55
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:35
|
||||
//line app/vlselect/logsql/query_response.qtpl:58
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
//line app/vlselect/logsql/query_response.qtpl:60
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
//line app/vlselect/logsql/query_response.qtpl:61
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
StreamJSONRows(qw422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
func JSONRows(rows [][]logstorage.Field) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
WriteJSONRows(qb422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
|
|||
## tip
|
||||
|
||||
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
|
||||
* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||
|
||||
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
|
||||
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).
|
||||
|
|
|
|||
|
|
@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
|
|||
defer putLogfmtParser(p)
|
||||
|
||||
p.parse(s)
|
||||
result := MarshalFieldsToJSON(nil, p.fields)
|
||||
result := MarshalFieldsToLogfmt(nil, p.fields)
|
||||
if string(result) != resultExpected {
|
||||
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(``, `{}`)
|
||||
f(`foo=bar`, `{"foo":"bar"}`)
|
||||
f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`)
|
||||
f(`foo=`, `{"foo":""}`)
|
||||
f(`foo`, `{"foo":""}`)
|
||||
f(`foo bar`, `{"foo":"","bar":""}`)
|
||||
f(`foo bar=baz`, `{"foo":"","bar":"baz"}`)
|
||||
f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`)
|
||||
f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`)
|
||||
f(``, ``)
|
||||
f(`foo=bar`, `foo=bar`)
|
||||
f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
|
||||
f(`foo=`, `foo=`)
|
||||
f(`foo`, `foo=`)
|
||||
f(`foo bar`, `foo= bar=`)
|
||||
f(`foo bar=baz`, `foo= bar=baz`)
|
||||
f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
|
||||
f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
|
|||
{"_msg", `x`},
|
||||
{"foo", `abc`},
|
||||
{"bar", `cde`},
|
||||
{"a", `{"foo":"abc","baz":""}`},
|
||||
{"a", `{"foo":"abc"}`},
|
||||
},
|
||||
{
|
||||
{"a", `{"foo":"","baz":""}`},
|
||||
{"a", `{}`},
|
||||
{"c", "d"},
|
||||
},
|
||||
})
|
||||
|
|
|
|||
|
|
@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
|
|||
}
|
||||
|
||||
func (f *Field) marshalToLogfmt(dst []byte) []byte {
|
||||
dst = append(dst, f.Name...)
|
||||
name := f.Name
|
||||
if name == "" {
|
||||
name = "_msg"
|
||||
}
|
||||
dst = append(dst, name...)
|
||||
dst = append(dst, '=')
|
||||
if needLogfmtQuoting(f.Value) {
|
||||
dst = quicktemplate.AppendJSONString(dst, f.Value, true)
|
||||
|
|
@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
|
|||
|
||||
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
|
||||
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
|
||||
fields = SkipLeadingFieldsWithoutValues(fields)
|
||||
dst = append(dst, '{')
|
||||
if len(fields) > 0 {
|
||||
dst = fields[0].marshalToJSON(dst)
|
||||
fields = fields[1:]
|
||||
for i := range fields {
|
||||
f := &fields[i]
|
||||
if f.Value == "" {
|
||||
// Skip fields without values
|
||||
continue
|
||||
}
|
||||
dst = append(dst, ',')
|
||||
dst = fields[i].marshalToJSON(dst)
|
||||
dst = f.marshalToJSON(dst)
|
||||
}
|
||||
}
|
||||
dst = append(dst, '}')
|
||||
|
|
@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
|
|||
return dst
|
||||
}
|
||||
|
||||
// SkipLeadingFieldsWithoutValues skips leading fields without values.
|
||||
func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
|
||||
i := 0
|
||||
for i < len(fields) && fields[i].Value == "" {
|
||||
i++
|
||||
}
|
||||
return fields[i:]
|
||||
}
|
||||
|
||||
func appendFields(a *arena, dst, src []Field) []Field {
|
||||
for _, f := range src {
|
||||
dst = append(dst, Field{
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"2","x":"","b":"3"}`},
|
||||
{"x", `{"a":"2","b":"3"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
|
|||
{
|
||||
{"a", "1"},
|
||||
{"b", "3"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "1"},
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"3","x":"","b":"54"}`},
|
||||
{"x", `{"a":"3","b":"54"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
|
|||
},
|
||||
}, [][]Field{
|
||||
{
|
||||
{"x", `{"a":"2","x":"","b":"3"}`},
|
||||
{"x", `{"a":"2","b":"3"}`},
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
|
|||
}, [][]Field{
|
||||
{
|
||||
{"a", "1"},
|
||||
{"x", `{"c":""}`},
|
||||
{"x", `{}`},
|
||||
},
|
||||
{
|
||||
{"a", "3"},
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
|
|||
defer PutSyslogParser(p)
|
||||
|
||||
p.Parse(s)
|
||||
result := MarshalFieldsToJSON(nil, p.Fields)
|
||||
result := MarshalFieldsToLogfmt(nil, p.Fields)
|
||||
if string(result) != resultExpected {
|
||||
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
|
||||
}
|
||||
|
|
@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
|
|||
|
||||
// RFC 3164
|
||||
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
|
||||
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
|
||||
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
|
||||
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
|
||||
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
|
||||
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`)
|
||||
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
|
||||
|
||||
// RFC 5424
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
|
||||
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
|
||||
`{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
|
||||
`format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
|
||||
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
|
||||
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
|
||||
|
||||
// Incomplete RFC 3164
|
||||
f("", time.UTC, `{}`)
|
||||
f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`)
|
||||
f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`)
|
||||
f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`)
|
||||
f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`)
|
||||
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`)
|
||||
f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`)
|
||||
f("", time.UTC, ``)
|
||||
f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
|
||||
f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
|
||||
f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
|
||||
f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
|
||||
f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
|
||||
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
|
||||
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
|
||||
f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
|
||||
|
||||
// Incomplete RFC 5424
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
|
||||
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
|
||||
f(`<165>1 `, time.UTC,
|
||||
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`)
|
||||
`priority=165 facility=20 severity=5 format=rfc5424`)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue