app/vlselect: do not show empty fields in query results

Empty fields are treated as non-existing fields by VictoriaLogs data model.
So there is no sense in returning empty fields in query results, since they may mislead and confuse users.

(cherry picked from commit bac193e50b)
This commit is contained in:
Aliaksandr Valialkin 2024-10-14 23:39:29 +02:00 committed by hagen1778
parent ac382d8b79
commit 0881e5fd5c
No known key found for this signature in database
GPG key ID: E92986095E0DD614
13 changed files with 203 additions and 125 deletions

View file

@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
msgField := "message"
rowsExpected := 4
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"}
{"_msg":"qwe rty","@timestamp":""}`
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"_msg":"baz"}
{"_msg":"xyz","x":"y"}
{"_msg":"qwe rty"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}

View file

@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
msgField := "message"
rowsExpected := 3
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"}`
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"_msg":"baz"}
{"_msg":"xyz","x":"y"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}

View file

@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
currentYear := 2023
rowsExpected := 3
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
}

View file

@ -6,15 +6,31 @@
// JSONRow creates JSON row from the given fields.
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
{
{% code c := &columns[0] %}
{% code
i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
%}
{% if len(columns) == 0 %}
{% return %}
{% endif %}
{
{% code c := &columns[0] %}
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% code columns = columns[1:] %}
{% for colIdx := range columns %}
{% code c := &columns[colIdx] %}
{% code
c := &columns[colIdx]
v := c.Values[rowIdx]
%}
{% if v == "" %}
{% continue %}
{% endif %}
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% endfor %}
}{% newline %}
}{% newline %}
{% endfunc %}
// JSONRows prints formatted rows
@ -23,7 +39,11 @@
{% return %}
{% endif %}
{% for _, fields := range rows %}
{
{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
{% if len(fields) == 0 %}
{% continue %}
{% endif %}
{
{% if len(fields) > 0 %}
{% code
f := fields[0]
@ -31,10 +51,13 @@
%}
{%q= f.Name %}:{%q= f.Value %}
{% for _, f := range fields %}
{% if f.Value == "" %}
{% continue %}
{% endif %}
,{%q= f.Name %}:{%q= f.Value %}
{% endfor %}
{% endif %}
}{% newline %}
}{% newline %}
{% endfor %}
{% endfunc %}

View file

@ -26,141 +26,176 @@ var (
//line app/vlselect/logsql/query_response.qtpl:8
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:8
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:10
i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
//line app/vlselect/logsql/query_response.qtpl:16
if len(columns) == 0 {
//line app/vlselect/logsql/query_response.qtpl:17
return
//line app/vlselect/logsql/query_response.qtpl:18
}
//line app/vlselect/logsql/query_response.qtpl:18
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:20
c := &columns[0]
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:12
//line app/vlselect/logsql/query_response.qtpl:22
columns = columns[1:]
//line app/vlselect/logsql/query_response.qtpl:13
//line app/vlselect/logsql/query_response.qtpl:23
for colIdx := range columns {
//line app/vlselect/logsql/query_response.qtpl:14
//line app/vlselect/logsql/query_response.qtpl:25
c := &columns[colIdx]
v := c.Values[rowIdx]
//line app/vlselect/logsql/query_response.qtpl:14
//line app/vlselect/logsql/query_response.qtpl:28
if v == "" {
//line app/vlselect/logsql/query_response.qtpl:29
continue
//line app/vlselect/logsql/query_response.qtpl:30
}
//line app/vlselect/logsql/query_response.qtpl:30
qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:15
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:15
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:15
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:16
//line app/vlselect/logsql/query_response.qtpl:32
}
//line app/vlselect/logsql/query_response.qtpl:16
//line app/vlselect/logsql/query_response.qtpl:32
qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:17
//line app/vlselect/logsql/query_response.qtpl:33
qw422016.N().S(`
`)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
StreamJSONRow(qw422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
WriteJSONRow(qb422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
return qs422016
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
// JSONRows prints formatted rows
//line app/vlselect/logsql/query_response.qtpl:21
//line app/vlselect/logsql/query_response.qtpl:37
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:22
//line app/vlselect/logsql/query_response.qtpl:38
if len(rows) == 0 {
//line app/vlselect/logsql/query_response.qtpl:23
//line app/vlselect/logsql/query_response.qtpl:39
return
//line app/vlselect/logsql/query_response.qtpl:24
//line app/vlselect/logsql/query_response.qtpl:40
}
//line app/vlselect/logsql/query_response.qtpl:25
//line app/vlselect/logsql/query_response.qtpl:41
for _, fields := range rows {
//line app/vlselect/logsql/query_response.qtpl:25
//line app/vlselect/logsql/query_response.qtpl:42
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
//line app/vlselect/logsql/query_response.qtpl:43
if len(fields) == 0 {
//line app/vlselect/logsql/query_response.qtpl:44
continue
//line app/vlselect/logsql/query_response.qtpl:45
}
//line app/vlselect/logsql/query_response.qtpl:45
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:27
//line app/vlselect/logsql/query_response.qtpl:47
if len(fields) > 0 {
//line app/vlselect/logsql/query_response.qtpl:29
//line app/vlselect/logsql/query_response.qtpl:49
f := fields[0]
fields = fields[1:]
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:33
//line app/vlselect/logsql/query_response.qtpl:53
for _, f := range fields {
//line app/vlselect/logsql/query_response.qtpl:33
//line app/vlselect/logsql/query_response.qtpl:54
if f.Value == "" {
//line app/vlselect/logsql/query_response.qtpl:55
continue
//line app/vlselect/logsql/query_response.qtpl:56
}
//line app/vlselect/logsql/query_response.qtpl:56
qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:35
//line app/vlselect/logsql/query_response.qtpl:58
}
//line app/vlselect/logsql/query_response.qtpl:36
//line app/vlselect/logsql/query_response.qtpl:59
}
//line app/vlselect/logsql/query_response.qtpl:36
//line app/vlselect/logsql/query_response.qtpl:59
qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:37
//line app/vlselect/logsql/query_response.qtpl:60
qw422016.N().S(`
`)
//line app/vlselect/logsql/query_response.qtpl:38
//line app/vlselect/logsql/query_response.qtpl:61
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
StreamJSONRows(qw422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
func JSONRows(rows [][]logstorage.Field) string {
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
WriteJSONRows(qb422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
return qs422016
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}

View file

@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).

View file

@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
defer putLogfmtParser(p)
p.parse(s)
result := MarshalFieldsToJSON(nil, p.fields)
result := MarshalFieldsToLogfmt(nil, p.fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
}
f(``, `{}`)
f(`foo=bar`, `{"foo":"bar"}`)
f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`)
f(`foo=`, `{"foo":""}`)
f(`foo`, `{"foo":""}`)
f(`foo bar`, `{"foo":"","bar":""}`)
f(`foo bar=baz`, `{"foo":"","bar":"baz"}`)
f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`)
f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`)
f(``, ``)
f(`foo=bar`, `foo=bar`)
f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
f(`foo=`, `foo=`)
f(`foo`, `foo=`)
f(`foo bar`, `foo= bar=`)
f(`foo bar=baz`, `foo= bar=baz`)
f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
}

View file

@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
{"_msg", `x`},
{"foo", `abc`},
{"bar", `cde`},
{"a", `{"foo":"abc","baz":""}`},
{"a", `{"foo":"abc"}`},
},
{
{"a", `{"foo":"","baz":""}`},
{"a", `{}`},
{"c", "d"},
},
})

View file

@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
}
func (f *Field) marshalToLogfmt(dst []byte) []byte {
dst = append(dst, f.Name...)
name := f.Name
if name == "" {
name = "_msg"
}
dst = append(dst, name...)
dst = append(dst, '=')
if needLogfmtQuoting(f.Value) {
dst = quicktemplate.AppendJSONString(dst, f.Value, true)
@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
fields = SkipLeadingFieldsWithoutValues(fields)
dst = append(dst, '{')
if len(fields) > 0 {
dst = fields[0].marshalToJSON(dst)
fields = fields[1:]
for i := range fields {
f := &fields[i]
if f.Value == "" {
// Skip fields without values
continue
}
dst = append(dst, ',')
dst = fields[i].marshalToJSON(dst)
dst = f.marshalToJSON(dst)
}
}
dst = append(dst, '}')
@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
return dst
}
// SkipLeadingFieldsWithoutValues skips leading fields without values.
func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
i := 0
for i < len(fields) && fields[i].Value == "" {
i++
}
return fields[i:]
}
func appendFields(a *arena, dst, src []Field) []Field {
for _, f := range src {
dst = append(dst, Field{

View file

@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"2","x":"","b":"3"}`},
{"x", `{"a":"2","b":"3"}`},
},
})
@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},
@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
{
{"a", "1"},
{"b", "3"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "1"},

View file

@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"3","x":"","b":"54"}`},
{"x", `{"a":"3","b":"54"}`},
},
})
@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},

View file

@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"2","x":"","b":"3"}`},
{"x", `{"a":"2","b":"3"}`},
},
})
@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},

View file

@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
defer PutSyslogParser(p)
p.Parse(s)
result := MarshalFieldsToJSON(nil, p.Fields)
result := MarshalFieldsToLogfmt(nil, p.Fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
// RFC 3164
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
`priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
// RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
`format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
// Incomplete RFC 3164
f("", time.UTC, `{}`)
f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`)
f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`)
f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`)
f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`)
f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`)
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`)
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`)
f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`)
f("", time.UTC, ``)
f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
// Incomplete RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
f(`<165>1 `, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`)
`priority=165 facility=20 severity=5 format=rfc5424`)
}