github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vlselect: do not show empty fields in query results

Browse source 
Empty fields are treated as non-existing fields by VictoriaLogs data model.
So there is no sense in returning empty fields in query results, since they may mislead and confuse users.
This commit is contained in:
Aliaksandr Valialkin 2024-10-14 23:39:29 +02:00
parent 343463fc0f
commit bac193e50b
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
13 changed files with 203 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
app/vlinsert/elasticsearch/elasticsearch_test.go
View file

@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
msgField := "message"
rowsExpected := 4
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"}
{"_msg":"qwe rty","@timestamp":""}`
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"_msg":"baz"}
{"_msg":"xyz","x":"y"}
{"_msg":"qwe rty"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}

6
app/vlinsert/jsonline/jsonline_test.go
View file

@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
msgField := "message"
rowsExpected := 3
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"@timestamp":"","_msg":"baz"}
{"_msg":"xyz","@timestamp":"","x":"y"}`
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
{"_msg":"baz"}
{"_msg":"xyz","x":"y"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}

6
app/vlinsert/syslog/syslog_test.go
View file

@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
currentYear := 2023
rowsExpected := 3
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
}

29
app/vlselect/logsql/query_response.qtpl
View file

@ -6,15 +6,31 @@
// JSONRow creates JSON row from the given fields.
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
{
{% code
i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
%}
{% if len(columns) == 0 %}
{% return %}
{% endif %}
{
{% code c := &columns[0] %}
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% code columns = columns[1:] %}
{% for colIdx := range columns %}
{% code c := &columns[colIdx] %}
{% code
c := &columns[colIdx]
v := c.Values[rowIdx]
%}
{% if v == "" %}
{% continue %}
{% endif %}
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% endfor %}
}{% newline %}
}{% newline %}
{% endfunc %}
// JSONRows prints formatted rows
@ -23,6 +39,10 @@
{% return %}
{% endif %}
{% for _, fields := range rows %}
{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
{% if len(fields) == 0 %}
{% continue %}
{% endif %}
{
{% if len(fields) > 0 %}
{% code
@ -31,6 +51,9 @@
%}
{%q= f.Name %}:{%q= f.Value %}
{% for _, f := range fields %}
{% if f.Value == "" %}
{% continue %}
{% endif %}
,{%q= f.Name %}:{%q= f.Value %}
{% endfor %}
{% endif %}

167
app/vlselect/logsql/query_response.qtpl.go
View file

@ -26,141 +26,176 @@ var (
//line app/vlselect/logsql/query_response.qtpl:8
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:8
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:10
i := 0
for i < len(columns) && columns[i].Values[rowIdx] == "" {
i++
}
columns = columns[i:]
//line app/vlselect/logsql/query_response.qtpl:16
if len(columns) == 0 {
//line app/vlselect/logsql/query_response.qtpl:17
return
//line app/vlselect/logsql/query_response.qtpl:18
}
//line app/vlselect/logsql/query_response.qtpl:18
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:20
c := &columns[0]
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:11
//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:12
//line app/vlselect/logsql/query_response.qtpl:22
columns = columns[1:]
//line app/vlselect/logsql/query_response.qtpl:13
//line app/vlselect/logsql/query_response.qtpl:23
for colIdx := range columns {
//line app/vlselect/logsql/query_response.qtpl:14
//line app/vlselect/logsql/query_response.qtpl:25
c := &columns[colIdx]
v := c.Values[rowIdx]
//line app/vlselect/logsql/query_response.qtpl:14
qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:15
qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:15
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:15
qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:16
//line app/vlselect/logsql/query_response.qtpl:28
if v == "" {
//line app/vlselect/logsql/query_response.qtpl:29
continue
//line app/vlselect/logsql/query_response.qtpl:30
}
//line app/vlselect/logsql/query_response.qtpl:16
//line app/vlselect/logsql/query_response.qtpl:30
qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Name)
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Values[rowIdx])
//line app/vlselect/logsql/query_response.qtpl:32
}
//line app/vlselect/logsql/query_response.qtpl:32
qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:17
//line app/vlselect/logsql/query_response.qtpl:33
qw422016.N().S(`
`)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
StreamJSONRow(qw422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
WriteJSONRow(qb422016, columns, rowIdx)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
return qs422016
//line app/vlselect/logsql/query_response.qtpl:18
//line app/vlselect/logsql/query_response.qtpl:34
}
// JSONRows prints formatted rows
//line app/vlselect/logsql/query_response.qtpl:21
//line app/vlselect/logsql/query_response.qtpl:37
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:22
//line app/vlselect/logsql/query_response.qtpl:38
if len(rows) == 0 {
//line app/vlselect/logsql/query_response.qtpl:23
//line app/vlselect/logsql/query_response.qtpl:39
return
//line app/vlselect/logsql/query_response.qtpl:24
//line app/vlselect/logsql/query_response.qtpl:40
}
//line app/vlselect/logsql/query_response.qtpl:25
//line app/vlselect/logsql/query_response.qtpl:41
for _, fields := range rows {
//line app/vlselect/logsql/query_response.qtpl:25
//line app/vlselect/logsql/query_response.qtpl:42
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
//line app/vlselect/logsql/query_response.qtpl:43
if len(fields) == 0 {
//line app/vlselect/logsql/query_response.qtpl:44
continue
//line app/vlselect/logsql/query_response.qtpl:45
}
//line app/vlselect/logsql/query_response.qtpl:45
qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:27
//line app/vlselect/logsql/query_response.qtpl:47
if len(fields) > 0 {
//line app/vlselect/logsql/query_response.qtpl:29
//line app/vlselect/logsql/query_response.qtpl:49
f := fields[0]
fields = fields[1:]
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:32
//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:33
//line app/vlselect/logsql/query_response.qtpl:53
for _, f := range fields {
//line app/vlselect/logsql/query_response.qtpl:33
//line app/vlselect/logsql/query_response.qtpl:54
if f.Value == "" {
//line app/vlselect/logsql/query_response.qtpl:55
continue
//line app/vlselect/logsql/query_response.qtpl:56
}
//line app/vlselect/logsql/query_response.qtpl:56
qw422016.N().S(`,`)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Name)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().S(`:`)
//line app/vlselect/logsql/query_response.qtpl:34
//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Value)
//line app/vlselect/logsql/query_response.qtpl:35
//line app/vlselect/logsql/query_response.qtpl:58
}
//line app/vlselect/logsql/query_response.qtpl:36
//line app/vlselect/logsql/query_response.qtpl:59
}
//line app/vlselect/logsql/query_response.qtpl:36
//line app/vlselect/logsql/query_response.qtpl:59
qw422016.N().S(`}`)
//line app/vlselect/logsql/query_response.qtpl:37
//line app/vlselect/logsql/query_response.qtpl:60
qw422016.N().S(`
`)
//line app/vlselect/logsql/query_response.qtpl:38
//line app/vlselect/logsql/query_response.qtpl:61
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
StreamJSONRows(qw422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseWriter(qw422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
func JSONRows(rows [][]logstorage.Field) string {
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qb422016 := qt422016.AcquireByteBuffer()
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
WriteJSONRows(qb422016, rows)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qs422016 := string(qb422016.B)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseByteBuffer(qb422016)
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
return qs422016
//line app/vlselect/logsql/query_response.qtpl:39
//line app/vlselect/logsql/query_response.qtpl:62
}

1
docs/VictoriaLogs/CHANGELOG.md
View file

@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).

20
lib/logstorage/logfmt_parser_test.go
View file

@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
defer putLogfmtParser(p)
p.parse(s)
result := MarshalFieldsToJSON(nil, p.fields)
result := MarshalFieldsToLogfmt(nil, p.fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
}
f(``, `{}`)
f(`foo=bar`, `{"foo":"bar"}`)
f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`)
f(`foo=`, `{"foo":""}`)
f(`foo`, `{"foo":""}`)
f(`foo bar`, `{"foo":"","bar":""}`)
f(`foo bar=baz`, `{"foo":"","bar":"baz"}`)
f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`)
f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`)
f(``, ``)
f(`foo=bar`, `foo=bar`)
f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
f(`foo=`, `foo=`)
f(`foo`, `foo=`)
f(`foo bar`, `foo= bar=`)
f(`foo bar=baz`, `foo= bar=baz`)
f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
}

4
lib/logstorage/pipe_pack_json_test.go
View file

@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
{"_msg", `x`},
{"foo", `abc`},
{"bar", `cde`},
{"a", `{"foo":"abc","baz":""}`},
{"a", `{"foo":"abc"}`},
},
{
{"a", `{"foo":"","baz":""}`},
{"a", `{}`},
{"c", "d"},
},
})

23
lib/logstorage/rows.go
View file

@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
}
func (f *Field) marshalToLogfmt(dst []byte) []byte {
dst = append(dst, f.Name...)
name := f.Name
if name == "" {
name = "_msg"
}
dst = append(dst, name...)
dst = append(dst, '=')
if needLogfmtQuoting(f.Value) {
dst = quicktemplate.AppendJSONString(dst, f.Value, true)
@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
fields = SkipLeadingFieldsWithoutValues(fields)
dst = append(dst, '{')
if len(fields) > 0 {
dst = fields[0].marshalToJSON(dst)
fields = fields[1:]
for i := range fields {
f := &fields[i]
if f.Value == "" {
// Skip fields without values
continue
}
dst = append(dst, ',')
dst = fields[i].marshalToJSON(dst)
dst = f.marshalToJSON(dst)
}
}
dst = append(dst, '}')
@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
return dst
}
// SkipLeadingFieldsWithoutValues skips leading fields without values.
func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
i := 0
for i < len(fields) && fields[i].Value == "" {
i++
}
return fields[i:]
}
func appendFields(a *arena, dst, src []Field) []Field {
for _, f := range src {
dst = append(dst, Field{

6
lib/logstorage/stats_row_any_test.go
View file

@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"2","x":"","b":"3"}`},
{"x", `{"a":"2","b":"3"}`},
},
})
@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},
@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
{
{"a", "1"},
{"b", "3"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "1"},

4
lib/logstorage/stats_row_max_test.go
View file

@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"3","x":"","b":"54"}`},
{"x", `{"a":"3","b":"54"}`},
},
})
@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},

4
lib/logstorage/stats_row_min_test.go
View file

@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
},
}, [][]Field{
{
{"x", `{"a":"2","x":"","b":"3"}`},
{"x", `{"a":"2","b":"3"}`},
},
})
@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
{"x", `{"c":""}`},
{"x", `{}`},
},
{
{"a", "3"},

52
lib/logstorage/syslog_parser_test.go
View file

@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
defer PutSyslogParser(p)
p.Parse(s)
result := MarshalFieldsToJSON(nil, p.Fields)
result := MarshalFieldsToLogfmt(nil, p.Fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
// RFC 3164
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
`priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
`{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`)
`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
// RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
`{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
`format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
// Incomplete RFC 3164
f("", time.UTC, `{}`)
f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`)
f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`)
f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`)
f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`)
f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`)
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`)
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`)
f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`)
f("", time.UTC, ``)
f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
// Incomplete RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`)
`priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
f(`<165>1 `, time.UTC,
`{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`)
`priority=165 facility=20 severity=5 format=rfc5424`)
}