From bac193e50b171dbb2d27965c06784e9239e2cb54 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 14 Oct 2024 23:39:29 +0200 Subject: [PATCH] app/vlselect: do not show empty fields in query results Empty fields are treated as non-existing fields by VictoriaLogs data model. So there is no sense in returning empty fields in query results, since they may mislead and confuse users. --- .../elasticsearch/elasticsearch_test.go | 8 +- app/vlinsert/jsonline/jsonline_test.go | 6 +- app/vlinsert/syslog/syslog_test.go | 6 +- app/vlselect/logsql/query_response.qtpl | 35 +++- app/vlselect/logsql/query_response.qtpl.go | 159 +++++++++++------- docs/VictoriaLogs/CHANGELOG.md | 1 + lib/logstorage/logfmt_parser_test.go | 20 +-- lib/logstorage/pipe_pack_json_test.go | 4 +- lib/logstorage/rows.go | 23 ++- lib/logstorage/stats_row_any_test.go | 6 +- lib/logstorage/stats_row_max_test.go | 4 +- lib/logstorage/stats_row_min_test.go | 4 +- lib/logstorage/syslog_parser_test.go | 52 +++--- 13 files changed, 203 insertions(+), 125 deletions(-) diff --git a/app/vlinsert/elasticsearch/elasticsearch_test.go b/app/vlinsert/elasticsearch/elasticsearch_test.go index bfb1cd52e..4370fb669 100644 --- a/app/vlinsert/elasticsearch/elasticsearch_test.go +++ b/app/vlinsert/elasticsearch/elasticsearch_test.go @@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) { msgField := "message" rowsExpected := 4 timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000} - resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} -{"@timestamp":"","_msg":"baz"} -{"_msg":"xyz","@timestamp":"","x":"y"} -{"_msg":"qwe rty","@timestamp":""}` + resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} +{"_msg":"baz"} +{"_msg":"xyz","x":"y"} +{"_msg":"qwe rty"}` f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected) } diff --git a/app/vlinsert/jsonline/jsonline_test.go b/app/vlinsert/jsonline/jsonline_test.go index 068bfb92f..17dc0ad95 100644 --- a/app/vlinsert/jsonline/jsonline_test.go +++ b/app/vlinsert/jsonline/jsonline_test.go @@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) { msgField := "message" rowsExpected := 3 timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000} - resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} -{"@timestamp":"","_msg":"baz"} -{"_msg":"xyz","@timestamp":"","x":"y"}` + resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"} +{"_msg":"baz"} +{"_msg":"xyz","x":"y"}` f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected) } diff --git a/app/vlinsert/syslog/syslog_test.go b/app/vlinsert/syslog/syslog_test.go index 8c96ee664..78b8ca5de 100644 --- a/app/vlinsert/syslog/syslog_test.go +++ b/app/vlinsert/syslog/syslog_test.go @@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) { currentYear := 2023 rowsExpected := 3 timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000} - resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."} -{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"} -{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}` + resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."} +{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"} +{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}` f(data, currentYear, rowsExpected, timestampsExpected, resultExpected) } diff --git a/app/vlselect/logsql/query_response.qtpl b/app/vlselect/logsql/query_response.qtpl index 06205615e..e02b371b4 100644 --- a/app/vlselect/logsql/query_response.qtpl +++ b/app/vlselect/logsql/query_response.qtpl @@ -6,15 +6,31 @@ // JSONRow creates JSON row from the given fields. {% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %} -{ - {% code c := &columns[0] %} + {% code + i := 0 + for i < len(columns) && columns[i].Values[rowIdx] == "" { + i++ + } + columns = columns[i:] + %} + {% if len(columns) == 0 %} + {% return %} + {% endif %} + { + {% code c := &columns[0] %} {%q= c.Name %}:{%q= c.Values[rowIdx] %} {% code columns = columns[1:] %} {% for colIdx := range columns %} - {% code c := &columns[colIdx] %} + {% code + c := &columns[colIdx] + v := c.Values[rowIdx] + %} + {% if v == "" %} + {% continue %} + {% endif %} ,{%q= c.Name %}:{%q= c.Values[rowIdx] %} {% endfor %} -}{% newline %} + }{% newline %} {% endfunc %} // JSONRows prints formatted rows @@ -23,7 +39,11 @@ {% return %} {% endif %} {% for _, fields := range rows %} - { + {% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %} + {% if len(fields) == 0 %} + {% continue %} + {% endif %} + { {% if len(fields) > 0 %} {% code f := fields[0] @@ -31,10 +51,13 @@ %} {%q= f.Name %}:{%q= f.Value %} {% for _, f := range fields %} + {% if f.Value == "" %} + {% continue %} + {% endif %} ,{%q= f.Name %}:{%q= f.Value %} {% endfor %} {% endif %} - }{% newline %} + }{% newline %} {% endfor %} {% endfunc %} diff --git a/app/vlselect/logsql/query_response.qtpl.go b/app/vlselect/logsql/query_response.qtpl.go index dd3458c21..232755dda 100644 --- a/app/vlselect/logsql/query_response.qtpl.go +++ b/app/vlselect/logsql/query_response.qtpl.go @@ -26,141 +26,176 @@ var ( //line app/vlselect/logsql/query_response.qtpl:8 func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) { -//line app/vlselect/logsql/query_response.qtpl:8 - qw422016.N().S(`{`) //line app/vlselect/logsql/query_response.qtpl:10 + i := 0 + for i < len(columns) && columns[i].Values[rowIdx] == "" { + i++ + } + columns = columns[i:] + +//line app/vlselect/logsql/query_response.qtpl:16 + if len(columns) == 0 { +//line app/vlselect/logsql/query_response.qtpl:17 + return +//line app/vlselect/logsql/query_response.qtpl:18 + } +//line app/vlselect/logsql/query_response.qtpl:18 + qw422016.N().S(`{`) +//line app/vlselect/logsql/query_response.qtpl:20 c := &columns[0] -//line app/vlselect/logsql/query_response.qtpl:11 +//line app/vlselect/logsql/query_response.qtpl:21 qw422016.N().Q(c.Name) -//line app/vlselect/logsql/query_response.qtpl:11 +//line app/vlselect/logsql/query_response.qtpl:21 qw422016.N().S(`:`) -//line app/vlselect/logsql/query_response.qtpl:11 +//line app/vlselect/logsql/query_response.qtpl:21 qw422016.N().Q(c.Values[rowIdx]) -//line app/vlselect/logsql/query_response.qtpl:12 +//line app/vlselect/logsql/query_response.qtpl:22 columns = columns[1:] -//line app/vlselect/logsql/query_response.qtpl:13 +//line app/vlselect/logsql/query_response.qtpl:23 for colIdx := range columns { -//line app/vlselect/logsql/query_response.qtpl:14 +//line app/vlselect/logsql/query_response.qtpl:25 c := &columns[colIdx] + v := c.Values[rowIdx] -//line app/vlselect/logsql/query_response.qtpl:14 +//line app/vlselect/logsql/query_response.qtpl:28 + if v == "" { +//line app/vlselect/logsql/query_response.qtpl:29 + continue +//line app/vlselect/logsql/query_response.qtpl:30 + } +//line app/vlselect/logsql/query_response.qtpl:30 qw422016.N().S(`,`) -//line app/vlselect/logsql/query_response.qtpl:15 +//line app/vlselect/logsql/query_response.qtpl:31 qw422016.N().Q(c.Name) -//line app/vlselect/logsql/query_response.qtpl:15 +//line app/vlselect/logsql/query_response.qtpl:31 qw422016.N().S(`:`) -//line app/vlselect/logsql/query_response.qtpl:15 +//line app/vlselect/logsql/query_response.qtpl:31 qw422016.N().Q(c.Values[rowIdx]) -//line app/vlselect/logsql/query_response.qtpl:16 +//line app/vlselect/logsql/query_response.qtpl:32 } -//line app/vlselect/logsql/query_response.qtpl:16 +//line app/vlselect/logsql/query_response.qtpl:32 qw422016.N().S(`}`) -//line app/vlselect/logsql/query_response.qtpl:17 +//line app/vlselect/logsql/query_response.qtpl:33 qw422016.N().S(` `) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 } -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) { -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 StreamJSONRow(qw422016, columns, rowIdx) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 qt422016.ReleaseWriter(qw422016) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 } -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string { -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 qb422016 := qt422016.AcquireByteBuffer() -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 WriteJSONRow(qb422016, columns, rowIdx) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 qs422016 := string(qb422016.B) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 qt422016.ReleaseByteBuffer(qb422016) -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 return qs422016 -//line app/vlselect/logsql/query_response.qtpl:18 +//line app/vlselect/logsql/query_response.qtpl:34 } // JSONRows prints formatted rows -//line app/vlselect/logsql/query_response.qtpl:21 +//line app/vlselect/logsql/query_response.qtpl:37 func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) { -//line app/vlselect/logsql/query_response.qtpl:22 +//line app/vlselect/logsql/query_response.qtpl:38 if len(rows) == 0 { -//line app/vlselect/logsql/query_response.qtpl:23 +//line app/vlselect/logsql/query_response.qtpl:39 return -//line app/vlselect/logsql/query_response.qtpl:24 +//line app/vlselect/logsql/query_response.qtpl:40 } -//line app/vlselect/logsql/query_response.qtpl:25 +//line app/vlselect/logsql/query_response.qtpl:41 for _, fields := range rows { -//line app/vlselect/logsql/query_response.qtpl:25 +//line app/vlselect/logsql/query_response.qtpl:42 + fields = logstorage.SkipLeadingFieldsWithoutValues(fields) + +//line app/vlselect/logsql/query_response.qtpl:43 + if len(fields) == 0 { +//line app/vlselect/logsql/query_response.qtpl:44 + continue +//line app/vlselect/logsql/query_response.qtpl:45 + } +//line app/vlselect/logsql/query_response.qtpl:45 qw422016.N().S(`{`) -//line app/vlselect/logsql/query_response.qtpl:27 +//line app/vlselect/logsql/query_response.qtpl:47 if len(fields) > 0 { -//line app/vlselect/logsql/query_response.qtpl:29 +//line app/vlselect/logsql/query_response.qtpl:49 f := fields[0] fields = fields[1:] -//line app/vlselect/logsql/query_response.qtpl:32 +//line app/vlselect/logsql/query_response.qtpl:52 qw422016.N().Q(f.Name) -//line app/vlselect/logsql/query_response.qtpl:32 +//line app/vlselect/logsql/query_response.qtpl:52 qw422016.N().S(`:`) -//line app/vlselect/logsql/query_response.qtpl:32 +//line app/vlselect/logsql/query_response.qtpl:52 qw422016.N().Q(f.Value) -//line app/vlselect/logsql/query_response.qtpl:33 +//line app/vlselect/logsql/query_response.qtpl:53 for _, f := range fields { -//line app/vlselect/logsql/query_response.qtpl:33 +//line app/vlselect/logsql/query_response.qtpl:54 + if f.Value == "" { +//line app/vlselect/logsql/query_response.qtpl:55 + continue +//line app/vlselect/logsql/query_response.qtpl:56 + } +//line app/vlselect/logsql/query_response.qtpl:56 qw422016.N().S(`,`) -//line app/vlselect/logsql/query_response.qtpl:34 +//line app/vlselect/logsql/query_response.qtpl:57 qw422016.N().Q(f.Name) -//line app/vlselect/logsql/query_response.qtpl:34 +//line app/vlselect/logsql/query_response.qtpl:57 qw422016.N().S(`:`) -//line app/vlselect/logsql/query_response.qtpl:34 +//line app/vlselect/logsql/query_response.qtpl:57 qw422016.N().Q(f.Value) -//line app/vlselect/logsql/query_response.qtpl:35 +//line app/vlselect/logsql/query_response.qtpl:58 } -//line app/vlselect/logsql/query_response.qtpl:36 +//line app/vlselect/logsql/query_response.qtpl:59 } -//line app/vlselect/logsql/query_response.qtpl:36 +//line app/vlselect/logsql/query_response.qtpl:59 qw422016.N().S(`}`) -//line app/vlselect/logsql/query_response.qtpl:37 +//line app/vlselect/logsql/query_response.qtpl:60 qw422016.N().S(` `) -//line app/vlselect/logsql/query_response.qtpl:38 +//line app/vlselect/logsql/query_response.qtpl:61 } -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 } -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) { -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 StreamJSONRows(qw422016, rows) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 qt422016.ReleaseWriter(qw422016) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 } -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 func JSONRows(rows [][]logstorage.Field) string { -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 qb422016 := qt422016.AcquireByteBuffer() -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 WriteJSONRows(qb422016, rows) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 qs422016 := string(qb422016.B) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 qt422016.ReleaseByteBuffer(qb422016) -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 return qs422016 -//line app/vlselect/logsql/query_response.qtpl:39 +//line app/vlselect/logsql/query_response.qtpl:62 } diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index cfedbe96f..7cb34f221 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip * FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge). +* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). * BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries. * BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163). diff --git a/lib/logstorage/logfmt_parser_test.go b/lib/logstorage/logfmt_parser_test.go index ee0309f8f..8ee3e392c 100644 --- a/lib/logstorage/logfmt_parser_test.go +++ b/lib/logstorage/logfmt_parser_test.go @@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) { defer putLogfmtParser(p) p.parse(s) - result := MarshalFieldsToJSON(nil, p.fields) + result := MarshalFieldsToLogfmt(nil, p.fields) if string(result) != resultExpected { t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected) } } - f(``, `{}`) - f(`foo=bar`, `{"foo":"bar"}`) - f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`) - f(`foo=`, `{"foo":""}`) - f(`foo`, `{"foo":""}`) - f(`foo bar`, `{"foo":"","bar":""}`) - f(`foo bar=baz`, `{"foo":"","bar":"baz"}`) - f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`) - f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`) + f(``, ``) + f(`foo=bar`, `foo=bar`) + f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`) + f(`foo=`, `foo=`) + f(`foo`, `foo=`) + f(`foo bar`, `foo= bar=`) + f(`foo bar=baz`, `foo= bar=baz`) + f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`) + f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`) } diff --git a/lib/logstorage/pipe_pack_json_test.go b/lib/logstorage/pipe_pack_json_test.go index b57f150c8..a568b91b7 100644 --- a/lib/logstorage/pipe_pack_json_test.go +++ b/lib/logstorage/pipe_pack_json_test.go @@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) { {"_msg", `x`}, {"foo", `abc`}, {"bar", `cde`}, - {"a", `{"foo":"abc","baz":""}`}, + {"a", `{"foo":"abc"}`}, }, { - {"a", `{"foo":"","baz":""}`}, + {"a", `{}`}, {"c", "d"}, }, }) diff --git a/lib/logstorage/rows.go b/lib/logstorage/rows.go index a90e54a08..ca1638f86 100644 --- a/lib/logstorage/rows.go +++ b/lib/logstorage/rows.go @@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte { } func (f *Field) marshalToLogfmt(dst []byte) []byte { - dst = append(dst, f.Name...) + name := f.Name + if name == "" { + name = "_msg" + } + dst = append(dst, name...) dst = append(dst, '=') if needLogfmtQuoting(f.Value) { dst = quicktemplate.AppendJSONString(dst, f.Value, true) @@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) { // MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result. func MarshalFieldsToJSON(dst []byte, fields []Field) []byte { + fields = SkipLeadingFieldsWithoutValues(fields) dst = append(dst, '{') if len(fields) > 0 { dst = fields[0].marshalToJSON(dst) fields = fields[1:] for i := range fields { + f := &fields[i] + if f.Value == "" { + // Skip fields without values + continue + } dst = append(dst, ',') - dst = fields[i].marshalToJSON(dst) + dst = f.marshalToJSON(dst) } } dst = append(dst, '}') @@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte { return dst } +// SkipLeadingFieldsWithoutValues skips leading fields without values. +func SkipLeadingFieldsWithoutValues(fields []Field) []Field { + i := 0 + for i < len(fields) && fields[i].Value == "" { + i++ + } + return fields[i:] +} + func appendFields(a *arena, dst, src []Field) []Field { for _, f := range src { dst = append(dst, Field{ diff --git a/lib/logstorage/stats_row_any_test.go b/lib/logstorage/stats_row_any_test.go index 0312ce276..9f8b1d76d 100644 --- a/lib/logstorage/stats_row_any_test.go +++ b/lib/logstorage/stats_row_any_test.go @@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) { }, }, [][]Field{ { - {"x", `{"a":"2","x":"","b":"3"}`}, + {"x", `{"a":"2","b":"3"}`}, }, }) @@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) { }, [][]Field{ { {"a", "1"}, - {"x", `{"c":""}`}, + {"x", `{}`}, }, { {"a", "3"}, @@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) { { {"a", "1"}, {"b", "3"}, - {"x", `{"c":""}`}, + {"x", `{}`}, }, { {"a", "1"}, diff --git a/lib/logstorage/stats_row_max_test.go b/lib/logstorage/stats_row_max_test.go index 39fd82564..d7b6f4085 100644 --- a/lib/logstorage/stats_row_max_test.go +++ b/lib/logstorage/stats_row_max_test.go @@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) { }, }, [][]Field{ { - {"x", `{"a":"3","x":"","b":"54"}`}, + {"x", `{"a":"3","b":"54"}`}, }, }) @@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) { }, [][]Field{ { {"a", "1"}, - {"x", `{"c":""}`}, + {"x", `{}`}, }, { {"a", "3"}, diff --git a/lib/logstorage/stats_row_min_test.go b/lib/logstorage/stats_row_min_test.go index 67225c7f6..a1c06f463 100644 --- a/lib/logstorage/stats_row_min_test.go +++ b/lib/logstorage/stats_row_min_test.go @@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) { }, }, [][]Field{ { - {"x", `{"a":"2","x":"","b":"3"}`}, + {"x", `{"a":"2","b":"3"}`}, }, }) @@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) { }, [][]Field{ { {"a", "1"}, - {"x", `{"c":""}`}, + {"x", `{}`}, }, { {"a", "3"}, diff --git a/lib/logstorage/syslog_parser_test.go b/lib/logstorage/syslog_parser_test.go index 5bc07939a..ed14a607a 100644 --- a/lib/logstorage/syslog_parser_test.go +++ b/lib/logstorage/syslog_parser_test.go @@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) { defer PutSyslogParser(p) p.Parse(s) - result := MarshalFieldsToJSON(nil, p.Fields) + result := MarshalFieldsToLogfmt(nil, p.Fields) if string(result) != resultExpected { t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected) } @@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) { // RFC 3164 f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC, - `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`) + `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`) f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`) + `priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`) f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC, - `{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`) + `format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`) f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC, - `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`) + `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`) f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC, - `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`) + `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`) // RFC 5424 f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`) f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC, - `{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`) + `format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`) f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`) f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`) // Incomplete RFC 3164 - f("", time.UTC, `{}`) - f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`) - f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`) - f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`) - f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`) - f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`) - f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`) - f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`) - f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`) + f("", time.UTC, ``) + f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`) + f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`) + f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`) + f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`) + f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`) + f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`) + f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`) + f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`) // Incomplete RFC 5424 f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`) f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`) f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`) f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`) f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`) f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`) + `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`) f(`<165>1 `, time.UTC, - `{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`) + `priority=165 facility=20 severity=5 format=rfc5424`) }