From 0881e5fd5cbd15b55058a405099bc79c1f33b3bb Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 14 Oct 2024 23:39:29 +0200
Subject: [PATCH] app/vlselect: do not show empty fields in query results
Empty fields are treated as non-existing fields by VictoriaLogs data model.
So there is no sense in returning empty fields in query results, since they may mislead and confuse users.
(cherry picked from commit bac193e50b171dbb2d27965c06784e9239e2cb54)
---
.../elasticsearch/elasticsearch_test.go | 8 +-
app/vlinsert/jsonline/jsonline_test.go | 6 +-
app/vlinsert/syslog/syslog_test.go | 6 +-
app/vlselect/logsql/query_response.qtpl | 35 +++-
app/vlselect/logsql/query_response.qtpl.go | 159 +++++++++++-------
docs/VictoriaLogs/CHANGELOG.md | 1 +
lib/logstorage/logfmt_parser_test.go | 20 +--
lib/logstorage/pipe_pack_json_test.go | 4 +-
lib/logstorage/rows.go | 23 ++-
lib/logstorage/stats_row_any_test.go | 6 +-
lib/logstorage/stats_row_max_test.go | 4 +-
lib/logstorage/stats_row_min_test.go | 4 +-
lib/logstorage/syslog_parser_test.go | 52 +++---
13 files changed, 203 insertions(+), 125 deletions(-)
diff --git a/app/vlinsert/elasticsearch/elasticsearch_test.go b/app/vlinsert/elasticsearch/elasticsearch_test.go
index bfb1cd52e6..4370fb6690 100644
--- a/app/vlinsert/elasticsearch/elasticsearch_test.go
+++ b/app/vlinsert/elasticsearch/elasticsearch_test.go
@@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
msgField := "message"
rowsExpected := 4
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
- resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-{"_msg":"qwe rty","@timestamp":""}`
+ resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}
+{"_msg":"xyz","x":"y"}
+{"_msg":"qwe rty"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}
diff --git a/app/vlinsert/jsonline/jsonline_test.go b/app/vlinsert/jsonline/jsonline_test.go
index 068bfb92f0..17dc0ad950 100644
--- a/app/vlinsert/jsonline/jsonline_test.go
+++ b/app/vlinsert/jsonline/jsonline_test.go
@@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
msgField := "message"
rowsExpected := 3
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
- resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
+ resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}
+{"_msg":"xyz","x":"y"}`
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
}
diff --git a/app/vlinsert/syslog/syslog_test.go b/app/vlinsert/syslog/syslog_test.go
index 8c96ee6640..78b8ca5dee 100644
--- a/app/vlinsert/syslog/syslog_test.go
+++ b/app/vlinsert/syslog/syslog_test.go
@@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
currentYear := 2023
rowsExpected := 3
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
- resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+ resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
}
diff --git a/app/vlselect/logsql/query_response.qtpl b/app/vlselect/logsql/query_response.qtpl
index 06205615eb..e02b371b4b 100644
--- a/app/vlselect/logsql/query_response.qtpl
+++ b/app/vlselect/logsql/query_response.qtpl
@@ -6,15 +6,31 @@
// JSONRow creates JSON row from the given fields.
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-{
- {% code c := &columns[0] %}
+ {% code
+ i := 0
+ for i < len(columns) && columns[i].Values[rowIdx] == "" {
+ i++
+ }
+ columns = columns[i:]
+ %}
+ {% if len(columns) == 0 %}
+ {% return %}
+ {% endif %}
+ {
+ {% code c := &columns[0] %}
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% code columns = columns[1:] %}
{% for colIdx := range columns %}
- {% code c := &columns[colIdx] %}
+ {% code
+ c := &columns[colIdx]
+ v := c.Values[rowIdx]
+ %}
+ {% if v == "" %}
+ {% continue %}
+ {% endif %}
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
{% endfor %}
-}{% newline %}
+ }{% newline %}
{% endfunc %}
// JSONRows prints formatted rows
@@ -23,7 +39,11 @@
{% return %}
{% endif %}
{% for _, fields := range rows %}
- {
+ {% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
+ {% if len(fields) == 0 %}
+ {% continue %}
+ {% endif %}
+ {
{% if len(fields) > 0 %}
{% code
f := fields[0]
@@ -31,10 +51,13 @@
%}
{%q= f.Name %}:{%q= f.Value %}
{% for _, f := range fields %}
+ {% if f.Value == "" %}
+ {% continue %}
+ {% endif %}
,{%q= f.Name %}:{%q= f.Value %}
{% endfor %}
{% endif %}
- }{% newline %}
+ }{% newline %}
{% endfor %}
{% endfunc %}
diff --git a/app/vlselect/logsql/query_response.qtpl.go b/app/vlselect/logsql/query_response.qtpl.go
index dd3458c21b..232755ddaf 100644
--- a/app/vlselect/logsql/query_response.qtpl.go
+++ b/app/vlselect/logsql/query_response.qtpl.go
@@ -26,141 +26,176 @@ var (
//line app/vlselect/logsql/query_response.qtpl:8
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:8
- qw422016.N().S(`{`)
//line app/vlselect/logsql/query_response.qtpl:10
+ i := 0
+ for i < len(columns) && columns[i].Values[rowIdx] == "" {
+ i++
+ }
+ columns = columns[i:]
+
+//line app/vlselect/logsql/query_response.qtpl:16
+ if len(columns) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:17
+ return
+//line app/vlselect/logsql/query_response.qtpl:18
+ }
+//line app/vlselect/logsql/query_response.qtpl:18
+ qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:20
c := &columns[0]
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:12
+//line app/vlselect/logsql/query_response.qtpl:22
columns = columns[1:]
-//line app/vlselect/logsql/query_response.qtpl:13
+//line app/vlselect/logsql/query_response.qtpl:23
for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:14
+//line app/vlselect/logsql/query_response.qtpl:25
c := &columns[colIdx]
+ v := c.Values[rowIdx]
-//line app/vlselect/logsql/query_response.qtpl:14
+//line app/vlselect/logsql/query_response.qtpl:28
+ if v == "" {
+//line app/vlselect/logsql/query_response.qtpl:29
+ continue
+//line app/vlselect/logsql/query_response.qtpl:30
+ }
+//line app/vlselect/logsql/query_response.qtpl:30
qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:16
+//line app/vlselect/logsql/query_response.qtpl:32
}
-//line app/vlselect/logsql/query_response.qtpl:16
+//line app/vlselect/logsql/query_response.qtpl:32
qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:17
+//line app/vlselect/logsql/query_response.qtpl:33
qw422016.N().S(`
`)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
}
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
}
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
return qs422016
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
}
// JSONRows prints formatted rows
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:37
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:22
+//line app/vlselect/logsql/query_response.qtpl:38
if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:23
+//line app/vlselect/logsql/query_response.qtpl:39
return
-//line app/vlselect/logsql/query_response.qtpl:24
+//line app/vlselect/logsql/query_response.qtpl:40
}
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:41
for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:42
+ fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
+
+//line app/vlselect/logsql/query_response.qtpl:43
+ if len(fields) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:44
+ continue
+//line app/vlselect/logsql/query_response.qtpl:45
+ }
+//line app/vlselect/logsql/query_response.qtpl:45
qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:27
+//line app/vlselect/logsql/query_response.qtpl:47
if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:29
+//line app/vlselect/logsql/query_response.qtpl:49
f := fields[0]
fields = fields[1:]
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:53
for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:54
+ if f.Value == "" {
+//line app/vlselect/logsql/query_response.qtpl:55
+ continue
+//line app/vlselect/logsql/query_response.qtpl:56
+ }
+//line app/vlselect/logsql/query_response.qtpl:56
qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:35
+//line app/vlselect/logsql/query_response.qtpl:58
}
-//line app/vlselect/logsql/query_response.qtpl:36
+//line app/vlselect/logsql/query_response.qtpl:59
}
-//line app/vlselect/logsql/query_response.qtpl:36
+//line app/vlselect/logsql/query_response.qtpl:59
qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:37
+//line app/vlselect/logsql/query_response.qtpl:60
qw422016.N().S(`
`)
-//line app/vlselect/logsql/query_response.qtpl:38
+//line app/vlselect/logsql/query_response.qtpl:61
}
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
}
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
}
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
return qs422016
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
}
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index cfedbe96f9..7cb34f2218 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -16,6 +16,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: add support for forced merge. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-merge).
+* FEATURE: skip empty log fields in query results, since they are treated as non-existing fields in [VictoriaLogs data model](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
* BUGFIX: avoid possible panic when logs for a new day are ingested during execution of concurrent queries.
* BUGFIX: avoid panic at `lib/logstorage.(*blockResultColumn).forEachDictValue()` when [stats with additional filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters). The panic has been introduced in [v0.33.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.33.0-victorialogs) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a350be48b68330ee1a487e1fb09b002d3be45163).
diff --git a/lib/logstorage/logfmt_parser_test.go b/lib/logstorage/logfmt_parser_test.go
index ee0309f8f0..8ee3e392ca 100644
--- a/lib/logstorage/logfmt_parser_test.go
+++ b/lib/logstorage/logfmt_parser_test.go
@@ -12,19 +12,19 @@ func TestLogfmtParser(t *testing.T) {
defer putLogfmtParser(p)
p.parse(s)
- result := MarshalFieldsToJSON(nil, p.fields)
+ result := MarshalFieldsToLogfmt(nil, p.fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
}
- f(``, `{}`)
- f(`foo=bar`, `{"foo":"bar"}`)
- f(`foo="bar=baz x=y"`, `{"foo":"bar=baz x=y"}`)
- f(`foo=`, `{"foo":""}`)
- f(`foo`, `{"foo":""}`)
- f(`foo bar`, `{"foo":"","bar":""}`)
- f(`foo bar=baz`, `{"foo":"","bar":"baz"}`)
- f(`foo=bar baz="x y" a=b`, `{"foo":"bar","baz":"x y","a":"b"}`)
- f(` foo=bar baz=x =z qwe`, `{"foo":"bar","baz":"x","_msg":"z","qwe":""}`)
+ f(``, ``)
+ f(`foo=bar`, `foo=bar`)
+ f(`foo="bar=baz x=y"`, `foo="bar=baz x=y"`)
+ f(`foo=`, `foo=`)
+ f(`foo`, `foo=`)
+ f(`foo bar`, `foo= bar=`)
+ f(`foo bar=baz`, `foo= bar=baz`)
+ f(`foo=bar baz="x y" a=b`, `foo=bar baz="x y" a=b`)
+ f(` foo=bar baz=x =z qwe`, `foo=bar baz=x _msg=z qwe=`)
}
diff --git a/lib/logstorage/pipe_pack_json_test.go b/lib/logstorage/pipe_pack_json_test.go
index b57f150c89..a568b91b7b 100644
--- a/lib/logstorage/pipe_pack_json_test.go
+++ b/lib/logstorage/pipe_pack_json_test.go
@@ -96,10 +96,10 @@ func TestPipePackJSON(t *testing.T) {
{"_msg", `x`},
{"foo", `abc`},
{"bar", `cde`},
- {"a", `{"foo":"abc","baz":""}`},
+ {"a", `{"foo":"abc"}`},
},
{
- {"a", `{"foo":"","baz":""}`},
+ {"a", `{}`},
{"c", "d"},
},
})
diff --git a/lib/logstorage/rows.go b/lib/logstorage/rows.go
index a90e54a089..ca1638f860 100644
--- a/lib/logstorage/rows.go
+++ b/lib/logstorage/rows.go
@@ -70,7 +70,11 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
}
func (f *Field) marshalToLogfmt(dst []byte) []byte {
- dst = append(dst, f.Name...)
+ name := f.Name
+ if name == "" {
+ name = "_msg"
+ }
+ dst = append(dst, name...)
dst = append(dst, '=')
if needLogfmtQuoting(f.Value) {
dst = quicktemplate.AppendJSONString(dst, f.Value, true)
@@ -126,13 +130,19 @@ func RenameField(fields []Field, oldName, newName string) {
// MarshalFieldsToJSON appends JSON-marshaled fields to dst and returns the result.
func MarshalFieldsToJSON(dst []byte, fields []Field) []byte {
+ fields = SkipLeadingFieldsWithoutValues(fields)
dst = append(dst, '{')
if len(fields) > 0 {
dst = fields[0].marshalToJSON(dst)
fields = fields[1:]
for i := range fields {
+ f := &fields[i]
+ if f.Value == "" {
+ // Skip fields without values
+ continue
+ }
dst = append(dst, ',')
- dst = fields[i].marshalToJSON(dst)
+ dst = f.marshalToJSON(dst)
}
}
dst = append(dst, '}')
@@ -153,6 +163,15 @@ func MarshalFieldsToLogfmt(dst []byte, fields []Field) []byte {
return dst
}
+// SkipLeadingFieldsWithoutValues skips leading fields without values.
+func SkipLeadingFieldsWithoutValues(fields []Field) []Field {
+ i := 0
+ for i < len(fields) && fields[i].Value == "" {
+ i++
+ }
+ return fields[i:]
+}
+
func appendFields(a *arena, dst, src []Field) []Field {
for _, f := range src {
dst = append(dst, Field{
diff --git a/lib/logstorage/stats_row_any_test.go b/lib/logstorage/stats_row_any_test.go
index 0312ce276f..9f8b1d76df 100644
--- a/lib/logstorage/stats_row_any_test.go
+++ b/lib/logstorage/stats_row_any_test.go
@@ -63,7 +63,7 @@ func TestStatsRowAny(t *testing.T) {
},
}, [][]Field{
{
- {"x", `{"a":"2","x":"","b":"3"}`},
+ {"x", `{"a":"2","b":"3"}`},
},
})
@@ -138,7 +138,7 @@ func TestStatsRowAny(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
- {"x", `{"c":""}`},
+ {"x", `{}`},
},
{
{"a", "3"},
@@ -166,7 +166,7 @@ func TestStatsRowAny(t *testing.T) {
{
{"a", "1"},
{"b", "3"},
- {"x", `{"c":""}`},
+ {"x", `{}`},
},
{
{"a", "1"},
diff --git a/lib/logstorage/stats_row_max_test.go b/lib/logstorage/stats_row_max_test.go
index 39fd82564b..d7b6f40850 100644
--- a/lib/logstorage/stats_row_max_test.go
+++ b/lib/logstorage/stats_row_max_test.go
@@ -110,7 +110,7 @@ func TestStatsRowMax(t *testing.T) {
},
}, [][]Field{
{
- {"x", `{"a":"3","x":"","b":"54"}`},
+ {"x", `{"a":"3","b":"54"}`},
},
})
@@ -242,7 +242,7 @@ func TestStatsRowMax(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
- {"x", `{"c":""}`},
+ {"x", `{}`},
},
{
{"a", "3"},
diff --git a/lib/logstorage/stats_row_min_test.go b/lib/logstorage/stats_row_min_test.go
index 67225c7f69..a1c06f4634 100644
--- a/lib/logstorage/stats_row_min_test.go
+++ b/lib/logstorage/stats_row_min_test.go
@@ -110,7 +110,7 @@ func TestStatsRowMin(t *testing.T) {
},
}, [][]Field{
{
- {"x", `{"a":"2","x":"","b":"3"}`},
+ {"x", `{"a":"2","b":"3"}`},
},
})
@@ -241,7 +241,7 @@ func TestStatsRowMin(t *testing.T) {
}, [][]Field{
{
{"a", "1"},
- {"x", `{"c":""}`},
+ {"x", `{}`},
},
{
{"a", "3"},
diff --git a/lib/logstorage/syslog_parser_test.go b/lib/logstorage/syslog_parser_test.go
index 5bc07939ac..ed14a607a3 100644
--- a/lib/logstorage/syslog_parser_test.go
+++ b/lib/logstorage/syslog_parser_test.go
@@ -14,7 +14,7 @@ func TestSyslogParser(t *testing.T) {
defer PutSyslogParser(p)
p.Parse(s)
- result := MarshalFieldsToJSON(nil, p.Fields)
+ result := MarshalFieldsToLogfmt(nil, p.Fields)
if string(result) != resultExpected {
t.Fatalf("unexpected result when parsing [%s]; got\n%s\nwant\n%s\n", s, result, resultExpected)
}
@@ -22,50 +22,50 @@ func TestSyslogParser(t *testing.T) {
// RFC 3164
f("Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
- `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
+ `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("<165>Jun 3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"systemd","proc_id":"1","message":"Starting Update the local ESM caches..."}`)
+ `priority=165 facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
- `{"format":"rfc3164","timestamp":"2024-03-13T12:08:33.000Z","hostname":"abcd","app_name":"systemd","message":"Starting Update the local ESM caches..."}`)
+ `format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
- `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"-","message":"Starting Update the local ESM caches..."}`)
+ `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=- message="Starting Update the local ESM caches..."`)
f("Jun 3 12:08:33 - - Starting Update the local ESM caches...", time.UTC,
- `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"-","app_name":"-","message":"Starting Update the local ESM caches..."}`)
+ `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=- app_name=- message="Starting Update the local ESM caches..."`)
// RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
- `{"format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","message":"This is a test message with structured data."}`)
+ `format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","message":"This is a test message with structured data."}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"] This is a test message with structured data.`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:00.000Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123.iut":"3","bar@456.eventID":"11211","message":"This is a test message with structured data."}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 message="This is a test message with structured data."`)
// Incomplete RFC 3164
- f("", time.UTC, `{}`)
- f("Jun 3 12:08:33", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z"}`)
- f("Foo 3 12:08:33", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33"}`)
- f("Foo 3 12:08:33bar", time.UTC, `{"format":"rfc3164","message":"Foo 3 12:08:33bar"}`)
- f("Jun 3 12:08:33 abcd", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd"}`)
- f("Jun 3 12:08:33 abcd sudo", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo"}`)
- f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","proc_id":"123"}`)
- f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `{"format":"rfc3164","timestamp":"2024-06-03T12:08:33.000Z","hostname":"abcd","app_name":"sudo","message":"foobar"}`)
- f(`foo bar baz`, time.UTC, `{"format":"rfc3164","message":"foo bar baz"}`)
+ f("", time.UTC, ``)
+ f("Jun 3 12:08:33", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z`)
+ f("Foo 3 12:08:33", time.UTC, `format=rfc3164 message="Foo 3 12:08:33"`)
+ f("Foo 3 12:08:33bar", time.UTC, `format=rfc3164 message="Foo 3 12:08:33bar"`)
+ f("Jun 3 12:08:33 abcd", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd`)
+ f("Jun 3 12:08:33 abcd sudo", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo`)
+ f("Jun 3 12:08:33 abcd sudo[123]", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo proc_id=123`)
+ f("Jun 3 12:08:33 abcd sudo foobar", time.UTC, `format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=sudo message=foobar`)
+ f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
// Incomplete RFC 5424
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","foo@123":""}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com","app_name":"appname"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z","hostname":"mymachine.example.com"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424","timestamp":"2023-06-03T17:42:32.123456789Z"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
f(`<165>1 `, time.UTC,
- `{"priority":"165","facility":"20","severity":"5","format":"rfc5424"}`)
+ `priority=165 facility=20 severity=5 format=rfc5424`)
}