From 2ff9cf9f43f7f16384b0c4cc786263a17e18586e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 22 May 2024 17:41:45 +0200 Subject: [PATCH] wip --- docs/VictoriaLogs/LogsQL.md | 7 +++--- lib/logstorage/pattern.go | 33 ++++++++++++++++++++----- lib/logstorage/pattern_test.go | 15 +++++++++++ lib/logstorage/pipe_extract.go | 2 +- lib/logstorage/pipe_format.go | 7 +++++- lib/logstorage/pipe_format_test.go | 16 +++++++++++- lib/logstorage/pipe_unpack_json_test.go | 5 ---- 7 files changed, 68 insertions(+), 17 deletions(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 6cbee8e54..71de7e9d7 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1273,7 +1273,7 @@ See also: ### format pipe `| format "pattern" as result_field` [pipe](#format-pipe) combines [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) -according to the `pattern` and stores it to the `result_field`. +according to the `pattern` and stores it to the `result_field`. All the other fields remain unchanged after the `| format ...` pipe. For example, the following query stores `request from :` text into [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field), by substituting `` and `` with the corresponding [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names: @@ -1289,12 +1289,12 @@ then `as _msg` part can be omitted. The following query is equivalent to the pre _time:5m | format "request from :" ``` -If some field values must be put into double quotes before formatting, then add `:q` after the corresponding field name. +If some field values must be put into double quotes before formatting, then add `q:` in front of the corresponding field name. For example, the following command generates properly encoded JSON object from `_msg` and `stacktrace` [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) and stores it into `my_json` output field: ```logsql -_time:5m | format '{"_msg":<_msg:q>,"stacktrace":}' as my_json +_time:5m | format '{"_msg":,"stacktrace":}' as my_json ``` See also: @@ -1302,6 +1302,7 @@ See also: - [Conditional format](#conditional-format) - [`extract` pipe](#extract-pipe) + #### Conditional format If the [`format` pipe](#format-pipe) musn't be applied to every [log entry](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model), diff --git a/lib/logstorage/pattern.go b/lib/logstorage/pattern.go index e195a998d..ed1f7758f 100644 --- a/lib/logstorage/pattern.go +++ b/lib/logstorage/pattern.go @@ -29,6 +29,7 @@ type patternField struct { type patternStep struct { prefix string field string + opt string } func (ptn *pattern) clone() *pattern { @@ -154,6 +155,31 @@ func tryUnquoteString(s string) (string, int) { } func parsePatternSteps(s string) ([]patternStep, error) { + steps, err := parsePatternStepsInternal(s) + if err != nil { + return nil, err + } + + // Unescape prefixes + for i := range steps { + step := &steps[i] + step.prefix = html.UnescapeString(step.prefix) + } + + // extract options part from fields + for i := range steps { + step := &steps[i] + field := step.field + if n := strings.IndexByte(field, ':'); n >= 0 { + step.opt = field[:n] + step.field = field[n+1:] + } + } + + return steps, nil +} + +func parsePatternStepsInternal(s string) ([]patternStep, error) { if len(s) == 0 { return nil, nil } @@ -163,7 +189,7 @@ func parsePatternSteps(s string) ([]patternStep, error) { n := strings.IndexByte(s, '<') if n < 0 { steps = append(steps, patternStep{ - prefix: html.UnescapeString(s), + prefix: s, }) return steps, nil } @@ -199,10 +225,5 @@ func parsePatternSteps(s string) ([]patternStep, error) { s = s[n+1:] } - for i := range steps { - step := &steps[i] - step.prefix = html.UnescapeString(step.prefix) - } - return steps, nil } diff --git a/lib/logstorage/pattern_test.go b/lib/logstorage/pattern_test.go index a17ba3fa0..eee5a1a26 100644 --- a/lib/logstorage/pattern_test.go +++ b/lib/logstorage/pattern_test.go @@ -205,6 +205,21 @@ func TestParsePatternStepsSuccess(t *testing.T) { prefix: ">", }, }) + f("barf<:foo:bar:baz>", []patternStep{ + { + field: "foo", + opt: "q", + }, + { + prefix: "bar", + field: "baz:c:y", + opt: "abc", + }, + { + prefix: "f", + field: "foo:bar:baz", + }, + }) } diff --git a/lib/logstorage/pipe_extract.go b/lib/logstorage/pipe_extract.go index 4e172d240..b9b4c3ae1 100644 --- a/lib/logstorage/pipe_extract.go +++ b/lib/logstorage/pipe_extract.go @@ -125,7 +125,7 @@ func parsePipeExtract(lex *lexer) (*pipeExtract, error) { fromField: fromField, ptn: ptn, patternStr: patternStr, - iff: iff, + iff: iff, } return pe, nil diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go index 3f830b199..459400c4d 100644 --- a/lib/logstorage/pipe_format.go +++ b/lib/logstorage/pipe_format.go @@ -2,6 +2,7 @@ package logstorage import ( "fmt" + "strconv" "unsafe" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" @@ -136,7 +137,11 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult if step.field != "" { c := br.getColumnByName(step.field) v := c.getValueAtRow(br, rowIdx) - b = append(b, v...) + if step.opt == "q" { + b = strconv.AppendQuote(b, v) + } else { + b = append(b, v...) + } } } bb.B = b diff --git a/lib/logstorage/pipe_format_test.go b/lib/logstorage/pipe_format_test.go index d40f85bac..bda91e588 100644 --- a/lib/logstorage/pipe_format_test.go +++ b/lib/logstorage/pipe_format_test.go @@ -39,6 +39,20 @@ func TestPipeFormat(t *testing.T) { expectPipeResults(t, pipeStr, rows, rowsExpected) } + // plain string into a single field + f(`format '{"foo":,"bar":""}' as x`, [][]Field{ + { + {"foo", `"abc"`}, + {"bar", `cde`}, + }, + }, [][]Field{ + { + {"foo", `"abc"`}, + {"bar", `cde`}, + {"x", `{"foo":"\"abc\"","bar":"cde"}`}, + }, + }) + // plain string into a single field f(`format foo as x`, [][]Field{ { @@ -95,7 +109,7 @@ func TestPipeFormat(t *testing.T) { }) // format into existing field - f(`format "aaa<_msg>xxx" as _msg`, [][]Field{ + f(`format "aaa<_msg>xxx"`, [][]Field{ { {"_msg", `foobar`}, {"a", "b"}, diff --git a/lib/logstorage/pipe_unpack_json_test.go b/lib/logstorage/pipe_unpack_json_test.go index 06f3e60d0..4e7fbe2f3 100644 --- a/lib/logstorage/pipe_unpack_json_test.go +++ b/lib/logstorage/pipe_unpack_json_test.go @@ -233,11 +233,6 @@ func expectPipeResults(t *testing.T, pipeStr string, rows, rowsExpected [][]Fiel t.Fatalf("unexpected error when parsing %q: %s", pipeStr, err) } - pipeStrResult := p.String() - if pipeStrResult != pipeStr { - t.Fatalf("unexpected string representation for the pipe; got\n%s\nwant\n%s", pipeStrResult, pipeStr) - } - workersCount := 5 stopCh := make(chan struct{}) cancel := func() {}