wip

docs/VictoriaLogs/LogsQL.md

@@ -1273,7 +1273,7 @@ See also:
 ### format pipe
 
 `| format "pattern" as result_field` [pipe](#format-pipe) combines [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
-according to the `pattern` and stores it to the `result_field`.
+according to the `pattern` and stores it to the `result_field`. All the other fields remain unchanged after the `| format ...` pipe.
 
 For example, the following query stores `request from <ip>:<port>` text into [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
 by substituting `<ip>` and `<port>` with the corresponding [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names:
@@ -1289,12 +1289,12 @@ then `as _msg` part can be omitted. The following query is equivalent to the pre
 _time:5m | format "request from <ip>:<port>"
 ```
 
-If some field values must be put into double quotes before formatting, then add `:q` after the corresponding field name.
+If some field values must be put into double quotes before formatting, then add `q:` in front of the corresponding field name.
 For example, the following command generates properly encoded JSON object from `_msg` and `stacktrace` [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
 and stores it into `my_json` output field:
 
 ```logsql
-_time:5m | format '{"_msg":<_msg:q>,"stacktrace":<stacktrace:q>}' as my_json
+_time:5m | format '{"_msg":<q:_msg>,"stacktrace":<q:stacktrace>}' as my_json
 ```
 
 See also:
@@ -1302,6 +1302,7 @@ See also:
 - [Conditional format](#conditional-format)
 - [`extract` pipe](#extract-pipe)
 
+
 #### Conditional format
 
 If the [`format` pipe](#format-pipe) musn't be applied to every [log entry](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),

lib/logstorage/pattern.go

@@ -29,6 +29,7 @@ type patternField struct {
 type patternStep struct {
 	prefix string
 	field  string
+	opt    string
 }
 
 func (ptn *pattern) clone() *pattern {
@@ -154,6 +155,31 @@ func tryUnquoteString(s string) (string, int) {
 }
 
 func parsePatternSteps(s string) ([]patternStep, error) {
+	steps, err := parsePatternStepsInternal(s)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unescape prefixes
+	for i := range steps {
+		step := &steps[i]
+		step.prefix = html.UnescapeString(step.prefix)
+	}
+
+	// extract options part from fields
+	for i := range steps {
+		step := &steps[i]
+		field := step.field
+		if n := strings.IndexByte(field, ':'); n >= 0 {
+			step.opt = field[:n]
+			step.field = field[n+1:]
+		}
+	}
+
+	return steps, nil
+}
+
+func parsePatternStepsInternal(s string) ([]patternStep, error) {
 	if len(s) == 0 {
 		return nil, nil
 	}
@@ -163,7 +189,7 @@ func parsePatternSteps(s string) ([]patternStep, error) {
 	n := strings.IndexByte(s, '<')
 	if n < 0 {
 		steps = append(steps, patternStep{
-			prefix: html.UnescapeString(s),
+			prefix: s,
 		})
 		return steps, nil
 	}
@@ -199,10 +225,5 @@ func parsePatternSteps(s string) ([]patternStep, error) {
 		s = s[n+1:]
 	}
 
-	for i := range steps {
-		step := &steps[i]
-		step.prefix = html.UnescapeString(step.prefix)
-	}
-
 	return steps, nil
 }

lib/logstorage/pattern_test.go

@@ -205,6 +205,21 @@ func TestParsePatternStepsSuccess(t *testing.T) {
 			prefix: ">",
 		},
 	})
+	f("<q:foo>bar<abc:baz:c:y>f<:foo:bar:baz>", []patternStep{
+		{
+			field: "foo",
+			opt:   "q",
+		},
+		{
+			prefix: "bar",
+			field:  "baz:c:y",
+			opt:    "abc",
+		},
+		{
+			prefix: "f",
+			field:  "foo:bar:baz",
+		},
+	})
 
 }
 

lib/logstorage/pipe_extract.go

@@ -125,7 +125,7 @@ func parsePipeExtract(lex *lexer) (*pipeExtract, error) {
 		fromField:  fromField,
 		ptn:        ptn,
 		patternStr: patternStr,
-		iff: iff,
+		iff:        iff,
 	}
 
 	return pe, nil

lib/logstorage/pipe_format.go

@@ -2,6 +2,7 @@ package logstorage
 
 import (
 	"fmt"
+	"strconv"
 	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -136,7 +137,11 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult
 		if step.field != "" {
 			c := br.getColumnByName(step.field)
 			v := c.getValueAtRow(br, rowIdx)
-			b = append(b, v...)
+			if step.opt == "q" {
+				b = strconv.AppendQuote(b, v)
+			} else {
+				b = append(b, v...)
+			}
 		}
 	}
 	bb.B = b

lib/logstorage/pipe_format_test.go

@@ -39,6 +39,20 @@ func TestPipeFormat(t *testing.T) {
 		expectPipeResults(t, pipeStr, rows, rowsExpected)
 	}
 
+	// plain string into a single field
+	f(`format '{"foo":<q:foo>,"bar":"<bar>"}' as x`, [][]Field{
+		{
+			{"foo", `"abc"`},
+			{"bar", `cde`},
+		},
+	}, [][]Field{
+		{
+			{"foo", `"abc"`},
+			{"bar", `cde`},
+			{"x", `{"foo":"\"abc\"","bar":"cde"}`},
+		},
+	})
+
 	// plain string into a single field
 	f(`format foo as x`, [][]Field{
 		{
@@ -95,7 +109,7 @@ func TestPipeFormat(t *testing.T) {
 	})
 
 	// format into existing field
-	f(`format "a<foo>aa<_msg>xx<a>x" as _msg`, [][]Field{
+	f(`format "a<foo>aa<_msg>xx<a>x"`, [][]Field{
 		{
 			{"_msg", `foobar`},
 			{"a", "b"},

lib/logstorage/pipe_unpack_json_test.go

@@ -233,11 +233,6 @@ func expectPipeResults(t *testing.T, pipeStr string, rows, rowsExpected [][]Fiel
 		t.Fatalf("unexpected error when parsing %q: %s", pipeStr, err)
 	}
 
-	pipeStrResult := p.String()
-	if pipeStrResult != pipeStr {
-		t.Fatalf("unexpected string representation for the pipe; got\n%s\nwant\n%s", pipeStrResult, pipeStr)
-	}
-
 	workersCount := 5
 	stopCh := make(chan struct{})
 	cancel := func() {}