From c0caa6993903a748c8942c8c28b5901ee6d5f4d4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 5 Jul 2024 01:17:03 +0200 Subject: [PATCH] lib/logstorage: use quicktemplate.AppendJSONString instead of strconv.AppendQuote for encoding JSON strings The strconv.AppendQuote improperly encodes special chars such as \x1b . They must be encoded as \u001b . See https://github.com/VictoriaMetrics/victorialogs-datasource/issues/24 --- docs/VictoriaLogs/CHANGELOG.md | 2 + lib/logstorage/pipe_format.go | 5 ++- lib/logstorage/pipe_sort.go | 7 ++-- lib/logstorage/rows.go | 9 ++-- lib/logstorage/rows_test.go | 64 +++++++++++++++++++++++++++++ lib/logstorage/stats_uniq_values.go | 7 ++-- 6 files changed, 82 insertions(+), 12 deletions(-) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 67fd095d6..219f2d507 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* BUGFIX: properly JSON-encode strings with special chars in [HTTP querying API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) responses. This fixes the `error decode response: invalid character 'x' in string escape code` error in [VictoriaLogs datasource for Grafana](https://github.com/VictoriaMetrics/victorialogs-datasource/). See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/24). + ## [v0.27.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.27.0-victorialogs) Released at 2024-07-02 diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go index a47328063..b6e80573e 100644 --- a/lib/logstorage/pipe_format.go +++ b/lib/logstorage/pipe_format.go @@ -3,9 +3,10 @@ package logstorage import ( "fmt" "math" - "strconv" "unsafe" + "github.com/valyala/quicktemplate" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" ) @@ -192,7 +193,7 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult v := c.getValueAtRow(br, rowIdx) switch step.fieldOpt { case "q": - b = strconv.AppendQuote(b, v) + b = quicktemplate.AppendJSONString(b, v, true) case "time": nsecs, ok := tryParseInt64(v) if !ok { diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go index 849a721eb..07fa5d553 100644 --- a/lib/logstorage/pipe_sort.go +++ b/lib/logstorage/pipe_sort.go @@ -5,12 +5,13 @@ import ( "fmt" "math" "sort" - "strconv" "strings" "sync" "sync/atomic" "unsafe" + "github.com/valyala/quicktemplate" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" "github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil" @@ -892,8 +893,8 @@ func tryParseInt64(s string) (int64, bool) { } func marshalJSONKeyValue(dst []byte, k, v string) []byte { - dst = strconv.AppendQuote(dst, k) + dst = quicktemplate.AppendJSONString(dst, k, true) dst = append(dst, ':') - dst = strconv.AppendQuote(dst, v) + dst = quicktemplate.AppendJSONString(dst, v, true) return dst } diff --git a/lib/logstorage/rows.go b/lib/logstorage/rows.go index 8b444a768..bee9fbd8d 100644 --- a/lib/logstorage/rows.go +++ b/lib/logstorage/rows.go @@ -2,7 +2,8 @@ package logstorage import ( "fmt" - "strconv" + + "github.com/valyala/quicktemplate" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" @@ -62,9 +63,9 @@ func (f *Field) marshalToJSON(dst []byte) []byte { if name == "" { name = "_msg" } - dst = strconv.AppendQuote(dst, name) + dst = quicktemplate.AppendJSONString(dst, name, true) dst = append(dst, ':') - dst = strconv.AppendQuote(dst, f.Value) + dst = quicktemplate.AppendJSONString(dst, f.Value, true) return dst } @@ -72,7 +73,7 @@ func (f *Field) marshalToLogfmt(dst []byte) []byte { dst = append(dst, f.Name...) dst = append(dst, '=') if needLogfmtQuoting(f.Value) { - dst = strconv.AppendQuote(dst, f.Value) + dst = quicktemplate.AppendJSONString(dst, f.Value, true) } else { dst = append(dst, f.Value...) } diff --git a/lib/logstorage/rows_test.go b/lib/logstorage/rows_test.go index 8ddae6fce..e69a3c69a 100644 --- a/lib/logstorage/rows_test.go +++ b/lib/logstorage/rows_test.go @@ -5,6 +5,70 @@ import ( "testing" ) +func TestMarshalFieldsToJSON(t *testing.T) { + f := func(fields []Field, resultExpected string) { + t.Helper() + + result := MarshalFieldsToJSON(nil, fields) + if string(result) != resultExpected { + t.Fatalf("unexpected result\ngot\n%q\nwant\n%q", result, resultExpected) + } + } + + f(nil, "{}") + f([]Field{}, "{}") + + f([]Field{ + { + Name: "foo", + Value: "bar", + }, + }, `{"foo":"bar"}`) + + f([]Field{ + { + Name: "foo\nbar", + Value: " \u001b[32m ", + }, + { + Name: " \u001b[11m ", + Value: "АБв", + }, + }, `{"foo\nbar":" \u001b[32m "," \u001b[11m ":"АБв"}`) +} + +func TestMarshalFieldsToLogfmt(t *testing.T) { + f := func(fields []Field, resultExpected string) { + t.Helper() + + result := MarshalFieldsToLogfmt(nil, fields) + if string(result) != resultExpected { + t.Fatalf("unexpected result\ngot\n%q\nwant\n%q", result, resultExpected) + } + } + + f(nil, "") + f([]Field{}, "") + + f([]Field{ + { + Name: "foo", + Value: "bar", + }, + }, `foo=bar`) + + f([]Field{ + { + Name: "foo", + Value: " \u001b[32m ", + }, + { + Name: "bar", + Value: "АБв", + }, + }, `foo=" \u001b[32m " bar=АБв`) +} + func TestGetRowsSizeBytes(t *testing.T) { f := func(rows [][]Field, uncompressedSizeBytesExpected int) { t.Helper() diff --git a/lib/logstorage/stats_uniq_values.go b/lib/logstorage/stats_uniq_values.go index 4e07ab4fb..a266b2c02 100644 --- a/lib/logstorage/stats_uniq_values.go +++ b/lib/logstorage/stats_uniq_values.go @@ -3,10 +3,11 @@ package logstorage import ( "fmt" "slices" - "strconv" "strings" "unsafe" + "github.com/valyala/quicktemplate" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" ) @@ -228,10 +229,10 @@ func marshalJSONArray(items []string) string { b := make([]byte, 0, bufSize) b = append(b, '[') - b = strconv.AppendQuote(b, items[0]) + b = quicktemplate.AppendJSONString(b, items[0], true) for _, item := range items[1:] { b = append(b, ',') - b = strconv.AppendQuote(b, item) + b = quicktemplate.AppendJSONString(b, item, true) } b = append(b, ']')