lib/logstorage: use quicktemplate.AppendJSONString instead of strconv.AppendQuote for encoding JSON strings

The strconv.AppendQuote improperly encodes special chars such as \x1b . They must be encoded as \u001b .

See https://github.com/VictoriaMetrics/victorialogs-datasource/issues/24

docs/VictoriaLogs/CHANGELOG.md

@@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+* BUGFIX: properly JSON-encode strings with special chars in [HTTP querying API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) responses. This fixes the `error decode response: invalid character 'x' in string escape code` error in [VictoriaLogs datasource for Grafana](https://github.com/VictoriaMetrics/victorialogs-datasource/). See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/24).
+
 ## [v0.27.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.27.0-victorialogs)
 
 Released at 2024-07-02

lib/logstorage/pipe_format.go

@@ -3,9 +3,10 @@ package logstorage
 import (
 	"fmt"
 	"math"
-	"strconv"
 	"unsafe"
 
+	"github.com/valyala/quicktemplate"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
@@ -192,7 +193,7 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult
 			v := c.getValueAtRow(br, rowIdx)
 			switch step.fieldOpt {
 			case "q":
-				b = strconv.AppendQuote(b, v)
+				b = quicktemplate.AppendJSONString(b, v, true)
 			case "time":
 				nsecs, ok := tryParseInt64(v)
 				if !ok {

lib/logstorage/pipe_sort.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"math"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"unsafe"
 
+	"github.com/valyala/quicktemplate"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
@@ -892,8 +893,8 @@ func tryParseInt64(s string) (int64, bool) {
 }
 
 func marshalJSONKeyValue(dst []byte, k, v string) []byte {
-	dst = strconv.AppendQuote(dst, k)
+	dst = quicktemplate.AppendJSONString(dst, k, true)
 	dst = append(dst, ':')
-	dst = strconv.AppendQuote(dst, v)
+	dst = quicktemplate.AppendJSONString(dst, v, true)
 	return dst
 }

lib/logstorage/rows.go

@@ -2,7 +2,8 @@ package logstorage
 
 import (
 	"fmt"
-	"strconv"
+
+	"github.com/valyala/quicktemplate"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -62,9 +63,9 @@ func (f *Field) marshalToJSON(dst []byte) []byte {
 	if name == "" {
 		name = "_msg"
 	}
-	dst = strconv.AppendQuote(dst, name)
+	dst = quicktemplate.AppendJSONString(dst, name, true)
 	dst = append(dst, ':')
-	dst = strconv.AppendQuote(dst, f.Value)
+	dst = quicktemplate.AppendJSONString(dst, f.Value, true)
 	return dst
 }
 
@@ -72,7 +73,7 @@ func (f *Field) marshalToLogfmt(dst []byte) []byte {
 	dst = append(dst, f.Name...)
 	dst = append(dst, '=')
 	if needLogfmtQuoting(f.Value) {
-		dst = strconv.AppendQuote(dst, f.Value)
+		dst = quicktemplate.AppendJSONString(dst, f.Value, true)
 	} else {
 		dst = append(dst, f.Value...)
 	}

lib/logstorage/rows_test.go

@@ -5,6 +5,70 @@ import (
 	"testing"
 )
 
+func TestMarshalFieldsToJSON(t *testing.T) {
+	f := func(fields []Field, resultExpected string) {
+		t.Helper()
+
+		result := MarshalFieldsToJSON(nil, fields)
+		if string(result) != resultExpected {
+			t.Fatalf("unexpected result\ngot\n%q\nwant\n%q", result, resultExpected)
+		}
+	}
+
+	f(nil, "{}")
+	f([]Field{}, "{}")
+
+	f([]Field{
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+	}, `{"foo":"bar"}`)
+
+	f([]Field{
+		{
+			Name:  "foo\nbar",
+			Value: "  \u001b[32m ",
+		},
+		{
+			Name:  "  \u001b[11m ",
+			Value: "АБв",
+		},
+	}, `{"foo\nbar":"  \u001b[32m ","  \u001b[11m ":"АБв"}`)
+}
+
+func TestMarshalFieldsToLogfmt(t *testing.T) {
+	f := func(fields []Field, resultExpected string) {
+		t.Helper()
+
+		result := MarshalFieldsToLogfmt(nil, fields)
+		if string(result) != resultExpected {
+			t.Fatalf("unexpected result\ngot\n%q\nwant\n%q", result, resultExpected)
+		}
+	}
+
+	f(nil, "")
+	f([]Field{}, "")
+
+	f([]Field{
+		{
+			Name:  "foo",
+			Value: "bar",
+		},
+	}, `foo=bar`)
+
+	f([]Field{
+		{
+			Name:  "foo",
+			Value: "  \u001b[32m ",
+		},
+		{
+			Name:  "bar",
+			Value: "АБв",
+		},
+	}, `foo="  \u001b[32m " bar=АБв`)
+}
+
 func TestGetRowsSizeBytes(t *testing.T) {
 	f := func(rows [][]Field, uncompressedSizeBytesExpected int) {
 		t.Helper()

lib/logstorage/stats_uniq_values.go

@@ -3,10 +3,11 @@ package logstorage
 import (
 	"fmt"
 	"slices"
-	"strconv"
 	"strings"
 	"unsafe"
 
+	"github.com/valyala/quicktemplate"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
@@ -228,10 +229,10 @@ func marshalJSONArray(items []string) string {
 	b := make([]byte, 0, bufSize)
 
 	b = append(b, '[')
-	b = strconv.AppendQuote(b, items[0])
+	b = quicktemplate.AppendJSONString(b, items[0], true)
 	for _, item := range items[1:] {
 		b = append(b, ',')
-		b = strconv.AppendQuote(b, item)
+		b = quicktemplate.AppendJSONString(b, item, true)
 	}
 	b = append(b, ']')