Properly parse json when export import metric (#3180)

* app/vmselect: properly work when export import json from `api/v1/{export, import}` API

* app/vmselect: update convert function

* app/vmselect: export null if `math.IsNaN(v)`

* app/vmselect: get float from json

* lib/protoparser: add test

* docs: add change log

* lib/protoparser: make export import api compatible

app/vmselect/prometheus/export.qtpl

@@ -99,10 +99,10 @@
 		"values":[
 			{% if len(xb.values) > 0 %}
 				{% code values := xb.values %}
-				{%f= values[0] %}
+					{%= convertValueToSpecialJSON(values[0]) %}
 				{% code values = values[1:] %}
 				{% for _, v := range values %}
-				  ,{% if math.IsNaN(v) %}null{% else %}{%f= v %}{% endif %}
+					,{%= convertValueToSpecialJSON(v) %}
 				{% endfor %}
 			{% endif %}
 		],
@@ -158,4 +158,17 @@
 	}
 	{% endif %}
 {% endfunc %}
+
+{% func convertValueToSpecialJSON(v float64) %}
+	{% if math.IsNaN(v) %}
+		null
+	{% elseif math.IsInf(v, 1) %}
+		"Infinity"
+	{% elseif math.IsInf(v, -1) %}
+		"-Infinity"
+	{% else %}
+		{%f= v %}
+	{% endif %}
+{% endfunc %}
 {% endstripspace %}
+

app/vmselect/prometheus/export.qtpl.go

@@ -295,7 +295,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
 		values := xb.values
 
 //line app/vmselect/prometheus/export.qtpl:102
-		qw422016.N().F(values[0])
+		streamconvertValueToSpecialJSON(qw422016, values[0])
 //line app/vmselect/prometheus/export.qtpl:103
 		values = values[1:]
 
@@ -304,15 +304,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
 //line app/vmselect/prometheus/export.qtpl:104
 			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:105
-			if math.IsNaN(v) {
+			streamconvertValueToSpecialJSON(qw422016, v)
-//line app/vmselect/prometheus/export.qtpl:105
-				qw422016.N().S(`null`)
-//line app/vmselect/prometheus/export.qtpl:105
-			} else {
-//line app/vmselect/prometheus/export.qtpl:105
-				qw422016.N().F(v)
-//line app/vmselect/prometheus/export.qtpl:105
-			}
 //line app/vmselect/prometheus/export.qtpl:106
 		}
 //line app/vmselect/prometheus/export.qtpl:107
@@ -554,3 +546,52 @@ func prometheusMetricName(mn *storage.MetricName) string {
 	return qs422016
 //line app/vmselect/prometheus/export.qtpl:160
 }
+
+//line app/vmselect/prometheus/export.qtpl:187
+func streamconvertValueToSpecialJSON(qw422016 *qt422016.Writer, v float64) {
+//line app/vmselect/prometheus/export.qtpl:188
+	if math.IsNaN(v) {
+//line app/vmselect/prometheus/export.qtpl:188
+		qw422016.N().S(`"NaN"`)
+//line app/vmselect/prometheus/export.qtpl:190
+	} else if math.IsInf(v, 1) {
+//line app/vmselect/prometheus/export.qtpl:190
+		qw422016.N().S(`"Infinity"`)
+//line app/vmselect/prometheus/export.qtpl:192
+	} else if math.IsInf(v, -1) {
+//line app/vmselect/prometheus/export.qtpl:192
+		qw422016.N().S(`"-Infinity"`)
+//line app/vmselect/prometheus/export.qtpl:194
+	} else {
+//line app/vmselect/prometheus/export.qtpl:195
+		qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:196
+	}
+//line app/vmselect/prometheus/export.qtpl:197
+}
+
+//line app/vmselect/prometheus/export.qtpl:197
+func writeconvertValueToSpecialJSON(qq422016 qtio422016.Writer, v float64) {
+//line app/vmselect/prometheus/export.qtpl:197
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:197
+	streamconvertValueToSpecialJSON(qw422016, v)
+//line app/vmselect/prometheus/export.qtpl:197
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:197
+}
+
+//line app/vmselect/prometheus/export.qtpl:197
+func convertValueToSpecialJSON(v float64) string {
+//line app/vmselect/prometheus/export.qtpl:197
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:197
+	writeconvertValueToSpecialJSON(qb422016, v)
+//line app/vmselect/prometheus/export.qtpl:197
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:197
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:197
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:197
+}

docs/CHANGELOG.md

@@ -35,6 +35,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
 * FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038).
 * FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052).
 * FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090).
+* FEATURE: properly parse json when export import metrics via `api/v1/export` and `api/v1/import` API. Added support of the `["Infinity", "-Infinity", "NaN", null]` values. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): move the "Execute Query" and "Add Query" buttons below the query fields, change icon for remove query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3101).

lib/protoparser/vmimport/parser.go

@@ -2,6 +2,8 @@ package vmimport
 
 import (
 	"fmt"
+	"math"
+	"strconv"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -77,7 +79,7 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error {
 		return fmt.Errorf("missing `values` array")
 	}
 	for i, v := range values {
-		f, err := v.Float64()
+		f, err := getFloat64(v)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal value at position %d: %w", i, err)
 		}
@@ -103,6 +105,39 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error {
 	return nil
 }
 
+func getFloat64(value *fastjson.Value) (float64, error) {
+	if value == nil {
+		return 0, fmt.Errorf("value is empty")
+	}
+
+	switch value.Type() {
+	case fastjson.TypeNull:
+		return math.NaN(), nil
+	case fastjson.TypeString:
+		return getSpecialFloat64ValueFromString(value.String())
+	default:
+		return value.Float64()
+	}
+}
+
+func getSpecialFloat64ValueFromString(strVal string) (float64, error) {
+	str, err := strconv.Unquote(strings.ToLower(strVal))
+	if err != nil {
+		return 0, err
+	}
+
+	switch str {
+	case "infinity":
+		return math.Inf(1), nil
+	case "-infinity":
+		return math.Inf(-1), nil
+	case "null":
+		return math.NaN(), nil
+	default:
+		return 0, fmt.Errorf("got unsupported string: %q", str)
+	}
+}
+
 // Tag represents `/api/v1/import` tag.
 type Tag struct {
 	Key   []byte

lib/protoparser/vmimport/parser_test.go

@@ -47,6 +47,9 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`)
 	f(`{"metric":{"foo":"bar"},"timestamps":[3,4]}`)
 	f(`{"metric":{"foo":"bar"},"values":["foo"],"timestamps":[3]}`)
+	f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":"null","timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":["NaN"],"timestamps":[3,4]}`)
 
 	// Invalid timestamps
 	f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":3}`)
@@ -71,6 +74,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		t.Helper()
 		var rows Rows
 		rows.Unmarshal(s)
+
+		if containsNaN(rows) {
+			if !checkNaN(rows, rowsExpected) {
+				t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+				return
+			}
+			return
+		}
+
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
@@ -104,15 +116,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		}},
 	})
 
-	// Inf and nan values
+	// Inf and nan, null values
-	f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf],"timestamps":[456, 789]}`, &Rows{
+	f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, null, "null"],"timestamps":[456, 789, 123, 0, 1, 2, 3]}`, &Rows{
 		Rows: []Row{{
 			Tags: []Tag{{
 				Key:   []byte("foo"),
 				Value: []byte("bar"),
 			}},
-			Values:     []float64{math.Inf(1), math.Inf(-1)},
+			Values:     []float64{math.Inf(1), math.Inf(-1), math.Inf(1), math.Inf(-1), math.NaN(), math.NaN(), math.NaN()},
-			Timestamps: []int64{456, 789},
+			Timestamps: []int64{456, 789, 123, 0, 1, 2, 3},
 		}},
 	})
 
@@ -229,3 +241,57 @@ garbage here
 		},
 	})
 }
+
+func Test_getFloat64FromStringValue(t *testing.T) {
+	f := func(name, strVal string, want float64, wantErr bool) {
+		t.Run(name, func(t *testing.T) {
+			got, err := getSpecialFloat64ValueFromString(strVal)
+			if (err != nil) != wantErr {
+				t.Errorf("getSpecialFloat64ValueFromString() error = %v, wantErr %v", err, wantErr)
+				return
+			}
+
+			if math.IsNaN(want) {
+				if !math.IsNaN(got) {
+					t.Fatalf("unexpected result; got %v; want %v", got, want)
+					return
+				}
+				return
+			}
+
+			if got != want {
+				t.Errorf("getSpecialFloat64ValueFromString() got = %v, want %v", got, want)
+			}
+		})
+	}
+
+	f("empty string", "", 0, true)
+	f("unsupported string", "1", 0, true)
+	f("null string", "null", 0, true)
+	f("infinity string", "\"Infinity\"", math.Inf(1), false)
+	f("-infinity string", "\"-Infinity\"", math.Inf(-1), false)
+	f("null string", "\"null\"", math.NaN(), false)
+}
+
+func containsNaN(rows Rows) bool {
+	for _, row := range rows.Rows {
+		for _, f := range row.Values {
+			if math.IsNaN(f) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func checkNaN(rows Rows, expectedRows *Rows) bool {
+	for i, row := range rows.Rows {
+		r := expectedRows.Rows[i]
+		for j, f := range row.Values {
+			if math.IsNaN(f) && math.IsNaN(r.Values[j]) {
+				return true
+			}
+		}
+	}
+	return false
+}