diff --git a/app/vmselect/prometheus/export.qtpl b/app/vmselect/prometheus/export.qtpl index 06f675e54..bc738ca31 100644 --- a/app/vmselect/prometheus/export.qtpl +++ b/app/vmselect/prometheus/export.qtpl @@ -99,10 +99,10 @@ "values":[ {% if len(xb.values) > 0 %} {% code values := xb.values %} - {%f= values[0] %} + {%= convertValueToSpecialJSON(values[0]) %} {% code values = values[1:] %} {% for _, v := range values %} - ,{% if math.IsNaN(v) %}null{% else %}{%f= v %}{% endif %} + ,{%= convertValueToSpecialJSON(v) %} {% endfor %} {% endif %} ], @@ -158,4 +158,17 @@ } {% endif %} {% endfunc %} + +{% func convertValueToSpecialJSON(v float64) %} + {% if math.IsNaN(v) %} + null + {% elseif math.IsInf(v, 1) %} + "Infinity" + {% elseif math.IsInf(v, -1) %} + "-Infinity" + {% else %} + {%f= v %} + {% endif %} +{% endfunc %} {% endstripspace %} + diff --git a/app/vmselect/prometheus/export.qtpl.go b/app/vmselect/prometheus/export.qtpl.go index 7b20c7fa0..56f211469 100644 --- a/app/vmselect/prometheus/export.qtpl.go +++ b/app/vmselect/prometheus/export.qtpl.go @@ -295,7 +295,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) { values := xb.values //line app/vmselect/prometheus/export.qtpl:102 - qw422016.N().F(values[0]) + streamconvertValueToSpecialJSON(qw422016, values[0]) //line app/vmselect/prometheus/export.qtpl:103 values = values[1:] @@ -304,15 +304,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) { //line app/vmselect/prometheus/export.qtpl:104 qw422016.N().S(`,`) //line app/vmselect/prometheus/export.qtpl:105 - if math.IsNaN(v) { -//line app/vmselect/prometheus/export.qtpl:105 - qw422016.N().S(`null`) -//line app/vmselect/prometheus/export.qtpl:105 - } else { -//line app/vmselect/prometheus/export.qtpl:105 - qw422016.N().F(v) -//line app/vmselect/prometheus/export.qtpl:105 - } + streamconvertValueToSpecialJSON(qw422016, v) //line app/vmselect/prometheus/export.qtpl:106 } //line app/vmselect/prometheus/export.qtpl:107 @@ -554,3 +546,52 @@ func prometheusMetricName(mn *storage.MetricName) string { return qs422016 //line app/vmselect/prometheus/export.qtpl:160 } + +//line app/vmselect/prometheus/export.qtpl:187 +func streamconvertValueToSpecialJSON(qw422016 *qt422016.Writer, v float64) { +//line app/vmselect/prometheus/export.qtpl:188 + if math.IsNaN(v) { +//line app/vmselect/prometheus/export.qtpl:188 + qw422016.N().S(`"NaN"`) +//line app/vmselect/prometheus/export.qtpl:190 + } else if math.IsInf(v, 1) { +//line app/vmselect/prometheus/export.qtpl:190 + qw422016.N().S(`"Infinity"`) +//line app/vmselect/prometheus/export.qtpl:192 + } else if math.IsInf(v, -1) { +//line app/vmselect/prometheus/export.qtpl:192 + qw422016.N().S(`"-Infinity"`) +//line app/vmselect/prometheus/export.qtpl:194 + } else { +//line app/vmselect/prometheus/export.qtpl:195 + qw422016.N().F(v) +//line app/vmselect/prometheus/export.qtpl:196 + } +//line app/vmselect/prometheus/export.qtpl:197 +} + +//line app/vmselect/prometheus/export.qtpl:197 +func writeconvertValueToSpecialJSON(qq422016 qtio422016.Writer, v float64) { +//line app/vmselect/prometheus/export.qtpl:197 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/prometheus/export.qtpl:197 + streamconvertValueToSpecialJSON(qw422016, v) +//line app/vmselect/prometheus/export.qtpl:197 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/prometheus/export.qtpl:197 +} + +//line app/vmselect/prometheus/export.qtpl:197 +func convertValueToSpecialJSON(v float64) string { +//line app/vmselect/prometheus/export.qtpl:197 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/prometheus/export.qtpl:197 + writeconvertValueToSpecialJSON(qb422016, v) +//line app/vmselect/prometheus/export.qtpl:197 + qs422016 := string(qb422016.B) +//line app/vmselect/prometheus/export.qtpl:197 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/prometheus/export.qtpl:197 + return qs422016 +//line app/vmselect/prometheus/export.qtpl:197 +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 550e10af6..0ad843a58 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038). * FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052). * FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090). +* FEATURE: properly parse json when export import metrics via `api/v1/export` and `api/v1/import` API. Added support of the `["Infinity", "-Infinity", "NaN", null]` values. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): move the "Execute Query" and "Add Query" buttons below the query fields, change icon for remove query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3101). diff --git a/lib/protoparser/vmimport/parser.go b/lib/protoparser/vmimport/parser.go index 8bbd3b44c..715299ac1 100644 --- a/lib/protoparser/vmimport/parser.go +++ b/lib/protoparser/vmimport/parser.go @@ -2,6 +2,8 @@ package vmimport import ( "fmt" + "math" + "strconv" "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -77,7 +79,7 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error { return fmt.Errorf("missing `values` array") } for i, v := range values { - f, err := v.Float64() + f, err := getFloat64(v) if err != nil { return fmt.Errorf("cannot unmarshal value at position %d: %w", i, err) } @@ -103,6 +105,39 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error { return nil } +func getFloat64(value *fastjson.Value) (float64, error) { + if value == nil { + return 0, fmt.Errorf("value is empty") + } + + switch value.Type() { + case fastjson.TypeNull: + return math.NaN(), nil + case fastjson.TypeString: + return getSpecialFloat64ValueFromString(value.String()) + default: + return value.Float64() + } +} + +func getSpecialFloat64ValueFromString(strVal string) (float64, error) { + str, err := strconv.Unquote(strings.ToLower(strVal)) + if err != nil { + return 0, err + } + + switch str { + case "infinity": + return math.Inf(1), nil + case "-infinity": + return math.Inf(-1), nil + case "null": + return math.NaN(), nil + default: + return 0, fmt.Errorf("got unsupported string: %q", str) + } +} + // Tag represents `/api/v1/import` tag. type Tag struct { Key []byte diff --git a/lib/protoparser/vmimport/parser_test.go b/lib/protoparser/vmimport/parser_test.go index a3aaa70a3..28d6d4510 100644 --- a/lib/protoparser/vmimport/parser_test.go +++ b/lib/protoparser/vmimport/parser_test.go @@ -47,6 +47,9 @@ func TestRowsUnmarshalFailure(t *testing.T) { f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`) f(`{"metric":{"foo":"bar"},"timestamps":[3,4]}`) f(`{"metric":{"foo":"bar"},"values":["foo"],"timestamps":[3]}`) + f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`) + f(`{"metric":{"foo":"bar"},"values":"null","timestamps":[3,4]}`) + f(`{"metric":{"foo":"bar"},"values":["NaN"],"timestamps":[3,4]}`) // Invalid timestamps f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":3}`) @@ -71,6 +74,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) { t.Helper() var rows Rows rows.Unmarshal(s) + + if containsNaN(rows) { + if !checkNaN(rows, rowsExpected) { + t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) + return + } + return + } + if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) { t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) } @@ -104,15 +116,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) { }}, }) - // Inf and nan values - f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf],"timestamps":[456, 789]}`, &Rows{ + // Inf and nan, null values + f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, null, "null"],"timestamps":[456, 789, 123, 0, 1, 2, 3]}`, &Rows{ Rows: []Row{{ Tags: []Tag{{ Key: []byte("foo"), Value: []byte("bar"), }}, - Values: []float64{math.Inf(1), math.Inf(-1)}, - Timestamps: []int64{456, 789}, + Values: []float64{math.Inf(1), math.Inf(-1), math.Inf(1), math.Inf(-1), math.NaN(), math.NaN(), math.NaN()}, + Timestamps: []int64{456, 789, 123, 0, 1, 2, 3}, }}, }) @@ -229,3 +241,57 @@ garbage here }, }) } + +func Test_getFloat64FromStringValue(t *testing.T) { + f := func(name, strVal string, want float64, wantErr bool) { + t.Run(name, func(t *testing.T) { + got, err := getSpecialFloat64ValueFromString(strVal) + if (err != nil) != wantErr { + t.Errorf("getSpecialFloat64ValueFromString() error = %v, wantErr %v", err, wantErr) + return + } + + if math.IsNaN(want) { + if !math.IsNaN(got) { + t.Fatalf("unexpected result; got %v; want %v", got, want) + return + } + return + } + + if got != want { + t.Errorf("getSpecialFloat64ValueFromString() got = %v, want %v", got, want) + } + }) + } + + f("empty string", "", 0, true) + f("unsupported string", "1", 0, true) + f("null string", "null", 0, true) + f("infinity string", "\"Infinity\"", math.Inf(1), false) + f("-infinity string", "\"-Infinity\"", math.Inf(-1), false) + f("null string", "\"null\"", math.NaN(), false) +} + +func containsNaN(rows Rows) bool { + for _, row := range rows.Rows { + for _, f := range row.Values { + if math.IsNaN(f) { + return true + } + } + } + return false +} + +func checkNaN(rows Rows, expectedRows *Rows) bool { + for i, row := range rows.Rows { + r := expectedRows.Rows[i] + for j, f := range row.Values { + if math.IsNaN(f) && math.IsNaN(r.Values[j]) { + return true + } + } + } + return false +}