diff --git a/app/vmselect/prometheus/export.qtpl b/app/vmselect/prometheus/export.qtpl index bc738ca31..9925edd72 100644 --- a/app/vmselect/prometheus/export.qtpl +++ b/app/vmselect/prometheus/export.qtpl @@ -162,13 +162,15 @@ {% func convertValueToSpecialJSON(v float64) %} {% if math.IsNaN(v) %} null - {% elseif math.IsInf(v, 1) %} - "Infinity" - {% elseif math.IsInf(v, -1) %} - "-Infinity" + {% elseif math.IsInf(v, 0) %} + {% if v > 0 %} + "Infinity" + {% else %} + "-Infinity" + {% endif %} {% else %} {%f= v %} {% endif %} {% endfunc %} -{% endstripspace %} +{% endstripspace %} diff --git a/app/vmselect/prometheus/export.qtpl.go b/app/vmselect/prometheus/export.qtpl.go index 56f211469..47fe8561a 100644 --- a/app/vmselect/prometheus/export.qtpl.go +++ b/app/vmselect/prometheus/export.qtpl.go @@ -547,51 +547,55 @@ func prometheusMetricName(mn *storage.MetricName) string { //line app/vmselect/prometheus/export.qtpl:160 } -//line app/vmselect/prometheus/export.qtpl:187 +//line app/vmselect/prometheus/export.qtpl:162 func streamconvertValueToSpecialJSON(qw422016 *qt422016.Writer, v float64) { -//line app/vmselect/prometheus/export.qtpl:188 +//line app/vmselect/prometheus/export.qtpl:163 if math.IsNaN(v) { -//line app/vmselect/prometheus/export.qtpl:188 - qw422016.N().S(`"NaN"`) -//line app/vmselect/prometheus/export.qtpl:190 - } else if math.IsInf(v, 1) { -//line app/vmselect/prometheus/export.qtpl:190 - qw422016.N().S(`"Infinity"`) -//line app/vmselect/prometheus/export.qtpl:192 - } else if math.IsInf(v, -1) { -//line app/vmselect/prometheus/export.qtpl:192 - qw422016.N().S(`"-Infinity"`) -//line app/vmselect/prometheus/export.qtpl:194 +//line app/vmselect/prometheus/export.qtpl:163 + qw422016.N().S(`null`) +//line app/vmselect/prometheus/export.qtpl:165 + } else if math.IsInf(v, 0) { +//line app/vmselect/prometheus/export.qtpl:166 + if v > 0 { +//line app/vmselect/prometheus/export.qtpl:166 + qw422016.N().S(`"Infinity"`) +//line app/vmselect/prometheus/export.qtpl:168 + } else { +//line app/vmselect/prometheus/export.qtpl:168 + qw422016.N().S(`"-Infinity"`) +//line app/vmselect/prometheus/export.qtpl:170 + } +//line app/vmselect/prometheus/export.qtpl:171 } else { -//line app/vmselect/prometheus/export.qtpl:195 +//line app/vmselect/prometheus/export.qtpl:172 qw422016.N().F(v) -//line app/vmselect/prometheus/export.qtpl:196 +//line app/vmselect/prometheus/export.qtpl:173 } -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 } -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 func writeconvertValueToSpecialJSON(qq422016 qtio422016.Writer, v float64) { -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 streamconvertValueToSpecialJSON(qw422016, v) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 } -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 func convertValueToSpecialJSON(v float64) string { -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 writeconvertValueToSpecialJSON(qb422016, v) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 return qs422016 -//line app/vmselect/prometheus/export.qtpl:197 +//line app/vmselect/prometheus/export.qtpl:174 } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0ad843a58..76760955c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,7 +35,6 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038). * FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052). * FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090). -* FEATURE: properly parse json when export import metrics via `api/v1/export` and `api/v1/import` API. Added support of the `["Infinity", "-Infinity", "NaN", null]` values. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): move the "Execute Query" and "Add Query" buttons below the query fields, change icon for remove query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3101). @@ -58,6 +57,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying per-`-remoteWrite.url` limits for on-disk size for pending data via `-remoteWrite.maxDiskUsagePerURL` command-line flag. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071). * BUGFIX: do not export stale metrics via [/federate api](https://docs.victoriametrics.com/#federation) after the staleness markers. Previously such metrics were exported with `NaN` values. this could break some setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185). +* BUGFIX: export ininity numbers as `"Infinity"` strings at `api/v1/export`, so they can be parsed by standard JSON parsers. Previously infinity numbers were exported as `Inf` values, which couldn't be parsed by standard JSON parsers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161). * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): properly handle request paths ending with `/` such as `/vmui/`. Previously `vmui` was dropping the traling `/`, which could prevent from using `vmui` via `vmauth`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly parse relabel config when regex ending with escaped `$`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3131). diff --git a/lib/protoparser/vmimport/parser.go b/lib/protoparser/vmimport/parser.go index 715299ac1..4111d8fb4 100644 --- a/lib/protoparser/vmimport/parser.go +++ b/lib/protoparser/vmimport/parser.go @@ -3,9 +3,9 @@ package vmimport import ( "fmt" "math" - "strconv" "strings" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/metrics" "github.com/valyala/fastjson" @@ -79,9 +79,13 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error { return fmt.Errorf("missing `values` array") } for i, v := range values { - f, err := getFloat64(v) + f, err := v.Float64() if err != nil { - return fmt.Errorf("cannot unmarshal value at position %d: %w", i, err) + // Fall back to parsing special values + f, err = getSpecialFloat64(v) + if err != nil { + return fmt.Errorf("cannot unmarshal value at position %d: %w", i, err) + } } r.Values = append(r.Values, f) } @@ -105,36 +109,40 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error { return nil } -func getFloat64(value *fastjson.Value) (float64, error) { - if value == nil { - return 0, fmt.Errorf("value is empty") - } +var nan = math.NaN() - switch value.Type() { +func getSpecialFloat64(v *fastjson.Value) (float64, error) { + vt := v.Type() + switch vt { case fastjson.TypeNull: - return math.NaN(), nil + return nan, nil case fastjson.TypeString: - return getSpecialFloat64ValueFromString(value.String()) + b, _ := v.StringBytes() + s := bytesutil.ToUnsafeString(b) + return getSpecialFloat64FromString(s) default: - return value.Float64() + return 0, fmt.Errorf("unsupported value type: %s; value=%q", vt, v) } } -func getSpecialFloat64ValueFromString(strVal string) (float64, error) { - str, err := strconv.Unquote(strings.ToLower(strVal)) - if err != nil { - return 0, err - } +var inf = math.Inf(1) - switch str { - case "infinity": - return math.Inf(1), nil - case "-infinity": - return math.Inf(-1), nil - case "null": - return math.NaN(), nil +func getSpecialFloat64FromString(s string) (float64, error) { + minus := false + if strings.HasPrefix(s, "-") { + minus = true + s = s[1:] + } + switch s { + case "infinity", "Infinity", "Inf", "inf": + if minus { + return -inf, nil + } + return inf, nil + case "null", "Null", "nan", "NaN": + return nan, nil default: - return 0, fmt.Errorf("got unsupported string: %q", str) + return 0, fmt.Errorf("unsupported string: %q", s) } } diff --git a/lib/protoparser/vmimport/parser_test.go b/lib/protoparser/vmimport/parser_test.go index 28d6d4510..d37af11b0 100644 --- a/lib/protoparser/vmimport/parser_test.go +++ b/lib/protoparser/vmimport/parser_test.go @@ -1,6 +1,7 @@ package vmimport import ( + "fmt" "math" "reflect" "testing" @@ -49,7 +50,8 @@ func TestRowsUnmarshalFailure(t *testing.T) { f(`{"metric":{"foo":"bar"},"values":["foo"],"timestamps":[3]}`) f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`) f(`{"metric":{"foo":"bar"},"values":"null","timestamps":[3,4]}`) - f(`{"metric":{"foo":"bar"},"values":["NaN"],"timestamps":[3,4]}`) + f(`{"metric":{"foo":"bar"},"values":"NaN","timestamps":[3,4]}`) + f(`{"metric":{"foo":"bar"},"values":[["NaN"]],"timestamps":[3,4]}`) // Invalid timestamps f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":3}`) @@ -75,22 +77,14 @@ func TestRowsUnmarshalSuccess(t *testing.T) { var rows Rows rows.Unmarshal(s) - if containsNaN(rows) { - if !checkNaN(rows, rowsExpected) { - t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) - return - } - return - } - - if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) { - t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) + if err := compareRows(&rows, rowsExpected); err != nil { + t.Fatalf("unexpected rows: %s;\ngot\n%+v;\nwant\n%+v", err, rows.Rows, rowsExpected.Rows) } // Try unmarshaling again rows.Unmarshal(s) - if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) { - t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) + if err := compareRows(&rows, rowsExpected); err != nil { + t.Fatalf("unexpected rows at second unmarshal: %s;\ngot\n%+v;\nwant\n%+v", err, rows.Rows, rowsExpected.Rows) } rows.Reset() @@ -117,14 +111,14 @@ func TestRowsUnmarshalSuccess(t *testing.T) { }) // Inf and nan, null values - f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, null, "null"],"timestamps":[456, 789, 123, 0, 1, 2, 3]}`, &Rows{ + f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, "NaN", null, "null", 1.2],"timestamps":[456, 789, 123, 0, 1, 42, 2, 3, 7]}`, &Rows{ Rows: []Row{{ Tags: []Tag{{ Key: []byte("foo"), Value: []byte("bar"), }}, - Values: []float64{math.Inf(1), math.Inf(-1), math.Inf(1), math.Inf(-1), math.NaN(), math.NaN(), math.NaN()}, - Timestamps: []int64{456, 789, 123, 0, 1, 2, 3}, + Values: []float64{inf, -inf, inf, -inf, nan, nan, nan, nan, 1.2}, + Timestamps: []int64{456, 789, 123, 0, 1, 42, 2, 3, 7}, }}, }) @@ -242,56 +236,45 @@ garbage here }) } -func Test_getFloat64FromStringValue(t *testing.T) { - f := func(name, strVal string, want float64, wantErr bool) { - t.Run(name, func(t *testing.T) { - got, err := getSpecialFloat64ValueFromString(strVal) - if (err != nil) != wantErr { - t.Errorf("getSpecialFloat64ValueFromString() error = %v, wantErr %v", err, wantErr) - return - } - - if math.IsNaN(want) { - if !math.IsNaN(got) { - t.Fatalf("unexpected result; got %v; want %v", got, want) - return - } - return - } - - if got != want { - t.Errorf("getSpecialFloat64ValueFromString() got = %v, want %v", got, want) - } - }) +func compareRows(rows, rowsExpected *Rows) error { + if len(rows.Rows) != len(rowsExpected.Rows) { + return fmt.Errorf("unexpected number of rows; got %d; want %d", len(rows.Rows), len(rowsExpected.Rows)) } - - f("empty string", "", 0, true) - f("unsupported string", "1", 0, true) - f("null string", "null", 0, true) - f("infinity string", "\"Infinity\"", math.Inf(1), false) - f("-infinity string", "\"-Infinity\"", math.Inf(-1), false) - f("null string", "\"null\"", math.NaN(), false) -} - -func containsNaN(rows Rows) bool { - for _, row := range rows.Rows { - for _, f := range row.Values { - if math.IsNaN(f) { - return true - } - } - } - return false -} - -func checkNaN(rows Rows, expectedRows *Rows) bool { for i, row := range rows.Rows { - r := expectedRows.Rows[i] - for j, f := range row.Values { - if math.IsNaN(f) && math.IsNaN(r.Values[j]) { - return true - } + rowExpected := rowsExpected.Rows[i] + if err := compareSingleRow(&row, &rowExpected); err != nil { + return fmt.Errorf("unexpected row at position #%d: %w", i, err) } } - return false + return nil +} + +func compareSingleRow(row, rowExpected *Row) error { + if !reflect.DeepEqual(row.Tags, rowExpected.Tags) { + return fmt.Errorf("unexpected tags; got %q; want %q", row.Tags, rowExpected.Tags) + } + if !reflect.DeepEqual(row.Timestamps, rowExpected.Timestamps) { + return fmt.Errorf("unexpected timestamps; got %d; want %d", row.Timestamps, rowExpected.Timestamps) + } + if err := compareValues(row.Values, rowExpected.Values); err != nil { + return fmt.Errorf("unexpected values; got %v; want %v", row.Values, rowExpected.Values) + } + return nil +} + +func compareValues(values, valuesExpected []float64) error { + if len(values) != len(valuesExpected) { + return fmt.Errorf("unexpected number of values; got %d; want %d", len(values), len(valuesExpected)) + } + for i, v := range values { + vExpected := valuesExpected[i] + if math.IsNaN(v) { + if !math.IsNaN(vExpected) { + return fmt.Errorf("expecting NaN at position #%d; got %v", i, v) + } + } else if v != vExpected { + return fmt.Errorf("unepxected value at position #%d; got %v; want %v", i, v, vExpected) + } + } + return nil }