mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
app: follow-up after ec04fcac93
* Optimize fast path for /api/v1/import when importing numeric values * Move the docs about the change from features to bugfixes at docs/CHANGELOG.md * Update tests at lib/protoparser/vmimport Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161
This commit is contained in:
Aliaksandr Valialkin
parent
ec04fcac93
commit
d9282027e6
5 changed files with 119 additions and 122 deletions
|
|
@ -162,13 +162,15 @@
|
|||
{% func convertValueToSpecialJSON(v float64) %}
|
||||
{% if math.IsNaN(v) %}
|
||||
null
|
||||
{% elseif math.IsInf(v, 1) %}
|
||||
{% elseif math.IsInf(v, 0) %}
|
||||
{% if v > 0 %}
|
||||
"Infinity"
|
||||
{% elseif math.IsInf(v, -1) %}
|
||||
{% else %}
|
||||
"-Infinity"
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{%f= v %}
|
||||
{% endif %}
|
||||
{% endfunc %}
|
||||
{% endstripspace %}
|
||||
|
||||
{% endstripspace %}
|
||||
|
|
|
|||
|
|
@ -547,51 +547,55 @@ func prometheusMetricName(mn *storage.MetricName) string {
|
|||
//line app/vmselect/prometheus/export.qtpl:160
|
||||
}
|
||||
|
||||
//line app/vmselect/prometheus/export.qtpl:187
|
||||
//line app/vmselect/prometheus/export.qtpl:162
|
||||
func streamconvertValueToSpecialJSON(qw422016 *qt422016.Writer, v float64) {
|
||||
//line app/vmselect/prometheus/export.qtpl:188
|
||||
//line app/vmselect/prometheus/export.qtpl:163
|
||||
if math.IsNaN(v) {
|
||||
//line app/vmselect/prometheus/export.qtpl:188
|
||||
qw422016.N().S(`"NaN"`)
|
||||
//line app/vmselect/prometheus/export.qtpl:190
|
||||
} else if math.IsInf(v, 1) {
|
||||
//line app/vmselect/prometheus/export.qtpl:190
|
||||
//line app/vmselect/prometheus/export.qtpl:163
|
||||
qw422016.N().S(`null`)
|
||||
//line app/vmselect/prometheus/export.qtpl:165
|
||||
} else if math.IsInf(v, 0) {
|
||||
//line app/vmselect/prometheus/export.qtpl:166
|
||||
if v > 0 {
|
||||
//line app/vmselect/prometheus/export.qtpl:166
|
||||
qw422016.N().S(`"Infinity"`)
|
||||
//line app/vmselect/prometheus/export.qtpl:192
|
||||
} else if math.IsInf(v, -1) {
|
||||
//line app/vmselect/prometheus/export.qtpl:192
|
||||
qw422016.N().S(`"-Infinity"`)
|
||||
//line app/vmselect/prometheus/export.qtpl:194
|
||||
//line app/vmselect/prometheus/export.qtpl:168
|
||||
} else {
|
||||
//line app/vmselect/prometheus/export.qtpl:195
|
||||
qw422016.N().F(v)
|
||||
//line app/vmselect/prometheus/export.qtpl:196
|
||||
//line app/vmselect/prometheus/export.qtpl:168
|
||||
qw422016.N().S(`"-Infinity"`)
|
||||
//line app/vmselect/prometheus/export.qtpl:170
|
||||
}
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:171
|
||||
} else {
|
||||
//line app/vmselect/prometheus/export.qtpl:172
|
||||
qw422016.N().F(v)
|
||||
//line app/vmselect/prometheus/export.qtpl:173
|
||||
}
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
}
|
||||
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
func writeconvertValueToSpecialJSON(qq422016 qtio422016.Writer, v float64) {
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
streamconvertValueToSpecialJSON(qw422016, v)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
}
|
||||
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
func convertValueToSpecialJSON(v float64) string {
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
writeconvertValueToSpecialJSON(qb422016, v)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
return qs422016
|
||||
//line app/vmselect/prometheus/export.qtpl:197
|
||||
//line app/vmselect/prometheus/export.qtpl:174
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
|
|||
* FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038).
|
||||
* FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052).
|
||||
* FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090).
|
||||
* FEATURE: properly parse json when export import metrics via `api/v1/export` and `api/v1/import` API. Added support of the `["Infinity", "-Infinity", "NaN", null]` values. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): move the "Execute Query" and "Add Query" buttons below the query fields, change icon for remove query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3101).
|
||||
|
|
@ -58,6 +57,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
|
|||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying per-`-remoteWrite.url` limits for on-disk size for pending data via `-remoteWrite.maxDiskUsagePerURL` command-line flag. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071).
|
||||
|
||||
* BUGFIX: do not export stale metrics via [/federate api](https://docs.victoriametrics.com/#federation) after the staleness markers. Previously such metrics were exported with `NaN` values. this could break some setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185).
|
||||
* BUGFIX: export ininity numbers as `"Infinity"` strings at `api/v1/export`, so they can be parsed by standard JSON parsers. Previously infinity numbers were exported as `Inf` values, which couldn't be parsed by standard JSON parsers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3161).
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): properly handle request paths ending with `/` such as `/vmui/`. Previously `vmui` was dropping the traling `/`, which could prevent from using `vmui` via `vmauth`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly parse relabel config when regex ending with escaped `$`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3131).
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ package vmimport
|
|||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
|
@ -79,10 +79,14 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error {
|
|||
return fmt.Errorf("missing `values` array")
|
||||
}
|
||||
for i, v := range values {
|
||||
f, err := getFloat64(v)
|
||||
f, err := v.Float64()
|
||||
if err != nil {
|
||||
// Fall back to parsing special values
|
||||
f, err = getSpecialFloat64(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot unmarshal value at position %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
r.Values = append(r.Values, f)
|
||||
}
|
||||
|
||||
|
|
@ -105,36 +109,40 @@ func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func getFloat64(value *fastjson.Value) (float64, error) {
|
||||
if value == nil {
|
||||
return 0, fmt.Errorf("value is empty")
|
||||
}
|
||||
var nan = math.NaN()
|
||||
|
||||
switch value.Type() {
|
||||
func getSpecialFloat64(v *fastjson.Value) (float64, error) {
|
||||
vt := v.Type()
|
||||
switch vt {
|
||||
case fastjson.TypeNull:
|
||||
return math.NaN(), nil
|
||||
return nan, nil
|
||||
case fastjson.TypeString:
|
||||
return getSpecialFloat64ValueFromString(value.String())
|
||||
b, _ := v.StringBytes()
|
||||
s := bytesutil.ToUnsafeString(b)
|
||||
return getSpecialFloat64FromString(s)
|
||||
default:
|
||||
return value.Float64()
|
||||
return 0, fmt.Errorf("unsupported value type: %s; value=%q", vt, v)
|
||||
}
|
||||
}
|
||||
|
||||
func getSpecialFloat64ValueFromString(strVal string) (float64, error) {
|
||||
str, err := strconv.Unquote(strings.ToLower(strVal))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var inf = math.Inf(1)
|
||||
|
||||
switch str {
|
||||
case "infinity":
|
||||
return math.Inf(1), nil
|
||||
case "-infinity":
|
||||
return math.Inf(-1), nil
|
||||
case "null":
|
||||
return math.NaN(), nil
|
||||
func getSpecialFloat64FromString(s string) (float64, error) {
|
||||
minus := false
|
||||
if strings.HasPrefix(s, "-") {
|
||||
minus = true
|
||||
s = s[1:]
|
||||
}
|
||||
switch s {
|
||||
case "infinity", "Infinity", "Inf", "inf":
|
||||
if minus {
|
||||
return -inf, nil
|
||||
}
|
||||
return inf, nil
|
||||
case "null", "Null", "nan", "NaN":
|
||||
return nan, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("got unsupported string: %q", str)
|
||||
return 0, fmt.Errorf("unsupported string: %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package vmimport
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
|
@ -49,7 +50,8 @@ func TestRowsUnmarshalFailure(t *testing.T) {
|
|||
f(`{"metric":{"foo":"bar"},"values":["foo"],"timestamps":[3]}`)
|
||||
f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`)
|
||||
f(`{"metric":{"foo":"bar"},"values":"null","timestamps":[3,4]}`)
|
||||
f(`{"metric":{"foo":"bar"},"values":["NaN"],"timestamps":[3,4]}`)
|
||||
f(`{"metric":{"foo":"bar"},"values":"NaN","timestamps":[3,4]}`)
|
||||
f(`{"metric":{"foo":"bar"},"values":[["NaN"]],"timestamps":[3,4]}`)
|
||||
|
||||
// Invalid timestamps
|
||||
f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":3}`)
|
||||
|
|
@ -75,22 +77,14 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
|||
var rows Rows
|
||||
rows.Unmarshal(s)
|
||||
|
||||
if containsNaN(rows) {
|
||||
if !checkNaN(rows, rowsExpected) {
|
||||
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
|
||||
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
|
||||
if err := compareRows(&rows, rowsExpected); err != nil {
|
||||
t.Fatalf("unexpected rows: %s;\ngot\n%+v;\nwant\n%+v", err, rows.Rows, rowsExpected.Rows)
|
||||
}
|
||||
|
||||
// Try unmarshaling again
|
||||
rows.Unmarshal(s)
|
||||
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
|
||||
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
|
||||
if err := compareRows(&rows, rowsExpected); err != nil {
|
||||
t.Fatalf("unexpected rows at second unmarshal: %s;\ngot\n%+v;\nwant\n%+v", err, rows.Rows, rowsExpected.Rows)
|
||||
}
|
||||
|
||||
rows.Reset()
|
||||
|
|
@ -117,14 +111,14 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
|||
})
|
||||
|
||||
// Inf and nan, null values
|
||||
f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, null, "null"],"timestamps":[456, 789, 123, 0, 1, 2, 3]}`, &Rows{
|
||||
f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf, "Infinity", "-Infinity", NaN, "NaN", null, "null", 1.2],"timestamps":[456, 789, 123, 0, 1, 42, 2, 3, 7]}`, &Rows{
|
||||
Rows: []Row{{
|
||||
Tags: []Tag{{
|
||||
Key: []byte("foo"),
|
||||
Value: []byte("bar"),
|
||||
}},
|
||||
Values: []float64{math.Inf(1), math.Inf(-1), math.Inf(1), math.Inf(-1), math.NaN(), math.NaN(), math.NaN()},
|
||||
Timestamps: []int64{456, 789, 123, 0, 1, 2, 3},
|
||||
Values: []float64{inf, -inf, inf, -inf, nan, nan, nan, nan, 1.2},
|
||||
Timestamps: []int64{456, 789, 123, 0, 1, 42, 2, 3, 7},
|
||||
}},
|
||||
})
|
||||
|
||||
|
|
@ -242,56 +236,45 @@ garbage here
|
|||
})
|
||||
}
|
||||
|
||||
func Test_getFloat64FromStringValue(t *testing.T) {
|
||||
f := func(name, strVal string, want float64, wantErr bool) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := getSpecialFloat64ValueFromString(strVal)
|
||||
if (err != nil) != wantErr {
|
||||
t.Errorf("getSpecialFloat64ValueFromString() error = %v, wantErr %v", err, wantErr)
|
||||
return
|
||||
func compareRows(rows, rowsExpected *Rows) error {
|
||||
if len(rows.Rows) != len(rowsExpected.Rows) {
|
||||
return fmt.Errorf("unexpected number of rows; got %d; want %d", len(rows.Rows), len(rowsExpected.Rows))
|
||||
}
|
||||
|
||||
if math.IsNaN(want) {
|
||||
if !math.IsNaN(got) {
|
||||
t.Fatalf("unexpected result; got %v; want %v", got, want)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if got != want {
|
||||
t.Errorf("getSpecialFloat64ValueFromString() got = %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
f("empty string", "", 0, true)
|
||||
f("unsupported string", "1", 0, true)
|
||||
f("null string", "null", 0, true)
|
||||
f("infinity string", "\"Infinity\"", math.Inf(1), false)
|
||||
f("-infinity string", "\"-Infinity\"", math.Inf(-1), false)
|
||||
f("null string", "\"null\"", math.NaN(), false)
|
||||
}
|
||||
|
||||
func containsNaN(rows Rows) bool {
|
||||
for _, row := range rows.Rows {
|
||||
for _, f := range row.Values {
|
||||
if math.IsNaN(f) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkNaN(rows Rows, expectedRows *Rows) bool {
|
||||
for i, row := range rows.Rows {
|
||||
r := expectedRows.Rows[i]
|
||||
for j, f := range row.Values {
|
||||
if math.IsNaN(f) && math.IsNaN(r.Values[j]) {
|
||||
return true
|
||||
rowExpected := rowsExpected.Rows[i]
|
||||
if err := compareSingleRow(&row, &rowExpected); err != nil {
|
||||
return fmt.Errorf("unexpected row at position #%d: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
func compareSingleRow(row, rowExpected *Row) error {
|
||||
if !reflect.DeepEqual(row.Tags, rowExpected.Tags) {
|
||||
return fmt.Errorf("unexpected tags; got %q; want %q", row.Tags, rowExpected.Tags)
|
||||
}
|
||||
if !reflect.DeepEqual(row.Timestamps, rowExpected.Timestamps) {
|
||||
return fmt.Errorf("unexpected timestamps; got %d; want %d", row.Timestamps, rowExpected.Timestamps)
|
||||
}
|
||||
if err := compareValues(row.Values, rowExpected.Values); err != nil {
|
||||
return fmt.Errorf("unexpected values; got %v; want %v", row.Values, rowExpected.Values)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compareValues(values, valuesExpected []float64) error {
|
||||
if len(values) != len(valuesExpected) {
|
||||
return fmt.Errorf("unexpected number of values; got %d; want %d", len(values), len(valuesExpected))
|
||||
}
|
||||
for i, v := range values {
|
||||
vExpected := valuesExpected[i]
|
||||
if math.IsNaN(v) {
|
||||
if !math.IsNaN(vExpected) {
|
||||
return fmt.Errorf("expecting NaN at position #%d; got %v", i, v)
|
||||
}
|
||||
} else if v != vExpected {
|
||||
return fmt.Errorf("unepxected value at position #%d; got %v; want %v", i, v, vExpected)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue