From a7a04bd4e9142740dca683357dfa50ff554246a7 Mon Sep 17 00:00:00 2001 From: Alexander Marshalov <_@marshalov.org> Date: Thu, 15 Feb 2024 20:19:17 +0100 Subject: [PATCH] [lib/promutils, lib/httputils] fixed floating-point error when parsing time in RFC3339 format (#5801) --- docs/CHANGELOG.md | 1 + lib/httputils/time.go | 4 ++-- lib/httputils/time_test.go | 1 + lib/promutils/time.go | 46 ++++++++++++++++++++++++++++---------- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3c46f4d44..7cab491a5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The sandbox cluster installation is running under the constant load generated by ## tip * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): expose `vm_last_partition_parts` [metrics](https://docs.victoriametrics.com/#monitoring), which show the number of [parts in the latest partition](https://docs.victoriametrics.com/#storage). These metrics may help debugging query performance slowdown related to the increased number of parts in the last partition, since usually all the ingested data is written to the last partition and all the queries are performed over the recently ingested data, e.g. the last partition. +* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fixed floating-point error when parsing time in RFC3339 format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5801) for details. ## [v1.98.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.98.0) diff --git a/lib/httputils/time.go b/lib/httputils/time.go index e55591251..efdaff337 100644 --- a/lib/httputils/time.go +++ b/lib/httputils/time.go @@ -28,11 +28,11 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { return maxTimeMsecs, nil } // Parse argValue - secs, err := promutils.ParseTime(argValue) + ms, err := promutils.ParseTimeMs(argValue) if err != nil { return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) } - msecs := int64(secs * 1e3) + msecs := int64(ms) if msecs < minTimeMsecs { msecs = 0 } diff --git a/lib/httputils/time_test.go b/lib/httputils/time_test.go index 949b3e47d..73667e719 100644 --- a/lib/httputils/time_test.go +++ b/lib/httputils/time_test.go @@ -47,6 +47,7 @@ func TestGetTimeSuccess(t *testing.T) { f("2019-02-02T01:01:00", 1549069260000) f("2019-02-02T01:01:01", 1549069261000) f("2019-07-07T20:01:02Z", 1562529662000) + f("2020-02-21T16:07:49.433Z", 1582301269433) f("2019-07-07T20:47:40+03:00", 1562521660000) f("-292273086-05-16T16:47:06Z", minTimeMsecs) f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs) diff --git a/lib/promutils/time.go b/lib/promutils/time.go index 5b999c27f..bf9ec95a7 100644 --- a/lib/promutils/time.go +++ b/lib/promutils/time.go @@ -17,6 +17,16 @@ func ParseTime(s string) (float64, error) { return ParseTimeAt(s, currentTimestamp) } +// ParseTimeMs parses time s in different formats. +// +// See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats +// +// It returns unix timestamp in milliseconds. +func ParseTimeMs(s string) (float64, error) { + currentTimestampMs := float64(time.Now().UnixNano()) / 1e6 + return ParseTimeMsAt(s, currentTimestampMs) +} + const ( // time.UnixNano can only store maxInt64, which is 2262 maxValidYear = 2262 @@ -32,6 +42,18 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if s == "now" { return currentTimestamp, nil } + return ParseTimeMsAt(s, currentTimestamp*1e3) +} + +// ParseTimeMsAt parses time s in different formats, assuming the given currentTimestamp. +// +// See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats +// +// It returns unix timestamp in milliseconds. +func ParseTimeMsAt(s string, currentTimestampMs float64) (float64, error) { + if s == "now" { + return currentTimestampMs, nil + } sOrig := s tzOffset := float64(0) if len(sOrig) > 6 { @@ -47,7 +69,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, fmt.Errorf("cannot parse minute from timezone offset %q: %w", tz, err) } - tzOffset = float64(hour*3600 + minute*60) + tzOffset = float64(1000 * (hour*3600 + minute*60)) if isPlus { tzOffset = -tzOffset } @@ -65,7 +87,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if d > 0 { d = -d } - return currentTimestamp + float64(d)/1e9, nil + return currentTimestampMs + float64(d)/1e6, nil } if len(s) == 4 { // Parse YYYY @@ -77,7 +99,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if y > maxValidYear || y < minValidYear { return 0, fmt.Errorf("cannot parse year from %q: year must in range [%d, %d]", s, minValidYear, maxValidYear) } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } if !strings.Contains(sOrig, "-") { // Parse the timestamp in seconds or in milliseconds @@ -85,9 +107,9 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - if ts >= (1 << 32) { - // The timestamp is in milliseconds. Convert it to seconds. - ts /= 1000 + if ts < (1 << 32) { + // The timestamp is in seconds. Convert it to milliseconds. + ts *= 1000 } return ts, nil } @@ -97,7 +119,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } if len(s) == 10 { // Parse YYYY-MM-DD @@ -105,7 +127,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } if len(s) == 13 { // Parse YYYY-MM-DDTHH @@ -113,7 +135,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } if len(s) == 16 { // Parse YYYY-MM-DDTHH:MM @@ -121,7 +143,7 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } if len(s) == 19 { // Parse YYYY-MM-DDTHH:MM:SS @@ -129,12 +151,12 @@ func ParseTimeAt(s string, currentTimestamp float64) (float64, error) { if err != nil { return 0, err } - return tzOffset + float64(t.UnixNano())/1e9, nil + return tzOffset + float64(t.UnixNano())/1e6, nil } // Parse RFC3339 t, err := time.Parse(time.RFC3339, sOrig) if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return float64(t.UnixNano()) / 1e6, nil }