github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-11 15:34:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/decimal: increase float64->decimal conversion precision a bit

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/213
This commit is contained in:
Aliaksandr Valialkin 2019-10-30 02:03:40 +02:00
parent fa7c3ab93a
commit cfa5e279c2
2 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/decimal/decimal.go
View file

@ -302,7 +302,7 @@ func positiveFloatToDecimal(f float64) (int64, int16) {
u := uint64(f)
if float64(u) == f {
// Fast path for integers.
for u >= 1<<54 {
for u >= 1<<55 {
// Remove trailing garbage bits left after float64->uint64 conversion,
// since float64 contains only 53 significant bits.
// See https://en.wikipedia.org/wiki/Double-precision_floating-point_format

4
lib/decimal/decimal_test.go
View file

@ -25,10 +25,10 @@ func TestPositiveFloatToDecimal(t *testing.T) {
f(12345678901234567, 12345678901234568, 0)
f(1234567890123456789, 12345678901234567, 2)
f(12345678901234567890, 12345678901234567, 3)
f(18446744073670737131, 1844674407367073, 4)
f(18446744073670737131, 18446744073670737, 3)
f(123456789012345678901, 12345678901234568, 4)
f(1<<53, 1<<53, 0)
f(1<<54, 1801439850948198, 1)
f(1<<54, 18014398509481984, 0)
f(1<<55, 3602879701896396, 1)
f(1<<62, 4611686018427387, 3)
f(1<<63, 9223372036854775, 3)