diff --git a/lib/decimal/decimal.go b/lib/decimal/decimal.go index 8b02be0c8..5a526aa6d 100644 --- a/lib/decimal/decimal.go +++ b/lib/decimal/decimal.go @@ -3,6 +3,8 @@ package decimal import ( "math" "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fastnum" ) // CalibrateScale calibrates a and b with the corresponding exponents ae, be @@ -81,6 +83,13 @@ func ExtendInt64sCapacity(dst []int64, additionalItems int) []int64 { // AppendDecimalToFloat converts each item in va to f=v*10^e, appends it // to dst and returns the resulting dst. func AppendDecimalToFloat(dst []float64, va []int64, e int16) []float64 { + if fastnum.IsInt64Zeros(va) { + return fastnum.AppendFloat64Zeros(dst, len(va)) + } + if e == 0 && fastnum.IsInt64Ones(va) { + return fastnum.AppendFloat64Ones(dst, len(va)) + } + // Extend dst capacity in order to eliminate memory allocations below. dst = ExtendFloat64sCapacity(dst, len(va)) @@ -108,6 +117,14 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) { if len(src) == 0 { return dst, 0 } + if fastnum.IsFloat64Zeros(src) { + dst = fastnum.AppendInt64Zeros(dst, len(src)) + return dst, 0 + } + if fastnum.IsFloat64Ones(src) { + dst = fastnum.AppendInt64Ones(dst, len(src)) + return dst, 0 + } // Extend dst capacity in order to eliminate memory allocations below. dst = ExtendInt64sCapacity(dst, len(src)) diff --git a/lib/encoding/encoding.go b/lib/encoding/encoding.go index e93baaabe..06f6681fe 100644 --- a/lib/encoding/encoding.go +++ b/lib/encoding/encoding.go @@ -5,6 +5,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fastnum" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) @@ -201,6 +202,14 @@ func unmarshalInt64Array(dst []int64, src []byte, mt MarshalType, firstValue int if len(src) > 0 { return nil, fmt.Errorf("unexpected data left in const encoding: %d bytes", len(src)) } + if firstValue == 0 { + dst = fastnum.AppendInt64Zeros(dst, itemsCount) + return dst, nil + } + if firstValue == 1 { + dst = fastnum.AppendInt64Ones(dst, itemsCount) + return dst, nil + } for itemsCount > 0 { dst = append(dst, firstValue) itemsCount-- @@ -267,6 +276,14 @@ func isConst(a []int64) bool { if len(a) == 0 { return false } + if fastnum.IsInt64Zeros(a) { + // Fast path for array containing only zeros. + return true + } + if fastnum.IsInt64Ones(a) { + // Fast path for array containing only ones. + return true + } v1 := a[0] for _, v := range a { if v != v1 { diff --git a/lib/fastnum/fastnum.go b/lib/fastnum/fastnum.go new file mode 100644 index 000000000..736d7e3cf --- /dev/null +++ b/lib/fastnum/fastnum.go @@ -0,0 +1,144 @@ +package fastnum + +import ( + "bytes" + "unsafe" +) + +// AppendInt64Zeros appends items zeros to dst and returns the result. +// +// It is faster than the corresponding loop. +func AppendInt64Zeros(dst []int64, items int) []int64 { + return appendInt64Data(dst, items, int64Zeros[:]) +} + +// AppendInt64Ones appends items ones to dst and returns the result. +// +// It is faster than the correponding loop. +func AppendInt64Ones(dst []int64, items int) []int64 { + return appendInt64Data(dst, items, int64Ones[:]) +} + +// AppendFloat64Zeros appends items zeros to dst and returns the result. +// +// It is faster than the corresponding loop. +func AppendFloat64Zeros(dst []float64, items int) []float64 { + return appendFloat64Data(dst, items, float64Zeros[:]) +} + +// AppendFloat64Ones appends items ones to dst and returns the result. +// +// It is faster than the corresponding loop. +func AppendFloat64Ones(dst []float64, items int) []float64 { + return appendFloat64Data(dst, items, float64Ones[:]) +} + +// IsInt64Zeros checks whether a contains only zeros. +func IsInt64Zeros(a []int64) bool { + return isInt64Data(a, int64Zeros[:]) +} + +// IsInt64Ones checks whether a contains only ones. +func IsInt64Ones(a []int64) bool { + return isInt64Data(a, int64Ones[:]) +} + +// IsFloat64Zeros checks whether a contains only zeros. +func IsFloat64Zeros(a []float64) bool { + return isFloat64Data(a, float64Zeros[:]) +} + +// IsFloat64Ones checks whether a contains only ones. +func IsFloat64Ones(a []float64) bool { + return isFloat64Data(a, float64Ones[:]) +} + +func appendInt64Data(dst []int64, items int, src []int64) []int64 { + for items > 0 { + n := len(src) + if n > items { + n = items + } + dst = append(dst, src[:n]...) + items -= n + } + return dst +} + +func appendFloat64Data(dst []float64, items int, src []float64) []float64 { + for items > 0 { + n := len(src) + if n > items { + n = items + } + dst = append(dst, src[:n]...) + items -= n + } + return dst +} + +func isInt64Data(a, data []int64) bool { + if len(a) == 0 { + return true + } + if len(data) != 8*1024 { + panic("len(data) must equal to 8*1024") + } + b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0])) + for len(a) > 0 { + n := len(data) + if n > len(a) { + n = len(a) + } + x := a[:n] + a = a[n:] + xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0])) + xbLen := len(x) * 8 + if !bytes.Equal(xb[:xbLen], b[:xbLen]) { + return false + } + } + return true +} + +func isFloat64Data(a, data []float64) bool { + if len(a) == 0 { + return true + } + if len(data) != 8*1024 { + panic("len(data) must equal to 8*1024") + } + b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0])) + for len(a) > 0 { + n := len(data) + if n > len(a) { + n = len(a) + } + x := a[:n] + a = a[n:] + xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0])) + xbLen := len(x) * 8 + if !bytes.Equal(xb[:xbLen], b[:xbLen]) { + return false + } + } + return true +} + +var ( + int64Zeros [8 * 1024]int64 + int64Ones = func() (a [8 * 1024]int64) { + for i := 0; i < len(a); i++ { + a[i] = 1 + } + return a + }() + + float64Zeros [8 * 1024]float64 + float64Ones = func() (a [8 * 1024]float64) { + for i := 0; i < len(a); i++ { + a[i] = 1 + } + return a + }() +) diff --git a/lib/fastnum/fastnum_test.go b/lib/fastnum/fastnum_test.go new file mode 100644 index 000000000..ddb514a29 --- /dev/null +++ b/lib/fastnum/fastnum_test.go @@ -0,0 +1,192 @@ +package fastnum + +import ( + "fmt" + "testing" +) + +func TestIsInt64Zeros(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := make([]int64, n) + if !IsInt64Zeros(a) { + t.Fatalf("IsInt64Zeros must return true") + } + if len(a) > 0 { + a[len(a)-1] = 1 + if IsInt64Zeros(a) { + t.Fatalf("IsInt64Zeros must return false") + } + } + }) + } +} + +func TestIsInt64Ones(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := make([]int64, n) + for i := 0; i < n; i++ { + a[i] = 1 + } + if !IsInt64Ones(a) { + t.Fatalf("IsInt64Ones must return true") + } + if len(a) > 0 { + a[len(a)-1] = 0 + if IsInt64Ones(a) { + t.Fatalf("IsInt64Ones must return false") + } + } + }) + } +} + +func TestIsFloat64Zeros(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := make([]float64, n) + if !IsFloat64Zeros(a) { + t.Fatalf("IsInt64Zeros must return true") + } + if len(a) > 0 { + a[len(a)-1] = 1 + if IsFloat64Zeros(a) { + t.Fatalf("IsInt64Zeros must return false") + } + } + }) + } +} + +func TestIsFloat64Ones(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := make([]float64, n) + for i := 0; i < n; i++ { + a[i] = 1 + } + if !IsFloat64Ones(a) { + t.Fatalf("IsInt64Ones must return true") + } + if len(a) > 0 { + a[len(a)-1] = 0 + if IsFloat64Ones(a) { + t.Fatalf("IsInt64Ones must return false") + } + } + }) + } +} + +func TestAppendInt64Zeros(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := AppendInt64Zeros(nil, n) + if len(a) != n { + t.Fatalf("unexpected len(a); got %d; want %d", len(a), n) + } + if !IsInt64Zeros(a) { + t.Fatalf("IsInt64Zeros must return true") + } + + prefix := []int64{1, 2, 3} + a = AppendInt64Zeros(prefix, n) + if len(a) != len(prefix)+n { + t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n) + } + for i := 0; i < len(prefix); i++ { + if a[i] != prefix[i] { + t.Fatalf("unexpected prefix[%d]; got %d; want %d", i, a[i], prefix[i]) + } + } + if !IsInt64Zeros(a[len(prefix):]) { + t.Fatalf("IsInt64Zeros for prefixed a must return true") + } + }) + } +} + +func TestAppendInt64Ones(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := AppendInt64Ones(nil, n) + if len(a) != n { + t.Fatalf("unexpected len(a); got %d; want %d", len(a), n) + } + if !IsInt64Ones(a) { + t.Fatalf("IsInt64Ones must return true") + } + + prefix := []int64{1, 2, 3} + a = AppendInt64Ones(prefix, n) + if len(a) != len(prefix)+n { + t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n) + } + for i := 0; i < len(prefix); i++ { + if a[i] != prefix[i] { + t.Fatalf("unexpected prefix[%d]; got %d; want %d", i, a[i], prefix[i]) + } + } + if !IsInt64Ones(a[len(prefix):]) { + t.Fatalf("IsInt64Ones for prefixed a must return true") + } + }) + } +} + +func TestAppendFloat64Zeros(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := AppendFloat64Zeros(nil, n) + if len(a) != n { + t.Fatalf("unexpected len(a); got %d; want %d", len(a), n) + } + if !IsFloat64Zeros(a) { + t.Fatalf("IsFloat64Zeros must return true") + } + + prefix := []float64{1, 2, 3} + a = AppendFloat64Zeros(prefix, n) + if len(a) != len(prefix)+n { + t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n) + } + for i := 0; i < len(prefix); i++ { + if a[i] != prefix[i] { + t.Fatalf("unexpected prefix[%d]; got %f; want %f", i, a[i], prefix[i]) + } + } + if !IsFloat64Zeros(a[len(prefix):]) { + t.Fatalf("IsFloat64Zeros for prefixed a must return true") + } + }) + } +} + +func TestAppendFloat64Ones(t *testing.T) { + for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} { + t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) { + a := AppendFloat64Ones(nil, n) + if len(a) != n { + t.Fatalf("unexpected len(a); got %d; want %d", len(a), n) + } + if !IsFloat64Ones(a) { + t.Fatalf("IsFloat64Ones must return true") + } + + prefix := []float64{1, 2, 3} + a = AppendFloat64Ones(prefix, n) + if len(a) != len(prefix)+n { + t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n) + } + for i := 0; i < len(prefix); i++ { + if a[i] != prefix[i] { + t.Fatalf("unexpected prefix[%d]; got %f; want %f", i, a[i], prefix[i]) + } + } + if !IsFloat64Ones(a[len(prefix):]) { + t.Fatalf("IsFloat64Ones for prefixed a must return true") + } + }) + } +}