lib/encoding: follow-up for 49e3665d6d

Improve performance for typical cases of varint marshaling / unmarshaling further.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5721

lib/encoding/int.go

@@ -3,7 +3,6 @@ package encoding
 import (
 	"encoding/binary"
 	"fmt"
-	"math/bits"
 	"sync"
 )
 
@@ -91,36 +90,64 @@ func MarshalVarInt64(dst []byte, v int64) []byte {
 
 // MarshalVarInt64s appends marshaled vs to dst and returns the result.
 func MarshalVarInt64s(dst []byte, vs []int64) []byte {
+	dstLen := len(dst)
 	for _, v := range vs {
-		n := uint64((v << 1) ^ (v >> 63))
+		if v >= (1<<6) || v <= (-1<<6) {
-		if n < (1 << 7) {
+			return marshalVarInt64sSlow(dst[:dstLen], vs)
-			dst = append(dst, byte(n))
+		}
+		u := uint64((v << 1) ^ (v >> 63))
+		dst = append(dst, byte(u))
+	}
+	return dst
+}
+
+func marshalVarInt64sSlow(dst []byte, vs []int64) []byte {
+	for _, v := range vs {
+		u := uint64((v << 1) ^ (v >> 63))
+
+		// Cases below are sorted in the descending order of frequency on real data
+		if u < (1 << 7) {
+			dst = append(dst, byte(u))
 			continue
 		}
-		switch (64 - bits.LeadingZeros64(n>>1)) / 7 {
+		if u < (1 << (2 * 7)) {
-		case 0:
+			dst = append(dst, byte(u|0x80), byte(u>>7))
-			dst = append(dst, byte(n))
+			continue
-		case 1:
-			dst = append(dst, byte(n|0x80), byte(n>>7))
-		case 2:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte(n>>14))
-		case 3:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte(n>>21))
-		case 4:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28))
-		case 5:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28|0x80), byte(n>>35))
-		case 6:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28|0x80), byte(n>>35|0x80), byte(n>>42))
-		case 7:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28|0x80), byte(n>>35|0x80), byte(n>>42|0x80), byte(n>>49))
-		case 8:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28|0x80), byte(n>>35|0x80), byte(n>>42|0x80), byte(n>>49|0x80), byte(n>>56))
-		case 9:
-			fallthrough
-		default:
-			dst = append(dst, byte(n|0x80), byte((n>>7)|0x80), byte((n>>14)|0x80), byte((n>>21)|0x80), byte(n>>28|0x80), byte(n>>35|0x80), byte(n>>42|0x80), byte(n>>49|0x80), byte(n>>56|0x80), byte(n>>63))
 		}
+		if u < (1 << (3 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte(u>>(2*7)))
+			continue
+		}
+
+		if u >= (1 << (8 * 7)) {
+			if u < (1 << (9 * 7)) {
+				dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+					byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte((u>>(7*7))|0x80), byte(u>>(8*7)))
+				continue
+			}
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+				byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte((u>>(7*7))|0x80), byte((u>>(8*7))|0x80), 1)
+			continue
+		}
+
+		if u < (1 << (4 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte(u>>(3*7)))
+			continue
+		}
+		if u < (1 << (5 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte(u>>(4*7)))
+			continue
+		}
+		if u < (1 << (6 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80), byte(u>>(5*7)))
+			continue
+		}
+		if u < (1 << (7 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80), byte((u>>(5*7))|0x80), byte(u>>(6*7)))
+			continue
+		}
+		dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+			byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte(u>>(7*7)))
 	}
 	return dst
 }
@@ -136,6 +163,20 @@ func UnmarshalVarInt64(src []byte) ([]byte, int64, error) {
 // UnmarshalVarInt64s unmarshals len(dst) int64 values from src to dst
 // and returns the remaining tail from src.
 func UnmarshalVarInt64s(dst []int64, src []byte) ([]byte, error) {
+	if len(src) < len(dst) {
+		return src, fmt.Errorf("too small len(src)=%d; it must be bigger or equal to len(dst)=%d", len(src), len(dst))
+	}
+	for i := range dst {
+		c := src[i]
+		if c >= 0x80 {
+			return unmarshalVarInt64sSlow(dst, src)
+		}
+		dst[i] = int64(int8(c>>1) ^ (int8(c<<7) >> 7))
+	}
+	return src[len(dst):], nil
+}
+
+func unmarshalVarInt64sSlow(dst []int64, src []byte) ([]byte, error) {
 	idx := uint(0)
 	for i := range dst {
 		if idx >= uint(len(src)) {
@@ -144,62 +185,87 @@ func UnmarshalVarInt64s(dst []int64, src []byte) ([]byte, error) {
 		c := src[idx]
 		idx++
 		if c < 0x80 {
-			// Fast path
+			// Fast path for 1 byte
-			v := int8(c>>1) ^ (int8(c<<7) >> 7) // zig-zag decoding without branching.
+			dst[i] = int64(int8(c>>1) ^ (int8(c<<7) >> 7))
-			dst[i] = int64(v)
 			continue
 		}
-		if idx < uint(len(src)) && src[idx] < 0x80 {
+
-			// Fast path, for 2 bytes
+		if idx >= uint(len(src)) {
-			n := uint64(c&0x7f) | (uint64(src[idx]&0x7f) << 7)
+			return nil, fmt.Errorf("unexpected end of encoded varint at byte 1; src=%x", src[idx-1:])
-			dst[i] = int64(n>>1) ^ (int64(n<<63) >> 63)
+		}
+		d := src[idx]
+		idx++
+		if d < 0x80 {
+			// Fast path for 2 bytes
+			u := uint64(c&0x7f) | (uint64(d) << 7)
+			dst[i] = int64(u>>1) ^ (int64(u<<63) >> 63)
+			continue
+		}
+
+		if idx >= uint(len(src)) {
+			return nil, fmt.Errorf("unexpected end of encoded varint at byte 2; src=%x", src[idx-2:])
+		}
+		e := src[idx]
+		idx++
+		if e < 0x80 {
+			// Fast path for 3 bytes
+			u := uint64(c&0x7f) | (uint64(d&0x7f) << 7) | (uint64(e) << (2 * 7))
+			dst[i] = int64(u>>1) ^ (int64(u<<63) >> 63)
+			continue
+		}
+
+		u := uint64(c&0x7f) | (uint64(d&0x7f) << 7) | (uint64(e&0x7f) << (2 * 7))
+
+		// Slow path
+		j := idx
+		for {
+			if idx >= uint(len(src)) {
+				return nil, fmt.Errorf("unexpected end of encoded varint; src=%x", src[j-3:])
+			}
+			c := src[idx]
 			idx++
-			continue
+			if c < 0x80 {
-		}
-		j := idx + 1
-		for ; j < uint(len(src)); j++ { // find end loc
-			if src[j] < 0x80 {
 				break
 			}
 		}
-		if j-idx > 10 {
+
-			return nil, fmt.Errorf("cannot unmarshal varint, buffer too long, len=%d", j-idx)
+		// These are the most common cases
+		switch idx - j {
+		case 1:
+			u |= (uint64(src[j]) << (3 * 7))
+		case 2:
+			b := src[j : j+2 : j+2]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]) << (4 * 7))
+		case 3:
+			b := src[j : j+3 : j+3]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]) << (5 * 7))
+		case 4:
+			b := src[j : j+4 : j+4]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]) << (6 * 7))
+		case 5:
+			b := src[j : j+5 : j+5]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]) << (7 * 7))
+		case 6:
+			b := src[j : j+6 : j+6]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]&0x7f) << (7 * 7)) | (uint64(b[5]) << (8 * 7))
+		case 7:
+			b := src[j : j+7 : j+7]
+			if b[6] > 1 {
+				return src[idx:], fmt.Errorf("too big encoded varint; src=%x", src[j-3:])
+			}
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]&0x7f) << (7 * 7)) | (uint64(b[5]&0x7f) << (8 * 7)) | (1 << (9 * 7))
+		default:
+			return src[idx:], fmt.Errorf("too long encoded varint; the maximum allowed length is 10 bytes; got %d bytes; src=%x", idx-j+3, src[j-3:])
 		}
-		dst[i] = unmarshalVarInt64ForOne(src[idx-1 : j+1])
+
-		idx = j + 1
+		dst[i] = int64(u>>1) ^ (int64(u<<63) >> 63)
 	}
 	return src[idx:], nil
 }
 
-func unmarshalVarInt64ForOne(buf []byte) int64 {
-	var n uint64
-	switch len(buf) {
-	case 1:
-		n = uint64(buf[0] & 0x7F)
-	case 2:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7)
-	case 3:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14)
-	case 4:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21)
-	case 5:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28)
-	case 6:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28) | (uint64(buf[5]&0x7F) << 35)
-	case 7:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28) | (uint64(buf[5]&0x7F) << 35) | (uint64(buf[6]&0x7F) << 42)
-	case 8:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28) | (uint64(buf[5]&0x7F) << 35) | (uint64(buf[6]&0x7F) << 42) | (uint64(buf[7]&0x7F) << 49)
-	case 9:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28) | (uint64(buf[5]&0x7F) << 35) | (uint64(buf[6]&0x7F) << 42) | (uint64(buf[7]&0x7F) << 49) | (uint64(buf[8]&0x7F) << 56)
-	case 10:
-		n = uint64(buf[0]&0x7F) | (uint64(buf[1]&0x7F) << 7) | (uint64(buf[2]&0x7F) << 14) | (uint64(buf[3]&0x7F) << 21) | (uint64(buf[4]&0x7F) << 28) | (uint64(buf[5]&0x7F) << 35) | (uint64(buf[6]&0x7F) << 42) | (uint64(buf[7]&0x7F) << 49) | (uint64(buf[8]&0x7F) << 56) | (uint64(buf[9]&0x7F) << 63)
-	default:
-		panic("impossible error: buf length must be 1 to 10")
-	}
-	return int64(n>>1) ^ (int64(n<<63) >> 63)
-}
-
 // MarshalVarUint64 appends marshaled u to dst and returns the result.
 func MarshalVarUint64(dst []byte, u uint64) []byte {
 	var tmp [1]uint64
@@ -209,17 +275,61 @@ func MarshalVarUint64(dst []byte, u uint64) []byte {
 
 // MarshalVarUint64s appends marshaled us to dst and returns the result.
 func MarshalVarUint64s(dst []byte, us []uint64) []byte {
+	dstLen := len(dst)
 	for _, u := range us {
-		if u < 0x80 {
+		if u >= (1 << 7) {
-			// Fast path
+			return marshalVarUint64sSlow(dst[:dstLen], us)
+		}
+		dst = append(dst, byte(u))
+	}
+	return dst
+}
+
+func marshalVarUint64sSlow(dst []byte, us []uint64) []byte {
+	for _, u := range us {
+		// Cases below are sorted in the descending order of frequency on real data
+		if u < (1 << 7) {
 			dst = append(dst, byte(u))
 			continue
 		}
-		for u > 0x7f {
+		if u < (1 << (2 * 7)) {
-			dst = append(dst, 0x80|byte(u))
+			dst = append(dst, byte(u|0x80), byte(u>>7))
-			u >>= 7
+			continue
 		}
-		dst = append(dst, byte(u))
+		if u < (1 << (3 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte(u>>(2*7)))
+			continue
+		}
+
+		if u >= (1 << (8 * 7)) {
+			if u < (1 << (9 * 7)) {
+				dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+					byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte((u>>(7*7))|0x80), byte(u>>(8*7)))
+				continue
+			}
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+				byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte((u>>(7*7))|0x80), byte((u>>(8*7))|0x80), 1)
+			continue
+		}
+
+		if u < (1 << (4 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte(u>>(3*7)))
+			continue
+		}
+		if u < (1 << (5 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte(u>>(4*7)))
+			continue
+		}
+		if u < (1 << (6 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80), byte(u>>(5*7)))
+			continue
+		}
+		if u < (1 << (7 * 7)) {
+			dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80), byte((u>>(5*7))|0x80), byte(u>>(6*7)))
+			continue
+		}
+		dst = append(dst, byte(u|0x80), byte((u>>7)|0x80), byte((u>>(2*7))|0x80), byte((u>>(3*7))|0x80), byte((u>>(4*7))|0x80),
+			byte((u>>(5*7))|0x80), byte((u>>(6*7))|0x80), byte(u>>(7*7)))
 	}
 	return dst
 }
@@ -235,6 +345,20 @@ func UnmarshalVarUint64(src []byte) ([]byte, uint64, error) {
 // UnmarshalVarUint64s unmarshals len(dst) uint64 values from src to dst
 // and returns the remaining tail from src.
 func UnmarshalVarUint64s(dst []uint64, src []byte) ([]byte, error) {
+	if len(src) < len(dst) {
+		return src, fmt.Errorf("too small len(src)=%d; it must be bigger or equal to len(dst)=%d", len(src), len(dst))
+	}
+	for i := range dst {
+		c := src[i]
+		if c >= 0x80 {
+			return unmarshalVarUint64sSlow(dst, src)
+		}
+		dst[i] = uint64(c)
+	}
+	return src[len(dst):], nil
+}
+
+func unmarshalVarUint64sSlow(dst []uint64, src []byte) ([]byte, error) {
 	idx := uint(0)
 	for i := range dst {
 		if idx >= uint(len(src)) {
@@ -243,28 +367,80 @@ func UnmarshalVarUint64s(dst []uint64, src []byte) ([]byte, error) {
 		c := src[idx]
 		idx++
 		if c < 0x80 {
-			// Fast path
+			// Fast path for 1 byte
 			dst[i] = uint64(c)
 			continue
 		}
 
-		// Slow path
+		if idx >= uint(len(src)) {
-		u := uint64(c & 0x7f)
+			return nil, fmt.Errorf("unexpected end of encoded varuint at byte 1; src=%x", src[idx-1:])
-		startIdx := idx - 1
-		shift := uint8(0)
-		for c >= 0x80 {
-			if idx >= uint(len(src)) {
-				return nil, fmt.Errorf("unexpected end of encoded varint at byte %d; src=%x", idx-startIdx, src[startIdx:])
-			}
-			if idx-startIdx > 9 {
-				return src[idx:], fmt.Errorf("too long encoded varint; the maximum allowed length is 10 bytes; got %d bytes; src=%x",
-					(idx-startIdx)+1, src[startIdx:])
-			}
-			c = src[idx]
-			idx++
-			shift += 7
-			u |= uint64(c&0x7f) << shift
 		}
+		d := src[idx]
+		idx++
+		if d < 0x80 {
+			// Fast path for 2 bytes
+			dst[i] = uint64(c&0x7f) | (uint64(d) << 7)
+			continue
+		}
+
+		if idx >= uint(len(src)) {
+			return nil, fmt.Errorf("unexpected end of encoded varuint at byte 2; src=%x", src[idx-2:])
+		}
+		e := src[idx]
+		idx++
+		if e < 0x80 {
+			// Fast path for 3 bytes
+			dst[i] = uint64(c&0x7f) | (uint64(d&0x7f) << 7) | (uint64(e) << (2 * 7))
+			continue
+		}
+
+		u := uint64(c&0x7f) | (uint64(d&0x7f) << 7) | (uint64(e&0x7f) << (2 * 7))
+
+		// Slow path
+		j := idx
+		for {
+			if idx >= uint(len(src)) {
+				return nil, fmt.Errorf("unexpected end of encoded varint; src=%x", src[j-3:])
+			}
+			c := src[idx]
+			idx++
+			if c < 0x80 {
+				break
+			}
+		}
+
+		// These are the most common cases
+		switch idx - j {
+		case 1:
+			u |= (uint64(src[j]) << (3 * 7))
+		case 2:
+			b := src[j : j+2 : j+2]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]) << (4 * 7))
+		case 3:
+			b := src[j : j+3 : j+3]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]) << (5 * 7))
+		case 4:
+			b := src[j : j+4 : j+4]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]) << (6 * 7))
+		case 5:
+			b := src[j : j+5 : j+5]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]) << (7 * 7))
+		case 6:
+			b := src[j : j+6 : j+6]
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]&0x7f) << (7 * 7)) | (uint64(b[5]) << (8 * 7))
+		case 7:
+			b := src[j : j+7 : j+7]
+			if b[6] > 1 {
+				return src[idx:], fmt.Errorf("too big encoded varuint; src=%x", src[j-3:])
+			}
+			u |= (uint64(b[0]&0x7f) << (3 * 7)) | (uint64(b[1]&0x7f) << (4 * 7)) | (uint64(b[2]&0x7f) << (5 * 7)) | (uint64(b[3]&0x7f) << (6 * 7)) |
+				(uint64(b[4]&0x7f) << (7 * 7)) | (uint64(b[5]&0x7f) << (8 * 7)) | (1 << (9 * 7))
+		default:
+			return src[idx:], fmt.Errorf("too long encoded varuint; the maximum allowed length is 10 bytes; got %d bytes; src=%x", idx-j+3, src[j-3:])
+		}
+
 		dst[i] = u
 	}
 	return src[idx:], nil

lib/encoding/int_test.go

@@ -187,6 +187,15 @@ func TestMarshalUnmarshalVarInt64(t *testing.T) {
 	testMarshalUnmarshalVarInt64(t, 0)
 	testMarshalUnmarshalVarInt64(t, 1)
 	testMarshalUnmarshalVarInt64(t, -1)
+	testMarshalUnmarshalVarInt64(t, (1<<6)-1)
+	testMarshalUnmarshalVarInt64(t, (-1<<6)+1)
+	testMarshalUnmarshalVarInt64(t, 1<<6)
+	testMarshalUnmarshalVarInt64(t, -1<<6)
+	testMarshalUnmarshalVarInt64(t, (1<<13)-1)
+	testMarshalUnmarshalVarInt64(t, (-1<<13)+1)
+	testMarshalUnmarshalVarInt64(t, 1<<13)
+	testMarshalUnmarshalVarInt64(t, (1<<13)+1)
+	testMarshalUnmarshalVarInt64(t, -1<<13)
 	testMarshalUnmarshalVarInt64(t, -1<<63)
 	testMarshalUnmarshalVarInt64(t, (-1<<63)+1)
 	testMarshalUnmarshalVarInt64(t, (1<<63)-1)
@@ -202,6 +211,8 @@ func TestMarshalUnmarshalVarInt64(t *testing.T) {
 		testMarshalUnmarshalVarInt64(t, -i<<23)
 		testMarshalUnmarshalVarInt64(t, i<<33)
 		testMarshalUnmarshalVarInt64(t, -i<<33)
+		testMarshalUnmarshalVarInt64(t, i<<35)
+		testMarshalUnmarshalVarInt64(t, -i<<35)
 		testMarshalUnmarshalVarInt64(t, i<<43)
 		testMarshalUnmarshalVarInt64(t, -i<<43)
 		testMarshalUnmarshalVarInt64(t, i<<53)
@@ -237,6 +248,11 @@ func testMarshalUnmarshalVarInt64(t *testing.T, v int64) {
 func TestMarshalUnmarshalVarUint64(t *testing.T) {
 	testMarshalUnmarshalVarUint64(t, 0)
 	testMarshalUnmarshalVarUint64(t, 1)
+	testMarshalUnmarshalVarInt64(t, (1<<6)-1)
+	testMarshalUnmarshalVarInt64(t, 1<<6)
+	testMarshalUnmarshalVarInt64(t, (1<<13)-1)
+	testMarshalUnmarshalVarInt64(t, 1<<13)
+	testMarshalUnmarshalVarInt64(t, (1<<13)+1)
 	testMarshalUnmarshalVarUint64(t, (1<<63)-1)
 
 	for i := uint64(0); i < 1024; i++ {
@@ -245,6 +261,7 @@ func TestMarshalUnmarshalVarUint64(t *testing.T) {
 		testMarshalUnmarshalVarUint64(t, i<<16)
 		testMarshalUnmarshalVarUint64(t, i<<23)
 		testMarshalUnmarshalVarUint64(t, i<<33)
+		testMarshalUnmarshalVarUint64(t, i<<35)
 		testMarshalUnmarshalVarUint64(t, i<<41)
 		testMarshalUnmarshalVarUint64(t, i<<49)
 		testMarshalUnmarshalVarUint64(t, i<<54)