wip

lib/encoding/encoding.go

@@ -231,12 +231,12 @@ func unmarshalInt64Array(dst []int64, src []byte, mt MarshalType, firstValue int
 		return dst, nil
 	case MarshalTypeDeltaConst:
 		v := firstValue
-		tail, d, err := UnmarshalVarInt64(src)
+		d, nLen := UnmarshalVarInt64(src)
-		if err != nil {
+		if nLen <= 0 {
 			return nil, fmt.Errorf("cannot unmarshal delta value for delta const: %w", err)
 		}
-		if len(tail) > 0 {
+		if nLen < len(src) {
-			return nil, fmt.Errorf("unexpected trailing data after delta const (d=%d): %d bytes", d, len(tail))
+			return nil, fmt.Errorf("unexpected trailing data after delta const (d=%d): %d bytes", d, len(src)-nLen)
 		}
 		for itemsCount > 0 {
 			dst = append(dst, v)

lib/encoding/int.go

@@ -167,16 +167,13 @@ func marshalVarInt64sSlow(dst []byte, vs []int64) []byte {
 	return dst
 }
 
-// UnmarshalVarInt64 returns unmarshaled int64 from src and returns the remaining tail from src.
+// UnmarshalVarInt64 returns unmarshaled int64 from src and its size in bytes.
-func UnmarshalVarInt64(src []byte) ([]byte, int64, error) {
+func UnmarshalVarInt64(src []byte) (int64, int) {
 	// TODO substitute binary.Uvarint with binary.Varint when benchmark results will show it is faster.
 	// It is slower on amd64/linux Go1.22.
-	u64, offset := binary.Uvarint(src)
+	u64, nSize := binary.Uvarint(src)
-	if offset <= 0 {
-		return src, 0, fmt.Errorf("cannot unmarshal varint")
-	}
 	i64 := int64(int64(u64>>1) ^ (int64(u64<<63) >> 63))
-	return src[offset:], i64, nil
+	return i64, nSize
 }
 
 // UnmarshalVarInt64s unmarshals len(dst) int64 values from src to dst and returns the remaining tail from src.
@@ -363,13 +360,9 @@ func marshalVarUint64sSlow(dst []byte, us []uint64) []byte {
 	return dst
 }
 
-// UnmarshalVarUint64 returns unmarshaled uint64 from src and returns the remaining tail from src.
+// UnmarshalVarUint64 returns unmarshaled uint64 from src and its size in bytes
-func UnmarshalVarUint64(src []byte) ([]byte, uint64, error) {
+func UnmarshalVarUint64(src []byte) (uint64, int) {
-	u64, offset := binary.Uvarint(src)
+	return binary.Uvarint(src)
-	if offset <= 0 {
-		return src, 0, fmt.Errorf("cannot read varuint")
-	}
-	return src[offset:], u64, nil
 }
 
 // UnmarshalVarUint64s unmarshals len(dst) uint64 values from src to dst and returns the remaining tail from src.
@@ -498,7 +491,7 @@ func MarshalBytes(dst, b []byte) []byte {
 
 // UnmarshalBytes returns unmarshaled bytes from src.
 func UnmarshalBytes(src []byte) ([]byte, []byte, error) {
-	n, nSize := binary.Uvarint(src)
+	n, nSize := UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return nil, nil, fmt.Errorf("cannot unmarshal string size from uvarint")
 	}

lib/encoding/int_test.go

@@ -224,10 +224,11 @@ func testMarshalUnmarshalVarInt64(t *testing.T, v int64) {
 	t.Helper()
 
 	b := MarshalVarInt64(nil, v)
-	tail, vNew, err := UnmarshalVarInt64(b)
+	vNew, nSize := UnmarshalVarInt64(b)
-	if err != nil {
+	if nSize <= 0 {
-		t.Fatalf("unexpected error when unmarshaling v=%d from b=%x: %s", v, b, err)
+		t.Fatalf("unexpected error when unmarshaling v=%d from b=%x", v, b)
 	}
+	tail := b[nSize:]
 	if vNew != v {
 		t.Fatalf("unexpected vNew from b=%x; got %d; expecting %d", b, vNew, v)
 	}
@@ -272,10 +273,11 @@ func testMarshalUnmarshalVarUint64(t *testing.T, u uint64) {
 	t.Helper()
 
 	b := MarshalVarUint64(nil, u)
-	tail, uNew, err := UnmarshalVarUint64(b)
+	uNew, nSize := UnmarshalVarUint64(b)
-	if err != nil {
+	if nSize <= 0 {
-		t.Fatalf("unexpected error when unmarshaling u=%d from b=%x: %s", u, b, err)
+		t.Fatalf("unexpected error when unmarshaling u=%d from b=%x", u, b)
 	}
+	tail := b[nSize:]
 	if uNew != u {
 		t.Fatalf("unexpected uNew from b=%x; got %d; expecting %d", b, uNew, u)
 	}

lib/encoding/int_timing_test.go

@@ -173,12 +173,12 @@ func benchmarkUnmarshalVarUint64(b *testing.B, maxValue uint64) {
 		for pb.Next() {
 			src := data
 			for len(src) > 0 {
-				tail, n, err := UnmarshalVarUint64(src)
+				n, nSize := UnmarshalVarUint64(src)
-				if err != nil {
+				if nSize <= 0 {
-					panic(fmt.Errorf("unexpected error: %w", err))
+					panic(fmt.Errorf("unexpected error"))
 				}
+				src = src[nSize:]
 				sink += n
-				src = tail
 			}
 		}
 		Sink.Add(sink)
@@ -265,12 +265,12 @@ func benchmarkUnmarshalVarInt64(b *testing.B, maxValue int64) {
 		for pb.Next() {
 			src := data
 			for len(src) > 0 {
-				tail, n, err := UnmarshalVarInt64(src)
+				n, nSize := UnmarshalVarInt64(src)
-				if err != nil {
+				if nSize <= 0 {
-					panic(fmt.Errorf("unexpected error: %w", err))
+					panic(fmt.Errorf("unexpected error"))
 				}
+				src = src[nSize:]
 				sink += uint64(n)
-				src = tail
 			}
 		}
 		Sink.Add(sink)

lib/logstorage/block_header.go

@@ -1,7 +1,6 @@
 package logstorage
 
 import (
-	"encoding/binary"
 	"fmt"
 	"math"
 	"sync"
@@ -86,7 +85,7 @@ func (bh *blockHeader) unmarshal(src []byte) ([]byte, error) {
 	src = tail
 
 	// unmarshal bh.uncompressedSizeBytes
-	n, nSize := binary.Uvarint(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal uncompressedSizeBytes from uvarint")
 	}
@@ -94,7 +93,7 @@ func (bh *blockHeader) unmarshal(src []byte) ([]byte, error) {
 	bh.uncompressedSizeBytes = n
 
 	// unmarshal bh.rowsCount
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal rowsCount from uvarint")
 	}
@@ -112,7 +111,7 @@ func (bh *blockHeader) unmarshal(src []byte) ([]byte, error) {
 	src = tail
 
 	// unmarshal columnsHeaderOffset
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal columnsHeaderOffset from uvarint")
 	}
@@ -120,7 +119,7 @@ func (bh *blockHeader) unmarshal(src []byte) ([]byte, error) {
 	bh.columnsHeaderOffset = n
 
 	// unmarshal columnsHeaderSize
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal columnsHeaderSize from uvarint")
 	}
@@ -297,7 +296,7 @@ func (csh *columnsHeader) unmarshal(a *arena, src []byte) error {
 	csh.reset()
 
 	// unmarshal columnHeaders
-	n, nSize := binary.Uvarint(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return fmt.Errorf("cannot unmarshal columnHeaders len from uvarint")
 	}
@@ -316,7 +315,7 @@ func (csh *columnsHeader) unmarshal(a *arena, src []byte) error {
 	csh.columnHeaders = chs
 
 	// unmarshal constColumns
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return fmt.Errorf("cannot unmarshal constColumns len from uvarint")
 	}
@@ -660,14 +659,14 @@ func (ch *columnHeader) unmarshalValuesAndBloomFilters(src []byte) ([]byte, erro
 func (ch *columnHeader) unmarshalValues(src []byte) ([]byte, error) {
 	srcOrig := src
 
-	n, nSize := binary.Uvarint(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal valuesOffset from uvarint")
 	}
 	src = src[nSize:]
 	ch.valuesOffset = n
 
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal valuesSize from uvarint")
 	}
@@ -683,14 +682,14 @@ func (ch *columnHeader) unmarshalValues(src []byte) ([]byte, error) {
 func (ch *columnHeader) unmarshalBloomFilters(src []byte) ([]byte, error) {
 	srcOrig := src
 
-	n, nSize := binary.Uvarint(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal bloomFilterOffset from uvarint")
 	}
 	src = src[nSize:]
 	ch.bloomFilterOffset = n
 
-	n, nSize = binary.Uvarint(src)
+	n, nSize = encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal bloomFilterSize from uvarint")
 	}

lib/logstorage/encoding.go

@@ -1,7 +1,6 @@
 package logstorage
 
 import (
-	"encoding/binary"
 	"fmt"
 	"sync"
 
@@ -280,7 +279,7 @@ func unmarshalBytesBlock(dst, src []byte) ([]byte, []byte, error) {
 		// Compressed block
 
 		// Read block length
-		blockLen, nSize := binary.Uvarint(src)
+		blockLen, nSize := encoding.UnmarshalVarUint64(src)
 		if nSize <= 0 {
 			return dst, src, fmt.Errorf("cannot unmarshal compressed block size from uvarint")
 		}

lib/logstorage/indexdb.go

@@ -2,7 +2,6 @@ package logstorage
 
 import (
 	"bytes"
-	"encoding/binary"
 	"fmt"
 	"io"
 	"sort"
@@ -508,7 +507,7 @@ func (idb *indexdb) loadStreamIDsFromCache(tenantIDs []TenantID, sf *StreamFilte
 		return nil, false
 	}
 	// Cache hit - unpack streamIDs from data.
-	n, nSize := binary.Uvarint(data)
+	n, nSize := encoding.UnmarshalVarUint64(data)
 	if nSize <= 0 {
 		logger.Panicf("BUG: cannot unmarshal the number of streamIDs from cache")
 	}

lib/logstorage/stream_tags.go

@@ -2,7 +2,6 @@ package logstorage
 
 import (
 	"bytes"
-	"encoding/binary"
 	"fmt"
 	"sort"
 	"strconv"
@@ -120,7 +119,7 @@ func (st *StreamTags) UnmarshalCanonical(src []byte) ([]byte, error) {
 
 	srcOrig := src
 
-	n, nSize := binary.Uvarint(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		return srcOrig, fmt.Errorf("cannot unmarshal tags len from uvarint")
 	}

lib/promutils/labelscompressor.go

@@ -91,10 +91,11 @@ func cloneLabel(label prompbmarshal.Label) prompbmarshal.Label {
 //
 // It is safe calling Decompress from concurrent goroutines.
 func (lc *LabelsCompressor) Decompress(dst []prompbmarshal.Label, src []byte) []prompbmarshal.Label {
-	tail, labelsLen, err := encoding.UnmarshalVarUint64(src)
+	labelsLen, nSize := encoding.UnmarshalVarUint64(src)
-	if err != nil {
+	if nSize <= 0 {
-		logger.Panicf("BUG: cannot unmarshal labels length: %s", err)
+		logger.Panicf("BUG: cannot unmarshal labels length from uvarint")
 	}
+	tail := src[nSize:]
 	if labelsLen == 0 {
 		// fast path - nothing to decode
 		if len(tail) > 0 {
@@ -104,6 +105,7 @@ func (lc *LabelsCompressor) Decompress(dst []prompbmarshal.Label, src []byte) []
 	}
 
 	a := encoding.GetUint64s(int(labelsLen))
+	var err error
 	tail, err = encoding.UnmarshalVarUint64s(a.A, tail)
 	if err != nil {
 		logger.Panicf("BUG: cannot unmarshal label indexes: %s", err)

lib/storage/block_header.go

@@ -167,33 +167,42 @@ func (bh *blockHeader) marshalPortable(dst []byte) []byte {
 }
 
 func (bh *blockHeader) unmarshalPortable(src []byte) ([]byte, error) {
-	src, minTimestamp, err := encoding.UnmarshalVarInt64(src)
+	minTimestamp, nSize := encoding.UnmarshalVarInt64(src)
-	if err != nil {
+	if nSize <= 0 {
-		return src, fmt.Errorf("cannot unmarshal firstTimestamp: %w", err)
+		return src, fmt.Errorf("cannot unmarshal firstTimestamp from varint")
 	}
+	src = src[nSize:]
 	bh.MinTimestamp = minTimestamp
-	src, maxTimestamp, err := encoding.UnmarshalVarInt64(src)
+
-	if err != nil {
+	maxTimestamp, nSize := encoding.UnmarshalVarInt64(src)
-		return src, fmt.Errorf("cannot unmarshal firstTimestamp: %w", err)
+	if nSize <= 0 {
+		return src, fmt.Errorf("cannot unmarshal firstTimestamp rom varint")
 	}
+	src = src[nSize:]
 	bh.MaxTimestamp = maxTimestamp
-	src, firstValue, err := encoding.UnmarshalVarInt64(src)
+
-	if err != nil {
+	firstValue, nSize := encoding.UnmarshalVarInt64(src)
-		return src, fmt.Errorf("cannot unmarshal firstValue: %w", err)
+	if nSize <= 0 {
+		return src, fmt.Errorf("cannot unmarshal firstValue from varint")
 	}
+	src = src[nSize:]
 	bh.FirstValue = firstValue
-	src, rowsCount, err := encoding.UnmarshalVarUint64(src)
+
-	if err != nil {
+	rowsCount, nSize := encoding.UnmarshalVarUint64(src)
-		return src, fmt.Errorf("cannot unmarshal rowsCount: %w", err)
+	if nSize <= 0 {
+		return src, fmt.Errorf("cannot unmarshal rowsCount from varuint")
 	}
+	src = src[nSize:]
 	if rowsCount > math.MaxUint32 {
 		return src, fmt.Errorf("got too big rowsCount=%d; it mustn't exceed %d", rowsCount, uint32(math.MaxUint32))
 	}
 	bh.RowsCount = uint32(rowsCount)
-	src, scale, err := encoding.UnmarshalVarInt64(src)
+
-	if err != nil {
+	scale, nSize := encoding.UnmarshalVarInt64(src)
-		return src, fmt.Errorf("cannot unmarshal scale: %w", err)
+	if nSize <= 0 {
+		return src, fmt.Errorf("cannot unmarshal scale from varint")
 	}
+	src = src[nSize:]
 	if scale < math.MinInt16 {
 		return src, fmt.Errorf("got too small scale=%d; it mustn't be smaller than %d", scale, math.MinInt16)
 	}
@@ -204,6 +213,7 @@ func (bh *blockHeader) unmarshalPortable(src []byte) ([]byte, error) {
 	if len(src) < 1 {
 		return src, fmt.Errorf("cannot unmarshal marshalType for timestamps from %d bytes; need at least %d bytes", len(src), 1)
 	}
+
 	bh.TimestampsMarshalType = encoding.MarshalType(src[0])
 	src = src[1:]
 	if len(src) < 1 {

lib/storage/index_db.go

@@ -2015,11 +2015,11 @@ func removeCompositeTagFilters(tfs []*tagFilter, prefix []byte) []*tagFilter {
 			continue
 		}
 		tagKey = tagKey[1:]
-		var nameLen uint64
+		nameLen, nSize := encoding.UnmarshalVarUint64(tagKey)
-		tagKey, nameLen, err = encoding.UnmarshalVarUint64(tagKey)
+		if nSize <= 0 {
-		if err != nil {
+			logger.Panicf("BUG: cannot unmarshal nameLen from tagKey %q", tagKey)
-			logger.Panicf("BUG: cannot unmarshal nameLen from tagKey %q: %s", tagKey, err)
 		}
+		tagKey = tagKey[nSize:]
 		if nameLen == 0 {
 			logger.Panicf("BUG: nameLen must be greater than 0")
 		}
@@ -2830,11 +2830,11 @@ func unmarshalCompositeTagKey(src []byte) ([]byte, []byte, error) {
 		return nil, nil, fmt.Errorf("missing composite tag key prefix in %q", src)
 	}
 	src = src[1:]
-	tail, n, err := encoding.UnmarshalVarUint64(src)
+	n, nSize := encoding.UnmarshalVarUint64(src)
-	if err != nil {
+	if nSize <= 0 {
-		return nil, nil, fmt.Errorf("cannot unmarshal metric name length from composite tag key: %w", err)
+		return nil, nil, fmt.Errorf("cannot unmarshal metric name length from composite tag key")
 	}
-	src = tail
+	src = src[nSize:]
 	if uint64(len(src)) < n {
 		return nil, nil, fmt.Errorf("missing metric name with length %d in composite tag key %q", n, src)
 	}

lib/storage/search.go

@@ -396,33 +396,33 @@ func (sq *SearchQuery) Marshal(dst []byte) []byte {
 
 // Unmarshal unmarshals sq from src and returns the tail.
 func (sq *SearchQuery) Unmarshal(src []byte) ([]byte, error) {
-	tail, minTs, err := encoding.UnmarshalVarInt64(src)
+	minTs, nSize := encoding.UnmarshalVarInt64(src)
-	if err != nil {
+	if nSize <= 0 {
-		return src, fmt.Errorf("cannot unmarshal MinTimestamp: %w", err)
+		return src, fmt.Errorf("cannot unmarshal MinTimestamp from varint")
 	}
+	src = src[nSize:]
 	sq.MinTimestamp = minTs
-	src = tail
 
-	tail, maxTs, err := encoding.UnmarshalVarInt64(src)
+	maxTs, nSize := encoding.UnmarshalVarInt64(src)
-	if err != nil {
+	if nSize <= 0 {
-		return src, fmt.Errorf("cannot unmarshal MaxTimestamp: %w", err)
+		return src, fmt.Errorf("cannot unmarshal MaxTimestamp from varint")
 	}
+	src = src[nSize:]
 	sq.MaxTimestamp = maxTs
-	src = tail
 
-	tail, tfssCount, err := encoding.UnmarshalVarUint64(src)
+	tfssCount, nSize := encoding.UnmarshalVarUint64(src)
-	if err != nil {
+	if nSize <= 0 {
-		return src, fmt.Errorf("cannot unmarshal the count of TagFilterss: %w", err)
+		return src, fmt.Errorf("cannot unmarshal the count of TagFilterss from uvarint")
 	}
+	src = src[nSize:]
 	sq.TagFilterss = slicesutil.SetLength(sq.TagFilterss, int(tfssCount))
-	src = tail
 
 	for i := 0; i < int(tfssCount); i++ {
-		tail, tfsCount, err := encoding.UnmarshalVarUint64(src)
+		tfsCount, nSize := encoding.UnmarshalVarUint64(src)
-		if err != nil {
+		if nSize <= 0 {
-			return src, fmt.Errorf("cannot unmarshal the count of TagFilters: %w", err)
+			return src, fmt.Errorf("cannot unmarshal the count of TagFilters from uvarint")
 		}
-		src = tail
+		src = src[nSize:]
 
 		tagFilters := sq.TagFilterss[i]
 		tagFilters = slicesutil.SetLength(tagFilters, int(tfsCount))

lib/streamaggr/streamaggr.go

@@ -867,22 +867,23 @@ func decompressLabels(dst []prompbmarshal.Label, key string) []prompbmarshal.Lab
 
 func getOutputKey(key string) string {
 	src := bytesutil.ToUnsafeBytes(key)
-	tail, inputKeyLen, err := encoding.UnmarshalVarUint64(src)
+	inputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
-	if err != nil {
+	if nSize <= 0 {
-		logger.Panicf("BUG: cannot unmarshal inputKeyLen: %s", err)
+		logger.Panicf("BUG: cannot unmarshal inputKeyLen from uvarint")
 	}
-	outputKey := tail[inputKeyLen:]
+	outputKey := src[inputKeyLen:]
 	return bytesutil.ToUnsafeString(outputKey)
 }
 
 func getInputOutputKey(key string) (string, string) {
 	src := bytesutil.ToUnsafeBytes(key)
-	tail, inputKeyLen, err := encoding.UnmarshalVarUint64(src)
+	inputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
-	if err != nil {
+	if nSize <= 0 {
-		logger.Panicf("BUG: cannot unmarshal inputKeyLen: %s", err)
+		logger.Panicf("BUG: cannot unmarshal inputKeyLen from uvarint")
 	}
-	inputKey := tail[:inputKeyLen]
+	src = src[nSize:]
-	outputKey := tail[inputKeyLen:]
+	inputKey := src[:inputKeyLen]
+	outputKey := src[inputKeyLen:]
 	return bytesutil.ToUnsafeString(inputKey), bytesutil.ToUnsafeString(outputKey)
 }