diff --git a/lib/encoding/int.go b/lib/encoding/int.go index 110ecaed9..2e31d791c 100644 --- a/lib/encoding/int.go +++ b/lib/encoding/int.go @@ -489,17 +489,19 @@ func MarshalBytes(dst, b []byte) []byte { return dst } -// UnmarshalBytes returns unmarshaled bytes from src. -func UnmarshalBytes(src []byte) ([]byte, []byte, error) { +// UnmarshalBytes returns unmarshaled bytes from src and the size of the unmarshaled bytes. +// +// It returns 0 or negative value if it is impossible to unmarshal bytes from src. +func UnmarshalBytes(src []byte) ([]byte, int) { n, nSize := UnmarshalVarUint64(src) if nSize <= 0 { - return nil, nil, fmt.Errorf("cannot unmarshal string size from uvarint") + return nil, 0 } src = src[nSize:] if uint64(len(src)) < n { - return nil, nil, fmt.Errorf("src is too short for reading string with size %d; len(src)=%d", n, len(src)) + return nil, 0 } - return src[n:], src[:n], nil + return src[:n], nSize + int(n) } // GetInt64s returns an int64 slice with the given size. diff --git a/lib/encoding/int_test.go b/lib/encoding/int_test.go index 72349287f..f8b75c94c 100644 --- a/lib/encoding/int_test.go +++ b/lib/encoding/int_test.go @@ -312,10 +312,11 @@ func testMarshalUnmarshalBytes(t *testing.T, s string) { t.Helper() b := MarshalBytes(nil, []byte(s)) - tail, bNew, err := UnmarshalBytes(b) - if err != nil { - t.Fatalf("unexpected error when unmarshaling s=%q from b=%x: %s", s, b, err) + bNew, nSize := UnmarshalBytes(b) + if nSize <= 0 { + t.Fatalf("unexpected error when unmarshaling s=%q from b=%x", s, b) } + tail := b[nSize:] if string(bNew) != s { t.Fatalf("unexpected sNew from b=%x; got %q; expecting %q", b, bNew, s) } diff --git a/lib/logstorage/block_header.go b/lib/logstorage/block_header.go index 37b471b3c..12011921d 100644 --- a/lib/logstorage/block_header.go +++ b/lib/logstorage/block_header.go @@ -504,12 +504,12 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { srcOrig := src // Unmarshal column name - tail, data, err := encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot unmarshal column name: %w", err) + data, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot unmarshal column name") } + src = src[nSize:] ch.name = a.copyBytesToString(data) - src = tail // Unmarshal value type if len(src) < 1 { @@ -521,13 +521,13 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { // Unmarshal the rest of data depending on valueType switch ch.valueType { case valueTypeString: - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeString for column %q: %w", ch.name, err) } src = tail case valueTypeDict: - tail, err = ch.valuesDict.unmarshal(a, src) + tail, err := ch.valuesDict.unmarshal(a, src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal dict at valueTypeDict for column %q: %w", ch.name, err) } @@ -546,7 +546,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = uint64(src[1]) src = src[2:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeUint8 for column %q: %w", ch.name, err) } @@ -559,7 +559,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = uint64(encoding.UnmarshalUint16(src[2:])) src = src[4:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeUint16 for column %q: %w", ch.name, err) } @@ -572,7 +572,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = uint64(encoding.UnmarshalUint32(src[4:])) src = src[8:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeUint32 for column %q: %w", ch.name, err) } @@ -585,7 +585,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = encoding.UnmarshalUint64(src[8:]) src = src[16:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeUint64 for column %q: %w", ch.name, err) } @@ -599,7 +599,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = encoding.UnmarshalUint64(src[8:]) src = src[16:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeFloat64 for column %q: %w", ch.name, err) } @@ -612,7 +612,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = uint64(encoding.UnmarshalUint32(src[4:])) src = src[8:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeIPv4 for column %q: %w", ch.name, err) } @@ -626,7 +626,7 @@ func (ch *columnHeader) unmarshal(a *arena, src []byte) ([]byte, error) { ch.maxValue = encoding.UnmarshalUint64(src[8:]) src = src[16:] - tail, err = ch.unmarshalValuesAndBloomFilters(src) + tail, err := ch.unmarshalValuesAndBloomFilters(src) if err != nil { return srcOrig, fmt.Errorf("cannot unmarshal values and bloom filters at valueTypeTimestampISO8601 for column %q: %w", ch.name, err) } diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index 88a0bf8d3..3a9d8980b 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -398,12 +398,12 @@ func (psp *pipeStatsProcessor) flush() error { values = values[:0] keyBuf := bytesutil.ToUnsafeBytes(key) for len(keyBuf) > 0 { - tail, v, err := encoding.UnmarshalBytes(keyBuf) - if err != nil { - logger.Panicf("BUG: cannot unmarshal value from keyBuf=%q: %w", keyBuf, err) + v, nSize := encoding.UnmarshalBytes(keyBuf) + if nSize <= 0 { + logger.Panicf("BUG: cannot unmarshal value from keyBuf=%q", keyBuf) } + keyBuf = keyBuf[nSize:] values = append(values, bytesutil.ToUnsafeString(v)) - keyBuf = tail } if len(values) != len(byFields) { logger.Panicf("BUG: unexpected number of values decoded from keyBuf; got %d; want %d", len(values), len(byFields)) diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go index 6c171b931..6cdd46fc7 100644 --- a/lib/logstorage/pipe_uniq.go +++ b/lib/logstorage/pipe_uniq.go @@ -238,17 +238,17 @@ func (pup *pipeUniqProcessor) flush() error { rowFields = rowFields[:0] keyBuf := bytesutil.ToUnsafeBytes(k) for len(keyBuf) > 0 { - tail, name, err := encoding.UnmarshalBytes(keyBuf) - if err != nil { - logger.Panicf("BUG: cannot unmarshal field name: %s", err) + name, nSize := encoding.UnmarshalBytes(keyBuf) + if nSize <= 0 { + logger.Panicf("BUG: cannot unmarshal field name") } - keyBuf = tail + keyBuf = keyBuf[nSize:] - tail, value, err := encoding.UnmarshalBytes(keyBuf) - if err != nil { - logger.Panicf("BUG: cannot unmarshal field value: %s", err) + value, nSize := encoding.UnmarshalBytes(keyBuf) + if nSize <= 0 { + logger.Panicf("BUG: cannot unmarshal field value") } - keyBuf = tail + keyBuf = keyBuf[nSize:] rowFields = append(rowFields, Field{ Name: bytesutil.ToUnsafeString(name), @@ -269,11 +269,11 @@ func (pup *pipeUniqProcessor) flush() error { keyBuf := bytesutil.ToUnsafeBytes(k) fieldIdx := 0 for len(keyBuf) > 0 { - tail, value, err := encoding.UnmarshalBytes(keyBuf) - if err != nil { - logger.Panicf("BUG: cannot unmarshal field value: %s", err) + value, nSize := encoding.UnmarshalBytes(keyBuf) + if nSize <= 0 { + logger.Panicf("BUG: cannot unmarshal field value") } - keyBuf = tail + keyBuf = keyBuf[nSize:] rowFields = append(rowFields, Field{ Name: byFields[fieldIdx], diff --git a/lib/logstorage/rows.go b/lib/logstorage/rows.go index 62ab3c53f..fe7c268fd 100644 --- a/lib/logstorage/rows.go +++ b/lib/logstorage/rows.go @@ -38,20 +38,20 @@ func (f *Field) unmarshal(a *arena, src []byte) ([]byte, error) { srcOrig := src // Unmarshal field name - tail, b, err := encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot unmarshal field name: %w", err) + b, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot unmarshal field name") } + src = src[nSize:] f.Name = a.copyBytesToString(b) - src = tail // Unmarshal field value - tail, b, err = encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot unmarshal field value: %w", err) + b, nSize = encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot unmarshal field value") } + src = src[nSize:] f.Value = a.copyBytesToString(b) - src = tail return src, nil } diff --git a/lib/logstorage/stream_tags.go b/lib/logstorage/stream_tags.go index 38a97f9f7..796bb315b 100644 --- a/lib/logstorage/stream_tags.go +++ b/lib/logstorage/stream_tags.go @@ -125,17 +125,17 @@ func (st *StreamTags) UnmarshalCanonical(src []byte) ([]byte, error) { } src = src[nSize:] for i := uint64(0); i < n; i++ { - tail, name, err := encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot unmarshal tag name: %w", err) + name, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot unmarshal tag name") } - src = tail + src = src[nSize:] - tail, value, err := encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot unmarshal tag value: %w", err) + value, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot unmarshal tag value") } - src = tail + src = src[nSize:] sName := bytesutil.ToUnsafeString(name) sValue := bytesutil.ToUnsafeString(value) diff --git a/lib/logstorage/values_encoder.go b/lib/logstorage/values_encoder.go index cb0cee7b8..8d323aae3 100644 --- a/lib/logstorage/values_encoder.go +++ b/lib/logstorage/values_encoder.go @@ -1145,11 +1145,11 @@ func (vd *valuesDict) unmarshal(a *arena, src []byte) ([]byte, error) { dictLen := int(src[0]) src = src[1:] for i := 0; i < dictLen; i++ { - tail, data, err := encoding.UnmarshalBytes(src) - if err != nil { - return srcOrig, fmt.Errorf("cannot umarshal value %d out of %d from dict: %w", i, dictLen, err) + data, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return srcOrig, fmt.Errorf("cannot umarshal value %d out of %d from dict", i, dictLen) } - src = tail + src = src[nSize:] v := a.copyBytesToString(data) vd.values = append(vd.values, v) diff --git a/lib/mergeset/block_header.go b/lib/mergeset/block_header.go index 817fee7e6..a0fb1af2c 100644 --- a/lib/mergeset/block_header.go +++ b/lib/mergeset/block_header.go @@ -77,20 +77,20 @@ func (bh *blockHeader) Marshal(dst []byte) []byte { func (bh *blockHeader) UnmarshalNoCopy(src []byte) ([]byte, error) { bh.noCopy = true // Unmarshal commonPrefix - tail, cp, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal commonPrefix: %w", err) + cp, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal commonPrefix") } + src = src[nSize:] bh.commonPrefix = cp[:len(cp):len(cp)] - src = tail // Unmarshal firstItem - tail, fi, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal firstItem: %w", err) + fi, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal firstItem") } + src = src[nSize:] bh.firstItem = fi[:len(fi):len(fi)] - src = tail // Unmarshal marshalType if len(src) == 0 { diff --git a/lib/mergeset/metaindex_row.go b/lib/mergeset/metaindex_row.go index e80ba19a4..6e3e02ad5 100644 --- a/lib/mergeset/metaindex_row.go +++ b/lib/mergeset/metaindex_row.go @@ -41,12 +41,12 @@ func (mr *metaindexRow) Marshal(dst []byte) []byte { func (mr *metaindexRow) Unmarshal(src []byte) ([]byte, error) { // Unmarshal firstItem - tail, fi, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal firstItem: %w", err) + fi, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal firstItem") } + src = src[nSize:] mr.firstItem = append(mr.firstItem[:0], fi...) - src = tail // Unmarshal blockHeadersCount if len(src) < 4 { diff --git a/lib/storage/block.go b/lib/storage/block.go index 43635f651..0aa4e0616 100644 --- a/lib/storage/block.go +++ b/lib/storage/block.go @@ -369,15 +369,18 @@ func (b *Block) UnmarshalPortable(src []byte) ([]byte, error) { if err != nil { return src, err } - src, timestampsData, err := encoding.UnmarshalBytes(src) - if err != nil { - return src, fmt.Errorf("cannot read timestampsData: %w", err) + timestampsData, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot read timestampsData") } + src = src[nSize:] b.timestampsData = append(b.timestampsData[:0], timestampsData...) - src, valuesData, err := encoding.UnmarshalBytes(src) - if err != nil { - return src, fmt.Errorf("cannot read valuesData: %w", err) + + valuesData, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot read valuesData") } + src = src[nSize:] b.valuesData = append(b.valuesData[:0], valuesData...) if err := b.bh.validate(); err != nil { diff --git a/lib/storage/search.go b/lib/storage/search.go index 6f93547b0..bd7d87982 100644 --- a/lib/storage/search.go +++ b/lib/storage/search.go @@ -322,19 +322,19 @@ func (tf *TagFilter) Marshal(dst []byte) []byte { // Unmarshal unmarshals tf from src and returns the tail. func (tf *TagFilter) Unmarshal(src []byte) ([]byte, error) { - tail, k, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal Key: %w", err) + k, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal Key") } + src = src[nSize:] tf.Key = append(tf.Key[:0], k...) - src = tail - tail, v, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal Value: %w", err) + v, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal Value") } + src = src[nSize:] tf.Value = append(tf.Value[:0], v...) - src = tail if len(src) < 1 { return src, fmt.Errorf("cannot unmarshal IsNegative+IsRegexp from empty src") diff --git a/lib/storage/storage.go b/lib/storage/storage.go index c3f050f2c..bc3d4a288 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1575,25 +1575,26 @@ func (mr *MetricRow) Marshal(dst []byte) []byte { // // mr refers to src, so it remains valid until src changes. func (mr *MetricRow) UnmarshalX(src []byte) ([]byte, error) { - tail, metricNameRaw, err := encoding.UnmarshalBytes(src) - if err != nil { - return tail, fmt.Errorf("cannot unmarshal MetricName: %w", err) + metricNameRaw, nSize := encoding.UnmarshalBytes(src) + if nSize <= 0 { + return src, fmt.Errorf("cannot unmarshal MetricName") } + tail := src[nSize:] mr.MetricNameRaw = metricNameRaw if len(tail) < 8 { return tail, fmt.Errorf("cannot unmarshal Timestamp: want %d bytes; have %d bytes", 8, len(tail)) } timestamp := encoding.UnmarshalUint64(tail) - mr.Timestamp = int64(timestamp) tail = tail[8:] + mr.Timestamp = int64(timestamp) if len(tail) < 8 { return tail, fmt.Errorf("cannot unmarshal Value: want %d bytes; have %d bytes", 8, len(tail)) } value := encoding.UnmarshalUint64(tail) - mr.Value = math.Float64frombits(value) tail = tail[8:] + mr.Value = math.Float64frombits(value) return tail, nil }