mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-31 15:06:26 +00:00
wip
This commit is contained in:
parent
6f798c628f
commit
639b3091b5
16 changed files with 562 additions and 259 deletions
|
@ -1,7 +1,6 @@
|
||||||
package logstorage
|
package logstorage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
|
||||||
"math"
|
"math"
|
||||||
"slices"
|
"slices"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
@ -524,7 +523,7 @@ func (br *blockResult) getBucketedTimestampValues(bf *byStatsField) []string {
|
||||||
}
|
}
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalTimestampRFC3339Nano(buf, timestamps[i])
|
buf = marshalTimestampRFC3339NanoString(buf, timestamps[i])
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -560,7 +559,7 @@ func (br *blockResult) getBucketedTimestampValues(bf *byStatsField) []string {
|
||||||
timestampPrev = timestamp
|
timestampPrev = timestamp
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalTimestampRFC3339Nano(buf, timestamp)
|
buf = marshalTimestampRFC3339NanoString(buf, timestamp)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -625,9 +624,9 @@ func (br *blockResult) getBucketedUint8Values(encodedValues []string, bf *byStat
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
n := uint64(v[0])
|
n := unmarshalUint8(v)
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint8String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -645,7 +644,7 @@ func (br *blockResult) getBucketedUint8Values(encodedValues []string, bf *byStat
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
n := uint64(v[0])
|
n := uint64(unmarshalUint8(v))
|
||||||
n -= bucketOffsetInt
|
n -= bucketOffsetInt
|
||||||
n -= n % bucketSizeInt
|
n -= n % bucketSizeInt
|
||||||
n += bucketOffsetInt
|
n += bucketOffsetInt
|
||||||
|
@ -657,7 +656,7 @@ func (br *blockResult) getBucketedUint8Values(encodedValues []string, bf *byStat
|
||||||
nPrev = n
|
nPrev = n
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint64String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -683,10 +682,9 @@ func (br *blockResult) getBucketedUint16Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint16(v)
|
||||||
n := uint64(encoding.UnmarshalUint16(b))
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint16String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -704,8 +702,7 @@ func (br *blockResult) getBucketedUint16Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := uint64(unmarshalUint16(v))
|
||||||
n := uint64(encoding.UnmarshalUint16(b))
|
|
||||||
n -= bucketOffsetInt
|
n -= bucketOffsetInt
|
||||||
n -= n % bucketSizeInt
|
n -= n % bucketSizeInt
|
||||||
n += bucketOffsetInt
|
n += bucketOffsetInt
|
||||||
|
@ -717,7 +714,7 @@ func (br *blockResult) getBucketedUint16Values(encodedValues []string, bf *bySta
|
||||||
nPrev = n
|
nPrev = n
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint64String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -743,10 +740,9 @@ func (br *blockResult) getBucketedUint32Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint32(v)
|
||||||
n := uint64(encoding.UnmarshalUint32(b))
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint32String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -764,8 +760,7 @@ func (br *blockResult) getBucketedUint32Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := uint64(unmarshalUint32(v))
|
||||||
n := uint64(encoding.UnmarshalUint32(b))
|
|
||||||
n -= bucketOffsetInt
|
n -= bucketOffsetInt
|
||||||
n -= n % bucketSizeInt
|
n -= n % bucketSizeInt
|
||||||
n += bucketOffsetInt
|
n += bucketOffsetInt
|
||||||
|
@ -777,7 +772,7 @@ func (br *blockResult) getBucketedUint32Values(encodedValues []string, bf *bySta
|
||||||
nPrev = n
|
nPrev = n
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint64String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -803,10 +798,9 @@ func (br *blockResult) getBucketedUint64Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint64String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -824,8 +818,7 @@ func (br *blockResult) getBucketedUint64Values(encodedValues []string, bf *bySta
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
n -= bucketOffsetInt
|
n -= bucketOffsetInt
|
||||||
n -= n % bucketSizeInt
|
n -= n % bucketSizeInt
|
||||||
n += bucketOffsetInt
|
n += bucketOffsetInt
|
||||||
|
@ -837,7 +830,7 @@ func (br *blockResult) getBucketedUint64Values(encodedValues []string, bf *bySta
|
||||||
nPrev = n
|
nPrev = n
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalUint64(buf, n)
|
buf = marshalUint64String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -863,12 +856,10 @@ func (br *blockResult) getBucketedFloat64Values(encodedValues []string, bf *bySt
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
f := unmarshalFloat64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
f := math.Float64frombits(n)
|
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalFloat64(buf, f)
|
buf = marshalFloat64String(buf, f)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -889,9 +880,7 @@ func (br *blockResult) getBucketedFloat64Values(encodedValues []string, bf *bySt
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
f := unmarshalFloat64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
f := math.Float64frombits(n)
|
|
||||||
|
|
||||||
f -= bf.bucketOffset
|
f -= bf.bucketOffset
|
||||||
|
|
||||||
|
@ -909,7 +898,7 @@ func (br *blockResult) getBucketedFloat64Values(encodedValues []string, bf *bySt
|
||||||
fPrev = f
|
fPrev = f
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalFloat64(buf, f)
|
buf = marshalFloat64String(buf, f)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -935,8 +924,9 @@ func (br *blockResult) getBucketedIPv4Values(encodedValues []string, bf *byStats
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ip := unmarshalIPv4(v)
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = toIPv4String(buf, v)
|
buf = marshalIPv4String(buf, ip)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -954,8 +944,7 @@ func (br *blockResult) getBucketedIPv4Values(encodedValues []string, bf *byStats
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalIPv4(v)
|
||||||
n := binary.BigEndian.Uint32(b)
|
|
||||||
n -= bucketOffsetInt
|
n -= bucketOffsetInt
|
||||||
n -= n % bucketSizeInt
|
n -= n % bucketSizeInt
|
||||||
n += bucketOffsetInt
|
n += bucketOffsetInt
|
||||||
|
@ -967,7 +956,7 @@ func (br *blockResult) getBucketedIPv4Values(encodedValues []string, bf *byStats
|
||||||
nPrev = n
|
nPrev = n
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalIPv4(buf, n)
|
buf = marshalIPv4String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -993,11 +982,10 @@ func (br *blockResult) getBucketedTimestampISO8601Values(encodedValues []string,
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalTimestampISO8601(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalTimestampISO8601(buf, int64(n))
|
buf = marshalTimestampISO8601String(buf, n)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -1016,8 +1004,7 @@ func (br *blockResult) getBucketedTimestampISO8601Values(encodedValues []string,
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
timestamp := unmarshalTimestampISO8601(v)
|
||||||
timestamp := int64(encoding.UnmarshalUint64(b))
|
|
||||||
timestamp -= bucketOffsetInt
|
timestamp -= bucketOffsetInt
|
||||||
if bf.bucketSizeStr == "month" {
|
if bf.bucketSizeStr == "month" {
|
||||||
timestamp = truncateTimestampToMonth(timestamp)
|
timestamp = truncateTimestampToMonth(timestamp)
|
||||||
|
@ -1036,7 +1023,7 @@ func (br *blockResult) getBucketedTimestampISO8601Values(encodedValues []string,
|
||||||
timestampPrev = timestamp
|
timestampPrev = timestamp
|
||||||
|
|
||||||
bufLen := len(buf)
|
bufLen := len(buf)
|
||||||
buf = marshalTimestampISO8601(buf, int64(timestamp))
|
buf = marshalTimestampISO8601String(buf, timestamp)
|
||||||
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
s = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||||
valuesBuf = append(valuesBuf, s)
|
valuesBuf = append(valuesBuf, s)
|
||||||
}
|
}
|
||||||
|
@ -1082,7 +1069,7 @@ func (br *blockResult) getBucketedValue(s string, bf *byStatsField) string {
|
||||||
f += bf.bucketOffset
|
f += bf.bucketOffset
|
||||||
|
|
||||||
bufLen := len(br.buf)
|
bufLen := len(br.buf)
|
||||||
br.buf = marshalFloat64(br.buf, f)
|
br.buf = marshalFloat64String(br.buf, f)
|
||||||
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1104,7 +1091,7 @@ func (br *blockResult) getBucketedValue(s string, bf *byStatsField) string {
|
||||||
timestamp += bucketOffset
|
timestamp += bucketOffset
|
||||||
|
|
||||||
bufLen := len(br.buf)
|
bufLen := len(br.buf)
|
||||||
br.buf = marshalTimestampISO8601(br.buf, timestamp)
|
br.buf = marshalTimestampISO8601String(br.buf, timestamp)
|
||||||
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1126,7 +1113,7 @@ func (br *blockResult) getBucketedValue(s string, bf *byStatsField) string {
|
||||||
timestamp += bucketOffset
|
timestamp += bucketOffset
|
||||||
|
|
||||||
bufLen := len(br.buf)
|
bufLen := len(br.buf)
|
||||||
br.buf = marshalTimestampRFC3339Nano(br.buf, timestamp)
|
br.buf = marshalTimestampRFC3339NanoString(br.buf, timestamp)
|
||||||
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1142,7 +1129,7 @@ func (br *blockResult) getBucketedValue(s string, bf *byStatsField) string {
|
||||||
n += bucketOffset
|
n += bucketOffset
|
||||||
|
|
||||||
bufLen := len(br.buf)
|
bufLen := len(br.buf)
|
||||||
br.buf = marshalIPv4(br.buf, n)
|
br.buf = marshalIPv4String(br.buf, n)
|
||||||
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
return bytesutil.ToUnsafeString(br.buf[bufLen:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1488,20 +1475,20 @@ func (c *blockResultColumn) getFloatValueAtRow(rowIdx int) (float64, bool) {
|
||||||
v := c.dictValues[dictIdx]
|
v := c.dictValues[dictIdx]
|
||||||
return tryParseFloat64(v)
|
return tryParseFloat64(v)
|
||||||
case valueTypeUint8:
|
case valueTypeUint8:
|
||||||
return float64(c.encodedValues[rowIdx][0]), true
|
v := c.encodedValues[rowIdx]
|
||||||
|
return float64(unmarshalUint8(v)), true
|
||||||
case valueTypeUint16:
|
case valueTypeUint16:
|
||||||
b := bytesutil.ToUnsafeBytes(c.encodedValues[rowIdx])
|
v := c.encodedValues[rowIdx]
|
||||||
return float64(encoding.UnmarshalUint16(b)), true
|
return float64(unmarshalUint16(v)), true
|
||||||
case valueTypeUint32:
|
case valueTypeUint32:
|
||||||
b := bytesutil.ToUnsafeBytes(c.encodedValues[rowIdx])
|
v := c.encodedValues[rowIdx]
|
||||||
return float64(encoding.UnmarshalUint32(b)), true
|
return float64(unmarshalUint32(v)), true
|
||||||
case valueTypeUint64:
|
case valueTypeUint64:
|
||||||
b := bytesutil.ToUnsafeBytes(c.encodedValues[rowIdx])
|
v := c.encodedValues[rowIdx]
|
||||||
return float64(encoding.UnmarshalUint64(b)), true
|
return float64(unmarshalUint64(v)), true
|
||||||
case valueTypeFloat64:
|
case valueTypeFloat64:
|
||||||
b := bytesutil.ToUnsafeBytes(c.encodedValues[rowIdx])
|
v := c.encodedValues[rowIdx]
|
||||||
n := encoding.UnmarshalUint64(b)
|
f := unmarshalFloat64(v)
|
||||||
f := math.Float64frombits(n)
|
|
||||||
return f, !math.IsNaN(f)
|
return f, !math.IsNaN(f)
|
||||||
case valueTypeIPv4:
|
case valueTypeIPv4:
|
||||||
return 0, false
|
return 0, false
|
||||||
|
@ -1617,36 +1604,31 @@ func (c *blockResultColumn) sumValues(br *blockResult) (float64, int) {
|
||||||
case valueTypeUint8:
|
case valueTypeUint8:
|
||||||
sum := uint64(0)
|
sum := uint64(0)
|
||||||
for _, v := range c.encodedValues {
|
for _, v := range c.encodedValues {
|
||||||
sum += uint64(v[0])
|
sum += uint64(unmarshalUint8(v))
|
||||||
}
|
}
|
||||||
return float64(sum), len(br.timestamps)
|
return float64(sum), len(br.timestamps)
|
||||||
case valueTypeUint16:
|
case valueTypeUint16:
|
||||||
sum := uint64(0)
|
sum := uint64(0)
|
||||||
for _, v := range c.encodedValues {
|
for _, v := range c.encodedValues {
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
sum += uint64(unmarshalUint16(v))
|
||||||
sum += uint64(encoding.UnmarshalUint16(b))
|
|
||||||
}
|
}
|
||||||
return float64(sum), len(br.timestamps)
|
return float64(sum), len(br.timestamps)
|
||||||
case valueTypeUint32:
|
case valueTypeUint32:
|
||||||
sum := uint64(0)
|
sum := uint64(0)
|
||||||
for _, v := range c.encodedValues {
|
for _, v := range c.encodedValues {
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
sum += uint64(unmarshalUint32(v))
|
||||||
sum += uint64(encoding.UnmarshalUint32(b))
|
|
||||||
}
|
}
|
||||||
return float64(sum), len(br.timestamps)
|
return float64(sum), len(br.timestamps)
|
||||||
case valueTypeUint64:
|
case valueTypeUint64:
|
||||||
sum := float64(0)
|
sum := float64(0)
|
||||||
for _, v := range c.encodedValues {
|
for _, v := range c.encodedValues {
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
sum += float64(unmarshalUint64(v))
|
||||||
sum += float64(encoding.UnmarshalUint64(b))
|
|
||||||
}
|
}
|
||||||
return sum, len(br.timestamps)
|
return sum, len(br.timestamps)
|
||||||
case valueTypeFloat64:
|
case valueTypeFloat64:
|
||||||
sum := float64(0)
|
sum := float64(0)
|
||||||
for _, v := range c.encodedValues {
|
for _, v := range c.encodedValues {
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
f := unmarshalFloat64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
f := math.Float64frombits(n)
|
|
||||||
if !math.IsNaN(f) {
|
if !math.IsNaN(f) {
|
||||||
sum += f
|
sum += f
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,7 +91,7 @@ func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *b
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return matchExactPrefix(s, prefix)
|
return matchExactPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -108,7 +108,7 @@ func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefi
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchExactPrefix(s, prefix)
|
return matchExactPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -126,7 +126,7 @@ func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pr
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchExactPrefix(s, prefix)
|
return matchExactPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
|
@ -3,8 +3,6 @@ package logstorage
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,9 +16,9 @@ type filterIPv4Range struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fr *filterIPv4Range) String() string {
|
func (fr *filterIPv4Range) String() string {
|
||||||
minValue := string(encoding.MarshalUint32(nil, fr.minValue))
|
minValue := marshalIPv4String(nil, fr.minValue)
|
||||||
maxValue := string(encoding.MarshalUint32(nil, fr.maxValue))
|
maxValue := marshalIPv4String(nil, fr.maxValue)
|
||||||
return fmt.Sprintf("%sipv4_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), toIPv4String(nil, minValue), toIPv4String(nil, maxValue))
|
return fmt.Sprintf("%sipv4_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), minValue, maxValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fr *filterIPv4Range) apply(bs *blockSearch, bm *bitmap) {
|
func (fr *filterIPv4Range) apply(bs *blockSearch, bm *bitmap) {
|
||||||
|
@ -108,8 +106,7 @@ func matchIPv4ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, m
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of IPv4: got %d; want 4", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of IPv4: got %d; want 4", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalIPv4(v)
|
||||||
n := encoding.UnmarshalUint32(b)
|
|
||||||
return n >= minValue && n <= maxValue
|
return n >= minValue && n <= maxValue
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,7 +3,6 @@ package logstorage
|
||||||
import (
|
import (
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -89,7 +88,7 @@ func matchIPv4ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLen,
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchLenRange(s, minLen, maxLen)
|
return matchLenRange(s, minLen, maxLen)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -103,7 +102,7 @@ func matchFloat64ByLenRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minLe
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchLenRange(s, minLen, maxLen)
|
return matchLenRange(s, minLen, maxLen)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -191,12 +190,10 @@ func matchMinMaxValueLen(ch *columnHeader, minLen, maxLen uint64) bool {
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
defer bbPool.Put(bb)
|
defer bbPool.Put(bb)
|
||||||
|
|
||||||
bb.B = marshalUint64(bb.B[:0], ch.minValue)
|
bb.B = marshalUint64String(bb.B[:0], ch.minValue)
|
||||||
s := bytesutil.ToUnsafeString(bb.B)
|
if maxLen < uint64(len(bb.B)) {
|
||||||
if maxLen < uint64(len(s)) {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
bb.B = marshalUint64(bb.B[:0], ch.maxValue)
|
bb.B = marshalUint64String(bb.B[:0], ch.maxValue)
|
||||||
s = bytesutil.ToUnsafeString(bb.B)
|
return minLen <= uint64(len(bb.B))
|
||||||
return minLen <= uint64(len(s))
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -107,7 +107,7 @@ func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return matchPhrase(s, phrase)
|
return matchPhrase(s, phrase)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -131,7 +131,7 @@ func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase str
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchPhrase(s, phrase)
|
return matchPhrase(s, phrase)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -160,7 +160,7 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchPhrase(s, phrase)
|
return matchPhrase(s, phrase)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -294,26 +294,29 @@ func isMsgFieldName(fieldName string) bool {
|
||||||
return fieldName == "" || fieldName == "_msg"
|
return fieldName == "" || fieldName == "_msg"
|
||||||
}
|
}
|
||||||
|
|
||||||
func toFloat64StringExt(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
func toFloat64String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
bb.B = toFloat64String(bb.B[:0], v)
|
f := unmarshalFloat64(v)
|
||||||
|
bb.B = marshalFloat64String(bb.B[:0], f)
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
||||||
func toIPv4StringExt(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
func toIPv4String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of IPv4: got %d; want 4", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of IPv4: got %d; want 4", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
bb.B = toIPv4String(bb.B[:0], v)
|
ip := unmarshalIPv4(v)
|
||||||
|
bb.B = marshalIPv4String(bb.B[:0], ip)
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
||||||
func toTimestampISO8601StringExt(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
func toTimestampISO8601String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of ISO8601 timestamp: got %d; want 8", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of ISO8601 timestamp: got %d; want 8", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
bb.B = toTimestampISO8601String(bb.B[:0], v)
|
timestamp := unmarshalTimestampISO8601(v)
|
||||||
|
bb.B = marshalTimestampISO8601String(bb.B[:0], timestamp)
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -102,7 +101,7 @@ func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return matchPrefix(s, prefix)
|
return matchPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -123,7 +122,7 @@ func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix str
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchPrefix(s, prefix)
|
return matchPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -151,7 +150,7 @@ func matchFloat64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchPrefix(s, prefix)
|
return matchPrefix(s, prefix)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -321,8 +320,8 @@ func toUint8String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string {
|
||||||
if len(v) != 1 {
|
if len(v) != 1 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
n := uint64(v[0])
|
n := unmarshalUint8(v)
|
||||||
bb.B = marshalUint64(bb.B[:0], n)
|
bb.B = marshalUint8String(bb.B[:0], n)
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -330,9 +329,8 @@ func toUint16String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string
|
||||||
if len(v) != 2 {
|
if len(v) != 2 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint16(v)
|
||||||
n := uint64(encoding.UnmarshalUint16(b))
|
bb.B = marshalUint16String(bb.B[:0], n)
|
||||||
bb.B = marshalUint64(bb.B[:0], n)
|
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -340,9 +338,8 @@ func toUint32String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint32 number: got %d; want 4", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint32 number: got %d; want 4", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint32(v)
|
||||||
n := uint64(encoding.UnmarshalUint32(b))
|
bb.B = marshalUint32String(bb.B[:0], n)
|
||||||
bb.B = marshalUint64(bb.B[:0], n)
|
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -350,8 +347,7 @@ func toUint64String(bs *blockSearch, bb *bytesutil.ByteBuffer, v string) string
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint64 number: got %d; want 8", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint64 number: got %d; want 8", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
bb.B = marshalUint64String(bb.B[:0], n)
|
||||||
bb.B = marshalUint64(bb.B[:0], n)
|
|
||||||
return bytesutil.ToUnsafeString(bb.B)
|
return bytesutil.ToUnsafeString(bb.B)
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,8 +3,6 @@ package logstorage
|
||||||
import (
|
import (
|
||||||
"math"
|
"math"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -83,9 +81,7 @@ func matchFloat64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of floating-point number: got %d; want 8", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
f := unmarshalFloat64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
f := math.Float64frombits(n)
|
|
||||||
return f >= minValue && f <= maxValue
|
return f >= minValue && f <= maxValue
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -118,7 +114,7 @@ func matchUint8ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue,
|
||||||
if len(v) != 1 {
|
if len(v) != 1 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 1", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
n := uint64(v[0])
|
n := uint64(unmarshalUint8(v))
|
||||||
return n >= minValueUint && n <= maxValueUint
|
return n >= minValueUint && n <= maxValueUint
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -135,8 +131,7 @@ func matchUint16ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue,
|
||||||
if len(v) != 2 {
|
if len(v) != 2 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint16 number: got %d; want 2", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := uint64(unmarshalUint16(v))
|
||||||
n := uint64(encoding.UnmarshalUint16(b))
|
|
||||||
return n >= minValueUint && n <= maxValueUint
|
return n >= minValueUint && n <= maxValueUint
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -153,8 +148,7 @@ func matchUint32ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue,
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 4", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 4", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := uint64(unmarshalUint32(v))
|
||||||
n := uint64(encoding.UnmarshalUint32(b))
|
|
||||||
return n >= minValueUint && n <= maxValueUint
|
return n >= minValueUint && n <= maxValueUint
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -171,8 +165,7 @@ func matchUint64ByRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue,
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 8", bs.partPath(), len(v))
|
logger.Panicf("FATAL: %s: unexpected length for binary representation of uint8 number: got %d; want 8", bs.partPath(), len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
return n >= minValueUint && n <= maxValueUint
|
return n >= minValueUint && n <= maxValueUint
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
|
@ -69,7 +69,7 @@ func (fr *filterRegexp) apply(bs *blockSearch, bm *bitmap) {
|
||||||
func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return re.MatchString(s)
|
return re.MatchString(s)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -78,7 +78,7 @@ func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap
|
||||||
func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return re.MatchString(s)
|
return re.MatchString(s)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -87,7 +87,7 @@ func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp
|
||||||
func matchFloat64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
func matchFloat64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexp.Regexp) {
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return re.MatchString(s)
|
return re.MatchString(s)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
|
@ -124,7 +124,7 @@ func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitm
|
||||||
// Slow path - phrases contain incomplete timestamp. Search over string representation of the timestamp.
|
// Slow path - phrases contain incomplete timestamp. Search over string representation of the timestamp.
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return matchSequence(s, phrases)
|
return matchSequence(s, phrases)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -145,7 +145,7 @@ func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases,
|
||||||
// the ip to string before searching for prefix there.
|
// the ip to string before searching for prefix there.
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchSequence(s, phrases)
|
return matchSequence(s, phrases)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -163,7 +163,7 @@ func matchFloat64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phras
|
||||||
// of floating-point numbers :(
|
// of floating-point numbers :(
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchSequence(s, phrases)
|
return matchSequence(s, phrases)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
|
@ -81,7 +81,7 @@ func matchTimestampISO8601ByStringRange(bs *blockSearch, ch *columnHeader, bm *b
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toTimestampISO8601StringExt(bs, bb, v)
|
s := toTimestampISO8601String(bs, bb, v)
|
||||||
return matchStringRange(s, minValue, maxValue)
|
return matchStringRange(s, minValue, maxValue)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -95,7 +95,7 @@ func matchIPv4ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minVa
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toIPv4StringExt(bs, bb, v)
|
s := toIPv4String(bs, bb, v)
|
||||||
return matchStringRange(s, minValue, maxValue)
|
return matchStringRange(s, minValue, maxValue)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
@ -109,7 +109,7 @@ func matchFloat64ByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, mi
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
visitValues(bs, ch, bm, func(v string) bool {
|
visitValues(bs, ch, bm, func(v string) bool {
|
||||||
s := toFloat64StringExt(bs, bb, v)
|
s := toFloat64String(bs, bb, v)
|
||||||
return matchStringRange(s, minValue, maxValue)
|
return matchStringRange(s, minValue, maxValue)
|
||||||
})
|
})
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
|
@ -208,8 +208,8 @@ func (q *Query) String() string {
|
||||||
|
|
||||||
// AddTimeFilter adds global filter _time:[start ... end] to q.
|
// AddTimeFilter adds global filter _time:[start ... end] to q.
|
||||||
func (q *Query) AddTimeFilter(start, end int64) {
|
func (q *Query) AddTimeFilter(start, end int64) {
|
||||||
startStr := marshalTimestampRFC3339Nano(nil, start)
|
startStr := marshalTimestampRFC3339NanoString(nil, start)
|
||||||
endStr := marshalTimestampRFC3339Nano(nil, end)
|
endStr := marshalTimestampRFC3339NanoString(nil, end)
|
||||||
ft := &filterTime{
|
ft := &filterTime{
|
||||||
minTimestamp: start,
|
minTimestamp: start,
|
||||||
maxTimestamp: end,
|
maxTimestamp: end,
|
||||||
|
|
|
@ -1,11 +1,13 @@
|
||||||
package logstorage
|
package logstorage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"math"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
type statsMax struct {
|
type statsMax struct {
|
||||||
|
@ -61,14 +63,14 @@ func (smp *statsMaxProcessor) updateStatsForRow(br *blockResult, rowIdx int) int
|
||||||
// Find the minimum value across all the fields for the given row
|
// Find the minimum value across all the fields for the given row
|
||||||
for _, c := range br.getColumns() {
|
for _, c := range br.getColumns() {
|
||||||
v := c.getValueAtRow(br, rowIdx)
|
v := c.getValueAtRow(br, rowIdx)
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Find the minimum value across the requested fields for the given row
|
// Find the minimum value across the requested fields for the given row
|
||||||
for _, field := range smp.sm.fields {
|
for _, field := range smp.sm.fields {
|
||||||
c := br.getColumnByName(field)
|
c := br.getColumnByName(field)
|
||||||
v := c.getValueAtRow(br, rowIdx)
|
v := c.getValueAtRow(br, rowIdx)
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,17 +80,18 @@ func (smp *statsMaxProcessor) updateStatsForRow(br *blockResult, rowIdx int) int
|
||||||
func (smp *statsMaxProcessor) mergeState(sfp statsProcessor) {
|
func (smp *statsMaxProcessor) mergeState(sfp statsProcessor) {
|
||||||
src := sfp.(*statsMaxProcessor)
|
src := sfp.(*statsMaxProcessor)
|
||||||
if src.hasMax {
|
if src.hasMax {
|
||||||
smp.updateState(src.max)
|
smp.updateStateString(src.max)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (smp *statsMaxProcessor) updateStateForColumn(br *blockResult, c *blockResultColumn) {
|
func (smp *statsMaxProcessor) updateStateForColumn(br *blockResult, c *blockResultColumn) {
|
||||||
|
if len(br.timestamps) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if c.isTime {
|
if c.isTime {
|
||||||
// Special case for time column
|
// Special case for time column
|
||||||
timestamps := br.timestamps
|
timestamps := br.timestamps
|
||||||
if len(timestamps) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
maxTimestamp := timestamps[len(timestamps)-1]
|
maxTimestamp := timestamps[len(timestamps)-1]
|
||||||
for _, timestamp := range timestamps[:len(timestamps)-1] {
|
for _, timestamp := range timestamps[:len(timestamps)-1] {
|
||||||
if timestamp > maxTimestamp {
|
if timestamp > maxTimestamp {
|
||||||
|
@ -97,9 +100,8 @@ func (smp *statsMaxProcessor) updateStateForColumn(br *blockResult, c *blockResu
|
||||||
}
|
}
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
bb.B = marshalTimestampRFC3339Nano(bb.B[:0], maxTimestamp)
|
bb.B = marshalTimestampRFC3339NanoString(bb.B[:0], maxTimestamp)
|
||||||
v := bytesutil.ToUnsafeString(bb.B)
|
smp.updateStateBytes(bb.B)
|
||||||
smp.updateState(v)
|
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -107,16 +109,114 @@ func (smp *statsMaxProcessor) updateStateForColumn(br *blockResult, c *blockResu
|
||||||
if c.isConst {
|
if c.isConst {
|
||||||
// Special case for const column
|
// Special case for const column
|
||||||
v := c.encodedValues[0]
|
v := c.encodedValues[0]
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, v := range c.getValues(br) {
|
switch c.valueType {
|
||||||
smp.updateState(v)
|
case valueTypeString:
|
||||||
|
for _, v := range c.encodedValues {
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
case valueTypeDict:
|
||||||
|
for _, v := range c.dictValues {
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
case valueTypeUint8:
|
||||||
|
maxN := unmarshalUint8(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint8(v)
|
||||||
|
if n > maxN {
|
||||||
|
maxN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint8String(bb.B[:0], maxN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint16:
|
||||||
|
maxN := unmarshalUint16(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint16(v)
|
||||||
|
if n > maxN {
|
||||||
|
maxN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint16String(bb.B[:0], maxN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint32:
|
||||||
|
maxN := unmarshalUint32(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint32(v)
|
||||||
|
if n > maxN {
|
||||||
|
maxN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint32String(bb.B[:0], maxN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint64:
|
||||||
|
maxN := unmarshalUint64(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint64(v)
|
||||||
|
if n > maxN {
|
||||||
|
maxN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint64String(bb.B[:0], maxN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeFloat64:
|
||||||
|
maxF := unmarshalFloat64(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
f := unmarshalFloat64(v)
|
||||||
|
if math.IsNaN(maxF) || f > maxF {
|
||||||
|
maxF = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalFloat64String(bb.B[:0], maxF)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeIPv4:
|
||||||
|
maxIP := unmarshalIPv4(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
ip := unmarshalIPv4(v)
|
||||||
|
if ip > maxIP {
|
||||||
|
maxIP = ip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalIPv4String(bb.B[:0], maxIP)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeTimestampISO8601:
|
||||||
|
maxTimestamp := unmarshalTimestampISO8601(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
timestamp := unmarshalTimestampISO8601(v)
|
||||||
|
if timestamp > maxTimestamp {
|
||||||
|
maxTimestamp = timestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalTimestampISO8601String(bb.B[:0], maxTimestamp)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
default:
|
||||||
|
logger.Panicf("BUG: unknown valueType=%d", c.valueType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (smp *statsMaxProcessor) updateState(v string) {
|
func (smp *statsMaxProcessor) updateStateBytes(b []byte) {
|
||||||
|
v := bytesutil.ToUnsafeString(b)
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (smp *statsMaxProcessor) updateStateString(v string) {
|
||||||
if smp.hasMax && !lessString(smp.max, v) {
|
if smp.hasMax && !lessString(smp.max, v) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,13 @@
|
||||||
package logstorage
|
package logstorage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"math"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
type statsMin struct {
|
type statsMin struct {
|
||||||
|
@ -61,14 +63,14 @@ func (smp *statsMinProcessor) updateStatsForRow(br *blockResult, rowIdx int) int
|
||||||
// Find the minimum value across all the fields for the given row
|
// Find the minimum value across all the fields for the given row
|
||||||
for _, c := range br.getColumns() {
|
for _, c := range br.getColumns() {
|
||||||
v := c.getValueAtRow(br, rowIdx)
|
v := c.getValueAtRow(br, rowIdx)
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Find the minimum value across the requested fields for the given row
|
// Find the minimum value across the requested fields for the given row
|
||||||
for _, field := range smp.sm.fields {
|
for _, field := range smp.sm.fields {
|
||||||
c := br.getColumnByName(field)
|
c := br.getColumnByName(field)
|
||||||
v := c.getValueAtRow(br, rowIdx)
|
v := c.getValueAtRow(br, rowIdx)
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,17 +80,18 @@ func (smp *statsMinProcessor) updateStatsForRow(br *blockResult, rowIdx int) int
|
||||||
func (smp *statsMinProcessor) mergeState(sfp statsProcessor) {
|
func (smp *statsMinProcessor) mergeState(sfp statsProcessor) {
|
||||||
src := sfp.(*statsMinProcessor)
|
src := sfp.(*statsMinProcessor)
|
||||||
if src.hasMin {
|
if src.hasMin {
|
||||||
smp.updateState(src.min)
|
smp.updateStateString(src.min)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (smp *statsMinProcessor) updateStateForColumn(br *blockResult, c *blockResultColumn) {
|
func (smp *statsMinProcessor) updateStateForColumn(br *blockResult, c *blockResultColumn) {
|
||||||
|
if len(br.timestamps) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if c.isTime {
|
if c.isTime {
|
||||||
// Special case for time column
|
// Special case for time column
|
||||||
timestamps := br.timestamps
|
timestamps := br.timestamps
|
||||||
if len(timestamps) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
minTimestamp := timestamps[0]
|
minTimestamp := timestamps[0]
|
||||||
for _, timestamp := range timestamps[1:] {
|
for _, timestamp := range timestamps[1:] {
|
||||||
if timestamp < minTimestamp {
|
if timestamp < minTimestamp {
|
||||||
|
@ -97,9 +100,8 @@ func (smp *statsMinProcessor) updateStateForColumn(br *blockResult, c *blockResu
|
||||||
}
|
}
|
||||||
|
|
||||||
bb := bbPool.Get()
|
bb := bbPool.Get()
|
||||||
bb.B = marshalTimestampRFC3339Nano(bb.B[:0], minTimestamp)
|
bb.B = marshalTimestampRFC3339NanoString(bb.B[:0], minTimestamp)
|
||||||
v := bytesutil.ToUnsafeString(bb.B)
|
smp.updateStateBytes(bb.B)
|
||||||
smp.updateState(v)
|
|
||||||
bbPool.Put(bb)
|
bbPool.Put(bb)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -107,16 +109,114 @@ func (smp *statsMinProcessor) updateStateForColumn(br *blockResult, c *blockResu
|
||||||
if c.isConst {
|
if c.isConst {
|
||||||
// Special case for const column
|
// Special case for const column
|
||||||
v := c.encodedValues[0]
|
v := c.encodedValues[0]
|
||||||
smp.updateState(v)
|
smp.updateStateString(v)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, v := range c.getValues(br) {
|
switch c.valueType {
|
||||||
smp.updateState(v)
|
case valueTypeString:
|
||||||
|
for _, v := range c.encodedValues {
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
case valueTypeDict:
|
||||||
|
for _, v := range c.dictValues {
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
case valueTypeUint8:
|
||||||
|
minN := unmarshalUint8(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint8(v)
|
||||||
|
if n < minN {
|
||||||
|
minN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint8String(bb.B[:0], minN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint16:
|
||||||
|
minN := unmarshalUint16(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint16(v)
|
||||||
|
if n < minN {
|
||||||
|
minN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint16String(bb.B[:0], minN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint32:
|
||||||
|
minN := unmarshalUint32(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint32(v)
|
||||||
|
if n < minN {
|
||||||
|
minN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint32String(bb.B[:0], minN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeUint64:
|
||||||
|
minN := unmarshalUint64(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
n := unmarshalUint64(v)
|
||||||
|
if n < minN {
|
||||||
|
minN = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalUint64String(bb.B[:0], minN)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeFloat64:
|
||||||
|
minF := unmarshalFloat64(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
f := unmarshalFloat64(v)
|
||||||
|
if math.IsNaN(minF) || f < minF {
|
||||||
|
minF = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalFloat64String(bb.B[:0], minF)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeIPv4:
|
||||||
|
minIP := unmarshalIPv4(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
ip := unmarshalIPv4(v)
|
||||||
|
if ip < minIP {
|
||||||
|
minIP = ip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalIPv4String(bb.B[:0], minIP)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
case valueTypeTimestampISO8601:
|
||||||
|
minTimestamp := unmarshalTimestampISO8601(c.encodedValues[0])
|
||||||
|
for _, v := range c.encodedValues[1:] {
|
||||||
|
timestamp := unmarshalTimestampISO8601(v)
|
||||||
|
if timestamp < minTimestamp {
|
||||||
|
minTimestamp = timestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bb := bbPool.Get()
|
||||||
|
bb.B = marshalTimestampISO8601String(bb.B[:0], minTimestamp)
|
||||||
|
smp.updateStateBytes(bb.B)
|
||||||
|
bbPool.Put(bb)
|
||||||
|
default:
|
||||||
|
logger.Panicf("BUG: unknown valueType=%d", c.valueType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (smp *statsMinProcessor) updateState(v string) {
|
func (smp *statsMinProcessor) updateStateBytes(b []byte) {
|
||||||
|
v := bytesutil.ToUnsafeString(b)
|
||||||
|
smp.updateStateString(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (smp *statsMinProcessor) updateStateString(v string) {
|
||||||
if smp.hasMin && !lessString(v, smp.min) {
|
if smp.hasMin && !lessString(v, smp.min) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -168,9 +168,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 1 {
|
if len(v) != 1 {
|
||||||
return fmt.Errorf("unexpected value length for uint8; got %d; want 1", len(v))
|
return fmt.Errorf("unexpected value length for uint8; got %d; want 1", len(v))
|
||||||
}
|
}
|
||||||
n := uint64(v[0])
|
n := unmarshalUint8(v)
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = marshalUint64(dstBuf, n)
|
dstBuf = marshalUint8String(dstBuf, n)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeUint16:
|
case valueTypeUint16:
|
||||||
|
@ -178,10 +178,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 2 {
|
if len(v) != 2 {
|
||||||
return fmt.Errorf("unexpected value length for uint16; got %d; want 2", len(v))
|
return fmt.Errorf("unexpected value length for uint16; got %d; want 2", len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint16(v)
|
||||||
n := uint64(encoding.UnmarshalUint16(b))
|
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = marshalUint64(dstBuf, n)
|
dstBuf = marshalUint16String(dstBuf, n)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeUint32:
|
case valueTypeUint32:
|
||||||
|
@ -189,10 +188,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
return fmt.Errorf("unexpected value length for uint32; got %d; want 4", len(v))
|
return fmt.Errorf("unexpected value length for uint32; got %d; want 4", len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint32(v)
|
||||||
n := uint64(encoding.UnmarshalUint32(b))
|
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = marshalUint64(dstBuf, n)
|
dstBuf = marshalUint32String(dstBuf, n)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeUint64:
|
case valueTypeUint64:
|
||||||
|
@ -200,10 +198,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
||||||
}
|
}
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
n := unmarshalUint64(v)
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = marshalUint64(dstBuf, n)
|
dstBuf = marshalUint64String(dstBuf, n)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeFloat64:
|
case valueTypeFloat64:
|
||||||
|
@ -211,8 +208,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
||||||
}
|
}
|
||||||
|
f := unmarshalFloat64(v)
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = toFloat64String(dstBuf, v)
|
dstBuf = marshalFloat64String(dstBuf, f)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeIPv4:
|
case valueTypeIPv4:
|
||||||
|
@ -220,8 +218,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 4 {
|
if len(v) != 4 {
|
||||||
return fmt.Errorf("unexpected value length for ipv4; got %d; want 4", len(v))
|
return fmt.Errorf("unexpected value length for ipv4; got %d; want 4", len(v))
|
||||||
}
|
}
|
||||||
|
ip := unmarshalIPv4(v)
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = toIPv4String(dstBuf, v)
|
dstBuf = marshalIPv4String(dstBuf, ip)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
case valueTypeTimestampISO8601:
|
case valueTypeTimestampISO8601:
|
||||||
|
@ -229,8 +228,9 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
if len(v) != 8 {
|
if len(v) != 8 {
|
||||||
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
return fmt.Errorf("unexpected value length for uint64; got %d; want 8", len(v))
|
||||||
}
|
}
|
||||||
|
timestamp := unmarshalTimestampISO8601(v)
|
||||||
dstLen := len(dstBuf)
|
dstLen := len(dstBuf)
|
||||||
dstBuf = toTimestampISO8601String(dstBuf, v)
|
dstBuf = marshalTimestampISO8601String(dstBuf, timestamp)
|
||||||
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
values[i] = bytesutil.ToUnsafeString(dstBuf[dstLen:])
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
@ -241,32 +241,6 @@ func (vd *valuesDecoder) decodeInplace(values []string, vt valueType, dictValues
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func toTimestampISO8601String(dst []byte, v string) []byte {
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
dst = marshalTimestampISO8601(dst, int64(n))
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func toIPv4String(dst []byte, v string) []byte {
|
|
||||||
dst = marshalUint64(dst, uint64(v[0]))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64(v[1]))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64(v[2]))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64(v[3]))
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func toFloat64String(dst []byte, v string) []byte {
|
|
||||||
b := bytesutil.ToUnsafeBytes(v)
|
|
||||||
n := encoding.UnmarshalUint64(b)
|
|
||||||
f := math.Float64frombits(n)
|
|
||||||
dst = marshalFloat64(dst, f)
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func getValuesDecoder() *valuesDecoder {
|
func getValuesDecoder() *valuesDecoder {
|
||||||
v := valuesDecoderPool.Get()
|
v := valuesDecoderPool.Get()
|
||||||
if v == nil {
|
if v == nil {
|
||||||
|
@ -356,11 +330,6 @@ func tryParseTimestampRFC3339Nano(s string) (int64, bool) {
|
||||||
return nsecs, true
|
return nsecs, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// marshalTimestampRFC3339Nano appends RFC3339Nano-formatted nsecs to dst and returns the result.
|
|
||||||
func marshalTimestampRFC3339Nano(dst []byte, nsecs int64) []byte {
|
|
||||||
return time.Unix(0, nsecs).UTC().AppendFormat(dst, time.RFC3339Nano)
|
|
||||||
}
|
|
||||||
|
|
||||||
// tryParseTimestampISO8601 parses 'YYYY-MM-DDThh:mm:ss.mssZ' and returns unix timestamp in nanoseconds.
|
// tryParseTimestampISO8601 parses 'YYYY-MM-DDThh:mm:ss.mssZ' and returns unix timestamp in nanoseconds.
|
||||||
//
|
//
|
||||||
// The returned timestamp can be negative if s is smaller than 1970 year.
|
// The returned timestamp can be negative if s is smaller than 1970 year.
|
||||||
|
@ -404,13 +373,6 @@ func tryParseTimestampISO8601(s string) (int64, bool) {
|
||||||
return nsecs, true
|
return nsecs, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// marshalTimestampISO8601 appends ISO8601-formatted nsecs to dst and returns the result.
|
|
||||||
func marshalTimestampISO8601(dst []byte, nsecs int64) []byte {
|
|
||||||
return time.Unix(0, nsecs).UTC().AppendFormat(dst, iso8601Timestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
const iso8601Timestamp = "2006-01-02T15:04:05.000Z"
|
|
||||||
|
|
||||||
// tryParseTimestampSecs parses YYYY-MM-DDTHH:mm:ss into unix timestamp in seconds.
|
// tryParseTimestampSecs parses YYYY-MM-DDTHH:mm:ss into unix timestamp in seconds.
|
||||||
func tryParseTimestampSecs(s string) (int64, bool, string) {
|
func tryParseTimestampSecs(s string) (int64, bool, string) {
|
||||||
// Parse year
|
// Parse year
|
||||||
|
@ -517,11 +479,6 @@ func tryParseUint64(s string) (uint64, bool) {
|
||||||
return n, true
|
return n, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// marshalUint64 appends string representation of n to dst and returns the result.
|
|
||||||
func marshalUint64(dst []byte, n uint64) []byte {
|
|
||||||
return strconv.AppendUint(dst, n, 10)
|
|
||||||
}
|
|
||||||
|
|
||||||
func tryIPv4Encoding(dstBuf []byte, dstValues, srcValues []string) ([]byte, []string, valueType, uint64, uint64) {
|
func tryIPv4Encoding(dstBuf []byte, dstValues, srcValues []string) ([]byte, []string, valueType, uint64, uint64) {
|
||||||
u32s := encoding.GetUint32s(len(srcValues))
|
u32s := encoding.GetUint32s(len(srcValues))
|
||||||
defer encoding.PutUint32s(u32s)
|
defer encoding.PutUint32s(u32s)
|
||||||
|
@ -607,18 +564,6 @@ func tryParseIPv4(s string) (uint32, bool) {
|
||||||
return ipv4, true
|
return ipv4, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// marshalIPv4 appends string representation of IPv4 address in n to dst and returns the result.
|
|
||||||
func marshalIPv4(dst []byte, n uint32) []byte {
|
|
||||||
dst = marshalUint64(dst, uint64(n>>24))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64((n>>16)&0xff))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64((n>>8)&0xff))
|
|
||||||
dst = append(dst, '.')
|
|
||||||
dst = marshalUint64(dst, uint64(n&0xff))
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func tryFloat64Encoding(dstBuf []byte, dstValues, srcValues []string) ([]byte, []string, valueType, uint64, uint64) {
|
func tryFloat64Encoding(dstBuf []byte, dstValues, srcValues []string) ([]byte, []string, valueType, uint64, uint64) {
|
||||||
u64s := encoding.GetUint64s(len(srcValues))
|
u64s := encoding.GetUint64s(len(srcValues))
|
||||||
defer encoding.PutUint64s(u64s)
|
defer encoding.PutUint64s(u64s)
|
||||||
|
@ -710,11 +655,6 @@ func tryParseFloat64(s string) (float64, bool) {
|
||||||
return f, true
|
return f, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// marshalFloat64 appends formatted f to dst and returns the result.
|
|
||||||
func marshalFloat64(dst []byte, f float64) []byte {
|
|
||||||
return strconv.AppendFloat(dst, f, 'f', -1, 64)
|
|
||||||
}
|
|
||||||
|
|
||||||
// tryParseBytes parses user-readable bytes representation in s.
|
// tryParseBytes parses user-readable bytes representation in s.
|
||||||
//
|
//
|
||||||
// Supported suffixes:
|
// Supported suffixes:
|
||||||
|
@ -935,53 +875,53 @@ func marshalDuration(dst []byte, nsecs int64) []byte {
|
||||||
if nsecs >= nsecsPerWeek {
|
if nsecs >= nsecsPerWeek {
|
||||||
weeks := nsecs / nsecsPerWeek
|
weeks := nsecs / nsecsPerWeek
|
||||||
nsecs -= weeks * nsecsPerWeek
|
nsecs -= weeks * nsecsPerWeek
|
||||||
dst = marshalUint64(dst, uint64(weeks))
|
dst = marshalUint64String(dst, uint64(weeks))
|
||||||
dst = append(dst, 'w')
|
dst = append(dst, 'w')
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerDay {
|
if nsecs >= nsecsPerDay {
|
||||||
days := nsecs / nsecsPerDay
|
days := nsecs / nsecsPerDay
|
||||||
nsecs -= days * nsecsPerDay
|
nsecs -= days * nsecsPerDay
|
||||||
dst = marshalUint64(dst, uint64(days))
|
dst = marshalUint8String(dst, uint8(days))
|
||||||
dst = append(dst, 'd')
|
dst = append(dst, 'd')
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerHour {
|
if nsecs >= nsecsPerHour {
|
||||||
hours := nsecs / nsecsPerHour
|
hours := nsecs / nsecsPerHour
|
||||||
nsecs -= hours * nsecsPerHour
|
nsecs -= hours * nsecsPerHour
|
||||||
dst = marshalUint64(dst, uint64(hours))
|
dst = marshalUint8String(dst, uint8(hours))
|
||||||
dst = append(dst, 'h')
|
dst = append(dst, 'h')
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerMinute {
|
if nsecs >= nsecsPerMinute {
|
||||||
minutes := nsecs / nsecsPerMinute
|
minutes := nsecs / nsecsPerMinute
|
||||||
nsecs -= minutes * nsecsPerMinute
|
nsecs -= minutes * nsecsPerMinute
|
||||||
dst = marshalUint64(dst, uint64(minutes))
|
dst = marshalUint8String(dst, uint8(minutes))
|
||||||
dst = append(dst, 'm')
|
dst = append(dst, 'm')
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerSecond {
|
if nsecs >= nsecsPerSecond {
|
||||||
if formatFloat64Seconds {
|
if formatFloat64Seconds {
|
||||||
seconds := float64(nsecs) / nsecsPerSecond
|
seconds := float64(nsecs) / nsecsPerSecond
|
||||||
dst = marshalFloat64(dst, seconds)
|
dst = marshalFloat64String(dst, seconds)
|
||||||
dst = append(dst, 's')
|
dst = append(dst, 's')
|
||||||
return dst
|
return dst
|
||||||
}
|
}
|
||||||
seconds := nsecs / nsecsPerSecond
|
seconds := nsecs / nsecsPerSecond
|
||||||
nsecs -= seconds * nsecsPerSecond
|
nsecs -= seconds * nsecsPerSecond
|
||||||
dst = marshalUint64(dst, uint64(seconds))
|
dst = marshalUint8String(dst, uint8(seconds))
|
||||||
dst = append(dst, 's')
|
dst = append(dst, 's')
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerMillisecond {
|
if nsecs >= nsecsPerMillisecond {
|
||||||
msecs := nsecs / nsecsPerMillisecond
|
msecs := nsecs / nsecsPerMillisecond
|
||||||
nsecs -= msecs * nsecsPerMillisecond
|
nsecs -= msecs * nsecsPerMillisecond
|
||||||
dst = marshalUint64(dst, uint64(msecs))
|
dst = marshalUint16String(dst, uint16(msecs))
|
||||||
dst = append(dst, "ms"...)
|
dst = append(dst, "ms"...)
|
||||||
}
|
}
|
||||||
if nsecs >= nsecsPerMicrosecond {
|
if nsecs >= nsecsPerMicrosecond {
|
||||||
usecs := nsecs / nsecsPerMicrosecond
|
usecs := nsecs / nsecsPerMicrosecond
|
||||||
nsecs -= usecs * nsecsPerMicrosecond
|
nsecs -= usecs * nsecsPerMicrosecond
|
||||||
dst = marshalUint64(dst, uint64(usecs))
|
dst = marshalUint16String(dst, uint16(usecs))
|
||||||
dst = append(dst, "µs"...)
|
dst = append(dst, "µs"...)
|
||||||
}
|
}
|
||||||
if nsecs > 0 {
|
if nsecs > 0 {
|
||||||
dst = marshalUint64(dst, uint64(nsecs))
|
dst = marshalUint16String(dst, uint16(nsecs))
|
||||||
dst = append(dst, "ns"...)
|
dst = append(dst, "ns"...)
|
||||||
}
|
}
|
||||||
return dst
|
return dst
|
||||||
|
@ -1156,3 +1096,96 @@ func (vd *valuesDict) unmarshal(a *arena, src []byte) ([]byte, error) {
|
||||||
}
|
}
|
||||||
return src, nil
|
return src, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func unmarshalUint8(v string) uint8 {
|
||||||
|
return uint8(v[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalUint16(v string) uint16 {
|
||||||
|
b := bytesutil.ToUnsafeBytes(v)
|
||||||
|
return encoding.UnmarshalUint16(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalUint32(v string) uint32 {
|
||||||
|
b := bytesutil.ToUnsafeBytes(v)
|
||||||
|
return encoding.UnmarshalUint32(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalUint64(v string) uint64 {
|
||||||
|
b := bytesutil.ToUnsafeBytes(v)
|
||||||
|
return encoding.UnmarshalUint64(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalFloat64(v string) float64 {
|
||||||
|
n := unmarshalUint64(v)
|
||||||
|
return math.Float64frombits(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalIPv4(v string) uint32 {
|
||||||
|
return unmarshalUint32(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalTimestampISO8601(v string) int64 {
|
||||||
|
n := unmarshalUint64(v)
|
||||||
|
return int64(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalUint8String(dst []byte, n uint8) []byte {
|
||||||
|
if n < 10 {
|
||||||
|
return append(dst, '0'+n)
|
||||||
|
}
|
||||||
|
if n < 100 {
|
||||||
|
return append(dst, '0'+n/10, '0'+n%10)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n < 200 {
|
||||||
|
dst = append(dst, '1')
|
||||||
|
n -= 100
|
||||||
|
} else {
|
||||||
|
dst = append(dst, '2')
|
||||||
|
n -= 200
|
||||||
|
}
|
||||||
|
if n < 10 {
|
||||||
|
return append(dst, '0', '0'+n)
|
||||||
|
}
|
||||||
|
return append(dst, '0'+n/10, '0'+n%10)
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalUint16String(dst []byte, n uint16) []byte {
|
||||||
|
return marshalUint64String(dst, uint64(n))
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalUint32String(dst []byte, n uint32) []byte {
|
||||||
|
return marshalUint64String(dst, uint64(n))
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalUint64String(dst []byte, n uint64) []byte {
|
||||||
|
return strconv.AppendUint(dst, n, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalFloat64String(dst []byte, f float64) []byte {
|
||||||
|
return strconv.AppendFloat(dst, f, 'f', -1, 64)
|
||||||
|
}
|
||||||
|
|
||||||
|
func marshalIPv4String(dst []byte, n uint32) []byte {
|
||||||
|
dst = marshalUint8String(dst, uint8(n>>24))
|
||||||
|
dst = append(dst, '.')
|
||||||
|
dst = marshalUint8String(dst, uint8(n>>16))
|
||||||
|
dst = append(dst, '.')
|
||||||
|
dst = marshalUint8String(dst, uint8(n>>8))
|
||||||
|
dst = append(dst, '.')
|
||||||
|
dst = marshalUint8String(dst, uint8(n))
|
||||||
|
return dst
|
||||||
|
}
|
||||||
|
|
||||||
|
// marshalTimestampISO8601String appends ISO8601-formatted nsecs to dst and returns the result.
|
||||||
|
func marshalTimestampISO8601String(dst []byte, nsecs int64) []byte {
|
||||||
|
return time.Unix(0, nsecs).UTC().AppendFormat(dst, iso8601Timestamp)
|
||||||
|
}
|
||||||
|
|
||||||
|
const iso8601Timestamp = "2006-01-02T15:04:05.000Z"
|
||||||
|
|
||||||
|
// marshalTimestampRFC3339NanoString appends RFC3339Nano-formatted nsecs to dst and returns the result.
|
||||||
|
func marshalTimestampRFC3339NanoString(dst []byte, nsecs int64) []byte {
|
||||||
|
return time.Unix(0, nsecs).UTC().AppendFormat(dst, time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -96,7 +97,7 @@ func TestValuesEncoder(t *testing.T) {
|
||||||
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
|
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTryParseIPv4_Success(t *testing.T) {
|
func TestTryParseIPv4String_Success(t *testing.T) {
|
||||||
f := func(s string) {
|
f := func(s string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
@ -104,7 +105,7 @@ func TestTryParseIPv4_Success(t *testing.T) {
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("cannot parse %q", s)
|
t.Fatalf("cannot parse %q", s)
|
||||||
}
|
}
|
||||||
data := marshalIPv4(nil, n)
|
data := marshalIPv4String(nil, n)
|
||||||
if string(data) != s {
|
if string(data) != s {
|
||||||
t.Fatalf("unexpected ip; got %q; want %q", data, s)
|
t.Fatalf("unexpected ip; got %q; want %q", data, s)
|
||||||
}
|
}
|
||||||
|
@ -147,14 +148,14 @@ func TestTryParseIPv4_Failure(t *testing.T) {
|
||||||
f("127.127.127.-1")
|
f("127.127.127.-1")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTryParseTimestampRFC3339Nano_Success(t *testing.T) {
|
func TestTryParseTimestampRFC3339NanoString_Success(t *testing.T) {
|
||||||
f := func(s string) {
|
f := func(s string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
nsecs, ok := tryParseTimestampRFC3339Nano(s)
|
nsecs, ok := tryParseTimestampRFC3339Nano(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("cannot parse timestamp %q", s)
|
t.Fatalf("cannot parse timestamp %q", s)
|
||||||
}
|
}
|
||||||
data := marshalTimestampRFC3339Nano(nil, nsecs)
|
data := marshalTimestampRFC3339NanoString(nil, nsecs)
|
||||||
if string(data) != s {
|
if string(data) != s {
|
||||||
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
|
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
|
||||||
}
|
}
|
||||||
|
@ -236,14 +237,14 @@ func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
|
||||||
f("2023-01-23T23:33:ssZ")
|
f("2023-01-23T23:33:ssZ")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTryParseTimestampISO8601_Success(t *testing.T) {
|
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
|
||||||
f := func(s string) {
|
f := func(s string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
nsecs, ok := tryParseTimestampISO8601(s)
|
nsecs, ok := tryParseTimestampISO8601(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("cannot parse timestamp %q", s)
|
t.Fatalf("cannot parse timestamp %q", s)
|
||||||
}
|
}
|
||||||
data := marshalTimestampISO8601(nil, nsecs)
|
data := marshalTimestampISO8601String(nil, nsecs)
|
||||||
if string(data) != s {
|
if string(data) != s {
|
||||||
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
|
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
|
||||||
}
|
}
|
||||||
|
@ -554,11 +555,11 @@ func TestTryParseFloat64_Failure(t *testing.T) {
|
||||||
f("12-5")
|
f("12-5")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMarshalFloat64(t *testing.T) {
|
func TestMarshalFloat64String(t *testing.T) {
|
||||||
f := func(f float64, resultExpected string) {
|
f := func(f float64, resultExpected string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
result := marshalFloat64(nil, f)
|
result := marshalFloat64String(nil, f)
|
||||||
if string(result) != resultExpected {
|
if string(result) != resultExpected {
|
||||||
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
||||||
}
|
}
|
||||||
|
@ -618,11 +619,79 @@ func TestTryParseUint64_Failure(t *testing.T) {
|
||||||
f("foo")
|
f("foo")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMarshalUint64(t *testing.T) {
|
func TestMarshalUint8String(t *testing.T) {
|
||||||
|
f := func(n uint8, resultExpected string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
result := marshalUint8String(nil, n)
|
||||||
|
if string(result) != resultExpected {
|
||||||
|
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < 256; i++ {
|
||||||
|
resultExpected := strconv.Itoa(i)
|
||||||
|
f(uint8(i), resultExpected)
|
||||||
|
}
|
||||||
|
|
||||||
|
// the maximum possible value
|
||||||
|
f(math.MaxUint8, "255")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUint16String(t *testing.T) {
|
||||||
|
f := func(n uint16, resultExpected string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
result := marshalUint16String(nil, n)
|
||||||
|
if string(result) != resultExpected {
|
||||||
|
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
f(0, "0")
|
||||||
|
f(1, "1")
|
||||||
|
f(10, "10")
|
||||||
|
f(12, "12")
|
||||||
|
f(120, "120")
|
||||||
|
f(1203, "1203")
|
||||||
|
f(12345, "12345")
|
||||||
|
|
||||||
|
// the maximum possible value
|
||||||
|
f(math.MaxUint16, "65535")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUint32String(t *testing.T) {
|
||||||
|
f := func(n uint32, resultExpected string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
result := marshalUint32String(nil, n)
|
||||||
|
if string(result) != resultExpected {
|
||||||
|
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
f(0, "0")
|
||||||
|
f(1, "1")
|
||||||
|
f(10, "10")
|
||||||
|
f(12, "12")
|
||||||
|
f(120, "120")
|
||||||
|
f(1203, "1203")
|
||||||
|
f(12034, "12034")
|
||||||
|
f(123456, "123456")
|
||||||
|
f(1234567, "1234567")
|
||||||
|
f(12345678, "12345678")
|
||||||
|
f(123456789, "123456789")
|
||||||
|
f(1234567890, "1234567890")
|
||||||
|
|
||||||
|
// the maximum possible value
|
||||||
|
f(math.MaxUint32, "4294967295")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUint64String(t *testing.T) {
|
||||||
f := func(n uint64, resultExpected string) {
|
f := func(n uint64, resultExpected string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
result := marshalUint64(nil, n)
|
result := marshalUint64String(nil, n)
|
||||||
if string(result) != resultExpected {
|
if string(result) != resultExpected {
|
||||||
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
|
||||||
}
|
}
|
||||||
|
@ -632,8 +701,7 @@ func TestMarshalUint64(t *testing.T) {
|
||||||
f(123456, "123456")
|
f(123456, "123456")
|
||||||
|
|
||||||
// the maximum possible value
|
// the maximum possible value
|
||||||
f(18446744073709551615, "18446744073709551615")
|
f(math.MaxUint64, "18446744073709551615")
|
||||||
f(18_446_744_073_709_551_615, "18446744073709551615")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTryParseIPv4Mask_Success(t *testing.T) {
|
func TestTryParseIPv4Mask_Success(t *testing.T) {
|
||||||
|
|
|
@ -2,6 +2,7 @@ package logstorage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -17,14 +18,17 @@ func BenchmarkTryParseTimestampRFC3339Nano(b *testing.B) {
|
||||||
b.SetBytes(int64(len(a)))
|
b.SetBytes(int64(len(a)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
nSum := int64(0)
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
for _, s := range a {
|
for _, s := range a {
|
||||||
_, ok := tryParseTimestampRFC3339Nano(s)
|
n, ok := tryParseTimestampRFC3339Nano(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
||||||
}
|
}
|
||||||
|
nSum += n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
GlobalSink.Add(uint64(nSum))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,14 +44,17 @@ func BenchmarkTryParseTimestampISO8601(b *testing.B) {
|
||||||
b.SetBytes(int64(len(a)))
|
b.SetBytes(int64(len(a)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
nSum := int64(0)
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
for _, s := range a {
|
for _, s := range a {
|
||||||
_, ok := tryParseTimestampISO8601(s)
|
n, ok := tryParseTimestampISO8601(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
||||||
}
|
}
|
||||||
|
nSum += n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
GlobalSink.Add(uint64(nSum))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -63,14 +70,17 @@ func BenchmarkTryParseIPv4(b *testing.B) {
|
||||||
b.SetBytes(int64(len(a)))
|
b.SetBytes(int64(len(a)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
nSum := uint32(0)
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
for _, s := range a {
|
for _, s := range a {
|
||||||
_, ok := tryParseIPv4(s)
|
n, ok := tryParseIPv4(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Errorf("cannot parse ipv4 %q", s))
|
panic(fmt.Errorf("cannot parse ipv4 %q", s))
|
||||||
}
|
}
|
||||||
|
nSum += n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
GlobalSink.Add(uint64(nSum))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,14 +96,17 @@ func BenchmarkTryParseUint64(b *testing.B) {
|
||||||
b.SetBytes(int64(len(a)))
|
b.SetBytes(int64(len(a)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
var nSum uint64
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
for _, s := range a {
|
for _, s := range a {
|
||||||
_, ok := tryParseUint64(s)
|
n, ok := tryParseUint64(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Errorf("cannot parse uint %q", s))
|
panic(fmt.Errorf("cannot parse uint %q", s))
|
||||||
}
|
}
|
||||||
|
nSum += n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
GlobalSink.Add(nSum)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -109,13 +122,34 @@ func BenchmarkTryParseFloat64(b *testing.B) {
|
||||||
b.SetBytes(int64(len(a)))
|
b.SetBytes(int64(len(a)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
var fSum float64
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
for _, s := range a {
|
for _, s := range a {
|
||||||
_, ok := tryParseFloat64(s)
|
f, ok := tryParseFloat64(s)
|
||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Errorf("cannot parse float64 %q", s))
|
panic(fmt.Errorf("cannot parse float64 %q", s))
|
||||||
}
|
}
|
||||||
|
fSum += f
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
GlobalSink.Add(uint64(fSum))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalUint8String(b *testing.B) {
|
||||||
|
b.SetBytes(256)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
var buf []byte
|
||||||
|
n := 0
|
||||||
|
for pb.Next() {
|
||||||
|
for i := 0; i < 256; i++ {
|
||||||
|
buf = marshalUint8String(buf[:0], uint8(i))
|
||||||
|
n += len(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
GlobalSink.Add(uint64(n))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
var GlobalSink atomic.Uint64
|
||||||
|
|
Loading…
Reference in a new issue