mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/encoding: improve gauge series detection
- Series with negative values are always gauges - Counters may only have increasing values with possible counter resets This should improve compression ratio for gauge series which were previously mistakenly detected as counters.
This commit is contained in:
parent
b335a811c3
commit
3fae34eeb4
2 changed files with 49 additions and 43 deletions
|
@ -296,7 +296,7 @@ func isDeltaConst(a []int64) bool {
|
||||||
// i.e. arbitrary changing values.
|
// i.e. arbitrary changing values.
|
||||||
//
|
//
|
||||||
// It is OK if a few gauges aren't detected (i.e. detected as counters),
|
// It is OK if a few gauges aren't detected (i.e. detected as counters),
|
||||||
// since misdetected counters as gauges are much worse condition.
|
// since misdetected counters as gauges leads to worser compression ratio.
|
||||||
func isGauge(a []int64) bool {
|
func isGauge(a []int64) bool {
|
||||||
// Check all the items in a, since a part of items may lead
|
// Check all the items in a, since a part of items may lead
|
||||||
// to incorrect gauge detection.
|
// to incorrect gauge detection.
|
||||||
|
@ -305,32 +305,36 @@ func isGauge(a []int64) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
extremes := 0
|
resets := 0
|
||||||
plus := a[0] <= a[1]
|
vPrev := a[0]
|
||||||
v1 := a[1]
|
if vPrev < 0 {
|
||||||
for _, v2 := range a[2:] {
|
// Counter values cannot be negative.
|
||||||
if plus {
|
return true
|
||||||
if v2 < v1 {
|
|
||||||
extremes++
|
|
||||||
plus = false
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if v2 > v1 {
|
|
||||||
extremes++
|
|
||||||
plus = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
v1 = v2
|
|
||||||
}
|
}
|
||||||
if extremes <= 2 {
|
for _, v := range a[1:] {
|
||||||
// Probably counter reset.
|
if v < vPrev {
|
||||||
|
if v < 0 {
|
||||||
|
// Counter values cannot be negative.
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if v > (vPrev >> 3) {
|
||||||
|
// Decreasing sequence detected.
|
||||||
|
// This is a gauge.
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Possible counter reset.
|
||||||
|
resets++
|
||||||
|
}
|
||||||
|
vPrev = v
|
||||||
|
}
|
||||||
|
if resets <= 2 {
|
||||||
|
// Counter with a few resets.
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// A few extremes may indicate counter resets.
|
// Let it be a gauge if resets exceeds len(a)/8,
|
||||||
// Let it be a gauge if extremes exceed len(a)/32,
|
// otherwise assume counter.
|
||||||
// otherwise assume counter reset.
|
return resets > (len(a) >> 3)
|
||||||
return extremes > (len(a) >> 5)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func getCompressLevel(itemsCount int) int {
|
func getCompressLevel(itemsCount int) int {
|
||||||
|
|
|
@ -43,27 +43,29 @@ func TestIsDeltaConst(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIsGauge(t *testing.T) {
|
func TestIsGauge(t *testing.T) {
|
||||||
testIsGauge(t, []int64{}, false)
|
f := func(a []int64, okExpected bool) {
|
||||||
testIsGauge(t, []int64{0}, false)
|
t.Helper()
|
||||||
testIsGauge(t, []int64{1, 2}, false)
|
ok := isGauge(a)
|
||||||
testIsGauge(t, []int64{0, 1, 2, 3, 4, 5}, false)
|
if ok != okExpected {
|
||||||
testIsGauge(t, []int64{0, -1, -2, -3, -4}, false)
|
t.Fatalf("unexpected result for isGauge(%d); got %v; expecting %v", a, ok, okExpected)
|
||||||
testIsGauge(t, []int64{0, 0, 0, 0, 0, 0, 0}, false)
|
}
|
||||||
testIsGauge(t, []int64{1, 1, 1, 1, 1}, false)
|
|
||||||
testIsGauge(t, []int64{1, 1, 2, 2, 2, 2}, false)
|
|
||||||
testIsGauge(t, []int64{1, 5, 2, 3}, false) // a single counter reset
|
|
||||||
testIsGauge(t, []int64{1, 5, 2, 3, 2}, true)
|
|
||||||
testIsGauge(t, []int64{-1, -5, -2, -3}, false) // a single counter reset
|
|
||||||
testIsGauge(t, []int64{-1, -5, -2, -3, -2}, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testIsGauge(t *testing.T, a []int64, okExpected bool) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ok := isGauge(a)
|
|
||||||
if ok != okExpected {
|
|
||||||
t.Fatalf("unexpected result for isGauge(%d); got %v; expecting %v", a, ok, okExpected)
|
|
||||||
}
|
}
|
||||||
|
f([]int64{}, false)
|
||||||
|
f([]int64{0}, false)
|
||||||
|
f([]int64{1, 2}, false)
|
||||||
|
f([]int64{0, 1, 2, 3, 4, 5}, false)
|
||||||
|
f([]int64{0, -1, -2, -3, -4}, true)
|
||||||
|
f([]int64{0, 0, 0, 0, 0, 0, 0}, false)
|
||||||
|
f([]int64{1, 1, 1, 1, 1}, false)
|
||||||
|
f([]int64{1, 1, 2, 2, 2, 2}, false)
|
||||||
|
f([]int64{1, 17, 2, 3}, false) // a single counter reset
|
||||||
|
f([]int64{1, 5, 2, 3}, true)
|
||||||
|
f([]int64{1, 5, 2, 3, 2}, true)
|
||||||
|
f([]int64{-1, -5, -2, -3}, true)
|
||||||
|
f([]int64{-1, -5, -2, -3, -2}, true)
|
||||||
|
f([]int64{5, 6, 4, 3, 2}, true)
|
||||||
|
f([]int64{4, 5, 6, 5, 4, 3, 2}, true)
|
||||||
|
f([]int64{1064, 1132, 1083, 1062, 856, 747}, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnsureNonDecreasingSequence(t *testing.T) {
|
func TestEnsureNonDecreasingSequence(t *testing.T) {
|
||||||
|
|
Loading…
Reference in a new issue