lib/encoding: improve gauge series detection

- Series with negative values are always gauges
- Counters may only have increasing values with possible counter resets

This should improve compression ratio for gauge series which
were previously mistakenly detected as counters.
This commit is contained in:
Aliaksandr Valialkin 2019-07-20 14:02:51 +03:00
parent b335a811c3
commit 3fae34eeb4
2 changed files with 49 additions and 43 deletions

View file

@ -296,7 +296,7 @@ func isDeltaConst(a []int64) bool {
// i.e. arbitrary changing values. // i.e. arbitrary changing values.
// //
// It is OK if a few gauges aren't detected (i.e. detected as counters), // It is OK if a few gauges aren't detected (i.e. detected as counters),
// since misdetected counters as gauges are much worse condition. // since misdetected counters as gauges leads to worser compression ratio.
func isGauge(a []int64) bool { func isGauge(a []int64) bool {
// Check all the items in a, since a part of items may lead // Check all the items in a, since a part of items may lead
// to incorrect gauge detection. // to incorrect gauge detection.
@ -305,32 +305,36 @@ func isGauge(a []int64) bool {
return false return false
} }
extremes := 0 resets := 0
plus := a[0] <= a[1] vPrev := a[0]
v1 := a[1] if vPrev < 0 {
for _, v2 := range a[2:] { // Counter values cannot be negative.
if plus { return true
if v2 < v1 {
extremes++
plus = false
}
} else {
if v2 > v1 {
extremes++
plus = true
}
}
v1 = v2
} }
if extremes <= 2 { for _, v := range a[1:] {
// Probably counter reset. if v < vPrev {
if v < 0 {
// Counter values cannot be negative.
return true
}
if v > (vPrev >> 3) {
// Decreasing sequence detected.
// This is a gauge.
return true
}
// Possible counter reset.
resets++
}
vPrev = v
}
if resets <= 2 {
// Counter with a few resets.
return false return false
} }
// A few extremes may indicate counter resets. // Let it be a gauge if resets exceeds len(a)/8,
// Let it be a gauge if extremes exceed len(a)/32, // otherwise assume counter.
// otherwise assume counter reset. return resets > (len(a) >> 3)
return extremes > (len(a) >> 5)
} }
func getCompressLevel(itemsCount int) int { func getCompressLevel(itemsCount int) int {

View file

@ -43,27 +43,29 @@ func TestIsDeltaConst(t *testing.T) {
} }
func TestIsGauge(t *testing.T) { func TestIsGauge(t *testing.T) {
testIsGauge(t, []int64{}, false) f := func(a []int64, okExpected bool) {
testIsGauge(t, []int64{0}, false) t.Helper()
testIsGauge(t, []int64{1, 2}, false) ok := isGauge(a)
testIsGauge(t, []int64{0, 1, 2, 3, 4, 5}, false) if ok != okExpected {
testIsGauge(t, []int64{0, -1, -2, -3, -4}, false) t.Fatalf("unexpected result for isGauge(%d); got %v; expecting %v", a, ok, okExpected)
testIsGauge(t, []int64{0, 0, 0, 0, 0, 0, 0}, false) }
testIsGauge(t, []int64{1, 1, 1, 1, 1}, false)
testIsGauge(t, []int64{1, 1, 2, 2, 2, 2}, false)
testIsGauge(t, []int64{1, 5, 2, 3}, false) // a single counter reset
testIsGauge(t, []int64{1, 5, 2, 3, 2}, true)
testIsGauge(t, []int64{-1, -5, -2, -3}, false) // a single counter reset
testIsGauge(t, []int64{-1, -5, -2, -3, -2}, true)
}
func testIsGauge(t *testing.T, a []int64, okExpected bool) {
t.Helper()
ok := isGauge(a)
if ok != okExpected {
t.Fatalf("unexpected result for isGauge(%d); got %v; expecting %v", a, ok, okExpected)
} }
f([]int64{}, false)
f([]int64{0}, false)
f([]int64{1, 2}, false)
f([]int64{0, 1, 2, 3, 4, 5}, false)
f([]int64{0, -1, -2, -3, -4}, true)
f([]int64{0, 0, 0, 0, 0, 0, 0}, false)
f([]int64{1, 1, 1, 1, 1}, false)
f([]int64{1, 1, 2, 2, 2, 2}, false)
f([]int64{1, 17, 2, 3}, false) // a single counter reset
f([]int64{1, 5, 2, 3}, true)
f([]int64{1, 5, 2, 3, 2}, true)
f([]int64{-1, -5, -2, -3}, true)
f([]int64{-1, -5, -2, -3, -2}, true)
f([]int64{5, 6, 4, 3, 2}, true)
f([]int64{4, 5, 6, 5, 4, 3, 2}, true)
f([]int64{1064, 1132, 1083, 1062, 856, 747}, true)
} }
func TestEnsureNonDecreasingSequence(t *testing.T) { func TestEnsureNonDecreasingSequence(t *testing.T) {