mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
7c0ae3a86a
The change leaves raw sample with the biggest value for identical timestamps per each `-dedup.minScrapeInterval` discrete interval when the deduplication is enabled. ``` benchstat old.txt new.txt name old time/op new time/op delta DeduplicateSamples/minScrapeInterval=1s-10 817ns ± 2% 832ns ± 3% ~ (p=0.052 n=10+10) DeduplicateSamples/minScrapeInterval=2s-10 1.56µs ± 1% 2.12µs ± 0% +35.19% (p=0.000 n=9+7) DeduplicateSamples/minScrapeInterval=5s-10 1.32µs ± 3% 1.65µs ± 2% +25.57% (p=0.000 n=10+10) DeduplicateSamples/minScrapeInterval=10s-10 1.13µs ± 2% 1.50µs ± 1% +32.85% (p=0.000 n=10+10) name old speed new speed delta DeduplicateSamples/minScrapeInterval=1s-10 10.0GB/s ± 2% 9.9GB/s ± 3% ~ (p=0.052 n=10+10) DeduplicateSamples/minScrapeInterval=2s-10 5.24GB/s ± 1% 3.87GB/s ± 0% -26.03% (p=0.000 n=9+7) DeduplicateSamples/minScrapeInterval=5s-10 6.22GB/s ± 3% 4.96GB/s ± 2% -20.37% (p=0.000 n=10+10) DeduplicateSamples/minScrapeInterval=10s-10 7.28GB/s ± 2% 5.48GB/s ± 1% -24.74% (p=0.000 n=10+10) ``` https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333 Signed-off-by: hagen1778 <roman@victoriametrics.com> Signed-off-by: hagen1778 <roman@victoriametrics.com>
179 lines
4.1 KiB
Go
179 lines
4.1 KiB
Go
package netstorage
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
)
|
|
|
|
func TestMergeSortBlocks(t *testing.T) {
|
|
f := func(blocks []*sortBlock, dedupInterval int64, expectedResult *Result) {
|
|
t.Helper()
|
|
var result Result
|
|
mergeSortBlocks(&result, blocks, dedupInterval)
|
|
if !reflect.DeepEqual(result.Values, expectedResult.Values) {
|
|
t.Fatalf("unexpected values;\ngot\n%v\nwant\n%v", result.Values, expectedResult.Values)
|
|
}
|
|
if !reflect.DeepEqual(result.Timestamps, expectedResult.Timestamps) {
|
|
t.Fatalf("unexpected timestamps;\ngot\n%v\nwant\n%v", result.Timestamps, expectedResult.Timestamps)
|
|
}
|
|
}
|
|
|
|
// Zero blocks
|
|
f(nil, 1, &Result{})
|
|
|
|
// Single block without samples
|
|
f([]*sortBlock{{}}, 1, &Result{})
|
|
|
|
// Single block with a single samples.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1},
|
|
Values: []float64{4.2},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1},
|
|
Values: []float64{4.2},
|
|
})
|
|
|
|
// Single block with multiple samples.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 3},
|
|
Values: []float64{4.2, 2.1, 10},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1, 2, 3},
|
|
Values: []float64{4.2, 2.1, 10},
|
|
})
|
|
|
|
// Single block with multiple samples with deduplication.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 3},
|
|
Values: []float64{4.2, 2.1, 10},
|
|
},
|
|
}, 2, &Result{
|
|
Timestamps: []int64{2, 3},
|
|
Values: []float64{2.1, 10},
|
|
})
|
|
|
|
// Multiple blocks without time range intersection.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{3, 5},
|
|
Values: []float64{5.2, 6.1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2},
|
|
Values: []float64{4.2, 2.1},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1, 2, 3, 5},
|
|
Values: []float64{4.2, 2.1, 5.2, 6.1},
|
|
})
|
|
|
|
// Multiple blocks with time range intersection.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{3, 5},
|
|
Values: []float64{5.2, 6.1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{4.2, 2.1, 42},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1, 2, 3, 4, 5},
|
|
Values: []float64{4.2, 2.1, 5.2, 42, 6.1},
|
|
})
|
|
|
|
// Multiple blocks with time range inclusion.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{0, 3, 5},
|
|
Values: []float64{9, 5.2, 6.1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{4.2, 2.1, 42},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{0, 1, 2, 3, 4, 5},
|
|
Values: []float64{9, 4.2, 2.1, 5.2, 42, 6.1},
|
|
})
|
|
|
|
// Multiple blocks with identical timestamps.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{9, 5.2, 6.1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{4.2, 2.1, 42},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{4.2, 2.1, 42},
|
|
})
|
|
|
|
// Multiple blocks with identical timestamps, disabled deduplication.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{9, 5.2, 6.1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4},
|
|
Values: []float64{4.2, 2.1, 42},
|
|
},
|
|
}, 0, &Result{
|
|
Timestamps: []int64{1, 1, 2, 2, 4, 4},
|
|
Values: []float64{9, 4.2, 2.1, 5.2, 6.1, 42},
|
|
})
|
|
|
|
// Multiple blocks with identical timestamp ranges.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 5, 10, 11},
|
|
Values: []float64{9, 8, 7, 6, 5},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4, 10, 11, 12},
|
|
Values: []float64{21, 22, 23, 24, 25, 26},
|
|
},
|
|
}, 1, &Result{
|
|
Timestamps: []int64{1, 2, 4, 5, 10, 11, 12},
|
|
Values: []float64{21, 22, 23, 7, 24, 25, 26},
|
|
})
|
|
|
|
// Multiple blocks with identical timestamp ranges, no deduplication.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 5, 10, 11},
|
|
Values: []float64{9, 8, 7, 6, 5},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4, 10, 11, 12},
|
|
Values: []float64{21, 22, 23, 24, 25, 26},
|
|
},
|
|
}, 0, &Result{
|
|
Timestamps: []int64{1, 1, 2, 2, 4, 5, 10, 10, 11, 11, 12},
|
|
Values: []float64{9, 21, 22, 8, 23, 7, 6, 24, 25, 5, 26},
|
|
})
|
|
|
|
// Multiple blocks with identical timestamp ranges with deduplication.
|
|
f([]*sortBlock{
|
|
{
|
|
Timestamps: []int64{1, 2, 5, 10, 11},
|
|
Values: []float64{9, 8, 7, 6, 5},
|
|
},
|
|
{
|
|
Timestamps: []int64{1, 2, 4, 10, 11, 12},
|
|
Values: []float64{21, 22, 23, 24, 25, 26},
|
|
},
|
|
}, 5, &Result{
|
|
Timestamps: []int64{5, 10, 12},
|
|
Values: []float64{7, 24, 26},
|
|
})
|
|
}
|