github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-01 14:47:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmselect/promql: properly calculate vm_rows_scanned_per_query histogram for rollup functions, which take into account only a few samples on the provided lookbehind window

Browse source
This commit is contained in:
Aliaksandr Valialkin 2022-10-06 23:21:45 +03:00
parent 6c66848056
commit cba8d34dab
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
3 changed files with 143 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
app/vmselect/promql/rollup.go
View file

@ -167,6 +167,8 @@ var rollupFuncsCanAdjustWindow = map[string]bool{
"timestamp": true, "timestamp": true,
} }
// rollupFuncsRemoveCounterResets contains functions, which need to call removeCounterResets
// over input samples before calling the corresponding rollup functions.
var rollupFuncsRemoveCounterResets = map[string]bool{ var rollupFuncsRemoveCounterResets = map[string]bool{
"increase": true, "increase": true,
"increase_prometheus": true, "increase_prometheus": true,
@ -177,6 +179,36 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
"rollup_rate": true, "rollup_rate": true,
} }
// rollupFuncsSamplesScannedPerCall contains functions, which scan lower number of samples
// than is passed to the rollup func.
//
// It is expected that the remaining rollupFuncs scan all the samples passed to them.
var rollupFuncsSamplesScannedPerCall = map[string]int{
"absent_over_time": 1,
"count_over_time": 1,
"default_rollup": 1,
"delta": 2,
"delta_prometheus": 2,
"deriv_fast": 2,
"first_over_time": 1,
"idelta": 2,
"ideriv": 2,
"increase": 2,
"increase_prometheus": 2,
"increase_pure": 2,
"irate": 2,
"lag": 1,
"last_over_time": 1,
"lifetime": 2,
"present_over_time": 1,
"rate": 2,
"scrape_interval": 2,
"tfirst_over_time": 1,
"timestamp": 1,
"timestamp_with_name": 1,
"tlast_over_time": 1,
}
// These functions don't change physical meaning of input time series, // These functions don't change physical meaning of input time series,
// so they don't drop metric name // so they don't drop metric name
var rollupFuncsKeepMetricName = map[string]bool{ var rollupFuncsKeepMetricName = map[string]bool{
@ -248,26 +280,29 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
return aggrFuncNames, nil return aggrFuncNames, nil
} }
func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) ( func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) (
func(values []float64, timestamps []int64), []*rollupConfig, error) { func(values []float64, timestamps []int64), []*rollupConfig, error) {
preFunc := func(values []float64, timestamps []int64) {} preFunc := func(values []float64, timestamps []int64) {}
if rollupFuncsRemoveCounterResets[name] { funcName = strings.ToLower(funcName)
if rollupFuncsRemoveCounterResets[funcName] {
preFunc = func(values []float64, timestamps []int64) { preFunc = func(values []float64, timestamps []int64) {
removeCounterResets(values) removeCounterResets(values)
} }
} }
samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName]
newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig { newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig {
return &rollupConfig{ return &rollupConfig{
TagValue: tagValue, TagValue: tagValue,
Func: rf, Func: rf,
Start: start, Start: start,
End: end, End: end,
Step: step, Step: step,
Window: window, Window: window,
MayAdjustWindow: rollupFuncsCanAdjustWindow[name], MayAdjustWindow: rollupFuncsCanAdjustWindow[funcName],
LookbackDelta: lookbackDelta, LookbackDelta: lookbackDelta,
Timestamps: sharedTimestamps, Timestamps: sharedTimestamps,
isDefaultRollup: name == "default_rollup", isDefaultRollup: funcName == "default_rollup",
samplesScannedPerCall: samplesScannedPerCall,
} }
} }
appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig { appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
@ -277,7 +312,7 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
return dst return dst
} }
var rcs []*rollupConfig var rcs []*rollupConfig
switch name { switch funcName {
case "rollup": case "rollup":
rcs = appendRollupConfigs(rcs) rcs = appendRollupConfigs(rcs)
case "rollup_rate", "rollup_deriv": case "rollup_rate", "rollup_deriv":
@ -414,6 +449,11 @@ type rollupConfig struct {
// Whether default_rollup is used. // Whether default_rollup is used.
isDefaultRollup bool isDefaultRollup bool
// The estimated number of samples scanned per Func call.
//
// If zero, then it is considered that Func scans all the samples passed to it.
samplesScannedPerCall int
} }
func (rc *rollupConfig) String() string { func (rc *rollupConfig) String() string {
@ -552,7 +592,8 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
ni := 0 ni := 0
nj := 0 nj := 0
f := rc.Func f := rc.Func
var samplesScanned uint64 samplesScanned := uint64(len(values))
samplesScannedPerCall := uint64(rc.samplesScannedPerCall)
for _, tEnd := range rc.Timestamps { for _, tEnd := range rc.Timestamps {
tStart := tEnd - window tStart := tEnd - window
ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni)
@ -584,7 +625,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
rfa.currTimestamp = tEnd rfa.currTimestamp = tEnd
value := f(rfa) value := f(rfa)
rfa.idx++ rfa.idx++
samplesScanned += uint64(len(rfa.values)) if samplesScannedPerCall > 0 {
samplesScanned += samplesScannedPerCall
} else {
samplesScanned += uint64(len(rfa.values))
}
dstValues = append(dstValues, value) dstValues = append(dstValues, value)
} }
putRollupFuncArg(rfa) putRollupFuncArg(rfa)

164
app/vmselect/promql/rollup_test.go
View file

@ -586,8 +586,8 @@ func TestRollupNoWindowNoPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned != 0 { if samplesScanned != 12 {
t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, nan, nan, nan, nan} valuesExpected := []float64{nan, nan, nan, nan, nan}
timestampsExpected := []int64{0, 1, 2, 3, 4} timestampsExpected := []int64{0, 1, 2, 3, 4}
@ -623,8 +623,8 @@ func TestRollupWindowNoPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned != 0 { if samplesScanned != 12 {
t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, nan, nan, nan, nan} valuesExpected := []float64{nan, nan, nan, nan, nan}
timestampsExpected := []int64{0, 1, 2, 3, 4} timestampsExpected := []int64{0, 1, 2, 3, 4}
@ -640,8 +640,8 @@ func TestRollupWindowNoPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned != 0 { if samplesScanned != 12 {
t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, nan, nan, nan} valuesExpected := []float64{nan, nan, nan, nan}
timestampsExpected := []int64{161, 171, 181, 191} timestampsExpected := []int64{161, 171, 181, 191}
@ -660,8 +660,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 15 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 123, nan, 34, nan, 44} valuesExpected := []float64{nan, 123, nan, 34, nan, 44}
timestampsExpected := []int64{0, 5, 10, 15, 20, 25} timestampsExpected := []int64{0, 5, 10, 15, 20, 25}
@ -677,8 +677,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 16 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{44, 32, 34, nan} valuesExpected := []float64{44, 32, 34, nan}
timestampsExpected := []int64{100, 120, 140, 160} timestampsExpected := []int64{100, 120, 140, 160}
@ -694,8 +694,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, nan, 123, 34, 32} valuesExpected := []float64{nan, nan, 123, 34, 32}
timestampsExpected := []int64{-50, 0, 50, 100, 150} timestampsExpected := []int64{-50, 0, 50, 100, 150}
@ -714,8 +714,8 @@ func TestRollupWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 16 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 123, 123, 34, 34} valuesExpected := []float64{nan, 123, 123, 34, 34}
timestampsExpected := []int64{0, 5, 10, 15, 20} timestampsExpected := []int64{0, 5, 10, 15, 20}
@ -731,8 +731,8 @@ func TestRollupWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 16 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{44, 34, 34, nan} valuesExpected := []float64{44, 34, 34, nan}
timestampsExpected := []int64{100, 120, 140, 160} timestampsExpected := []int64{100, 120, 140, 160}
@ -748,8 +748,8 @@ func TestRollupWindowPartialPoints(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 15 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 54, 44, nan} valuesExpected := []float64{nan, 54, 44, nan}
timestampsExpected := []int64{0, 50, 100, 150} timestampsExpected := []int64{0, 50, 100, 150}
@ -768,8 +768,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 18 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
@ -785,8 +785,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 18 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
@ -802,8 +802,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 18 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
@ -822,8 +822,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 123, 54, 44, 34} valuesExpected := []float64{nan, 123, 54, 44, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -839,8 +839,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 4, 4, 3, 1} valuesExpected := []float64{nan, 4, 4, 3, 1}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -856,8 +856,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 21, 12, 32, 34} valuesExpected := []float64{nan, 21, 12, 32, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -873,8 +873,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 123, 99, 44, 34} valuesExpected := []float64{nan, 123, 99, 44, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -890,8 +890,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 222, 199, 110, 34} valuesExpected := []float64{nan, 222, 199, 110, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -907,8 +907,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 21, -9, 22, 0} valuesExpected := []float64{nan, 21, -9, 22, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -924,8 +924,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, -102, -42, -10, nan} valuesExpected := []float64{nan, -102, -42, -10, nan}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -941,8 +941,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{123, 33, -87, 0} valuesExpected := []float64{123, 33, -87, 0}
timestampsExpected := []int64{10, 50, 90, 130} timestampsExpected := []int64{10, 50, 90, 130}
@ -958,8 +958,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -975,8 +975,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -992,8 +992,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 47 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 47 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1009,8 +1009,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1026,8 +1026,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 35 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1043,8 +1043,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 4, 4, 3, 0} valuesExpected := []float64{nan, 4, 4, 3, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1060,8 +1060,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 3, 3, 2, 0} valuesExpected := []float64{nan, 3, 3, 2, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1077,8 +1077,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 16 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 1, 1, 1, 1, 0} valuesExpected := []float64{nan, 1, 1, 1, 1, 0}
timestampsExpected := []int64{0, 9, 18, 27, 36, 45} timestampsExpected := []int64{0, 9, 18, 27, 36, 45}
@ -1094,8 +1094,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 2, 2, 1, 0} valuesExpected := []float64{nan, 2, 2, 1, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1111,8 +1111,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1128,8 +1128,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0} valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1145,8 +1145,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 14 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 14 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, nan, nan, 0, -8900, 0} valuesExpected := []float64{nan, nan, nan, 0, -8900, 0}
timestampsExpected := []int64{0, 4, 8, 12, 16, 20} timestampsExpected := []int64{0, 4, 8, 12, 16, 20}
@ -1162,8 +1162,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1179,8 +1179,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0} valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1196,8 +1196,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1213,8 +1213,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 24 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 4, 4, 3, 1} valuesExpected := []float64{nan, 4, 4, 3, 1}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1230,8 +1230,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 35 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 4, 7, 6, 3} valuesExpected := []float64{nan, 4, 7, 6, 3}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1247,8 +1247,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 35 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 21, 34, 34, 34} valuesExpected := []float64{nan, 21, 34, 34, 34}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1264,8 +1264,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 35 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, 2775, 5262.5, 3862.5, 1800} valuesExpected := []float64{nan, 2775, 5262.5, 3862.5, 1800}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1281,8 +1281,8 @@ func TestRollupFuncsNoWindow(t *testing.T) {
} }
rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
values, samplesScanned := rc.Do(nil, testValues, testTimestamps) values, samplesScanned := rc.Do(nil, testValues, testTimestamps)
if samplesScanned == 0 { if samplesScanned != 35 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan} valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan}
timestampsExpected := []int64{0, 40, 80, 120, 160} timestampsExpected := []int64{0, 40, 80, 120, 160}
@ -1306,8 +1306,8 @@ func TestRollupBigNumberOfValues(t *testing.T) {
srcTimestamps[i] = int64(i / 2) srcTimestamps[i] = int64(i / 2)
} }
values, samplesScanned := rc.Do(nil, srcValues, srcTimestamps) values, samplesScanned := rc.Do(nil, srcValues, srcTimestamps)
if samplesScanned == 0 { if samplesScanned != 22002 {
t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") t.Fatalf("expecting 22002 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
} }
valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan}
timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000}

1
docs/CHANGELOG.md
View file

@ -31,6 +31,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
ange. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). ange. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119).
* BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.79.3](https://docs.victoriametrics.com/CHANGELOG.html#v1793) wasn't complete. * BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.79.3](https://docs.victoriametrics.com/CHANGELOG.html#v1793) wasn't complete.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval.
* BUGFIX: properly calculate `vm_rows_scanned_per_query` histogram exported at `/metrics` page of `vmselect` and single-node VictoriaMetrics. Previously it could return misleadingly high numbers for [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions), which scan only a few samples on the provided lookbehind window in square brackets. For example, `increase(m[1d])` always scans only 2 rows (aka `raw samples`) per each returned time series.
## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3) ## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3)