From cba8d34dab0f55db0099a11cbfcaf3f3dc484e1d Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 6 Oct 2022 23:21:45 +0300 Subject: [PATCH] app/vmselect/promql: properly calculate `vm_rows_scanned_per_query` histogram for rollup functions, which take into account only a few samples on the provided lookbehind window --- app/vmselect/promql/rollup.go | 75 ++++++++++--- app/vmselect/promql/rollup_test.go | 164 ++++++++++++++--------------- docs/CHANGELOG.md | 1 + 3 files changed, 143 insertions(+), 97 deletions(-) diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index af2b32228..7aa74964f 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -167,6 +167,8 @@ var rollupFuncsCanAdjustWindow = map[string]bool{ "timestamp": true, } +// rollupFuncsRemoveCounterResets contains functions, which need to call removeCounterResets +// over input samples before calling the corresponding rollup functions. var rollupFuncsRemoveCounterResets = map[string]bool{ "increase": true, "increase_prometheus": true, @@ -177,6 +179,36 @@ var rollupFuncsRemoveCounterResets = map[string]bool{ "rollup_rate": true, } +// rollupFuncsSamplesScannedPerCall contains functions, which scan lower number of samples +// than is passed to the rollup func. +// +// It is expected that the remaining rollupFuncs scan all the samples passed to them. +var rollupFuncsSamplesScannedPerCall = map[string]int{ + "absent_over_time": 1, + "count_over_time": 1, + "default_rollup": 1, + "delta": 2, + "delta_prometheus": 2, + "deriv_fast": 2, + "first_over_time": 1, + "idelta": 2, + "ideriv": 2, + "increase": 2, + "increase_prometheus": 2, + "increase_pure": 2, + "irate": 2, + "lag": 1, + "last_over_time": 1, + "lifetime": 2, + "present_over_time": 1, + "rate": 2, + "scrape_interval": 2, + "tfirst_over_time": 1, + "timestamp": 1, + "timestamp_with_name": 1, + "tlast_over_time": 1, +} + // These functions don't change physical meaning of input time series, // so they don't drop metric name var rollupFuncsKeepMetricName = map[string]bool{ @@ -248,26 +280,29 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) { return aggrFuncNames, nil } -func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) ( +func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) ( func(values []float64, timestamps []int64), []*rollupConfig, error) { preFunc := func(values []float64, timestamps []int64) {} - if rollupFuncsRemoveCounterResets[name] { + funcName = strings.ToLower(funcName) + if rollupFuncsRemoveCounterResets[funcName] { preFunc = func(values []float64, timestamps []int64) { removeCounterResets(values) } } + samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName] newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig { return &rollupConfig{ - TagValue: tagValue, - Func: rf, - Start: start, - End: end, - Step: step, - Window: window, - MayAdjustWindow: rollupFuncsCanAdjustWindow[name], - LookbackDelta: lookbackDelta, - Timestamps: sharedTimestamps, - isDefaultRollup: name == "default_rollup", + TagValue: tagValue, + Func: rf, + Start: start, + End: end, + Step: step, + Window: window, + MayAdjustWindow: rollupFuncsCanAdjustWindow[funcName], + LookbackDelta: lookbackDelta, + Timestamps: sharedTimestamps, + isDefaultRollup: funcName == "default_rollup", + samplesScannedPerCall: samplesScannedPerCall, } } appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig { @@ -277,7 +312,7 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en return dst } var rcs []*rollupConfig - switch name { + switch funcName { case "rollup": rcs = appendRollupConfigs(rcs) case "rollup_rate", "rollup_deriv": @@ -414,6 +449,11 @@ type rollupConfig struct { // Whether default_rollup is used. isDefaultRollup bool + + // The estimated number of samples scanned per Func call. + // + // If zero, then it is considered that Func scans all the samples passed to it. + samplesScannedPerCall int } func (rc *rollupConfig) String() string { @@ -552,7 +592,8 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu ni := 0 nj := 0 f := rc.Func - var samplesScanned uint64 + samplesScanned := uint64(len(values)) + samplesScannedPerCall := uint64(rc.samplesScannedPerCall) for _, tEnd := range rc.Timestamps { tStart := tEnd - window ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) @@ -584,7 +625,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu rfa.currTimestamp = tEnd value := f(rfa) rfa.idx++ - samplesScanned += uint64(len(rfa.values)) + if samplesScannedPerCall > 0 { + samplesScanned += samplesScannedPerCall + } else { + samplesScanned += uint64(len(rfa.values)) + } dstValues = append(dstValues, value) } putRollupFuncArg(rfa) diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 02db21132..7450d09f4 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -586,8 +586,8 @@ func TestRollupNoWindowNoPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} @@ -623,8 +623,8 @@ func TestRollupWindowNoPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} @@ -640,8 +640,8 @@ func TestRollupWindowNoPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan} timestampsExpected := []int64{161, 171, 181, 191} @@ -660,8 +660,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 15 { + t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, nan, 34, nan, 44} timestampsExpected := []int64{0, 5, 10, 15, 20, 25} @@ -677,8 +677,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{44, 32, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} @@ -694,8 +694,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, 123, 34, 32} timestampsExpected := []int64{-50, 0, 50, 100, 150} @@ -714,8 +714,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 123, 34, 34} timestampsExpected := []int64{0, 5, 10, 15, 20} @@ -731,8 +731,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{44, 34, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} @@ -748,8 +748,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 15 { + t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 54, 44, nan} timestampsExpected := []int64{0, 50, 100, 150} @@ -768,8 +768,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -785,8 +785,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -802,8 +802,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -822,8 +822,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 54, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -839,8 +839,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -856,8 +856,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, 12, 32, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -873,8 +873,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 99, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -890,8 +890,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 222, 199, 110, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -907,8 +907,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, -9, 22, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -924,8 +924,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -102, -42, -10, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -941,8 +941,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{123, 33, -87, 0} timestampsExpected := []int64{10, 50, 90, 130} @@ -958,8 +958,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -975,8 +975,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -992,8 +992,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 47 { + t.Fatalf("expecting 47 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1009,8 +1009,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1026,8 +1026,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1043,8 +1043,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1060,8 +1060,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 3, 3, 2, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1077,8 +1077,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 1, 1, 1, 1, 0} timestampsExpected := []int64{0, 9, 18, 27, 36, 45} @@ -1094,8 +1094,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2, 2, 1, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1111,8 +1111,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1128,8 +1128,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1145,8 +1145,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 14 { + t.Fatalf("expecting 14 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, 0, -8900, 0} timestampsExpected := []int64{0, 4, 8, 12, 16, 20} @@ -1162,8 +1162,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1179,8 +1179,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1196,8 +1196,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1213,8 +1213,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1230,8 +1230,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 7, 6, 3} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1247,8 +1247,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, 34, 34, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1264,8 +1264,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2775, 5262.5, 3862.5, 1800} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1281,8 +1281,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1306,8 +1306,8 @@ func TestRollupBigNumberOfValues(t *testing.T) { srcTimestamps[i] = int64(i / 2) } values, samplesScanned := rc.Do(nil, srcValues, srcTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 22002 { + t.Fatalf("expecting 22002 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5deb50e0c..1dbce8f56 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -31,6 +31,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ange. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). * BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.79.3](https://docs.victoriametrics.com/CHANGELOG.html#v1793) wasn't complete. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +* BUGFIX: properly calculate `vm_rows_scanned_per_query` histogram exported at `/metrics` page of `vmselect` and single-node VictoriaMetrics. Previously it could return misleadingly high numbers for [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions), which scan only a few samples on the provided lookbehind window in square brackets. For example, `increase(m[1d])` always scans only 2 rows (aka `raw samples`) per each returned time series. ## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3)