From 217c192c88ae3569a8a634655cb927829d0fb1d0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 13 Oct 2020 12:07:59 +0300 Subject: [PATCH] app/vmselect/promql: improve time series staleness detection This should prevent from double counting for time series at the time when it changes label. The most common case is in K8S, which changes pod uid label with each new deployment. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 --- CHANGELOG.md | 2 ++ app/vmselect/promql/exec_test.go | 4 +-- app/vmselect/promql/rollup.go | 11 ++++++- app/vmselect/promql/rollup_test.go | 48 +++++++++++++++--------------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f92b3804..b645b6b446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,8 @@ * `predict_linear` See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674 +* BUGFIX: properly handle stale time series after K8S deployment. Previously such time series could be double-counted. + See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 * BUGFIX: vmalert: accept days, weeks and years in `for: ` part of config like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817 * BUGFIX: fix `mode_over_time(m[d])` calculations. Previously the function could return incorrect results. diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index cea3fc724f..c1f65733ed 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -4463,7 +4463,7 @@ func TestExecSuccess(t *testing.T) { q := `distinct_over_time((time() < 1700)[500s])` r1 := netstorage.Result{ MetricName: metricNameExpected, - Values: []float64{3, 3, 3, 3, 2, 1}, + Values: []float64{3, 3, 3, 3, nan, nan}, Timestamps: timestampsExpected, } resultExpected := []netstorage.Result{r1} @@ -4474,7 +4474,7 @@ func TestExecSuccess(t *testing.T) { q := `distinct_over_time((time() < 1700)[2.5i])` r1 := netstorage.Result{ MetricName: metricNameExpected, - Values: []float64{3, 3, 3, 3, 2, 1}, + Values: []float64{3, 3, 3, 3, nan, nan}, Timestamps: timestampsExpected, } resultExpected := []netstorage.Result{r1} diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 56d26e2c53..a9718f008c 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -500,6 +500,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu j := 0 ni := 0 nj := 0 + stalenessInterval := int64(float64(scrapeInterval) * 0.9) for _, tEnd := range rc.Timestamps { tStart := tEnd - window ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) @@ -516,9 +517,17 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu rfa.prevValue = values[i-1] rfa.prevTimestamp = timestamps[i-1] } - rfa.values = values[i:j] rfa.timestamps = timestamps[i:j] + if j == len(timestamps) && i < j && tEnd-timestamps[j-1] > stalenessInterval { + // Do not take into account the last data point in time series if the distance between this data point + // and tEnd exceeds stalenessInterval. + // This should prevent from double counting when a label changes in time series (for instance, + // during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 + rfa.prevValue = nan + rfa.values = nil + rfa.timestamps = nil + } rfa.currTimestamp = tEnd value := rc.Func(rfa) rfa.idx++ diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index b66136317d..fb7083d809 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -583,7 +583,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, nan, 123, 34, 32} + valuesExpected := []float64{nan, nan, 123, 34, nan} timestampsExpected := []int64{-50, 0, 50, 100, 150} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -690,7 +690,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 123, 54, 44, 34} + valuesExpected := []float64{nan, 123, 54, 44, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -704,7 +704,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 1} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -718,7 +718,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 21, 12, 32, 34} + valuesExpected := []float64{nan, 21, 12, 32, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -732,7 +732,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 123, 99, 44, 34} + valuesExpected := []float64{nan, 123, 99, 44, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -746,7 +746,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 222, 199, 110, 34} + valuesExpected := []float64{nan, 222, 199, 110, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -760,7 +760,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, nan, -9, 22, 0} + valuesExpected := []float64{nan, nan, -9, 22, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -788,7 +788,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} + valuesExpected := []float64{nan, 0.004, 0, 0, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -802,7 +802,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} + valuesExpected := []float64{nan, 0.031, 0.044, 0.04, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -816,7 +816,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} + valuesExpected := []float64{nan, 0.031, 0.075, 0.115, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -830,7 +830,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} + valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -844,7 +844,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} + valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -858,7 +858,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 0} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -886,7 +886,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2, 2, 1, 0} + valuesExpected := []float64{nan, 2, 2, 1, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -900,7 +900,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} + valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -914,7 +914,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, 0} + valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -942,7 +942,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} + valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -956,7 +956,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 5.830951894845301} + valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -970,7 +970,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} + valuesExpected := []float64{nan, 2.148, 1.593, 1.156, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -984,7 +984,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 1} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -998,7 +998,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 7, 6, 3} + valuesExpected := []float64{nan, 4, 7, 6, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1012,7 +1012,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 21, 34, 34, 34} + valuesExpected := []float64{nan, 21, 34, 34, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1026,7 +1026,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, 2880} + valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1062,7 +1062,7 @@ func TestRollupBigNumberOfValues(t *testing.T) { srcTimestamps[i] = int64(i / 2) } values := rc.Do(nil, srcValues, srcTimestamps) - valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} + valuesExpected := []float64{1, 4001, 8001, nan, nan, nan} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }