diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f92b380..b645b6b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,8 @@ * `predict_linear` See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674 +* BUGFIX: properly handle stale time series after K8S deployment. Previously such time series could be double-counted. + See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 * BUGFIX: vmalert: accept days, weeks and years in `for: ` part of config like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817 * BUGFIX: fix `mode_over_time(m[d])` calculations. Previously the function could return incorrect results. diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index cea3fc724..c1f65733e 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -4463,7 +4463,7 @@ func TestExecSuccess(t *testing.T) { q := `distinct_over_time((time() < 1700)[500s])` r1 := netstorage.Result{ MetricName: metricNameExpected, - Values: []float64{3, 3, 3, 3, 2, 1}, + Values: []float64{3, 3, 3, 3, nan, nan}, Timestamps: timestampsExpected, } resultExpected := []netstorage.Result{r1} @@ -4474,7 +4474,7 @@ func TestExecSuccess(t *testing.T) { q := `distinct_over_time((time() < 1700)[2.5i])` r1 := netstorage.Result{ MetricName: metricNameExpected, - Values: []float64{3, 3, 3, 3, 2, 1}, + Values: []float64{3, 3, 3, 3, nan, nan}, Timestamps: timestampsExpected, } resultExpected := []netstorage.Result{r1} diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 56d26e2c5..a9718f008 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -500,6 +500,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu j := 0 ni := 0 nj := 0 + stalenessInterval := int64(float64(scrapeInterval) * 0.9) for _, tEnd := range rc.Timestamps { tStart := tEnd - window ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) @@ -516,9 +517,17 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu rfa.prevValue = values[i-1] rfa.prevTimestamp = timestamps[i-1] } - rfa.values = values[i:j] rfa.timestamps = timestamps[i:j] + if j == len(timestamps) && i < j && tEnd-timestamps[j-1] > stalenessInterval { + // Do not take into account the last data point in time series if the distance between this data point + // and tEnd exceeds stalenessInterval. + // This should prevent from double counting when a label changes in time series (for instance, + // during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 + rfa.prevValue = nan + rfa.values = nil + rfa.timestamps = nil + } rfa.currTimestamp = tEnd value := rc.Func(rfa) rfa.idx++ diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index b66136317..fb7083d80 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -583,7 +583,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, nan, 123, 34, 32} + valuesExpected := []float64{nan, nan, 123, 34, nan} timestampsExpected := []int64{-50, 0, 50, 100, 150} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -690,7 +690,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 123, 54, 44, 34} + valuesExpected := []float64{nan, 123, 54, 44, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -704,7 +704,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 1} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -718,7 +718,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 21, 12, 32, 34} + valuesExpected := []float64{nan, 21, 12, 32, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -732,7 +732,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 123, 99, 44, 34} + valuesExpected := []float64{nan, 123, 99, 44, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -746,7 +746,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 222, 199, 110, 34} + valuesExpected := []float64{nan, 222, 199, 110, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -760,7 +760,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, nan, -9, 22, 0} + valuesExpected := []float64{nan, nan, -9, 22, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -788,7 +788,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} + valuesExpected := []float64{nan, 0.004, 0, 0, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -802,7 +802,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} + valuesExpected := []float64{nan, 0.031, 0.044, 0.04, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -816,7 +816,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} + valuesExpected := []float64{nan, 0.031, 0.075, 0.115, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -830,7 +830,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} + valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -844,7 +844,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} + valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -858,7 +858,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 0} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -886,7 +886,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2, 2, 1, 0} + valuesExpected := []float64{nan, 2, 2, 1, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -900,7 +900,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} + valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -914,7 +914,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, 0} + valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -942,7 +942,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} + valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -956,7 +956,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 5.830951894845301} + valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -970,7 +970,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} + valuesExpected := []float64{nan, 2.148, 1.593, 1.156, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -984,7 +984,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 4, 3, 1} + valuesExpected := []float64{nan, 4, 4, 3, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -998,7 +998,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 4, 7, 6, 3} + valuesExpected := []float64{nan, 4, 7, 6, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1012,7 +1012,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 21, 34, 34, 34} + valuesExpected := []float64{nan, 21, 34, 34, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1026,7 +1026,7 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) values := rc.Do(nil, testValues, testTimestamps) - valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, 2880} + valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }) @@ -1062,7 +1062,7 @@ func TestRollupBigNumberOfValues(t *testing.T) { srcTimestamps[i] = int64(i / 2) } values := rc.Do(nil, srcValues, srcTimestamps) - valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} + valuesExpected := []float64{1, 4001, 8001, nan, nan, nan} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) }