github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: follow-up after a50d63c376 (#4289)

Browse source 
* lib/storage: follow-up after a50d63c376

- ensure retentionMsecs is rounded to day
- remove localTimeOffset in test as localOffset is ignored when using `UnixMilli`

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

* lib/storage: restore retention timezone offset effect on retention deadline

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

---------

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
This commit is contained in:
Zakhar Bessarab 2023-05-16 19:14:08 +04:00 committed by GitHub
parent 3a65f4a733
commit 242050ba94
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
lib/storage/storage.go
View file

@ -1032,17 +1032,18 @@ func nextRetentionDuration(retentionMsecs int64) time.Duration {
}
func nextRetentionDurationAt(atMsecs int64, retentionMsecs int64) time.Duration {
// Schedule the deadline to +4 hours from the next retention period start.
// This should prevent from possible double deletion of indexdb
// due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 .
retentionOffsetMsecs := retentionTimezoneOffsetMsecs - int64(4*3600*1000)
// Round retentionMsecs to days. This guarantees that per-day inverted index works as expected
deadline := ((atMsecs + retentionMsecs + retentionOffsetMsecs - 1) / retentionMsecs) * retentionMsecs
retentionMsecs = ((retentionMsecs + msecPerDay - 1) / msecPerDay) * msecPerDay
// The effect of time zone on retention period is moved out.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574
deadline -= retentionOffsetMsecs
deadline := ((atMsecs + retentionMsecs + retentionTimezoneOffsetMsecs - 1) / retentionMsecs) * retentionMsecs
// Schedule the deadline to +4 hours from the next retention period start.
// This should prevent from possible double deletion of indexdb
// due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 .
deadline += int64(4 * 3600 * 1000)
deadline -= retentionTimezoneOffsetMsecs
return time.Duration(deadline-atMsecs) * time.Millisecond
}

46
lib/storage/storage_test.go
View file

@ -409,40 +409,44 @@ func TestMetricRowMarshalUnmarshal(t *testing.T) {
}
func TestNextRetentionDuration(t *testing.T) {
validateRetention := func(now time.Time, retention float64) {
validateRetention := func(retention int64) {
t.Helper()
nowMsecs := now.UnixMilli()
d := nextRetentionDurationAt(nowMsecs, int64(retention*msecsPerMonth))
if d <= 0 {
nextTime := now.Add(d)
t.Fatalf("unexpected retention duration for retentionMonths=%f; got %s; must be %s + %f months", retention, nextTime, now, retention)
validateRetentionAt := func(now time.Time, retention int64) {
nowMsecs := now.UnixMilli()
d := nextRetentionDurationAt(nowMsecs, retention)
if d <= 0 {
nextTime := now.Add(d)
retentionHuman := time.Duration(retention) * time.Millisecond
t.Errorf("unexpected retention duration for retention=%s; got %s(%s); must be %s + %s; offset: %s", retentionHuman, nextTime, d, now, retentionHuman, time.Duration(retentionTimezoneOffsetMsecs)*time.Millisecond)
}
}
}
for retentionMonths := float64(0.1); retentionMonths < 120; retentionMonths += 0.3 {
// UTC offsets are in range [-12 hours, +14 hours].
// Verify that any legit combination of retention timezone and local time
// will return valid retention duration.
// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4207
for retentionOffset := -12; retentionOffset <= 14; retentionOffset++ {
for localTimeOffset := -12; localTimeOffset <= 14; localTimeOffset++ {
SetRetentionTimezoneOffset(time.Duration(retentionOffset) * time.Hour)
tz := time.FixedZone("", -1*localTimeOffset*60*60)
now := time.Now().In(tz)
validateRetention(now, retentionMonths)
SetRetentionTimezoneOffset(time.Duration(retentionOffset) * time.Hour)
validateRetentionAt(time.Now().UTC(), retention)
now = time.Date(2023, 4, 27, 3, 58, 0, 0, tz)
validateRetention(now, retentionMonths)
now := time.Date(2023, 4, 27, 23, 58, 0, 0, time.UTC)
validateRetentionAt(now, retention)
now = time.Date(2023, 4, 27, 4, 1, 0, 0, tz)
validateRetention(now, retentionMonths)
now = time.Date(2023, 4, 27, 0, 1, 0, 0, time.UTC)
validateRetentionAt(now, retention)
now = time.Date(2023, 4, 27, 6, 0, 0, 0, tz)
validateRetention(now, retentionMonths)
}
now = time.Date(2023, 4, 27, 0, 0, 0, 0, time.UTC)
validateRetentionAt(now, retention)
}
}
for retentionDays := 0.3; retentionDays < 3; retentionDays += 0.3 {
validateRetention(int64(retentionDays * msecPerDay))
}
for retentionMonths := float64(0.1); retentionMonths < 120; retentionMonths += 0.3 {
validateRetention(int64(retentionMonths * msecsPerMonth))
}
}
func TestStorageOpenClose(t *testing.T) {