lib/{storage,mergeset}: do not allow setting dataFlushInterval to values smaller than pending{Items,Rows}FlushInterval

Pending rows and items unconditionally remain in memory for up to pending{Items,Rows}FlushInterval,
so there is no any sense in setting dataFlushInterval (the interval for guaranteed flush of in-memory data to disk)
to values smaller than pending{Items,Rows}FlushInterval, since this doesn't affect the interval
for flushing pending rows and items from memory to disk.

This is a follow-up for 4c80b17027

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6221
This commit is contained in:
Aliaksandr Valialkin 2024-07-15 10:08:12 +02:00
parent c3d2351948
commit 4878152678
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
2 changed files with 15 additions and 9 deletions

View file

@ -47,8 +47,7 @@ const maxPartSize = 400e9
// The interval for flushing buffered data to parts, so it becomes visible to search. // The interval for flushing buffered data to parts, so it becomes visible to search.
const pendingItemsFlushInterval = time.Second const pendingItemsFlushInterval = time.Second
// The interval for guaranteed flush of recently ingested data from memory to on-disk parts, // The interval for guaranteed flush of recently ingested data from memory to on-disk parts so they survive process crash.
// so they survive process crash.
var dataFlushInterval = 5 * time.Second var dataFlushInterval = 5 * time.Second
// SetDataFlushInterval sets the interval for guaranteed flush of recently ingested data from memory to disk. // SetDataFlushInterval sets the interval for guaranteed flush of recently ingested data from memory to disk.
@ -57,9 +56,13 @@ var dataFlushInterval = 5 * time.Second
// //
// This function must be called before initializing the indexdb. // This function must be called before initializing the indexdb.
func SetDataFlushInterval(d time.Duration) { func SetDataFlushInterval(d time.Duration) {
if d >= time.Second { if d < pendingItemsFlushInterval {
dataFlushInterval = d // There is no sense in setting dataFlushInterval to values smaller than pendingItemsFlushInterval,
// since pending rows unconditionally remain in memory for up to pendingItemsFlushInterval.
d = pendingItemsFlushInterval
} }
dataFlushInterval = d
} }
// maxItemsPerCachedPart is the maximum items per created part by the merge, // maxItemsPerCachedPart is the maximum items per created part by the merge,

View file

@ -48,8 +48,7 @@ var rawRowsShardsPerPartition = cgroup.AvailableCPUs()
// The interval for flushing buffered rows into parts, so they become visible to search. // The interval for flushing buffered rows into parts, so they become visible to search.
const pendingRowsFlushInterval = 2 * time.Second const pendingRowsFlushInterval = 2 * time.Second
// The interval for guaranteed flush of recently ingested data from memory to on-disk parts, // The interval for guaranteed flush of recently ingested data from memory to on-disk parts, so they survive process crash.
// so they survive process crash.
var dataFlushInterval = 5 * time.Second var dataFlushInterval = 5 * time.Second
// SetDataFlushInterval sets the interval for guaranteed flush of recently ingested data from memory to disk. // SetDataFlushInterval sets the interval for guaranteed flush of recently ingested data from memory to disk.
@ -58,11 +57,15 @@ var dataFlushInterval = 5 * time.Second
// //
// This function must be called before initializing the storage. // This function must be called before initializing the storage.
func SetDataFlushInterval(d time.Duration) { func SetDataFlushInterval(d time.Duration) {
if d >= time.Second { if d < pendingRowsFlushInterval {
// There is no sense in setting dataFlushInterval to values smaller than pendingRowsFlushInterval,
// since pending rows unconditionally remain in memory for up to pendingRowsFlushInterval.
d = pendingRowsFlushInterval
}
dataFlushInterval = d dataFlushInterval = d
mergeset.SetDataFlushInterval(d) mergeset.SetDataFlushInterval(d)
} }
}
// The maximum number of rawRow items in rawRowsShard. // The maximum number of rawRow items in rawRowsShard.
// //