github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: prevent excessive loops when storage is in RO (#2962)

Browse source 
* lib/storage: prevent excessive loops when storage is in RO

Returning nil error when storage is in RO mode results
into excessive loops and function calls which could
result into CPU exhaustion. Returning an err instead
will trigger delays in the for loop and save some resources.

Signed-off-by: hagen1778 <roman@victoriametrics.com>

* document the change

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Roman Khavronenko 2022-08-09 11:17:00 +02:00 committed by Aliaksandr Valialkin
parent f90f654cf2
commit f42853275f
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
2 changed files with 7 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode).
## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0)

10
lib/storage/partition.go
View file

@ -569,7 +569,7 @@ func (pt *partition) addRowsPart(rows []rawRow) {
atomic.AddUint64(&pt.smallAssistedMerges, 1)
return
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot merge small parts: %s", err)
@ -956,7 +956,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
// The merger has been stopped.
return nil
}
if !errors.Is(err, errNothingToMerge) {
if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) {
return err
}
if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
@ -1012,11 +1012,13 @@ func (pt *partition) canBackgroundMerge() bool {
return atomic.LoadUint32(pt.isReadOnly) == 0
}
var errReadOnlyMode = fmt.Errorf("storage is in readonly mode")
func (pt *partition) mergeBigParts(isFinal bool) error {
if !pt.canBackgroundMerge() {
// Do not perform merge in read-only mode, since this may result in disk space shortage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
return nil
return errReadOnlyMode
}
maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)
@ -1032,7 +1034,7 @@ func (pt *partition) mergeSmallParts(isFinal bool) error {
if !pt.canBackgroundMerge() {
// Do not perform merge in read-only mode, since this may result in disk space shortage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
return nil
return errReadOnlyMode
}
// Try merging small parts to a big part at first.
maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)