From d503ba0c42b083f8d8e285a75d7c8c314cc3f5c8 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 26 Sep 2022 15:39:56 +0200 Subject: [PATCH] lib/mergeset: follow-up after a0e7432e4248405440675bde3b98554f4aa73a81 (#3145) * lib/mergeset: follow-up after a0e7432e4248405440675bde3b98554f4aa73a81 Signed-off-by: hagen1778 * Apply suggestions from code review Signed-off-by: hagen1778 Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 1 + lib/mergeset/table.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a7517370b..03e8bda6b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,6 +26,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `RangeError: Maximum call stack size exceeded` error when the query returns too many data points at `Table` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3092/files). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each alert evaluation. Previously, annotations were evaluated only on alert's value ch ange. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). +* BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.79.3](https://docs.victoriametrics.com/CHANGELOG.html#v1793) wasn't complete. ## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3) diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index eea3bf93c..08416a194 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -710,7 +710,7 @@ func (tb *Table) mergeRawItemsBlocks(ibs []*inmemoryBlock, isFinal bool) { atomic.AddUint64(&tb.assistedMerges, 1) continue } - if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) { + if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) { return } logger.Panicf("FATAL: cannot merge small parts: %s", err) @@ -805,12 +805,14 @@ func (tb *Table) canBackgroundMerge() bool { return atomic.LoadUint32(tb.isReadOnly) == 0 } +var errReadOnlyMode = fmt.Errorf("storage is in readonly mode") + func (tb *Table) mergeExistingParts(isFinal bool) error { if !tb.canBackgroundMerge() { // Do not perform background merge in read-only mode // in order to prevent from disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } n := fs.MustGetFreeSpace(tb.path) // Divide free space by the max number of concurrent merges. @@ -849,7 +851,7 @@ func (tb *Table) partMerger() error { // The merger has been stopped. return nil } - if !errors.Is(err, errNothingToMerge) { + if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) { return err } if fasttime.UnixTimestamp()-lastMergeTime > 30 {