From fe71c73fe1cca522f03127f4b060b1bd9bdc159a Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 26 Sep 2022 15:39:56 +0200 Subject: [PATCH] lib/mergeset: follow-up after a0e7432e4248405440675bde3b98554f4aa73a81 (#3145) * lib/mergeset: follow-up after a0e7432e4248405440675bde3b98554f4aa73a81 Signed-off-by: hagen1778 * Apply suggestions from code review Signed-off-by: hagen1778 Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 3 ++- lib/mergeset/table.go | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e1eeebceb..8d83ce527 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -38,7 +38,8 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly calculate query results at `vmselect`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067). The issue has been introduced in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810). * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): log clear error when multiple identical `-storageNode` command-line flags are passed to `vmselect` or to `vminsert`. Previously these components were crashed with cryptic panic `metric ... is already registered` in this case. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3076). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `RangeError: Maximum call stack size exceeded` error when the query returns too many data points at `Table` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3092/files). -* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). +* BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810) wasn't complete. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. ## [v1.81.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.2) diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go index 3b3ceb8f9..ed44357f7 100644 --- a/lib/mergeset/table.go +++ b/lib/mergeset/table.go @@ -709,7 +709,7 @@ func (tb *Table) mergeRawItemsBlocks(ibs []*inmemoryBlock, isFinal bool) { atomic.AddUint64(&tb.assistedMerges, 1) continue } - if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) { + if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) { return } logger.Panicf("FATAL: cannot merge small parts: %s", err) @@ -788,12 +788,14 @@ func (tb *Table) canBackgroundMerge() bool { return atomic.LoadUint32(tb.isReadOnly) == 0 } +var errReadOnlyMode = fmt.Errorf("storage is in readonly mode") + func (tb *Table) mergeExistingParts(isFinal bool) error { if !tb.canBackgroundMerge() { // Do not perform background merge in read-only mode // in order to prevent from disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } n := fs.MustGetFreeSpace(tb.path) // Divide free space by the max number of concurrent merges. @@ -832,7 +834,7 @@ func (tb *Table) partMerger() error { // The merger has been stopped. return nil } - if !errors.Is(err, errNothingToMerge) { + if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) { return err } if fasttime.UnixTimestamp()-lastMergeTime > 30 {