From e1bf8440eba155cd8d57bdbffb9cd45903019ab4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 8 Feb 2024 12:50:25 +0200 Subject: [PATCH] lib/mergeset: prevent from possible `too big indexBlockSize` panic This panic could occur when samples with too long label values are ingested into VictoriaMetrics. This could result in too long fistItem and commonPrefix values at blockHeader (up to 64kb each). This may inflate the maximum index block size by 4 * maxIndexBlockSize. --- docs/CHANGELOG.md | 1 + lib/mergeset/metaindex_row.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 88f43ccbb..c9185b8f7 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly propagate [label filters](https://docs.victoriametrics.com/keyconcepts/#filtering) from multiple arguments passed to [aggregate functions](https://docs.victoriametrics.com/metricsql/#aggregate-functions). For example, `sum({job="foo"}, {job="bar"}) by (job) + a` was improperly optimized to `sum({job="foo"}, {job="bar"}) by (job) + a{job="foo"}` before being executed. This could lead to unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5604). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle precision errors when calculating [changes](https://docs.victoriametrics.com/metricsql/#changes), [changes_prometheus](https://docs.victoriametrics.com/metricsql/#changes_prometheus), [increases_over_time](https://docs.victoriametrics.com/metricsql/#increases_over_time) and [resets](https://docs.victoriametrics.com/metricsql/#resets) functions. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767). * BUGFIX: all VictoriaMetrics components: consistently return 200 http status code from [`/-/reload` endpoint](https://docs.victoriametrics.com/vmagent/#configuration-update). Previously [single-node VictoriaMetrics](https://docs.victoriametrics.com/) was returning 204 http status code. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5774). +* BUGFIX: prevent from possible `too big indexBlockSize` panic. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the graph dragging for Firefox and Safari. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5764). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix handling invalid timezone. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5732). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the bug where the select does not open. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5728). diff --git a/lib/mergeset/metaindex_row.go b/lib/mergeset/metaindex_row.go index c06a4a194..e80ba19a4 100644 --- a/lib/mergeset/metaindex_row.go +++ b/lib/mergeset/metaindex_row.go @@ -72,8 +72,11 @@ func (mr *metaindexRow) Unmarshal(src []byte) ([]byte, error) { if mr.blockHeadersCount <= 0 { return src, fmt.Errorf("blockHeadersCount must be bigger than 0; got %d", mr.blockHeadersCount) } - if mr.indexBlockSize > 2*maxIndexBlockSize { - return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 2*maxIndexBlockSize) + if mr.indexBlockSize > 4*maxIndexBlockSize { + // The index block size can exceed maxIndexBlockSize by up to 4x, + // since it can contain commonPrefix and firstItem at blockHeader + // with the maximum length of maxIndexBlockSize per each field. + return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 4*maxIndexBlockSize) } return src, nil