From 054bff1f39c542ba6502456b69e8e01e3c8a7c06 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 8 Feb 2024 12:50:25 +0200 Subject: [PATCH] lib/mergeset: prevent from possible `too big indexBlockSize` panic This panic could occur when samples with too long label values are ingested into VictoriaMetrics. This could result in too long fistItem and commonPrefix values at blockHeader (up to 64kb each). This may inflate the maximum index block size by 4 * maxIndexBlockSize. --- docs/CHANGELOG.md | 1 + lib/mergeset/metaindex_row.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c97143ae7..65b4ae06c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly propagate [label filters](https://docs.victoriametrics.com/keyconcepts/#filtering) from multiple arguments passed to [aggregate functions](https://docs.victoriametrics.com/metricsql/#aggregate-functions). For example, `sum({job="foo"}, {job="bar"}) by (job) + a` was improperly optimized to `sum({job="foo"}, {job="bar"}) by (job) + a{job="foo"}` before being executed. This could lead to unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5604). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle precision errors when calculating [changes](https://docs.victoriametrics.com/metricsql/#changes), [changes_prometheus](https://docs.victoriametrics.com/metricsql/#changes_prometheus), [increases_over_time](https://docs.victoriametrics.com/metricsql/#increases_over_time) and [resets](https://docs.victoriametrics.com/metricsql/#resets) functions. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767). +* BUGFIX: prevent from possible `too big indexBlockSize` panic. ## [v1.93.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.11) diff --git a/lib/mergeset/metaindex_row.go b/lib/mergeset/metaindex_row.go index c06a4a194..e80ba19a4 100644 --- a/lib/mergeset/metaindex_row.go +++ b/lib/mergeset/metaindex_row.go @@ -72,8 +72,11 @@ func (mr *metaindexRow) Unmarshal(src []byte) ([]byte, error) { if mr.blockHeadersCount <= 0 { return src, fmt.Errorf("blockHeadersCount must be bigger than 0; got %d", mr.blockHeadersCount) } - if mr.indexBlockSize > 2*maxIndexBlockSize { - return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 2*maxIndexBlockSize) + if mr.indexBlockSize > 4*maxIndexBlockSize { + // The index block size can exceed maxIndexBlockSize by up to 4x, + // since it can contain commonPrefix and firstItem at blockHeader + // with the maximum length of maxIndexBlockSize per each field. + return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 4*maxIndexBlockSize) } return src, nil