From 5830ce270651265e45509710a6a6b8cf51c3e00b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 15 Jul 2021 16:03:26 +0300 Subject: [PATCH] app/vmselect/netstorage: add `-search.maxSamplesPerSeries` command-line option for limiting the number of samples a query can process per each series This should prevent from out of memory crashes like in https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1067 --- README.md | 2 ++ app/vmselect/netstorage/netstorage.go | 21 +++++++++++++++++---- docs/CHANGELOG.md | 2 ++ docs/Cluster-VictoriaMetrics.md | 2 ++ docs/README.md | 4 +++- docs/Single-server-VictoriaMetrics.md | 4 +++- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 437f813e6..3eaa1bdc4 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,8 @@ Below is the output for `/path/to/vmselect -help`: Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16384) -search.maxQueueDuration duration The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) + -search.maxSamplesPerSeries int + The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000) -search.maxStalenessInterval duration The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons -search.maxStatusRequestDuration duration diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index b597527b7..d54a604cc 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -31,8 +31,11 @@ import ( xxhash "github.com/cespare/xxhash/v2" ) -var replicationFactor = flag.Int("replicationFactor", 1, "How many copies of every time series is available on vmstorage nodes. "+ - "See -replicationFactor command-line flag for vminsert nodes") +var ( + replicationFactor = flag.Int("replicationFactor", 1, "How many copies of every time series is available on vmstorage nodes. "+ + "See -replicationFactor command-line flag for vminsert nodes") + maxSamplesPerSeries = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage") +) // Result is a single timeseries result. // @@ -376,6 +379,7 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage. pts.addrs = pts.addrs[:0] // Wait until work is complete + samples := 0 sbs := make([]*sortBlock, 0, addrsLen) var firstErr error for _, upw := range upws { @@ -384,8 +388,17 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage. firstErr = err } if firstErr == nil { - sbs = append(sbs, upw.sbs...) - } else { + for _, sb := range upw.sbs { + samples += len(sb.Timestamps) + } + if samples < *maxSamplesPerSeries { + sbs = append(sbs, upw.sbs...) + } else { + firstErr = fmt.Errorf("cannot process more than %d samples per series; either increase -search.maxSamplesPerSeries "+ + "or reduce time range for the query", *maxSamplesPerSeries) + } + } + if firstErr != nil { for _, sb := range upw.sbs { putSortBlock(sb) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8afcecf88..58b481330 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,8 @@ sort: 15 ## tip +* FEATURE: add `-search.maxSamplesPerSeries` command-line flag for limiting the number of raw samples a single query could process per each time series. This option can prevent from out of memory errors when a query processes tens of millions of raw samples per series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1067). + ## [v1.63.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.63.0) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 38241f645..c6521b343 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -654,6 +654,8 @@ Below is the output for `/path/to/vmselect -help`: Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16384) -search.maxQueueDuration duration The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) + -search.maxSamplesPerSeries int + The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000) -search.maxStalenessInterval duration The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons -search.maxStatusRequestDuration duration diff --git a/docs/README.md b/docs/README.md index eed405a0a..7cce8396a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1785,6 +1785,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16384) -search.maxQueueDuration duration The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) + -search.maxSamplesPerSeries int + The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000) -search.maxStalenessInterval duration The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons -search.maxStatusRequestDuration duration @@ -1798,7 +1800,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxTagValues int The maximum number of tag values returned from /api/v1/label//values (default 100000) -search.maxUniqueTimeseries int - The maximum number of unique time series each search can scan (default 300000) + The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' -search.queryStats.lastQueriesCount int diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 0fca0f605..b06bbf93e 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1789,6 +1789,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16384) -search.maxQueueDuration duration The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) + -search.maxSamplesPerSeries int + The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000) -search.maxStalenessInterval duration The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons -search.maxStatusRequestDuration duration @@ -1802,7 +1804,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxTagValues int The maximum number of tag values returned from /api/v1/label//values (default 100000) -search.maxUniqueTimeseries int - The maximum number of unique time series each search can scan (default 300000) + The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' -search.queryStats.lastQueriesCount int