From 8c50c38a80349bd54f6e5d6d1f13553b1ee5e31e Mon Sep 17 00:00:00 2001 From: Zhu Jiekun Date: Fri, 18 Oct 2024 13:41:43 +0200 Subject: [PATCH] vmstorage: auto calculate maxUniqueTimeseries based on resources (#6961) ### Describe Your Changes Add support for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6930 Calculate `-search.maxUniqueTimeseries` by `-search.maxConcurrentRequests` and remaining memory if it's **not set** or **less equal than 0**. The remaining memory is affected by `-memory.allowedPercent`, `-memory.allowedBytes` and cgroup memory limit. ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: hagen1778 Co-authored-by: Roman Khavronenko (cherry picked from commit 85f60237e25d324819f4fd23a63274660f5bd560) Signed-off-by: hagen1778 --- app/vmselect/main.go | 4 ++ app/vmselect/prometheus/prometheus.go | 53 +++++++++++++++++++--- app/vmselect/prometheus/prometheus_test.go | 27 +++++++++++ docs/Cluster-VictoriaMetrics.md | 14 +++--- docs/README.md | 5 +- docs/changelog/CHANGELOG.md | 4 ++ lib/storage/index_db.go | 2 +- 7 files changed, 94 insertions(+), 15 deletions(-) diff --git a/app/vmselect/main.go b/app/vmselect/main.go index ebbc724ed..85844ef90 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -60,6 +60,7 @@ func Init() { fs.RemoveDirContents(tmpDirPath) netstorage.InitTmpBlocksDir(tmpDirPath) promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult") + prometheus.InitMaxUniqueTimeseries(*maxConcurrentRequests) concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests) initVMAlertProxy() @@ -82,6 +83,9 @@ var ( _ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 { return float64(len(concurrencyLimitCh)) }) + _ = metrics.NewGauge(`vm_search_max_unique_timeseries`, func() float64 { + return float64(prometheus.GetMaxUniqueTimeSeries()) + }) ) //go:embed vmui diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 73afe7e21..91ac0c2fa 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -12,6 +12,10 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/metrics" + "github.com/VictoriaMetrics/metricsql" + "github.com/valyala/fastjson/fastfloat" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats" @@ -23,11 +27,10 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" - "github.com/VictoriaMetrics/metrics" - "github.com/VictoriaMetrics/metricsql" - "github.com/valyala/fastjson/fastfloat" ) var ( @@ -47,7 +50,8 @@ var ( maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+ "points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data") - maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage") + maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. "+ + "When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional).") maxFederateSeries = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage") maxExportSeries = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage") maxTSDBStatusSeries = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage") @@ -792,7 +796,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr End: start, Step: step, MaxPointsPerSeries: *maxPointsPerTimeseries, - MaxSeries: *maxUniqueTimeseries, + MaxSeries: GetMaxUniqueTimeSeries(), QuotedRemoteAddr: httpserver.GetQuotedRemoteAddr(r), Deadline: deadline, MayCache: mayCache, @@ -900,7 +904,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo End: end, Step: step, MaxPointsPerSeries: *maxPointsPerTimeseries, - MaxSeries: *maxUniqueTimeseries, + MaxSeries: GetMaxUniqueTimeSeries(), QuotedRemoteAddr: httpserver.GetQuotedRemoteAddr(r), Deadline: deadline, MayCache: mayCache, @@ -1246,3 +1250,40 @@ func (sw *scalableWriter) flush() error { }) return sw.bw.Flush() } + +var ( + maxUniqueTimeseriesValueOnce sync.Once + maxUniqueTimeseriesValue int +) + +// InitMaxUniqueTimeseries init the max metrics limit calculated by available resources. +// The calculation is split into calculateMaxUniqueTimeSeriesForResource for unit testing. +func InitMaxUniqueTimeseries(maxConcurrentRequests int) { + maxUniqueTimeseriesValueOnce.Do(func() { + maxUniqueTimeseriesValue = *maxUniqueTimeseries + if maxUniqueTimeseriesValue <= 0 { + maxUniqueTimeseriesValue = calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, memory.Remaining()) + } + }) +} + +// calculateMaxUniqueTimeSeriesForResource calculate the max metrics limit calculated by available resources. +func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMemory int) int { + if maxConcurrentRequests <= 0 { + // This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`. + // In such cases, fallback to unlimited. + logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests) + return 2e9 + } + + // Calculate the max metrics limit for a single request in the worst-case concurrent scenario. + // The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes. + mts := remainingMemory / 200 / maxConcurrentRequests + logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory) + return mts +} + +// GetMaxUniqueTimeSeries returns the max metrics limit calculated by available resources. +func GetMaxUniqueTimeSeries() int { + return maxUniqueTimeseriesValue +} diff --git a/app/vmselect/prometheus/prometheus_test.go b/app/vmselect/prometheus/prometheus_test.go index 607705447..6ed4cc373 100644 --- a/app/vmselect/prometheus/prometheus_test.go +++ b/app/vmselect/prometheus/prometheus_test.go @@ -4,6 +4,7 @@ import ( "math" "net/http" "reflect" + "runtime" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" @@ -229,3 +230,29 @@ func TestGetLatencyOffsetMillisecondsFailure(t *testing.T) { } f("http://localhost?latency_offset=foobar") } + +func TestCalculateMaxMetricsLimitByResource(t *testing.T) { + f := func(maxConcurrentRequest, remainingMemory, expect int) { + t.Helper() + maxMetricsLimit := calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequest, remainingMemory) + if maxMetricsLimit != expect { + t.Fatalf("unexpected max metrics limit: got %d, want %d", maxMetricsLimit, expect) + } + } + + // Skip when GOARCH=386 + if runtime.GOARCH != "386" { + // 8 CPU & 32 GiB + f(16, int(math.Round(32*1024*1024*1024*0.4)), 4294967) + // 4 CPU & 32 GiB + f(8, int(math.Round(32*1024*1024*1024*0.4)), 8589934) + } + + // 2 CPU & 4 GiB + f(4, int(math.Round(4*1024*1024*1024*0.4)), 2147483) + + // other edge cases + f(0, int(math.Round(4*1024*1024*1024*0.4)), 2e9) + f(4, 0, 0) + +} diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 2d49a76a6..e3a59bfa7 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -711,10 +711,12 @@ Some workloads may need fine-grained resource usage limits. In these cases the f Queries, which need more memory, are rejected. Heavy queries, which select big number of time series, may exceed the per-query memory limit by a small percent. The total memory limit for concurrently executed queries can be estimated as `-search.maxMemoryPerQuery` multiplied by `-search.maxConcurrentRequests`. -- `-search.maxUniqueTimeseries` at `vmselect` component limits the number of unique time series a single query can find and process. - `vmselect` passes the limit to `vmstorage` component, which keeps in memory some metainformation about the time series located - by each query and spends some CPU time for processing the found time series. This means that the maximum memory usage and CPU usage - a single query can use at `vmstorage` is proportional to `-search.maxUniqueTimeseries`. +- `-search.maxUniqueTimeseries` at `vmstorage` component limits the number of unique time series a single query can find and process. + This means that the maximum memory usage and CPU usage a single query can use at `vmstorage` is proportional to `-search.maxUniqueTimeseries`. + By default, `vmstorage` calculates this limit automatically based on the available memory and the maximum number of concurrent read requests (see `-search.maxConcurrentRequests`). + The calculated limit will be printed during process start-up logs and exposed as `vm_search_max_unique_timeseries` metric. +- `-search.maxUniqueTimeseries` at `vmselect` adjusts the limit with the same name at `vmstorage`. The vmstorage limit can be adjusted + only to **lower value** and can't exceed it. By default, vmselect doesn't apply limit adjustments. - `-search.maxQueryDuration` at `vmselect` limits the duration of a single query. If the query takes longer than the given duration, then it is canceled. This allows saving CPU and RAM at `vmselect` and `vmstorage` when executing unexpectedly heavy queries. The limit can be altered for each query by passing `timeout` GET parameter, but can't exceed the limit specified via `-search.maxQueryDuration` command-line flag. @@ -1620,7 +1622,7 @@ Below is the output for `/path/to/vmselect -help`: -search.maxTagValueSuffixesPerSearch int The maximum number of tag value suffixes returned from /metrics/find (default 100000) -search.maxUniqueTimeseries int - The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage (default 300000) + The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. The limit can't exceed the corresponding -search.maxUniqueTimeseries limit on vmstorage, it can be only set to lower values. (default 0) -search.maxWorkersPerQuery int The maximum number of CPU cores a single query can use. The default value should work good for most cases. The flag can be set to lower values for improving performance of big number of concurrently executed queries. The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). There is no sense in setting this flag to values bigger than the number of CPU cores available on the system (default 16) -search.minStalenessInterval duration @@ -1894,7 +1896,7 @@ Below is the output for `/path/to/vmstorage -help`: -search.maxTagValues int The maximum number of tag values returned per search. See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration (default 100000) -search.maxUniqueTimeseries int - The maximum number of unique time series, which can be scanned during every query. This allows protecting against heavy queries, which select unexpectedly high number of series. Zero means 'no limit'. See also -search.max* command-line flags at vmselect + The maximum number of unique time series, which can be scanned during every query. This allows protecting against heavy queries, which select unexpectedly high number of series. When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional). See also -search.max* command-line flags at vmselect. -smallMergeConcurrency int Deprecated: this flag does nothing -snapshotAuthKey value diff --git a/docs/README.md b/docs/README.md index 29c8f52a3..b648710c7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1707,7 +1707,8 @@ By default, VictoriaMetrics is tuned for an optimal resource usage under typical - `-search.maxMemoryPerQuery` limits the amounts of memory, which can be used for processing a single query. Queries, which need more memory, are rejected. Heavy queries, which select big number of time series, may exceed the per-query memory limit by a small percent. The total memory limit for concurrently executed queries can be estimated as `-search.maxMemoryPerQuery` multiplied by `-search.maxConcurrentRequests`. -- `-search.maxUniqueTimeseries` limits the number of unique time series a single query can find and process. VictoriaMetrics keeps in memory +- `-search.maxUniqueTimeseries` limits the number of unique time series a single query can find and process. By default, VictoriaMetrics calculates the limit automatically + based on the available memory and the maximum number of concurrent requests it can process (see `-search.maxConcurrentRequests`). VictoriaMetrics keeps in memory some metainformation about the time series located by each query and spends some CPU time for processing the found time series. This means that the maximum memory usage and CPU usage a single query can use is proportional to `-search.maxUniqueTimeseries`. - `-search.maxQueryDuration` limits the duration of a single query. If the query takes longer than the given duration, then it is canceled. @@ -3209,7 +3210,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxTagValues int The maximum number of tag values returned from /api/v1/label//values . See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration (default 100000) -search.maxUniqueTimeseries int - The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage (default 300000) + The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional). (default 0) -search.maxWorkersPerQuery int The maximum number of CPU cores a single query can use. The default value should work good for most cases. The flag can be set to lower values for improving performance of big number of concurrently executed queries. The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). There is no sense in setting this flag to values bigger than the number of CPU cores available on the system (default 16) -search.minStalenessInterval duration diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 55c42b646..d7c6e12eb 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -18,6 +18,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +**Update note 1: `-search.maxUniqueTimeseries` limit on `vmselect` can no longer exceed `-search.maxUniqueTimeseries` limit on `vmstorage`. If you don't set this flag at `vmstorage`, then it will be automatically calculated based on available resources. This can result into rejecting expensive read queries if they exceed auto-calculated limit. The limit can be overriden by manually setting `-search.maxUniqueTimeseries` at vmstorage, but for better reliability we recommend sticking to default values. Refer to the CHANGELOG below and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6930).** + * FEATURE: add Darwin binaries for [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) to the release flow. The binaries will be available in the new release. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): allow using HTTP/2 client for Kubernetes service discovery if `-promscrape.kubernetes.useHTTP2Client` cmd-line flag is set. This could help to reduce the amount of opened connections to the Kubernetes API server. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5971) for the details. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995). @@ -25,6 +27,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): disable stream processing mode for data [ingested via InfluxDB](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) HTTP endpoints by default. With this change, the data is processed in batches (see `-influx.maxRequestSize`) and user will get parsing errors immediately as they happen. This also improves users' experience and resiliency against thundering herd problems caused by clients without backoff policies like telegraf. To enable stream mode back, pass HTTP header `Stream-Mode: "1"` with each request. For data sent via TCP and UDP (see `-influxListenAddr`) protocols streaming processing remains enabled. * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): allow parsing `scope` claim parsing in array format. This is useful for cases when identity provider does encode claims in array format. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to cancel running queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7097). +* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): automatically set `-search.maxUniqueTimeseries` limit based on available memory and `-search.maxConcurrentRequests`. The more memory is available to the process and the lower is `-search.maxConcurrentRequests`, the higher will be `-search.maxUniqueTimeseries` limit. This should protect vmstorage from expensive queries without the need to manually set `-search.maxUniqueTimeseries`. The calculated limit will be printed during process start-up logs and exposed as `vm_search_max_unique_timeseries` metric. Set `-search.maxUniqueTimeseries` manually to override auto calculation. Please note, `-search.maxUniqueTimeseries` on vmselect can't exceed the same name limit on vmstorage, it can only be set to lower values. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6930). +* FEATURE: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): set default value for `-search.maxUniqueTimeseries` to `0`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6930). * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway/): fix possible panic during parsing of a token without `vm_access` claim. This issue was introduced in v1.104.0. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix error messages rendering from overflowing the screen with long messages. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7207). diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 1eeb1e680..c935e33b8 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -2271,7 +2271,7 @@ func (is *indexSearch) searchMetricIDs(qt *querytracer.Tracer, tfss []*TagFilter func errTooManyTimeseries(maxMetrics int) error { return fmt.Errorf("the number of matching timeseries exceeds %d; "+ - "either narrow down the search or increase -search.max* command-line flag values at vmselect "+ + "either narrow down the search or increase -search.max* command-line flag values "+ "(the most likely limit is -search.maxUniqueTimeseries); "+ "see https://docs.victoriametrics.com/#resource-usage-limits", maxMetrics) }