From 81789da731d00e08972b143041a4e2fc0e467e9c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 13 Jan 2021 12:15:36 +0200 Subject: [PATCH] lib/backup: increase backup chunk size from 128MB to 1GB This should reduce costs for object storage API calls by 8x. See https://cloud.google.com/storage/pricing#operations-pricing --- docs/CHANGELOG.md | 1 + lib/backup/common/part.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 30b414d25..0387608a6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ * FEATURE: add `tfirst_over_time(m[d])` and `tlast_over_time(m[d])` functions to [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) for returning timestamps for the first and the last data point in `m` over `d` duration. * FEATURE: enforce at least TLS v1.2 when accepting HTTPS requests if `-tls`, `-tlsCertFile` and `-tlsKeyFile` command-line flags are set, because older TLS protocols such as v1.0 and v1.1 have been deprecated due to security vulnerabilities. * FEATURE: support `extra_label` query arg for all HTTP-based [data ingestion protocols](https://victoriametrics.github.io/#how-to-import-time-series-data). This query arg can be used for specifying extra labels which should be added for the ingested data. +* FEATURE: vmbackup: increase backup chunk size from 128MB to 1GB. This should reduce the number of Object storage API calls during backups by 8x. This may also reduce costs, since object storage API calls usually have non-zero costs. See https://aws.amazon.com/s3/pricing/ and https://cloud.google.com/storage/pricing#operations-pricing . * BUGFIX: properly parse escaped unicode chars in MetricsQL metric names, label names and function names. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990 * BUGFIX: override user-provided labels with labels set in `extra_label` query args during data ingestion over HTTP-based protocols. diff --git a/lib/backup/common/part.go b/lib/backup/common/part.go index 276f2a436..bcf63c567 100644 --- a/lib/backup/common/part.go +++ b/lib/backup/common/part.go @@ -87,7 +87,7 @@ func (p *Part) ParseFromRemotePath(remotePath string) bool { // // The MaxPartSize reduces bandwidth usage during retires on network errors // when transferring multi-TB files. -const MaxPartSize = 128 * 1024 * 1024 +const MaxPartSize = 1024 * 1024 * 1024 // SortParts sorts parts by (Path, Offset) func SortParts(parts []Part) {