From ac836bcf6cb57abba478685ab48fcd91652fe48a Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Wed, 22 May 2024 13:58:39 +0200 Subject: [PATCH] lib/backup: add `-s3TLSInsecureSkipVerify` command-line flag (#6318) * The new flag can be used for for skipping TLS certificates verification when connecting to S3 endpoint. Affects vmbackup, vmrestore, vmbackupmanager. * replace deprecated `EndpointResolver` with `BaseEndpoint` Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1056 Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 1 + docs/vmbackup.md | 2 ++ docs/vmbackupmanager.md | 2 ++ docs/vmrestore.md | 2 ++ lib/backup/actions/util.go | 18 ++++++++++-------- lib/backup/s3remote/s3.go | 14 +++++++++++++- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e7d1a40cd..16bab3d6d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -48,6 +48,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): reduce CPU usage when evaluating high number of alerting and recording rules. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): speed up retrieving rules files from object storages by skipping unchanged objects during reloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6210). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support reading [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) records in `-datasource.url`, `-remoteWrite.url` and `-remoteRead.url` command-line option. For example, `-remoteWrite.url=http://srv+victoria-metrics` automatically resolves the `victoria-metrics` DNS SRV to a list of hostnames with TCP ports and then sends data to one of the addresses. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6053). +* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): add `-s3TLSInsecureSkipVerify` command-line flag for skipping TLS certificates verification when connecting to S3 endpoint. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) release. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix calendar display when `UTC+00:00` timezone is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6239). diff --git a/docs/vmbackup.md b/docs/vmbackup.md index 628e364bf..8b0b3db98 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -439,6 +439,8 @@ Run `vmbackup -help` in order to see all the available options: -s3StorageClass string The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html + -s3TLSInsecureSkipVerify + Whether to skip TLS verification when connecting to the S3 endpoint. -snapshot.createURL string VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set -snapshot.deleteURL string diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index 2bcd7dbdb..42a36655c 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -577,6 +577,8 @@ command-line flags: -s3StorageClass string The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html + -s3TLSInsecureSkipVerify + Whether to skip TLS verification when connecting to the S3 endpoint. -snapshot.createURL string VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snapshot/create -snapshot.deleteURL string diff --git a/docs/vmrestore.md b/docs/vmrestore.md index a8dec2ce1..50c7776bd 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -221,6 +221,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q -s3StorageClass string The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html + -s3TLSInsecureSkipVerify + Whether to skip TLS verification when connecting to the S3 endpoint. -skipBackupCompleteCheck Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file -src string diff --git a/lib/backup/actions/util.go b/lib/backup/actions/util.go index a6836c90d..464169da6 100644 --- a/lib/backup/actions/util.go +++ b/lib/backup/actions/util.go @@ -27,6 +27,7 @@ var ( s3StorageClass = flag.String("s3StorageClass", "", "The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, "+ "DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.\n"+ "See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html") + s3TLSInsecureSkipVerify = flag.Bool("s3TLSInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to the S3 endpoint.") ) func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error { @@ -240,14 +241,15 @@ func NewRemoteFS(path string) (common.RemoteFS, error) { bucket := dir[:n] dir = dir[n:] fs := &s3remote.FS{ - CredsFilePath: *credsFilePath, - ConfigFilePath: *configFilePath, - CustomEndpoint: *customS3Endpoint, - StorageClass: s3remote.StringToS3StorageClass(*s3StorageClass), - S3ForcePathStyle: *s3ForcePathStyle, - ProfileName: *configProfile, - Bucket: bucket, - Dir: dir, + CredsFilePath: *credsFilePath, + ConfigFilePath: *configFilePath, + CustomEndpoint: *customS3Endpoint, + TLSInsecureSkipVerify: *s3TLSInsecureSkipVerify, + StorageClass: s3remote.StringToS3StorageClass(*s3StorageClass), + S3ForcePathStyle: *s3ForcePathStyle, + ProfileName: *configProfile, + Bucket: bucket, + Dir: dir, } if err := fs.Init(); err != nil { return nil, fmt.Errorf("cannot initialize connection to s3: %w", err) diff --git a/lib/backup/s3remote/s3.go b/lib/backup/s3remote/s3.go index 120c9bcb1..173009c87 100644 --- a/lib/backup/s3remote/s3.go +++ b/lib/backup/s3remote/s3.go @@ -3,8 +3,10 @@ package s3remote import ( "bytes" "context" + "crypto/tls" "fmt" "io" + "net/http" "path" "strings" @@ -72,6 +74,9 @@ type FS struct { // The name of S3 config profile to use. ProfileName string + // Whether to use HTTP client with tls.InsecureSkipVerify setting + TLSInsecureSkipVerify bool + s3 *s3.Client uploader *manager.Uploader } @@ -112,12 +117,19 @@ func (fs *FS) Init() error { return err } + if fs.TLSInsecureSkipVerify { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + cfg.HTTPClient = &http.Client{Transport: tr} + } + var outerErr error fs.s3 = s3.NewFromConfig(cfg, func(o *s3.Options) { if len(fs.CustomEndpoint) > 0 { logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint) o.UsePathStyle = fs.S3ForcePathStyle - o.EndpointResolver = s3.EndpointResolverFromURL(fs.CustomEndpoint) + o.BaseEndpoint = &fs.CustomEndpoint } else { region, err := manager.GetBucketRegion(context.Background(), s3.NewFromConfig(cfg), fs.Bucket) if err != nil {