lib/backup: add -s3TLSInsecureSkipVerify command-line flag (#6318)

* The new flag can be used for for skipping TLS certificates
verification when connecting to S3 endpoint. Affects vmbackup,
vmrestore, vmbackupmanager.

* replace deprecated `EndpointResolver` with `BaseEndpoint`

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1056

Signed-off-by: hagen1778 <roman@victoriametrics.com>

docs/CHANGELOG.md

@@ -48,6 +48,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): reduce CPU usage when evaluating high number of alerting and recording rules.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): speed up retrieving rules files from object storages by skipping unchanged objects during reloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6210).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support reading [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) records in `-datasource.url`, `-remoteWrite.url` and `-remoteRead.url` command-line option. For example, `-remoteWrite.url=http://srv+victoria-metrics` automatically resolves the `victoria-metrics` DNS SRV to a list of hostnames with TCP ports and then sends data to one of the addresses. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6053).
+* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): add `-s3TLSInsecureSkipVerify` command-line flag for skipping TLS certificates verification when connecting to S3 endpoint.
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) release.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix calendar display when `UTC+00:00` timezone is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6239).

docs/vmbackup.md

@@ -439,6 +439,8 @@ Run `vmbackup -help` in order to see all the available options:
   -s3StorageClass string
      The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
      See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
+  -s3TLSInsecureSkipVerify
+     Whether to skip TLS verification when connecting to the S3 endpoint.
   -snapshot.createURL string
      VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
   -snapshot.deleteURL string

docs/vmbackupmanager.md

@@ -577,6 +577,8 @@ command-line flags:
   -s3StorageClass string
      The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
      See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
+  -s3TLSInsecureSkipVerify
+     Whether to skip TLS verification when connecting to the S3 endpoint.
   -snapshot.createURL string
      VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snapshot/create
   -snapshot.deleteURL string

docs/vmrestore.md

@@ -221,6 +221,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
   -s3StorageClass string
      The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
      See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
+  -s3TLSInsecureSkipVerify
+      Whether to skip TLS verification when connecting to the S3 endpoint.
   -skipBackupCompleteCheck
      Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
   -src string

lib/backup/actions/util.go

@@ -27,6 +27,7 @@ var (
 	s3StorageClass   = flag.String("s3StorageClass", "", "The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, "+
 		"DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.\n"+
 		"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html")
+	s3TLSInsecureSkipVerify = flag.Bool("s3TLSInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to the S3 endpoint.")
 )
 
 func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
@@ -240,14 +241,15 @@ func NewRemoteFS(path string) (common.RemoteFS, error) {
 		bucket := dir[:n]
 		dir = dir[n:]
 		fs := &s3remote.FS{
-			CredsFilePath:    *credsFilePath,
+			CredsFilePath:         *credsFilePath,
-			ConfigFilePath:   *configFilePath,
+			ConfigFilePath:        *configFilePath,
-			CustomEndpoint:   *customS3Endpoint,
+			CustomEndpoint:        *customS3Endpoint,
-			StorageClass:     s3remote.StringToS3StorageClass(*s3StorageClass),
+			TLSInsecureSkipVerify: *s3TLSInsecureSkipVerify,
-			S3ForcePathStyle: *s3ForcePathStyle,
+			StorageClass:          s3remote.StringToS3StorageClass(*s3StorageClass),
-			ProfileName:      *configProfile,
+			S3ForcePathStyle:      *s3ForcePathStyle,
-			Bucket:           bucket,
+			ProfileName:           *configProfile,
-			Dir:              dir,
+			Bucket:                bucket,
+			Dir:                   dir,
 		}
 		if err := fs.Init(); err != nil {
 			return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)

lib/backup/s3remote/s3.go

@@ -3,8 +3,10 @@ package s3remote
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
+	"net/http"
 	"path"
 	"strings"
 
@@ -72,6 +74,9 @@ type FS struct {
 	// The name of S3 config profile to use.
 	ProfileName string
 
+	// Whether to use HTTP client with tls.InsecureSkipVerify setting
+	TLSInsecureSkipVerify bool
+
 	s3       *s3.Client
 	uploader *manager.Uploader
 }
@@ -112,12 +117,19 @@ func (fs *FS) Init() error {
 		return err
 	}
 
+	if fs.TLSInsecureSkipVerify {
+		tr := &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		cfg.HTTPClient = &http.Client{Transport: tr}
+	}
+
 	var outerErr error
 	fs.s3 = s3.NewFromConfig(cfg, func(o *s3.Options) {
 		if len(fs.CustomEndpoint) > 0 {
 			logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint)
 			o.UsePathStyle = fs.S3ForcePathStyle
-			o.EndpointResolver = s3.EndpointResolverFromURL(fs.CustomEndpoint)
+			o.BaseEndpoint = &fs.CustomEndpoint
 		} else {
 			region, err := manager.GetBucketRegion(context.Background(), s3.NewFromConfig(cfg), fs.Bucket)
 			if err != nil {