mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/backup: cosmetic fixes after #243
This commit is contained in:
parent
15b7406f7b
commit
f733cb2186
4 changed files with 17 additions and 14 deletions
|
@ -6,6 +6,7 @@ Supported storage systems for backups:
|
||||||
|
|
||||||
* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
|
* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
|
||||||
* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
|
* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
|
||||||
|
* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio). See `-customS3Endpoint` command-line flag.
|
||||||
* Local filesystem. Example: `fs://</absolute/path/to/backup>`
|
* Local filesystem. Example: `fs://</absolute/path/to/backup>`
|
||||||
|
|
||||||
Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
|
Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
|
||||||
|
@ -129,13 +130,13 @@ Run `vmbackup -help` in order to see all the available options:
|
||||||
-configFilePath string
|
-configFilePath string
|
||||||
Path to file with S3 configs. Configs are loaded from default location if not set.
|
Path to file with S3 configs. Configs are loaded from default location if not set.
|
||||||
See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
||||||
|
-configProfile string
|
||||||
|
Profile name for S3 configs (default "default")
|
||||||
-credsFilePath string
|
-credsFilePath string
|
||||||
Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
|
Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
|
||||||
See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
||||||
-configProfile string
|
|
||||||
Profile name for S3 configs. (default "default")
|
|
||||||
-customS3Endpoint string
|
-customS3Endpoint string
|
||||||
Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO)
|
Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
|
||||||
-dst string
|
-dst string
|
||||||
Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
|
Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
|
||||||
-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
|
-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
|
||||||
|
|
|
@ -40,13 +40,13 @@ Run `vmrestore -help` in order to see all the available options:
|
||||||
-configFilePath string
|
-configFilePath string
|
||||||
Path to file with S3 configs. Configs are loaded from default location if not set.
|
Path to file with S3 configs. Configs are loaded from default location if not set.
|
||||||
See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
||||||
|
-configProfile string
|
||||||
|
Profile name for S3 configs (default "default")
|
||||||
-credsFilePath string
|
-credsFilePath string
|
||||||
Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
|
Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
|
||||||
See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
|
||||||
-configProfile string
|
|
||||||
Profile name for S3 configs. (default "default")
|
|
||||||
-customS3Endpoint string
|
-customS3Endpoint string
|
||||||
Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO)
|
Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
|
||||||
-loggerLevel string
|
-loggerLevel string
|
||||||
Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
|
Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
|
||||||
-maxBytesPerSecond int
|
-maxBytesPerSecond int
|
||||||
|
|
|
@ -18,8 +18,8 @@ var (
|
||||||
"See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
|
"See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
|
||||||
configFilePath = flag.String("configFilePath", "", "Path to file with S3 configs. Configs are loaded from default location if not set.\n"+
|
configFilePath = flag.String("configFilePath", "", "Path to file with S3 configs. Configs are loaded from default location if not set.\n"+
|
||||||
"See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
|
"See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
|
||||||
configProfile = flag.String("configProfile", "default", "Profile name for S3 configs. ")
|
configProfile = flag.String("configProfile", "default", "Profile name for S3 configs")
|
||||||
customS3Endpoint = flag.String("customS3Endpoint", "", "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO)")
|
customS3Endpoint = flag.String("customS3Endpoint", "", "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set")
|
||||||
)
|
)
|
||||||
|
|
||||||
func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
|
func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
|
||||||
|
|
|
@ -21,7 +21,7 @@ type FS struct {
|
||||||
// Path to S3 credentials file.
|
// Path to S3 credentials file.
|
||||||
CredsFilePath string
|
CredsFilePath string
|
||||||
|
|
||||||
// Pat to S3 configs file.
|
// Path to S3 configs file.
|
||||||
ConfigFilePath string
|
ConfigFilePath string
|
||||||
|
|
||||||
// GCS bucket to use.
|
// GCS bucket to use.
|
||||||
|
@ -30,10 +30,14 @@ type FS struct {
|
||||||
// Directory in the bucket to write to.
|
// Directory in the bucket to write to.
|
||||||
Dir string
|
Dir string
|
||||||
|
|
||||||
s3 *s3.S3
|
// Set for using S3-compatible enpoint such as MinIO etc.
|
||||||
uploader *s3manager.Uploader
|
|
||||||
CustomEndpoint string
|
CustomEndpoint string
|
||||||
ProfileName string
|
|
||||||
|
// The name of S3 config profile to use.
|
||||||
|
ProfileName string
|
||||||
|
|
||||||
|
s3 *s3.S3
|
||||||
|
uploader *s3manager.Uploader
|
||||||
}
|
}
|
||||||
|
|
||||||
// Init initializes fs.
|
// Init initializes fs.
|
||||||
|
@ -63,7 +67,6 @@ func (fs *FS) Init() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fs.CustomEndpoint) > 0 {
|
if len(fs.CustomEndpoint) > 0 {
|
||||||
|
|
||||||
// Use provided custom endpoint for S3
|
// Use provided custom endpoint for S3
|
||||||
logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint)
|
logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint)
|
||||||
sess.Config.WithEndpoint(fs.CustomEndpoint)
|
sess.Config.WithEndpoint(fs.CustomEndpoint)
|
||||||
|
@ -71,7 +74,6 @@ func (fs *FS) Init() error {
|
||||||
// Disable prefixing endpoint with bucket name
|
// Disable prefixing endpoint with bucket name
|
||||||
sess.Config.WithS3ForcePathStyle(true)
|
sess.Config.WithS3ForcePathStyle(true)
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
// Determine bucket region.
|
// Determine bucket region.
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
region, err := s3manager.GetBucketRegion(ctx, sess, fs.Bucket, "")
|
region, err := s3manager.GetBucketRegion(ctx, sess, fs.Bucket, "")
|
||||||
|
|
Loading…
Reference in a new issue