From d94024a3fe707235638e0a317991bdc93a9cb81b Mon Sep 17 00:00:00 2001 From: Daniel Carbone Date: Thu, 27 Jun 2024 18:56:59 -0500 Subject: [PATCH 1/2] wiring up s3 "UsePathStyle" client config option --- .gitignore | 2 ++ cmd/root.go | 10 ++++++---- pkg/config/local.go | 17 +++++++++++------ pkg/storage/s3/s3.go | 23 ++++++++++++++++------- sample-configs/local.yaml | 1 + 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index f4816cd5..adf1626a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +.idea/ dist/ tmp/ +vendor/ \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index bae60697..785bdb52 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,15 +5,16 @@ import ( "os" "strings" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/databacker/mysql-backup/pkg/config" "github.com/databacker/mysql-backup/pkg/core" "github.com/databacker/mysql-backup/pkg/database" databacklog "github.com/databacker/mysql-backup/pkg/log" "github.com/databacker/mysql-backup/pkg/storage/credentials" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/spf13/viper" ) type execs interface { @@ -185,6 +186,7 @@ func rootCmd(execs execs) (*cobra.Command, error) { pflags.String("aws-access-key-id", "", "Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-secret-access-key", "", "Secret Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-region", "", "Region for s3 and s3 interoperable systems; ignored if not using s3.") + pflags.Bool("aws-use-path-style", false, "Force the use of legacy path-style bucket routing; ignored if not using s3.") // smb options pflags.String("smb-user", "", "SMB username") diff --git a/pkg/config/local.go b/pkg/config/local.go index 3004d013..7847027b 100644 --- a/pkg/config/local.go +++ b/pkg/config/local.go @@ -3,13 +3,14 @@ package config import ( "fmt" + "gopkg.in/yaml.v3" + "github.com/databacker/mysql-backup/pkg/remote" "github.com/databacker/mysql-backup/pkg/storage" "github.com/databacker/mysql-backup/pkg/storage/credentials" "github.com/databacker/mysql-backup/pkg/storage/s3" "github.com/databacker/mysql-backup/pkg/storage/smb" "github.com/databacker/mysql-backup/pkg/util" - "gopkg.in/yaml.v3" ) type ConfigSpec struct { @@ -129,11 +130,12 @@ func (t *Target) UnmarshalYAML(n *yaml.Node) error { } type S3Target struct { - Type string `yaml:"type"` - URL string `yaml:"url"` - Region string `yaml:"region"` - Endpoint string `yaml:"endpoint"` - Credentials AWSCredentials `yaml:"credentials"` + Type string `yaml:"type"` + URL string `yaml:"url"` + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` + Credentials AWSCredentials `yaml:"credentials"` + UsePathStyle bool `yaml:"usePathStyle"` } func (s S3Target) Storage() (storage.Storage, error) { @@ -148,6 +150,9 @@ func (s S3Target) Storage() (storage.Storage, error) { if s.Endpoint != "" { opts = append(opts, s3.WithEndpoint(s.Endpoint)) } + if s.UsePathStyle { + opts = append(opts, s3.WithPathStyle()) + } if s.Credentials.AccessKeyId != "" { opts = append(opts, s3.WithAccessKeyId(s.Credentials.AccessKeyId)) } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 855256d2..355adc28 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -179,10 +179,13 @@ func (s *S3) Remove(target string, logger *log.Entry) error { func (s *S3) getClient(logger *log.Entry) (*s3.Client, error) { // Get the AWS config - var opts []func(*config.LoadOptions) error + var ( + cfgOpts []func(*config.LoadOptions) error + clientOpts []func(*s3.Options) + ) if s.endpoint != "" { cleanEndpoint := getEndpoint(s.endpoint) - opts = append(opts, + cfgOpts = append(cfgOpts, config.WithEndpointResolverWithOptions( aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { return aws.Endpoint{URL: cleanEndpoint}, nil @@ -191,27 +194,33 @@ func (s *S3) getClient(logger *log.Entry) (*s3.Client, error) { ) } if logger.Level == log.TraceLevel { - opts = append(opts, config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponse)) + cfgOpts = append(cfgOpts, config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponse)) } if s.region != "" { - opts = append(opts, config.WithRegion(s.region)) + cfgOpts = append(cfgOpts, config.WithRegion(s.region)) } if s.accessKeyId != "" { - opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfgOpts = append(cfgOpts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( s.accessKeyId, s.secretAccessKey, "", ))) } cfg, err := config.LoadDefaultConfig(context.TODO(), - opts..., + cfgOpts..., ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %v", err) } + // build client options list with path style config + clientOpts = append(clientOpts, func(opts *s3.Options) { + opts.UsePathStyle = s.pathStyle + }) + // Create a new S3 service client - return s3.NewFromConfig(cfg), nil + + return s3.NewFromConfig(cfg, clientOpts...), nil } // getEndpoint returns a clean (for AWS client) endpoint. Normally, this is unchanged, diff --git a/sample-configs/local.yaml b/sample-configs/local.yaml index 491b8418..ea5b5109 100644 --- a/sample-configs/local.yaml +++ b/sample-configs/local.yaml @@ -61,6 +61,7 @@ spec: details: region: us-west-1 endpoint: https://s3.us-west-1.amazonaws.com + usePathStyle: false accessKeyId: access_key_id secretAccessKey: secret_access_key file: From e47b4cc45c32af58446568879f07de619c436579 Mon Sep 17 00:00:00 2001 From: Daniel Carbone Date: Thu, 27 Jun 2024 20:38:35 -0500 Subject: [PATCH 2/2] initial attempt at enabling s3 path style usage --- cmd/root.go | 6 ++- docs/configuration.md | 67 ++++++++++++++++---------------- pkg/storage/credentials/creds.go | 1 + pkg/storage/parse.go | 3 ++ pkg/util/parse.go | 2 +- 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 785bdb52..1fe84c95 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,6 +57,9 @@ func rootCmd(execs execs) (*cobra.Command, error) { AWS_ACCESS_KEY_ID: AWS Key ID AWS_SECRET_ACCESS_KEY: AWS Secret Access Key AWS_REGION: Region in which the bucket resides + + It also supports one non-standard option: + AWS_S3_USE_PATH_STYLE: false `, PersistentPreRunE: func(c *cobra.Command, args []string) error { bindFlags(cmd, v) @@ -146,6 +149,7 @@ func rootCmd(execs execs) (*cobra.Command, error) { AccessKeyID: v.GetString("aws-access-key-id"), SecretAccessKey: v.GetString("aws-secret-access-key"), Region: v.GetString("aws-region"), + S3UsePathStyle: v.GetBool("aws-s3-use-path-style"), }, SMB: credentials.SMBCreds{ Username: v.GetString("smb-user"), @@ -186,7 +190,7 @@ func rootCmd(execs execs) (*cobra.Command, error) { pflags.String("aws-access-key-id", "", "Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-secret-access-key", "", "Secret Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-region", "", "Region for s3 and s3 interoperable systems; ignored if not using s3.") - pflags.Bool("aws-use-path-style", false, "Force the use of legacy path-style bucket routing; ignored if not using s3.") + pflags.Bool("aws-s3-use-path-style", false, "Force the use of legacy path-style bucket routing; ignored if not using s3.") // smb options pflags.String("smb-user", "", "SMB username") diff --git a/docs/configuration.md b/docs/configuration.md index 1edd382a..df77c94a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,39 +61,40 @@ Various sample configuration files are available in the [sample-configs](../samp The following are the environment variables, CLI flags and configuration file options for: backup(B), restore (R), prune (P). -| Purpose | Backup / Restore / Prune | CLI Flag | Env Var | Config Key | Default | -| --- | --- | --- | --- | --- | --- | -| config file path | BRP | `config` | `DB_DUMP_CONFIG` | | | -| hostname or unix domain socket path (starting with a slash) to connect to database. Required. | BR | `server` | `DB_SERVER` | `database.server` | | -| port to use to connect to database. Optional. | BR | `port` | `DB_PORT` | `database.port` | 3306 | -| username for the database | BR | `user` | `DB_USER` | `database.credentials.username` | | -| password for the database | BR | `pass` | `DB_PASS` | `database.credentials.password` | | -| names of databases to dump, comma-separated | B | `include` | `DB_NAMES` | `dump.include` | all databases in the server | -| names of databases to exclude from the dump | B | `exclude` | `DB_NAMES_EXCLUDE` | `dump.exclude` | | -| do not include `USE ;` statement in the dump | B | `no-database-name` | `NO_DATABASE_NAME` | `dump.noDatabaseName` | `false` | -| restore to a specific database | R | `restore --database` | `RESTORE_DATABASE` | `restore.database` | | -| how often to do a dump or prune, in minutes | BP | `dump --frequency` | `DB_DUMP_FREQ` | `dump.schedule.frequency` | `1440` (in minutes), i.e. once per day | -| what time to do the first dump or prune | BP | `dump --begin` | `DB_DUMP_BEGIN` | `dump.schedule.begin` | `0`, i.e. immediately | -| cron schedule for dumps or prunes | BP | `dump --cron` | `DB_DUMP_CRON` | `dump.schedule.cron` | | -| run the backup or prune a single time and exit | BP | `dump --once` | `RUN_ONCE` | `dump.schedule.once` | `false` | -| enable debug logging | BRP | `debug` | `DEBUG` | `logging` | `false` | -| where to put the dump file; see [backup](./backup.md) | BP | `dump --target` | `DB_DUMP_TARGET` | `dump.targets` | | -| where the restore file exists; see [restore](./restore.md) | R | `restore --target` | `DB_RESTORE_TARGET` | `restore.target` | | -| replace any `:` in the dump filename with `-` | BP | `dump --safechars` | `DB_DUMP_SAFECHARS` | `database.safechars` | `false` | -| AWS access key ID, used only if a target does not have one | BRP | `aws-access-key-id` | `AWS_ACCESS_KEY_ID` | `dump.targets[s3-target].accessKeyId` | | -| AWS secret access key, used only if a target does not have one | BRP | `aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | `dump.targets[s3-target].secretAccessKey` | | -| AWS default region, used only if a target does not have one | BRP | `aws-region` | `AWS_REGION` | `dump.targets[s3-target].region` | | -| alternative endpoint URL for S3-interoperable systems, used only if a target does not have one | BR | `aws-endpoint-url` | `AWS_ENDPOINT_URL` | `dump.targets[s3-target].endpoint` | | -| SMB username, used only if a target does not have one | BRP | `smb-user` | `SMB_USER` | `dump.targets[smb-target].username` | | -| SMB password, used only if a target does not have one | BRP | `smb-pass` | `SMB_PASS` | `dump.targets[smb-target].password` | | -| compression to use, one of: `bzip2`, `gzip` | BP | `compression` | `DB_DUMP_COMPRESSION` | `dump.compression` | `gzip` | -| when in container, run the dump or restore with `nice`/`ionice` | BR | `` | `NICE` | `` | `false` | -| filename to save the target backup file | B | `dump --filename-pattern` | `DB_DUMP_FILENAME_PATTERN` | `dump.filenamePattern` | | -| directory with scripts to execute before backup | B | `dump --pre-backup-scripts` | `DB_DUMP_PRE_BACKUP_SCRIPTS` | `dump.scripts.preBackup` | in container, `/scripts.d/pre-backup/` | -| directory with scripts to execute after backup | B | `dump --post-backup-scripts` | `DB_DUMP_POST_BACKUP_SCRIPTS` | `dump.scripts.postBackup` | in container, `/scripts.d/post-backup/` | -| directory with scripts to execute before restore | R | `restore --pre-restore-scripts` | `DB_DUMP_PRE_RESTORE_SCRIPTS` | `restore.scripts.preRestore` | in container, `/scripts.d/pre-restore/` | -| directory with scripts to execute after restore | R | `restore --post-restore-scripts` | `DB_DUMP_POST_RESTORE_SCRIPTS` | `restore.scripts.postRestore` | in container, `/scripts.d/post-restore/` | -| retention policy for backups | BP | `dump --retention` | `RETENTION` | `prune.retention` | Infinite | +| Purpose | Backup / Restore / Prune | CLI Flag | Env Var | Config Key | Default | +|------------------------------------------------------------------------------------------------|--------------------------|----------------------------------|--------------------------------|-------------------------------------------|------------------------------------------| +| config file path | BRP | `config` | `DB_DUMP_CONFIG` | | | +| hostname or unix domain socket path (starting with a slash) to connect to database. Required. | BR | `server` | `DB_SERVER` | `database.server` | | +| port to use to connect to database. Optional. | BR | `port` | `DB_PORT` | `database.port` | 3306 | +| username for the database | BR | `user` | `DB_USER` | `database.credentials.username` | | +| password for the database | BR | `pass` | `DB_PASS` | `database.credentials.password` | | +| names of databases to dump, comma-separated | B | `include` | `DB_NAMES` | `dump.include` | all databases in the server | +| names of databases to exclude from the dump | B | `exclude` | `DB_NAMES_EXCLUDE` | `dump.exclude` | | +| do not include `USE ;` statement in the dump | B | `no-database-name` | `NO_DATABASE_NAME` | `dump.noDatabaseName` | `false` | +| restore to a specific database | R | `restore --database` | `RESTORE_DATABASE` | `restore.database` | | +| how often to do a dump or prune, in minutes | BP | `dump --frequency` | `DB_DUMP_FREQ` | `dump.schedule.frequency` | `1440` (in minutes), i.e. once per day | +| what time to do the first dump or prune | BP | `dump --begin` | `DB_DUMP_BEGIN` | `dump.schedule.begin` | `0`, i.e. immediately | +| cron schedule for dumps or prunes | BP | `dump --cron` | `DB_DUMP_CRON` | `dump.schedule.cron` | | +| run the backup or prune a single time and exit | BP | `dump --once` | `RUN_ONCE` | `dump.schedule.once` | `false` | +| enable debug logging | BRP | `debug` | `DEBUG` | `logging` | `false` | +| where to put the dump file; see [backup](./backup.md) | BP | `dump --target` | `DB_DUMP_TARGET` | `dump.targets` | | +| where the restore file exists; see [restore](./restore.md) | R | `restore --target` | `DB_RESTORE_TARGET` | `restore.target` | | +| replace any `:` in the dump filename with `-` | BP | `dump --safechars` | `DB_DUMP_SAFECHARS` | `database.safechars` | `false` | +| AWS access key ID, used only if a target does not have one | BRP | `aws-access-key-id` | `AWS_ACCESS_KEY_ID` | `dump.targets[s3-target].accessKeyId` | | +| AWS secret access key, used only if a target does not have one | BRP | `aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | `dump.targets[s3-target].secretAccessKey` | | +| AWS default region, used only if a target does not have one | BRP | `aws-region` | `AWS_REGION` | `dump.targets[s3-target].region` | | +| Use legacy path style s3 bucket routing | BRP | `aws-s3-use-path-style` | `AWS_S3_USE_PATH_STYLE` | `dump.targets[s3-target].usePathStyle` | `false` | +| alternative endpoint URL for S3-interoperable systems, used only if a target does not have one | BR | `aws-endpoint-url` | `AWS_ENDPOINT_URL` | `dump.targets[s3-target].endpoint` | | +| SMB username, used only if a target does not have one | BRP | `smb-user` | `SMB_USER` | `dump.targets[smb-target].username` | | +| SMB password, used only if a target does not have one | BRP | `smb-pass` | `SMB_PASS` | `dump.targets[smb-target].password` | | +| compression to use, one of: `bzip2`, `gzip` | BP | `compression` | `DB_DUMP_COMPRESSION` | `dump.compression` | `gzip` | +| when in container, run the dump or restore with `nice`/`ionice` | BR | `` | `NICE` | `` | `false` | +| filename to save the target backup file | B | `dump --filename-pattern` | `DB_DUMP_FILENAME_PATTERN` | `dump.filenamePattern` | | +| directory with scripts to execute before backup | B | `dump --pre-backup-scripts` | `DB_DUMP_PRE_BACKUP_SCRIPTS` | `dump.scripts.preBackup` | in container, `/scripts.d/pre-backup/` | +| directory with scripts to execute after backup | B | `dump --post-backup-scripts` | `DB_DUMP_POST_BACKUP_SCRIPTS` | `dump.scripts.postBackup` | in container, `/scripts.d/post-backup/` | +| directory with scripts to execute before restore | R | `restore --pre-restore-scripts` | `DB_DUMP_PRE_RESTORE_SCRIPTS` | `restore.scripts.preRestore` | in container, `/scripts.d/pre-restore/` | +| directory with scripts to execute after restore | R | `restore --post-restore-scripts` | `DB_DUMP_POST_RESTORE_SCRIPTS` | `restore.scripts.postRestore` | in container, `/scripts.d/post-restore/` | +| retention policy for backups | BP | `dump --retention` | `RETENTION` | `prune.retention` | Infinite | ## Configuration File diff --git a/pkg/storage/credentials/creds.go b/pkg/storage/credentials/creds.go index 68745a5f..e4004180 100644 --- a/pkg/storage/credentials/creds.go +++ b/pkg/storage/credentials/creds.go @@ -16,4 +16,5 @@ type AWSCreds struct { SecretAccessKey string Endpoint string Region string + S3UsePathStyle bool } diff --git a/pkg/storage/parse.go b/pkg/storage/parse.go index 13975b51..e1750930 100644 --- a/pkg/storage/parse.go +++ b/pkg/storage/parse.go @@ -48,6 +48,9 @@ func ParseURL(url string, creds credentials.Creds) (Storage, error) { if creds.AWS.SecretAccessKey != "" { opts = append(opts, s3.WithSecretAccessKey(creds.AWS.SecretAccessKey)) } + if creds.AWS.S3UsePathStyle { + opts = append(opts, s3.WithPathStyle()) + } store = s3.New(*u, opts...) default: return nil, fmt.Errorf("unknown url protocol: %s", u.Scheme) diff --git a/pkg/util/parse.go b/pkg/util/parse.go index 7b7ba2b5..c16549eb 100644 --- a/pkg/util/parse.go +++ b/pkg/util/parse.go @@ -5,7 +5,7 @@ import ( "strings" ) -// smartParse parse a url, but convert "/" into "file:///" +// SmartParse parse a url, but convert "/" into "file:///" func SmartParse(raw string) (*url.URL, error) { if strings.HasPrefix(raw, "/") { raw = "file://" + raw