diff --git a/.lagoon.yml b/.lagoon.yml index 53b5a07..33ef0fe 100644 --- a/.lagoon.yml +++ b/.lagoon.yml @@ -3,11 +3,16 @@ docker-compose-yaml: docker-compose.yml project: "lagoon-sync" +# These ssh and api settings can be added to configure Lagoon outside of amazeeio.cloud infrastructure +# api: https://api.main.lagoon-core.cluster1.company.io/graphql +api: https://api.main.lagoon-core.test6.amazee.io/graphql +ssh: ssh.main.lagoon-core.test6.amazee.io:22 + lagoon-sync: ssh: - host: "example.ssh.lagoon.amazeeio.cloud" + host: "ssh.main.lagoon-core.test6.amazee.io" port: "22" - privateKey: "~/.ssh/example_id_rsa" + privateKey: "~/.ssh/id_rsa" verbose: true mariadb: config: @@ -58,4 +63,4 @@ lagoon-sync: local: overrides: config: - syncpath: "./config/sync" + syncpath: "./config/sync" \ No newline at end of file diff --git a/assets/lagoon.yml b/assets/lagoon.yml index ac58245..2432823 100644 --- a/assets/lagoon.yml +++ b/assets/lagoon.yml @@ -1,3 +1,6 @@ +api: https://api.lagoon.amazeeio.cloud/graphql +ssh: ssh.lagoon.amazeeio.cloud:32222 + lagoon-sync: ssh: host: "ssh.lagoon.amazeeio.cloud" diff --git a/cmd/root.go b/cmd/root.go index 2d24a08..da73482 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -153,7 +153,7 @@ func processConfig(cfgFile string) error { if !utils.FileExists(".lagoon-sync.yml") && !noCliInteraction { var response string // Ask user to safe-write config to '.lagoon-sync.yml' - fmt.Print("No configuration file found, do you want to add the file '.lagoon-sync.yml'? (y/n): ") + fmt.Print("No configuration file found, do you want to add the default config file '.lagoon-sync.yml'? (y/n): ") _, err = fmt.Scanln(&response) if err != nil { return fmt.Errorf("failed to read user input: %v", err) diff --git a/cmd/root_test.go b/cmd/root_test.go index 6c39c75..97fb04b 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -16,41 +16,19 @@ func Test_processConfigEnv(t *testing.T) { { name: "Initial test", args: args{ - paths: []string{"../test-resources/config-tests/initial-test"}, - DefaultConfigFileName: "intial-test-lagoon.yml", - }, - want: "intial-test-lagoon.yml", - wantErr: false, - }, - { - name: "Initial test - Empty path", - args: args{ - paths: []string{}, DefaultConfigFileName: "../test-resources/config-tests/initial-test/intial-test-lagoon.yml", }, - want: "../test-resources/config-tests/initial-test/intial-test-lagoon.yml", - wantErr: false, - }, - { - name: "Initial test - Multiple paths", - args: args{ - paths: []string{"../test-resources/sync-test/tests-defaults", "../test-resources/sync-test/tests-lagoon-yml", "../test-resources/config-tests/initial-test"}, - DefaultConfigFileName: ".lagoon.yml", - }, - want: ".lagoon.yml", + want: "intial-test-lagoon.yml", wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := processConfigEnv(tt.args.paths, tt.args.DefaultConfigFileName) + err := processConfig(tt.args.DefaultConfigFileName) if (err != nil) != tt.wantErr { - t.Errorf("processConfigEnv() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("processConfig() error = %v, wantErr %v", err, tt.wantErr) return } - if got != tt.want { - t.Errorf("processConfigEnv() = %v, want %v", got, tt.want) - } }) } } diff --git a/cmd/sync.go b/cmd/sync.go index e16ee15..0e6d3bb 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -1,18 +1,22 @@ package cmd import ( + "bufio" + "context" "fmt" "log" "os" "strings" - "github.com/mitchellh/mapstructure" - "github.com/manifoldco/promptui" + "github.com/mitchellh/mapstructure" "github.com/spf13/cobra" "github.com/spf13/viper" synchers "github.com/uselagoon/lagoon-sync/synchers" "github.com/uselagoon/lagoon-sync/utils" + "github.com/uselagoon/machinery/api/lagoon" + lclient "github.com/uselagoon/machinery/api/lagoon/client" + "github.com/uselagoon/machinery/utils/sshtoken" ) var ProjectName string @@ -21,6 +25,7 @@ var targetEnvironmentName string var SyncerType string var ServiceName string var configurationFile string +var Api string var SSHHost string var SSHPort string var SSHKey string @@ -34,7 +39,9 @@ var runSyncProcess synchers.RunSyncProcessFunctionType var skipSourceCleanup bool var skipTargetCleanup bool var skipTargetImport bool +var SkipAPI bool var localTransferResourceName string +var rsyncArgDefaults = "--omit-dir-times --no-perms --no-group --no-owner --chmod=ugo=rwX --recursive --compress" var namedTransferResource string var syncCmd = &cobra.Command{ @@ -45,9 +52,19 @@ var syncCmd = &cobra.Command{ Run: syncCommandRun, } +type Sync struct { + Source synchers.Environment + Target synchers.Environment + Type string + Config synchers.SyncherConfigRoot + EnableDebug bool +} + func syncCommandRun(cmd *cobra.Command, args []string) { + Sync := Sync{} SyncerType := args[0] viper.Set("syncer-type", args[0]) + Sync.Type = SyncerType var configRoot synchers.SyncherConfigRoot @@ -64,11 +81,11 @@ func syncCommandRun(cmd *cobra.Command, args []string) { if err != nil { log.Fatalf("There was an issue unmarshalling the sync configuration from %v: %v", viper.ConfigFileUsed(), err) } else { - // Update configRoot with loaded configRoot = loadedConfigRoot } } } + Sync.Config = configRoot // If no project flag is given, find project from env var. if ProjectName == "" { @@ -90,8 +107,8 @@ func syncCommandRun(cmd *cobra.Command, args []string) { ProjectName: ProjectName, EnvironmentName: sourceEnvironmentName, ServiceName: ServiceName, + SSH: synchers.SSHOptions{}, } - // We assume that the target environment is local if it's not passed as an argument if targetEnvironmentName == "" { targetEnvironmentName = synchers.LOCAL_ENVIRONMENT_NAME @@ -100,14 +117,16 @@ func syncCommandRun(cmd *cobra.Command, args []string) { ProjectName: ProjectName, EnvironmentName: targetEnvironmentName, ServiceName: ServiceName, + SSH: synchers.SSHOptions{}, } var lagoonSyncer synchers.Syncer + // Syncers are registered in their init() functions - so here we attempt to match // the syncer type with the argument passed through to this command // (e.g. if we're running `lagoon-sync sync mariadb --...options follow` the function // GetSyncersForTypeFromConfigRoot will return a prepared mariadb syncher object) - lagoonSyncer, err := synchers.GetSyncerForTypeFromConfigRoot(SyncerType, configRoot) + lagoonSyncer, err := synchers.GetSyncerForTypeFromConfigRoot(SyncerType, Sync.Config) if err != nil { utils.LogFatalError(err.Error(), nil) } @@ -116,6 +135,44 @@ func syncCommandRun(cmd *cobra.Command, args []string) { utils.LogFatalError("No Project name given", nil) } + // get api endpoint from config if found + if configRoot.Api != "" { + Sync.Config.Api = configRoot.Api + } + + // if no api endpoint found in config, ask the user if they want to set it + if !noCliInteraction { + if configRoot.Api == "" { + reader := bufio.NewReader(os.Stdin) + fmt.Print("\033[32mWe couldn't find a Lagoon API endpoint in your config.\n\033[0m") + fmt.Print("\033[32mDo you want to define that now, or use the default ('https://api.lagoon.amazeeio.cloud/graphql')? (yes/no): \033[0m") + input, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + input = strings.TrimSpace(strings.ToLower(input)) + if input == "yes" { + + fmt.Print("Enter Lagoon API Endpoint: ") + endpoint, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + Sync.Config.Api = strings.TrimSpace(endpoint) + } + } + } + + // use cli arg to override api endpoint if given + if Api != "" { + Sync.Config.Api = Api + } + + sourceEnvironment.SSH = Sync.GetSSHOptions(ProjectName, sourceEnvironment, Sync.Config) + utils.LogDebugInfo("Config that is used for source SSH", sourceEnvironment.SSH) + targetEnvironment.SSH = Sync.GetSSHOptions(ProjectName, targetEnvironment, Sync.Config) + utils.LogDebugInfo("Config that is used for target SSH", targetEnvironment.SSH) + if !noCliInteraction { confirmationResult, err := confirmPrompt(fmt.Sprintf("Project: %s - you are about to sync %s from %s to %s, is this correct", ProjectName, @@ -190,6 +247,150 @@ func syncCommandRun(cmd *cobra.Command, args []string) { } } +// Get SSH options based on the following prority: +// 1. CLI arguments given (--ssh-host, --ssh-port, for examples) +// 2. Cluster config variables such as 'LAGOON_CONFIG_SSH_HOST' and 'LAGOON_CONFIG_SSH_HOST' +// 3. Deploy Targets set for the environment from the Lagoon API +// 4. SSH defined fields in any config files (lagoon-sync.ssh, ssh) +// 5. User prompted input if none of the above is found +// 6. CLI defaults as fallback +func (s *Sync) GetSSHOptions(project string, environment synchers.Environment, configRoot synchers.SyncherConfigRoot) synchers.SSHOptions { + sshConfig := &synchers.SSHOptions{} + + // SSH Config from yaml + if s.Config.Ssh != "" { + sshString := strings.Split(s.Config.Ssh, ":") + if len(sshString) != 2 { + utils.LogFatalError("Invalid ssh host input format - should match 'host:port'.", nil) + } + + host := strings.TrimSpace(sshString[0]) + port := strings.TrimSpace(sshString[1]) + + sshConfig.Host = host + sshConfig.Port = port + } + if configRoot.LagoonSync["ssh"] != nil { + mapstructure.Decode(configRoot.LagoonSync["ssh"], &sshConfig) + } + + // if no ssh config is found, then ask user if they want to set it now or the defaults will be used + if !noCliInteraction { + if sshConfig.Host == "" || sshConfig.Port == "" { + reader := bufio.NewReader(os.Stdin) + + fmt.Printf("\033[32mWe couldn't find any Lagoon SSH config for environment '%s'.\n\033[0m", environment.EnvironmentName) + fmt.Print("\033[32mDo you want to define that now, or use the defaults? (yes/no): \033[0m") + input, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + input = strings.TrimSpace(strings.ToLower(input)) + if input == "yes" { + fmt.Printf("\033[32mEnter SSH Host for '%s': \033[0m", environment.EnvironmentName) + sshHost, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + sshConfig.Host = strings.TrimSpace(sshHost) + + fmt.Printf("\033[32mEnter SSH Port for '%s': \033[0m", environment.EnvironmentName) + sshPort, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + sshConfig.Port = strings.TrimSpace(sshPort) + + if sshConfig.PrivateKey == "" { + fmt.Printf("\033[32mEnter SSH Key for '%s': \033[0m", environment.EnvironmentName) + sshKey, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("Error reading user input: %v", err) + } + sshConfig.PrivateKey = strings.TrimSpace(sshKey) + } + } + } + } + + // Check lagoon api for ssh config based on deploy targets + sshConfig, err := s.fetchSSHPortalConfigFromAPI(project, environment.EnvironmentName, &synchers.SSHOptions{ + Host: sshConfig.Host, + PrivateKey: sshConfig.PrivateKey, + Port: sshConfig.Port, + Verbose: sshConfig.Verbose, + RsyncArgs: sshConfig.RsyncArgs, + }) + if err != nil { + utils.LogFatalError(err.Error(), nil) + } + + // Check LAGOOON_CONFIG_X env vars + lagoonSSHHost := os.Getenv("LAGOON_CONFIG_SSH_HOST") + if lagoonSSHHost != "" { + sshConfig.Host = lagoonSSHHost + } + + lagoonSSHPort := os.Getenv("LAGOON_CONFIG_SSH_PORT") + if lagoonSSHPort != "" { + sshConfig.Port = lagoonSSHPort + } + + // cli argument overrides config + if SSHHost != "" && SSHHost != "ssh.lagoon.amazeeio.cloud" { + sshConfig.Host = SSHHost + } + if SSHPort != "" && SSHPort != "32222" { + sshConfig.Port = SSHPort + } + if SSHKey != "" { + sshConfig.PrivateKey = SSHKey + } + if SSHVerbose { + sshConfig.Verbose = SSHVerbose + } + if RsyncArguments != "" && RsyncArguments != "--omit-dir-times --no-perms --no-group --no-owner --chmod=ugo=rwX --recursive --compress" { + sshConfig.RsyncArgs = RsyncArguments + } + + return *sshConfig +} + +func (s *Sync) fetchSSHPortalConfigFromAPI(project string, environment string, sshConfig *synchers.SSHOptions) (*synchers.SSHOptions, error) { + if SkipAPI { + return sshConfig, nil + } + + // Grab a lagoon token + token, err := sshtoken.RetrieveToken(sshConfig.PrivateKey, sshConfig.Host, sshConfig.Port) + if err != nil { + log.Println(fmt.Sprintf("ERROR: unable to generate token: %v", err)) + return nil, err + } + + lc := lclient.New(s.Config.Api, "lagoon-sync", &token, false) + ctx := context.TODO() + p, err := lagoon.GetSSHEndpointsByProject(ctx, project, lc) + if err != nil { + errMessage := fmt.Sprintf("Failed to get ssh config for '%s' at '%s': ", project, s.Config.Api) + utils.LogFatalError(errMessage, err.Error()) + return nil, err + } + + if p.Environments != nil { + for _, e := range p.Environments { + if e.Name == environment { + return &synchers.SSHOptions{ + Host: e.DeployTarget.SSHHost, + Port: e.DeployTarget.SSHPort, + }, nil + } + } + } + + return sshConfig, nil +} + func getServiceName(SyncerType string) string { if SyncerType == "mongodb" { return SyncerType @@ -217,16 +418,18 @@ func init() { syncCmd.PersistentFlags().StringVarP(&targetEnvironmentName, "target-environment-name", "t", "", "The target environment name (defaults to local)") syncCmd.PersistentFlags().StringVarP(&ServiceName, "service-name", "s", "", "The service name (default is 'cli'") syncCmd.MarkPersistentFlagRequired("remote-environment-name") - syncCmd.PersistentFlags().StringVarP(&SSHHost, "ssh-host", "H", "ssh.lagoon.amazeeio.cloud", "Specify your lagoon ssh host, defaults to 'ssh.lagoon.amazeeio.cloud'") - syncCmd.PersistentFlags().StringVarP(&SSHPort, "ssh-port", "P", "32222", "Specify your ssh port, defaults to '32222'") + syncCmd.PersistentFlags().StringVarP(&Api, "api", "A", "https://api.lagoon.amazeeio.cloud/graphql", "Specify your lagoon api endpoint") + syncCmd.PersistentFlags().StringVarP(&SSHHost, "ssh-host", "H", "ssh.lagoon.amazeeio.cloud", "Specify your lagoon ssh host") + syncCmd.PersistentFlags().StringVarP(&SSHPort, "ssh-port", "P", "32222", "Specify your ssh port") syncCmd.PersistentFlags().StringVarP(&SSHKey, "ssh-key", "i", "", "Specify path to a specific SSH key to use for authentication") syncCmd.PersistentFlags().BoolVar(&SSHVerbose, "verbose", false, "Run ssh commands in verbose (useful for debugging)") syncCmd.PersistentFlags().BoolVar(&noCliInteraction, "no-interaction", false, "Disallow interaction") syncCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Don't run the commands, just preview what will be run") - syncCmd.PersistentFlags().StringVarP(&RsyncArguments, "rsync-args", "r", "--omit-dir-times --no-perms --no-group --no-owner --chmod=ugo=rwX --recursive --compress", "Pass through arguments to change the behaviour of rsync") + syncCmd.PersistentFlags().StringVarP(&RsyncArguments, "rsync-args", "r", rsyncArgDefaults, "Pass through arguments to change the behaviour of rsync") syncCmd.PersistentFlags().BoolVar(&skipSourceCleanup, "skip-source-cleanup", false, "Don't clean up any of the files generated on the source") syncCmd.PersistentFlags().BoolVar(&skipTargetCleanup, "skip-target-cleanup", false, "Don't clean up any of the files generated on the target") syncCmd.PersistentFlags().BoolVar(&skipTargetImport, "skip-target-import", false, "This will skip the import step on the target, in combination with 'no-target-cleanup' this essentially produces a resource dump") + syncCmd.PersistentFlags().BoolVar(&SkipAPI, "skip-api", false, "This will skip checking the api for configuration and instead use the defaults") syncCmd.PersistentFlags().StringVarP(&namedTransferResource, "transfer-resource-name", "", "", "The name of the temporary file to be used to transfer generated resources (db dumps, etc) - random /tmp file otherwise") // By default, we hook up the syncers.RunSyncProcess function to the runSyncProcess variable diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 17900b2..5197455 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -3,9 +3,12 @@ package cmd import ( "errors" "fmt" + "reflect" + "testing" + "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/uselagoon/lagoon-sync/synchers" - "testing" ) func Test_syncCommandRun(t *testing.T) { @@ -52,6 +55,10 @@ func Test_syncCommandRun(t *testing.T) { }, }, runSyncProcess: func(args synchers.RunSyncProcessFunctionTypeArguments) error { + if args.SourceEnvironment.ProjectName != "lagoon-sync" { + return errors.New(fmt.Sprintf("Expecting project name 'lagoon-sync' - found: %v", args.SourceEnvironment.ProjectName)) + } + if args.SshOptions.Port != "777" { return errors.New(fmt.Sprintf("Expecting ssh port 777 - found: %v", args.SshOptions.Port)) } @@ -70,7 +77,76 @@ func Test_syncCommandRun(t *testing.T) { runSyncProcess = tt.runSyncProcess cfgFile = tt.lagoonYmlFile noCliInteraction = true + SkipAPI = true + + viper.SetConfigFile(cfgFile) + viper.AutomaticEnv() + syncCommandRun(tt.args.cmd, tt.args.args) }) } } + +func TestSetSSHOptions(t *testing.T) { + type args struct { + Project string + Source synchers.Environment + Target synchers.Environment + Type string + Config synchers.SyncherConfigRoot + EnableDebug bool + } + var tests = []struct { + name string + args args + want synchers.SSHOptions + }{ + { + name: "Tests config overrides", + args: args{ + Project: "high-cotton", + Source: synchers.Environment{EnvironmentName: "main"}, + Target: synchers.Environment{EnvironmentName: "dev"}, + Type: "mariadb", + Config: synchers.SyncherConfigRoot{ + LagoonSync: map[string]interface{}{ + "ssh": map[string]interface{}{ + "host": "main.lagoon.example.com", + "port": "111", + "privateKey": "~/.ssh/path/to/key", + "verbose": true, + "rsyncArgs": "-a", + }, + }, + Api: "https://api.lagoon.amazeeio.cloud/graphql", + }, + EnableDebug: false, + }, + want: synchers.SSHOptions{ + Host: "main.lagoon.example.com", + Port: "111", + Verbose: true, + PrivateKey: "~/.ssh/path/to/key", + RsyncArgs: "-a", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Sync{ + Source: tt.args.Source, + Target: tt.args.Target, + Type: tt.args.Type, + EnableDebug: tt.args.EnableDebug, + Config: tt.args.Config, + } + + SkipAPI = true + noCliInteraction = true + + if got := s.GetSSHOptions(tt.args.Project, tt.args.Source, tt.args.Config); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetSSHOptions() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod index c764f23..98ef702 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,14 @@ require ( github.com/mitchellh/mapstructure v1.4.3 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.9.0 + github.com/uselagoon/machinery v0.0.11 github.com/withmandala/go-log v0.1.0 - golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa + golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b golang.org/x/text v0.3.8 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.64.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) + +// local dev +replace github.com/uselagoon/machinery v0.0.6 => ../machinery diff --git a/go.sum b/go.sum index 1c6644c..39dbb02 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -136,6 +138,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -157,12 +160,16 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= +github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -180,6 +187,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -213,10 +222,14 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= +github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -249,6 +262,8 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -291,6 +306,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/uselagoon/machinery v0.0.11 h1:s6EhyU/pj1+C4FdS0EqmR6C0dLsoeCd9n+5xHL1YDag= +github.com/uselagoon/machinery v0.0.11/go.mod h1:IXLxlkahEAEgpCmu9Xa/Wmjo6ja4Aoq7tf8G7VrileE= github.com/withmandala/go-log v0.1.0 h1:wINmTEe7BQ6zEA8sE7lSsYeaxCLluK6RFjF/IB5tzkA= github.com/withmandala/go-log v0.1.0/go.mod h1:/V9xQUTW74VjYm3u2Liv/bIUGLWoL9z2GlHwtscp4vg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -323,8 +340,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0= +golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -399,6 +416,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/synchers/prerequisiteSyncUtils.go b/synchers/prerequisiteSyncUtils.go index 742c1ec..e4d99d4 100644 --- a/synchers/prerequisiteSyncUtils.go +++ b/synchers/prerequisiteSyncUtils.go @@ -14,7 +14,7 @@ import ( "github.com/uselagoon/lagoon-sync/utils" ) -func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType string, dryRun bool, sshOptions SSHOptions) (Environment, error) { +func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType string, dryRun bool) (Environment, error) { // We don't run prerequisite checks on these syncers for now. if syncerType == "files" || syncerType == "drupalconfig" { environment.RsyncPath = "rsync" @@ -34,7 +34,7 @@ func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType s if environment.EnvironmentName == LOCAL_ENVIRONMENT_NAME { execString = command } else { - execString = GenerateRemoteCommand(environment, command, sshOptions) + execString = GenerateRemoteCommand(environment, command) } utils.LogExecutionStep("Running the following prerequisite command", execString) @@ -82,7 +82,7 @@ func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType s if !dryRun && !environment.RsyncAvailable { // Add rsync to env - rsyncPath, err := createRsync(environment, syncer, lagoonSyncVersion, sshOptions) + rsyncPath, err := createRsync(environment, syncer, lagoonSyncVersion) if err != nil { fmt.Println(errstring) return environment, err @@ -96,7 +96,7 @@ func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType s return environment, nil } -func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool, sshOptions SSHOptions) error { +func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool) error { if rsyncPath == "" || rsyncPath == "rsync" || !strings.Contains(rsyncPath, "/tmp/") { return nil } @@ -106,7 +106,7 @@ func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool, execString := fmt.Sprintf("rm -r %s", rsyncPath) if environment.EnvironmentName != LOCAL_ENVIRONMENT_NAME { - execString = GenerateRemoteCommand(environment, execString, sshOptions) + execString = GenerateRemoteCommand(environment, execString) } utils.LogExecutionStep("Running the following", execString) @@ -126,7 +126,7 @@ func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool, var RsyncAssetPath = "/tmp/rsync" // will add bundled rsync onto environment and return the new rsync path as string -func createRsync(environment Environment, syncer Syncer, lagoonSyncVersion string, sshOptions SSHOptions) (string, error) { +func createRsync(environment Environment, syncer Syncer, lagoonSyncVersion string) (string, error) { utils.LogDebugInfo("%v environment doesn't have rsync", environment.EnvironmentName) utils.LogDebugInfo("Downloading rsync asset on", environment.EnvironmentName) @@ -168,7 +168,7 @@ func createRsync(environment Environment, syncer Syncer, lagoonSyncVersion strin } execString = fmt.Sprintf("ssh -t -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -p %s %s@%s %s %s", - sshOptions.Port, environment.GetOpenshiftProjectName(), sshOptions.Host, serviceArgument, command) + environment.SSH.Port, environment.GetOpenshiftProjectName(), environment.SSH.Host, serviceArgument, command) } utils.LogExecutionStep(fmt.Sprintf("Running the following for %s", environment.EnvironmentName), execString) diff --git a/synchers/syncdefs.go b/synchers/syncdefs.go index e81cfcb..ea3f9bb 100644 --- a/synchers/syncdefs.go +++ b/synchers/syncdefs.go @@ -45,23 +45,22 @@ type SyncerTransferResource struct { } type Environment struct { - ProjectName string `yaml:"projectName"` - EnvironmentName string `yaml:"environmentName"` - ServiceName string `yaml:"serviceName"` // This is used to determine which Lagoon service we need to rsync - RsyncAvailable bool `yaml:"rsyncAvailable"` - RsyncPath string `yaml:"rsyncPath"` - RsyncLocalPath string `yaml:"rsyncLocalPath"` + ProjectName string `yaml:"projectName"` + EnvironmentName string `yaml:"environmentName"` + ServiceName string `yaml:"serviceName"` // This is used to determine which Lagoon service we need to rsync + RsyncAvailable bool `yaml:"rsyncAvailable"` + RsyncPath string `yaml:"rsyncPath"` + RsyncLocalPath string `yaml:"rsyncLocalPath"` + SSH SSHOptions `yaml:"ssh,omitempty"` } // SyncherConfigRoot is used to unmarshall yaml config details generally type SyncherConfigRoot struct { Project string `yaml:"project" json:"project,omitempty"` - LagoonSync map[string]interface{} `yaml:"lagoon-sync" json:"lagoonSync,omitempty"` - Prerequisites []prerequisite.GatheredPrerequisite `yaml:"prerequisites" json:"prerequisites,omitempty"` -} - -type SSHConfig struct { - SSH SSHOptions `yaml:"ssh,omitempty" json:"SSH"` + LagoonSync map[string]interface{} `yaml:"lagoon-sync,omitempty" json:"lagoon-sync,omitempty"` + Api string `yaml:"api,omitempty" json:"api,omitempty"` + Ssh string `yaml:"ssh,omitempty" json:"ssh,omitempty"` + Prerequisites []prerequisite.GatheredPrerequisite `yaml:"prerequisites,omitempty" json:"prerequisites,omitempty"` } type SSHOptions struct { @@ -86,14 +85,14 @@ func UnmarshalIntoStruct(pluginIn interface{}, pluginOut interface{}) error { return yaml.Unmarshal(b, pluginOut) } -func GenerateRemoteCommand(remoteEnvironment Environment, command string, sshOptions SSHOptions) string { +func GenerateRemoteCommand(remoteEnvironment Environment, command string) string { var sshOptionsStr bytes.Buffer - if sshOptions.Verbose { + if remoteEnvironment.SSH.Verbose { sshOptionsStr.WriteString(" -v") } - if sshOptions.PrivateKey != "" { - sshOptionsStr.WriteString(fmt.Sprintf(" -i %s", sshOptions.PrivateKey)) + if remoteEnvironment.SSH.PrivateKey != "" { + sshOptionsStr.WriteString(fmt.Sprintf(" -i %s", remoteEnvironment.SSH.PrivateKey)) } serviceArgument := "" @@ -102,5 +101,5 @@ func GenerateRemoteCommand(remoteEnvironment Environment, command string, sshOpt } return fmt.Sprintf("ssh%s -tt -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -p %s %s@%s %s '%s'", - sshOptionsStr.String(), sshOptions.Port, remoteEnvironment.GetOpenshiftProjectName(), sshOptions.Host, serviceArgument, command) + sshOptionsStr.String(), remoteEnvironment.SSH.Port, remoteEnvironment.GetOpenshiftProjectName(), remoteEnvironment.SSH.Host, serviceArgument, command) } diff --git a/synchers/syncutils.go b/synchers/syncutils.go index dcdf648..cad583e 100644 --- a/synchers/syncutils.go +++ b/synchers/syncutils.go @@ -50,53 +50,53 @@ func RunSyncProcess(args RunSyncProcessFunctionTypeArguments) error { } //TODO: this can come out. - args.SourceEnvironment, err = RunPrerequisiteCommand(args.SourceEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptions) + args.SourceEnvironment, err = RunPrerequisiteCommand(args.SourceEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun) sourceRsyncPath := args.SourceEnvironment.RsyncPath if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun) return err } - err = SyncRunSourceCommand(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunSourceCommand(args.SourceEnvironment, args.LagoonSyncer, args.DryRun) if err != nil { - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun) return err } - args.TargetEnvironment, err = RunPrerequisiteCommand(args.TargetEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptions) + args.TargetEnvironment, err = RunPrerequisiteCommand(args.TargetEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun) targetRsyncPath := args.TargetEnvironment.RsyncPath if err != nil { - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun) return err } - err = SyncRunTransfer(args.SourceEnvironment, args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunTransfer(args.SourceEnvironment, args.TargetEnvironment, args.LagoonSyncer, args.DryRun) if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun) return err } if !args.SkipTargetImport { - err = SyncRunTargetCommand(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunTargetCommand(args.TargetEnvironment, args.LagoonSyncer, args.DryRun) if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun) + _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun) return err } } else { utils.LogProcessStep("Skipping target import step", nil) } - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun) if !args.SkipSourceCleanup { - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun) } if !args.SkipTargetCleanup { - _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun) } else { utils.LogProcessStep("File on the target saved as: "+args.LagoonSyncer.GetTransferResource(args.TargetEnvironment).Name, nil) } @@ -104,8 +104,7 @@ func RunSyncProcess(args RunSyncProcessFunctionTypeArguments) error { return nil } -func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { - +func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun bool) error { utils.LogProcessStep("Beginning export on source environment", remoteEnvironment.EnvironmentName) remoteCommands := syncer.GetRemoteCommand(remoteEnvironment) @@ -125,7 +124,7 @@ func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun b if remoteEnvironment.EnvironmentName == LOCAL_ENVIRONMENT_NAME { execString = command } else { - execString = GenerateRemoteCommand(remoteEnvironment, command, sshOptions) + execString = GenerateRemoteCommand(remoteEnvironment, command) } utils.LogExecutionStep("Running the following for source", execString) @@ -145,7 +144,7 @@ func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun b return nil } -func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environment, syncer Syncer, dryRun bool) error { utils.LogProcessStep("Beginning file transfer logic", nil) // If we're transferring to the same resource, we can skip this whole process. @@ -178,7 +177,6 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen rsyncRemoteSystemUsername := "" if sourceEnvironment.EnvironmentName != LOCAL_ENVIRONMENT_NAME { - //sourceEnvironmentName = fmt.Sprintf("%s@ssh.lagoon.amazeeio.cloud:%s", sourceEnvironment.GetOpenshiftProjectName(), sourceEnvironmentName) sourceEnvironmentName = fmt.Sprintf(":%s", sourceEnvironmentName) rsyncRemoteSystemUsername = sourceEnvironment.GetOpenshiftProjectName() if sourceEnvironment.ServiceName != "" { @@ -188,7 +186,6 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen targetEnvironmentName := syncer.GetTransferResource(targetEnvironment).Name if targetEnvironment.EnvironmentName != LOCAL_ENVIRONMENT_NAME && executeRsyncRemotelyOnTarget == false { - //targetEnvironmentName = fmt.Sprintf("%s@ssh.lagoon.amazeeio.cloud:%s", targetEnvironment.GetOpenshiftProjectName(), targetEnvironmentName) targetEnvironmentName = fmt.Sprintf(":%s", targetEnvironmentName) rsyncRemoteSystemUsername = targetEnvironment.GetOpenshiftProjectName() if targetEnvironment.ServiceName != "" { @@ -203,32 +200,32 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen var sshOptionsStr bytes.Buffer verboseFlag := "" - if sshOptions.Verbose { + if sourceEnvironment.SSH.Verbose || targetEnvironment.SSH.Verbose { verboseFlag = "-v" sshOptionsStr.WriteString(" -v") } - if sshOptions.PrivateKey != "" { - sshOptionsStr.WriteString(fmt.Sprintf(" -i %s", sshOptions.PrivateKey)) + if sourceEnvironment.SSH.PrivateKey != "" { + sshOptionsStr.WriteString(fmt.Sprintf(" -i %s", sourceEnvironment.SSH.PrivateKey)) } - rsyncArgs := sshOptions.RsyncArgs + rsyncArgs := sourceEnvironment.SSH.RsyncArgs execString := fmt.Sprintf("%s %s --rsync-path=%s %s -e \"ssh%s -o LogLevel=FATAL -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p %s -l %s %s service=%s\" %s %s %s", targetEnvironment.RsyncPath, rsyncArgs, sourceEnvironment.RsyncPath, verboseFlag, sshOptionsStr.String(), - sshOptions.Port, + sourceEnvironment.SSH.Port, rsyncRemoteSystemUsername, - sshOptions.Host, + sourceEnvironment.SSH.Host, lagoonRsyncService, syncExcludes, sourceEnvironmentName, targetEnvironmentName) if executeRsyncRemotelyOnTarget { - execString = GenerateRemoteCommand(targetEnvironment, execString, sshOptions) + execString = GenerateRemoteCommand(targetEnvironment, execString) } utils.LogExecutionStep(fmt.Sprintf("Running the following for target (%s)", targetEnvironment.EnvironmentName), execString) @@ -243,8 +240,7 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen return nil } -func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { - +func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun bool) error { utils.LogProcessStep("Beginning import on target environment", targetEnvironment.EnvironmentName) targetCommands := syncer.GetLocalCommand(targetEnvironment) @@ -264,7 +260,7 @@ func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun b if targetEnvironment.EnvironmentName == LOCAL_ENVIRONMENT_NAME { execString = targetCommands } else { - execString = GenerateRemoteCommand(targetEnvironment, targetCommands, sshOptions) + execString = GenerateRemoteCommand(targetEnvironment, targetCommands) } utils.LogExecutionStep(fmt.Sprintf("Running the following for target (%s)", targetEnvironment.EnvironmentName), execString) @@ -279,7 +275,7 @@ func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun b return nil } -func SyncCleanUp(environment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncCleanUp(environment Environment, syncer Syncer, dryRun bool) error { transferResouce := syncer.GetTransferResource(environment) if transferResouce.SkipCleanup == true { @@ -295,7 +291,7 @@ func SyncCleanUp(environment Environment, syncer Syncer, dryRun bool, sshOptions execString := fmt.Sprintf("rm -r %s || true", transferResourceName) if environment.EnvironmentName != LOCAL_ENVIRONMENT_NAME { - execString = GenerateRemoteCommand(environment, execString, sshOptions) + execString = GenerateRemoteCommand(environment, execString) } utils.LogExecutionStep("Running the following", execString) diff --git a/test-resources/sync-test/tests-lagoon-yml/.lagoon.yml b/test-resources/sync-test/tests-lagoon-yml/.lagoon.yml index 6b68b96..7b16649 100644 --- a/test-resources/sync-test/tests-lagoon-yml/.lagoon.yml +++ b/test-resources/sync-test/tests-lagoon-yml/.lagoon.yml @@ -3,6 +3,8 @@ docker-compose-yaml: docker-compose.yml project: "lagoon-sync" +api: "https://api.main.lagoon-core.test6.amazee.io/graphql" + lagoon-sync: ssh: host: "example.ssh.lagoon.amazeeio.cloud"