From 858236216d0692a242759aecd5f8d060fcc43045 Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Tue, 26 Mar 2024 06:50:45 +1300 Subject: [PATCH 1/2] Adds new SSHOptionWrapper --- synchers/sshOptionWrapper.go | 29 +++++++ synchers/sshOptionWrapper_test.go | 124 ++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 synchers/sshOptionWrapper.go create mode 100644 synchers/sshOptionWrapper_test.go diff --git a/synchers/sshOptionWrapper.go b/synchers/sshOptionWrapper.go new file mode 100644 index 0000000..86bee45 --- /dev/null +++ b/synchers/sshOptionWrapper.go @@ -0,0 +1,29 @@ +package synchers + +// sshOptionWrapper.go contains the logic for the new system for passing ssh portal data + +type SSHOptionWrapper struct { + ProjectName string // this is primarily used to ensure someone doesn't do something silly - it's an assertion + Options map[string]SSHOptions // a map off all named ssh options - environment => ssh config + Default SSHOptions // this will be returned if no explicit match is found in `Options` +} + +func NewSshOptionWrapper(projectName string, defaultSshOptions SSHOptions) *SSHOptionWrapper { + return &SSHOptionWrapper{ + ProjectName: projectName, + Options: map[string]SSHOptions{}, + Default: defaultSshOptions, + } +} + +func (receiver *SSHOptionWrapper) getSSHOptionsForEnvironment(environmentName string) SSHOptions { + sshOptionsMapValue, ok := receiver.Options[environmentName] + if ok { + return sshOptionsMapValue + } + return receiver.Default +} + +func (receiver *SSHOptionWrapper) addSsshOptionForEnvironment(environmentName string, sshOptions SSHOptions) { + receiver.Options[environmentName] = sshOptions +} diff --git a/synchers/sshOptionWrapper_test.go b/synchers/sshOptionWrapper_test.go new file mode 100644 index 0000000..683744c --- /dev/null +++ b/synchers/sshOptionWrapper_test.go @@ -0,0 +1,124 @@ +package synchers + +import ( + "reflect" + "testing" +) + +var testOptions = map[string]SSHOptions{ + "env1": { + Host: "env1s.host.com", // Note, we're only really setting the host to differentiate during the test + }, + "env2": { + Host: "env2s.host.com", + }, +} + +func TestSSHOptionWrapper_getSSHOptionsForEnvironment(t *testing.T) { + type fields struct { + ProjectName string + Options map[string]SSHOptions + Default SSHOptions + } + type args struct { + environmentName string + } + tests := []struct { + name string + fields fields + args args + want SSHOptions + }{ + { + name: "Falls back to default", + fields: fields{ + ProjectName: "test", + Options: testOptions, + Default: SSHOptions{ + Host: "defaulthost", + }, + }, + want: SSHOptions{ + Host: "defaulthost", + }, + args: args{environmentName: "shoulddefault"}, + }, + { + name: "Gets named environment ssh details", + fields: fields{ + ProjectName: "test", + Options: testOptions, + Default: SSHOptions{ + Host: "defaulthost", + }, + }, + want: SSHOptions{ + Host: "env1s.host.com", + }, + args: args{environmentName: "env1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + receiver := &SSHOptionWrapper{ + ProjectName: tt.fields.ProjectName, + Options: tt.fields.Options, + Default: tt.fields.Default, + } + if got := receiver.getSSHOptionsForEnvironment(tt.args.environmentName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSSHOptionsForEnvironment() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSSHOptionWrapper_addSsshOptionForEnvironment(t *testing.T) { + type fields struct { + ProjectName string + Options map[string]SSHOptions + Default SSHOptions + } + type args struct { + environmentName string + environmentSSHOptions SSHOptions + } + tests := []struct { + name string + fields fields + args args + want SSHOptions + }{ + { + name: "Adds a new item to the list", + fields: fields{ + ProjectName: "test", + Options: testOptions, + Default: SSHOptions{ + Host: "defaulthost", + }, + }, + want: SSHOptions{ + Host: "newItem.ssh.com", + }, + args: args{ + environmentSSHOptions: SSHOptions{ + Host: "newItem.ssh.com", + }, + environmentName: "newItem", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + receiver := &SSHOptionWrapper{ + ProjectName: tt.fields.ProjectName, + Options: tt.fields.Options, + Default: tt.fields.Default, + } + receiver.addSsshOptionForEnvironment(tt.args.environmentName, tt.args.environmentSSHOptions) + if got := receiver.getSSHOptionsForEnvironment(tt.args.environmentName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSSHOptionsForEnvironment() = %v, want %v", got, tt.want) + } + }) + } +} From 1851ef0e25d64739f7cc02bbfada712ca1f5b27f Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Tue, 26 Mar 2024 11:17:55 +1300 Subject: [PATCH 2/2] Uses new ssh options wrapper --- cmd/sync.go | 16 ++++--- cmd/sync_test.go | 18 ++++---- synchers/prerequisiteSyncUtils.go | 10 ++++- synchers/sshOptionWrapper.go | 1 + synchers/syncutils.go | 75 ++++++++++++++++++------------- 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 8916d2a..28a358d 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -156,6 +156,7 @@ func syncCommandRun(cmd *cobra.Command, args []string) { sshVerbose = sshConfig.Verbose } + // Here we have the default - let's add it to a wrapper sshOptions := synchers.SSHOptions{ Host: sshHost, PrivateKey: sshKey, @@ -165,6 +166,8 @@ func syncCommandRun(cmd *cobra.Command, args []string) { SkipAgent: SSHSkipAgent, } + sshOptionWrapper := synchers.NewSshOptionWrapper(ProjectName, sshOptions) + // let's update the named transfer resource if it is set if namedTransferResource != "" { err = lagoonSyncer.SetTransferResource(namedTransferResource) @@ -176,12 +179,13 @@ func syncCommandRun(cmd *cobra.Command, args []string) { utils.LogDebugInfo("Config that is used for SSH", sshOptions) err = runSyncProcess(synchers.RunSyncProcessFunctionTypeArguments{ - SourceEnvironment: sourceEnvironment, - TargetEnvironment: targetEnvironment, - LagoonSyncer: lagoonSyncer, - SyncerType: SyncerType, - DryRun: dryRun, - SshOptions: sshOptions, + SourceEnvironment: sourceEnvironment, + TargetEnvironment: targetEnvironment, + LagoonSyncer: lagoonSyncer, + SyncerType: SyncerType, + DryRun: dryRun, + //SshOptions: sshOptions, + SshOptionWrapper: sshOptionWrapper, SkipTargetCleanup: skipTargetCleanup, SkipSourceCleanup: skipSourceCleanup, SkipTargetImport: skipTargetImport, diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 17900b2..4705ae6 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -30,12 +30,13 @@ func Test_syncCommandRun(t *testing.T) { }, }, runSyncProcess: func(args synchers.RunSyncProcessFunctionTypeArguments) error { - if args.SshOptions.Port != "32222" { - return errors.New(fmt.Sprintf("Expecting ssh port 32222 - found: %v", args.SshOptions.Port)) + sshOptions := args.SshOptionWrapper.Default + if sshOptions.Port != "32222" { + return errors.New(fmt.Sprintf("Expecting ssh port 32222 - found: %v", sshOptions.Port)) } - if args.SshOptions.Host != "ssh.lagoon.amazeeio.cloud" { - return errors.New(fmt.Sprintf("Expecting ssh host ssh.lagoon.amazeeio.cloud - found: %v", args.SshOptions.Host)) + if sshOptions.Host != "ssh.lagoon.amazeeio.cloud" { + return errors.New(fmt.Sprintf("Expecting ssh host ssh.lagoon.amazeeio.cloud - found: %v", sshOptions.Host)) } return nil @@ -52,12 +53,13 @@ func Test_syncCommandRun(t *testing.T) { }, }, runSyncProcess: func(args synchers.RunSyncProcessFunctionTypeArguments) error { - if args.SshOptions.Port != "777" { - return errors.New(fmt.Sprintf("Expecting ssh port 777 - found: %v", args.SshOptions.Port)) + sshOptions := args.SshOptionWrapper.Default + if sshOptions.Port != "777" { + return errors.New(fmt.Sprintf("Expecting ssh port 777 - found: %v", sshOptions.Port)) } - if args.SshOptions.Host != "example.ssh.lagoon.amazeeio.cloud" { - return errors.New(fmt.Sprintf("Expecting ssh host ssh.lagoon.amazeeio.cloud - found: %v", args.SshOptions.Host)) + if sshOptions.Host != "example.ssh.lagoon.amazeeio.cloud" { + return errors.New(fmt.Sprintf("Expecting ssh host ssh.lagoon.amazeeio.cloud - found: %v", sshOptions.Host)) } return nil diff --git a/synchers/prerequisiteSyncUtils.go b/synchers/prerequisiteSyncUtils.go index f37eeca..53a3319 100644 --- a/synchers/prerequisiteSyncUtils.go +++ b/synchers/prerequisiteSyncUtils.go @@ -14,7 +14,10 @@ import ( "github.com/uselagoon/lagoon-sync/utils" ) -func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType string, dryRun bool, sshOptions SSHOptions) (Environment, error) { +func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType string, dryRun bool, sshOptionWrapper *SSHOptionWrapper) (Environment, error) { + + sshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(environment.EnvironmentName) + // We don't run prerequisite checks on these syncers for now. if syncerType == "files" || syncerType == "drupalconfig" { environment.RsyncPath = "rsync" @@ -104,7 +107,10 @@ func RunPrerequisiteCommand(environment Environment, syncer Syncer, syncerType s return environment, nil } -func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool, sshOptions SSHOptions) error { +func PrerequisiteCleanUp(environment Environment, rsyncPath string, dryRun bool, sshOptionWrapper *SSHOptionWrapper) error { + + sshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(environment.EnvironmentName) + if rsyncPath == "" || rsyncPath == "rsync" || !strings.Contains(rsyncPath, "/tmp/") { return nil } diff --git a/synchers/sshOptionWrapper.go b/synchers/sshOptionWrapper.go index 86bee45..8647af0 100644 --- a/synchers/sshOptionWrapper.go +++ b/synchers/sshOptionWrapper.go @@ -2,6 +2,7 @@ package synchers // sshOptionWrapper.go contains the logic for the new system for passing ssh portal data +// SSHOptionWrapper is passed around instead of specific SSHOptions - this allows resolution of the ssh endpoint when and where it's needed type SSHOptionWrapper struct { ProjectName string // this is primarily used to ensure someone doesn't do something silly - it's an assertion Options map[string]SSHOptions // a map off all named ssh options - environment => ssh config diff --git a/synchers/syncutils.go b/synchers/syncutils.go index a7b34a4..3102fdc 100644 --- a/synchers/syncutils.go +++ b/synchers/syncutils.go @@ -28,12 +28,13 @@ func UnmarshallLagoonYamlToLagoonSyncStructure(data []byte) (SyncherConfigRoot, } type RunSyncProcessFunctionTypeArguments struct { - SourceEnvironment Environment - TargetEnvironment Environment - LagoonSyncer Syncer - SyncerType string - DryRun bool - SshOptions SSHOptions + SourceEnvironment Environment + TargetEnvironment Environment + LagoonSyncer Syncer + SyncerType string + DryRun bool + //SshOptions SSHOptions + SshOptionWrapper *SSHOptionWrapper SkipSourceCleanup bool SkipTargetCleanup bool SkipTargetImport bool @@ -50,54 +51,54 @@ func RunSyncProcess(args RunSyncProcessFunctionTypeArguments) error { } //TODO: this can come out. - args.SourceEnvironment, err = RunPrerequisiteCommand(args.SourceEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptions) + args.SourceEnvironment, err = RunPrerequisiteCommand(args.SourceEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptionWrapper) sourceRsyncPath := "rsync" //args.SourceEnvironment.RsyncPath args.SourceEnvironment.RsyncPath = "rsync" if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptionWrapper) return err } - err = SyncRunSourceCommand(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunSourceCommand(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) if err != nil { - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) return err } - args.TargetEnvironment, err = RunPrerequisiteCommand(args.TargetEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptions) + args.TargetEnvironment, err = RunPrerequisiteCommand(args.TargetEnvironment, args.LagoonSyncer, args.SyncerType, args.DryRun, args.SshOptionWrapper) targetRsyncPath := args.TargetEnvironment.RsyncPath if err != nil { - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptionWrapper) return err } - err = SyncRunTransfer(args.SourceEnvironment, args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunTransfer(args.SourceEnvironment, args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptionWrapper) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) return err } if !args.SkipTargetImport { - err = SyncRunTargetCommand(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + err = SyncRunTargetCommand(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) if err != nil { - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) - _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptionWrapper) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptionWrapper) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) + _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) return err } } else { utils.LogProcessStep("Skipping target import step", nil) } - _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptions) - _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptions) + _ = PrerequisiteCleanUp(args.SourceEnvironment, sourceRsyncPath, args.DryRun, args.SshOptionWrapper) + _ = PrerequisiteCleanUp(args.TargetEnvironment, targetRsyncPath, args.DryRun, args.SshOptionWrapper) if !args.SkipSourceCleanup { - _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.SourceEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) } if !args.SkipTargetCleanup { - _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptions) + _ = SyncCleanUp(args.TargetEnvironment, args.LagoonSyncer, args.DryRun, args.SshOptionWrapper) } else { utils.LogProcessStep("File on the target saved as: "+args.LagoonSyncer.GetTransferResource(args.TargetEnvironment).Name, nil) } @@ -105,10 +106,12 @@ func RunSyncProcess(args RunSyncProcessFunctionTypeArguments) error { return nil } -func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun bool, sshOptionWrapper *SSHOptionWrapper) error { utils.LogProcessStep("Beginning export on source environment", remoteEnvironment.EnvironmentName) + sshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(remoteEnvironment.EnvironmentName) + remoteCommands := syncer.GetRemoteCommand(remoteEnvironment) for _, remoteCommand := range remoteCommands { if remoteCommand.NoOp { @@ -151,9 +154,13 @@ func SyncRunSourceCommand(remoteEnvironment Environment, syncer Syncer, dryRun b return nil } -func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptionWrapper *SSHOptionWrapper) error { utils.LogProcessStep("Beginning file transfer logic", nil) + // TODO: This is going to be the trickiest of the ssh option calculations. + // We need to determine ssh endpoints for both environments separately + sshOptions := sshOptionWrapper.Default + // If we're transferring to the same resource, we can skip this whole process. if sourceEnvironment.EnvironmentName == targetEnvironment.EnvironmentName { utils.LogDebugInfo("Source and target environments are the same, skipping transfer", nil) @@ -218,6 +225,7 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen sshOptionsStr.WriteString(fmt.Sprintf(" -i %s", sshOptions.PrivateKey)) } + sourceEnvSshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(sourceEnvironmentName) rsyncArgs := sshOptions.RsyncArgs execString := fmt.Sprintf("%s %s --rsync-path=%s %s -e \"ssh%s -o LogLevel=FATAL -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p %s -l %s %s service=%s\" %s %s %s", targetEnvironment.RsyncPath, @@ -225,9 +233,9 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen sourceEnvironment.RsyncPath, verboseFlag, sshOptionsStr.String(), - sshOptions.Port, + sourceEnvSshOptions.Port, rsyncRemoteSystemUsername, - sshOptions.Host, + sourceEnvSshOptions.Host, lagoonRsyncService, syncExcludes, sourceEnvironmentName, @@ -238,7 +246,8 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen if !dryRun { if executeRsyncRemotelyOnTarget { - err, output := utils.RemoteShellout(execString, targetEnvironment.GetOpenshiftProjectName(), sshOptions.Host, sshOptions.Port, sshOptions.PrivateKey, sshOptions.SkipAgent) + TargetEnvSshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(targetEnvironmentName) + err, output := utils.RemoteShellout(execString, targetEnvironment.GetOpenshiftProjectName(), TargetEnvSshOptions.Host, TargetEnvSshOptions.Port, TargetEnvSshOptions.PrivateKey, TargetEnvSshOptions.SkipAgent) utils.LogDebugInfo(output, nil) if err != nil { utils.LogFatalError("Unable to exec remote command: "+err.Error(), nil) @@ -255,10 +264,12 @@ func SyncRunTransfer(sourceEnvironment Environment, targetEnvironment Environmen return nil } -func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun bool, sshOptionWrapper *SSHOptionWrapper) error { utils.LogProcessStep("Beginning import on target environment", targetEnvironment.EnvironmentName) + sshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(targetEnvironment.EnvironmentName) + targetCommands := syncer.GetLocalCommand(targetEnvironment) for _, targetCommand := range targetCommands { @@ -295,9 +306,11 @@ func SyncRunTargetCommand(targetEnvironment Environment, syncer Syncer, dryRun b return nil } -func SyncCleanUp(environment Environment, syncer Syncer, dryRun bool, sshOptions SSHOptions) error { +func SyncCleanUp(environment Environment, syncer Syncer, dryRun bool, sshOptionWrapper *SSHOptionWrapper) error { transferResouce := syncer.GetTransferResource(environment) + sshOptions := sshOptionWrapper.getSSHOptionsForEnvironment(environment.EnvironmentName) + if transferResouce.SkipCleanup == true { log.Printf("Skipping cleanup for %v on %v environment", transferResouce.Name, environment.EnvironmentName) return nil