diff --git a/commands/cluster_command_launcher.go b/commands/cluster_command_launcher.go index 90d2682..f31dc22 100644 --- a/commands/cluster_command_launcher.go +++ b/commands/cluster_command_launcher.go @@ -80,6 +80,8 @@ const ( subclusterFlag = "subcluster" addNodeFlag = "new-hosts" sandboxFlag = "sandbox" + connFlag = "conn" + connKey = "conn" ) // Flag and key for database replication @@ -126,11 +128,20 @@ var flagKeyMap = map[string]string{ sourceTLSConfigFlag: sourceTLSConfigKey, } +// target database flags to viper key map +var targetFlagKeyMap = map[string]string{ + targetDBNameFlag: targetDBNameKey, + targetHostsFlag: targetHostsKey, + targetUserNameFlag: targetUserNameKey, + targetPasswordFileFlag: targetPasswordFileKey, +} + const ( createDBSubCmd = "create_db" stopDBSubCmd = "stop_db" reviveDBSubCmd = "revive_db" manageConfigSubCmd = "manage_config" + createConnectionSubCmd = "create_connection" configRecoverSubCmd = "recover" configShowSubCmd = "show" replicationSubCmd = "replication" @@ -159,6 +170,13 @@ type cmdGlobals struct { file *os.File keyFile string certFile string + + // Global variables for targetDB are used for the replication subcommand + targetHosts []string + targetPasswordFile string + targetDB string + targetUserName string + connFile string } var ( @@ -262,19 +280,44 @@ func setDBOptionsUsingViper(flag string) error { return nil } +// setTargetDBOptionsUsingViper can set the value of flag using the relevant key +// in viper +func setTargetDBOptionsUsingViper(flag string) error { + switch flag { + case targetDBNameFlag: + globals.targetDB = viper.GetString(targetDBNameKey) + case targetHostsFlag: + globals.targetHosts = viper.GetStringSlice(targetHostsKey) + case targetUserNameFlag: + globals.targetUserName = viper.GetString(targetUserNameKey) + case targetPasswordFileFlag: + globals.targetPasswordFile = viper.GetString(targetPasswordFileKey) + default: + return fmt.Errorf("cannot find the relevant target database option for flag %q", flag) + } + return nil +} + // configViper configures viper to load database options using this order: // user input -> environment variables -> vcluster config file func configViper(cmd *cobra.Command, flagsInConfig []string) error { // initialize config file initConfig() + // target-flags are only available for replication start command + if cmd.CalledAs() == startReplicationSubCmd { + for targetFlag := range targetFlagKeyMap { + flagsInConfig = append(flagsInConfig, targetFlag) + } + } // log-path is a flag that all the subcommands need flagsInConfig = append(flagsInConfig, logPathFlag) // cert-file and key-file are not available for // - manage_config // - manage_config show + // - create_connection if cmd.CalledAs() != manageConfigSubCmd && - cmd.CalledAs() != configShowSubCmd { + cmd.CalledAs() != configShowSubCmd && cmd.CalledAs() != createConnectionSubCmd { flagsInConfig = append(flagsInConfig, certFileFlag, keyFileFlag) } @@ -289,7 +332,21 @@ func configViper(cmd *cobra.Command, flagsInConfig []string) error { } } - // bind viper keys to env vars + // Bind viper keys to environment variables + if err := bindKeysToEnv(); err != nil { + return err + } + + // Load config options from file to viper + if err := loadConfig(cmd); err != nil { + return err + } + + return handleViperUserInput(flagsInConfig) +} + +// bind viper keys to env vars +func bindKeysToEnv() error { err := viper.BindEnv(logPathKey, vclusterLogPathEnv) if err != nil { return fmt.Errorf("fail to bind viper key %q to environment variable %q: %w", logPathKey, vclusterLogPathEnv, err) @@ -302,7 +359,11 @@ func configViper(cmd *cobra.Command, flagsInConfig []string) error { if err != nil { return fmt.Errorf("fail to bind viper key %q to environment variable %q: %w", certFileKey, vclusterCertFileEnv, err) } + return nil +} +// load db options from file to viper +func loadConfig(cmd *cobra.Command) (err error) { // load db options from config file to viper // note: config file is not available for create_db and revive_db // manage_config does not need viper to load config file info @@ -310,21 +371,29 @@ func configViper(cmd *cobra.Command, flagsInConfig []string) error { cmd.CalledAs() != reviveDBSubCmd && cmd.CalledAs() != configRecoverSubCmd && cmd.CalledAs() != configShowSubCmd { - err = loadConfigToViper() + err := loadConfigToViper() if err != nil { return err } } - return handleViperUserInput(flagsInConfig) + // load target db options from connection file to viper + // conn file is only available for replication subcommand + if cmd.CalledAs() == startReplicationSubCmd { + err := loadConnToViper() + if err != nil { + return err + } + } + return nil } func handleViperUserInput(flagsInConfig []string) error { - // if a flag is set in viper through user input, env var or config file, we assign its viper value + // if a flag is set in viper through user input, env var or config/connection file, we assign its viper value // to database options. viper can automatically retrieve the correct value following below order: // 1. user input // 2. environment variable - // 3. config file + // 3. config/connection file // if the flag is not set in viper, the default value of it will be used for _, flag := range flagsInConfig { if _, ok := flagKeyMap[flag]; !ok { @@ -332,9 +401,16 @@ func handleViperUserInput(flagsInConfig []string) error { continue } if viper.IsSet(flagKeyMap[flag]) { - err := setDBOptionsUsingViper(flag) - if err != nil { - return fmt.Errorf("fail to set flag %q using viper: %w", flag, err) + if _, ok := targetFlagKeyMap[flag]; !ok { + err := setDBOptionsUsingViper(flag) + if err != nil { + return fmt.Errorf("fail to set flag %q using viper: %w", flag, err) + } + } else { + err := setTargetDBOptionsUsingViper(flag) + if err != nil { + return fmt.Errorf("fail to set target flag %q using viper: %w", flag, err) + } } } } @@ -442,6 +518,7 @@ func constructCmds() []*cobra.Command { makeCmdScrutinize(), makeCmdManageConfig(), makeCmdReplication(), + makeCmdCreateConnection(), } } diff --git a/commands/cmd_base.go b/commands/cmd_base.go index 45eeaaf..2552008 100644 --- a/commands/cmd_base.go +++ b/commands/cmd_base.go @@ -92,8 +92,8 @@ func (c *CmdBase) setCommonFlags(cmd *cobra.Command, flags []string) { "Show the details of VCluster run in the console", ) // keyFile and certFile are flags that all subcommands require, - // except for manage_config and `manage_config show` - if cmd.Name() != configShowSubCmd { + // except for create_connection and manage_config show + if cmd.Name() != configShowSubCmd && cmd.Name() != createConnectionSubCmd { cmd.Flags().StringVar( &globals.keyFile, keyFileFlag, @@ -272,6 +272,17 @@ func (c *CmdBase) setDBPassword(opt *vclusterops.DatabaseOptions) error { return nil } + // hyphen(`-`) is used to indicate that input should come + // from stdin rather than from a file + if c.passwordFile == "-" { + password, err := readFromStdin() + if err != nil { + return err + } + *opt.Password = strings.TrimSuffix(password, "\n") + return nil + } + password, err := c.passwordFileHelper(c.passwordFile) if err != nil { return err @@ -284,15 +295,6 @@ func (c *CmdBase) passwordFileHelper(passwordFile string) (string, error) { if passwordFile == "" { return "", fmt.Errorf("password file path is empty") } - // hyphen(`-`) is used to indicate that input should come - // from stdin rather than from a file - if passwordFile == "-" { - password, err := readFromStdin() - if err != nil { - return "", err - } - return strings.TrimSuffix(password, "\n"), nil - } // Read password from file passwordBytes, err := os.ReadFile(passwordFile) diff --git a/commands/cmd_create_connection.go b/commands/cmd_create_connection.go new file mode 100644 index 0000000..23c3ab9 --- /dev/null +++ b/commands/cmd_create_connection.go @@ -0,0 +1,122 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/vertica/vcluster/vclusterops" + "github.com/vertica/vcluster/vclusterops/vlog" +) + +/* CmdCreateConnection + * + * Implements ClusterCommand interface + */ +type CmdCreateConnection struct { + connectionOptions *vclusterops.VReplicationDatabaseOptions + CmdBase +} + +func makeCmdCreateConnection() *cobra.Command { + newCmd := &CmdCreateConnection{} + opt := vclusterops.VReplicationDatabaseFactory() + newCmd.connectionOptions = &opt + opt.TargetPassword = new(string) + + cmd := makeBasicCobraCmd( + newCmd, + createConnectionSubCmd, + "create the content of the connection file", + `This subcommand is used to create the content of the connection file. + +You must specify the database name and host list. If the database has a +password, you need to provide password. If the database uses +trust authentication, the password can be ignored. + +Examples: + # create the connection file to /tmp/vertica_connection.yaml + vcluster create_connection --db-name platform_test_db --hosts 10.20.30.43 --db-user \ + dkr_dbadmin --password-file /tmp/password.txt --conn /tmp/vertica_connection.yaml +`, + []string{connFlag}, + ) + + // local flags + newCmd.setLocalFlags(cmd) + + markFlagsRequired(cmd, []string{dbNameFlag, hostsFlag, connFlag}) + return cmd +} + +// setLocalFlags will set the local flags the command has +func (c *CmdCreateConnection) setLocalFlags(cmd *cobra.Command) { + cmd.Flags().StringVar( + &c.connectionOptions.TargetDB, + dbNameFlag, + "", + "The name of the database", + ) + cmd.Flags().StringSliceVar( + &c.connectionOptions.TargetHosts, + hostsFlag, + []string{}, + "Comma-separated list of hosts in database") + cmd.Flags().StringVar( + &c.connectionOptions.TargetUserName, + dbUserFlag, + "", + "The username for connecting to the database", + ) + // password flags + cmd.Flags().StringVar( + c.connectionOptions.TargetPassword, + passwordFileFlag, + "", + "Path to the file to read the password from. ", + ) + cmd.Flags().StringVar( + &globals.connFile, + connFlag, + "", + "Path to the connection file") + markFlagsFileName(cmd, map[string][]string{connFlag: {"yaml"}}) +} + +func (c *CmdCreateConnection) Parse(inputArgv []string, logger vlog.Printer) error { + c.argv = inputArgv + logger.LogMaskedArgParse(c.argv) + + return nil +} + +func (c *CmdCreateConnection) Run(vcc vclusterops.ClusterCommands) error { + vcc.LogInfo("Called method Run()") + + // write target db info to vcluster connection file + err := writeConn(c.connectionOptions) + if err != nil { + return fmt.Errorf("fail to write connection file, details: %s", err) + } + fmt.Printf("Successfully write connection file in %s", globals.connFile) + return nil +} + +// SetDatabaseOptions will assign a vclusterops.DatabaseOptions instance +func (c *CmdCreateConnection) SetDatabaseOptions(opt *vclusterops.DatabaseOptions) { + c.connectionOptions.DatabaseOptions = *opt +} diff --git a/commands/cmd_start_replication.go b/commands/cmd_start_replication.go index bd36a6f..a84140f 100644 --- a/commands/cmd_start_replication.go +++ b/commands/cmd_start_replication.go @@ -33,7 +33,6 @@ type CmdStartReplication struct { startRepOptions *vclusterops.VReplicationDatabaseOptions CmdBase targetPasswordFile string - targetConnPath string } func makeCmdStartReplication() *cobra.Command { @@ -82,15 +81,15 @@ Examples: // Temporarily, the Vcluster CLI doesn't support a config file for this subcommand. // It will include all hosts from the config file. // VER-93450 will add 2 options for sandboxes, "source-sandbox" and "target-sandbox", to get the correct sourceHosts - []string{dbNameFlag, hostsFlag, ipv6Flag, configFlag, passwordFlag, dbUserFlag, eonModeFlag}, + []string{dbNameFlag, hostsFlag, ipv6Flag, configFlag, passwordFlag, dbUserFlag, eonModeFlag, connFlag}, ) // local flags newCmd.setLocalFlags(cmd) - // Temporarily, targetDBName and targetHost are required. - // They will be removed after target-conn is implemented in VER-93130 - markFlagsRequired(cmd, []string{targetDBNameFlag, targetHostsFlag}) + // either target dbname/hosts or connection file must be provided + cmd.MarkFlagsOneRequired(targetConnFlag, targetDBNameFlag) + cmd.MarkFlagsOneRequired(targetConnFlag, targetHostsFlag) // hide eon mode flag since we expect it to come from config file, not from user input hideLocalFlags(cmd, []string{eonModeFlag}) @@ -124,19 +123,17 @@ func (c *CmdStartReplication) setLocalFlags(cmd *cobra.Command) { ", must exist in the source database", ) cmd.Flags().StringVar( - &c.targetConnPath, + &globals.connFile, targetConnFlag, "", - "Path to the target connection file") - markFlagsFileName(cmd, map[string][]string{configFlag: {"yaml"}}) - + "Path to the connection file") + markFlagsFileName(cmd, map[string][]string{targetConnFlag: {"yaml"}}) // password flags cmd.Flags().StringVar( &c.targetPasswordFile, targetPasswordFileFlag, "", - "Path to the file to read the password for target database. "+ - "If - is passed, the password is read from stdin", + "Path to the file to read the password for target database. ", ) } @@ -195,7 +192,7 @@ func (c *CmdStartReplication) parseTargetHostList() error { func (c *CmdStartReplication) parseTargetPassword() error { options := c.startRepOptions - if !c.parser.Changed(targetPasswordFileFlag) { + if !viper.IsSet(targetPasswordFileKey) { // reset password option to nil if password is not provided in cli options.TargetPassword = nil return nil @@ -229,4 +226,8 @@ func (c *CmdStartReplication) Run(vcc vclusterops.ClusterCommands) error { // SetDatabaseOptions will assign a vclusterops.DatabaseOptions instance func (c *CmdStartReplication) SetDatabaseOptions(opt *vclusterops.DatabaseOptions) { c.startRepOptions.DatabaseOptions = *opt + c.startRepOptions.TargetUserName = globals.targetUserName + c.startRepOptions.TargetDB = globals.targetDB + c.startRepOptions.TargetHosts = globals.targetHosts + c.targetPasswordFile = globals.targetPasswordFile } diff --git a/commands/user_input_test.go b/commands/user_input_test.go index ed5fea7..e6cd8e4 100644 --- a/commands/user_input_test.go +++ b/commands/user_input_test.go @@ -17,12 +17,14 @@ package commands import ( "fmt" + "io" "log" "os" "strings" "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) var tempConfigFilePath = os.TempDir() + "/test_vertica_cluster.yaml" @@ -84,9 +86,41 @@ func TestManageReplication(t *testing.T) { assert.ErrorContains(t, err, `unknown command "test" for "vcluster replication start"`) } -func TestStartReplication(t *testing.T) { - // vcluster replication start should succeed - // since there is no op for this subcommand - err := simulateVClusterCli("vcluster replication start") - assert.ErrorContains(t, err, `required flag(s) "target-db-name", "target-hosts" not set`) +func TestCreateConnection(t *testing.T) { + var tempConnFilePath = os.TempDir() + "/vertica_connection.yaml" + dbName := "platform_test_db" + hosts := "192.168.1.101" + tempConfig, _ := os.Create(tempConnFilePath) + defer tempConfig.Close() + defer os.Remove(tempConnFilePath) + + // vcluster create_connection should succeed + err := simulateVClusterCli("vcluster create_connection --db-name " + dbName + " --hosts " + hosts + + " --conn " + tempConnFilePath) + assert.NoError(t, err) + + // verify the file content + file, err := os.Open(tempConnFilePath) + if err != nil { + fmt.Println("Error opening file:", err) + return + } + defer file.Close() + + buf := make([]byte, 1024) + n, err := file.Read(buf) + if err != nil && err != io.EOF { + fmt.Println("Error reading file:", err) + return + } + + var dbConn DatabaseConnection + err = yaml.Unmarshal([]byte(string(buf[:n])), &dbConn) + if err != nil { + fmt.Println("Error:", err) + return + } + + assert.Equal(t, dbName, dbConn.TargetDB) + assert.Equal(t, hosts, dbConn.TargetHosts[0]) } diff --git a/commands/vcluster_connection.go b/commands/vcluster_connection.go new file mode 100644 index 0000000..999535a --- /dev/null +++ b/commands/vcluster_connection.go @@ -0,0 +1,94 @@ +package commands + +import ( + "fmt" + "os" + + "github.com/spf13/viper" + "github.com/vertica/vcluster/vclusterops" + "gopkg.in/yaml.v3" +) + +type DatabaseConnection struct { + TargetPasswordFile string `yaml:"targetPasswordFile" mapstructure:"targetPasswordFile"` + TargetHosts []string `yaml:"targetHosts" mapstructure:"targetHosts"` + TargetDB string `yaml:"targetDB" mapstructure:"targetDB"` + TargetUserName string `yaml:"targetUserName" mapstructure:"targetUserName"` +} + +func MakeTargetDatabaseConn() DatabaseConnection { + return DatabaseConnection{} +} + +// loadConnToViper can fill viper keys using the connection file +func loadConnToViper() error { + // read connection file + viper.SetConfigFile(globals.connFile) + err := viper.ReadInConfig() + if err != nil { + fmt.Printf("Warning: fail to read connection file %q for viper: %v\n", globals.connFile, err) + return nil + } + + // retrieve dbconn info in viper + dbConn := MakeTargetDatabaseConn() + err = viper.Unmarshal(&dbConn) + if err != nil { + fmt.Printf("Warning: fail to unmarshal connection file %q: %v\n", globals.connFile, err) + return nil + } + + if !viper.IsSet(targetDBNameKey) { + viper.Set(targetDBNameKey, dbConn.TargetDB) + } + if !viper.IsSet(targetHostsKey) { + viper.Set(targetHostsKey, dbConn.TargetHosts) + } + if !viper.IsSet(targetUserNameKey) { + viper.Set(targetUserNameKey, dbConn.TargetUserName) + } + return nil +} + +// writeConn will save instructions for connecting to a database into a connection file. +func writeConn(targetdb *vclusterops.VReplicationDatabaseOptions) error { + if globals.connFile == "" { + return fmt.Errorf("conn path is empty") + } + + dbConn := readTargetDBToDBConn(targetdb) + + // write a connection file with the given target database info from create_connection + err := dbConn.write(globals.connFile) + if err != nil { + return err + } + + return nil +} + +// readTargetDBToDBConn converts target database to DatabaseConnection +func readTargetDBToDBConn(cnn *vclusterops.VReplicationDatabaseOptions) DatabaseConnection { + targetDBconn := MakeTargetDatabaseConn() + targetDBconn.TargetDB = cnn.TargetDB + targetDBconn.TargetHosts = cnn.TargetHosts + targetDBconn.TargetPasswordFile = *cnn.TargetPassword + targetDBconn.TargetUserName = cnn.TargetUserName + return targetDBconn +} + +// write writes connection information to connFilePath. It returns +// any write error encountered. The viper in-built write function cannot +// work well (the order of keys cannot be customized) so we used yaml.Marshal() +// and os.WriteFile() to write the connection file. +func (c *DatabaseConnection) write(connFilePath string) error { + configBytes, err := yaml.Marshal(*c) + if err != nil { + return fmt.Errorf("fail to marshal connection data, details: %w", err) + } + err = os.WriteFile(connFilePath, configBytes, configFilePerm) + if err != nil { + return fmt.Errorf("fail to write connection file, details: %w", err) + } + return nil +} diff --git a/vclusterops/replication.go b/vclusterops/replication.go index 9a47dcc..e833eb1 100644 --- a/vclusterops/replication.go +++ b/vclusterops/replication.go @@ -164,9 +164,9 @@ func (vcc VClusterCommands) produceDBReplicationInstructions(options *VReplicati } // verify the username for connecting to the target database - targetUserPassword := false + targetUsePassword := false if options.TargetPassword != nil { - targetUserPassword = true + targetUsePassword = true if options.TargetUserName == "" { username, e := util.GetCurrentUsername() if e != nil { @@ -190,7 +190,7 @@ func (vcc VClusterCommands) produceDBReplicationInstructions(options *VReplicati initiatorTargetHost := getInitiator(options.TargetHosts) httpsStartReplicationOp, err := makeHTTPSStartReplicationOp(options.DBName, options.Hosts, options.usePassword, - options.UserName, options.Password, targetUserPassword, options.TargetDB, options.TargetUserName, initiatorTargetHost, + options.UserName, options.Password, targetUsePassword, options.TargetDB, options.TargetUserName, initiatorTargetHost, options.TargetPassword, options.SourceTLSConfig) if err != nil { return instructions, err