From 1df66f8aefd933c586e931311cca8b7300ab271f Mon Sep 17 00:00:00 2001 From: Matt Spilchen Date: Tue, 30 Apr 2024 21:22:17 -0300 Subject: [PATCH] Sync from server repo (f5a3750d60b) --- commands/cluster_command_launcher.go | 10 ++--- commands/cmd_add_node.go | 6 +-- commands/cmd_add_subcluster.go | 2 +- commands/cmd_create_db.go | 12 +++--- commands/cmd_drop_db.go | 14 ++++--- commands/cmd_install_packages.go | 11 ++++-- commands/cmd_list_all_nodes.go | 16 ++++---- commands/cmd_manage_config.go | 4 +- commands/cmd_re_ip.go | 15 ++++---- commands/cmd_remove_node.go | 2 +- commands/cmd_replication.go | 3 +- commands/cmd_revive_db.go | 21 +++++------ commands/cmd_scrutinize.go | 15 ++++---- commands/cmd_show_restore_points.go | 10 ++--- commands/cmd_start_db.go | 8 +++- commands/cmd_stop_subcluster.go | 5 +-- commands/cmd_unsandbox.go | 2 - vclusterops/nma_vertica_version_op.go | 54 ++++++++++++++++++++++++--- vclusterops/start_db.go | 12 ++++-- 19 files changed, 140 insertions(+), 82 deletions(-) diff --git a/commands/cluster_command_launcher.go b/commands/cluster_command_launcher.go index f02ff20..c63ab5a 100644 --- a/commands/cluster_command_launcher.go +++ b/commands/cluster_command_launcher.go @@ -189,13 +189,13 @@ var ( rootCmd = &cobra.Command{ Use: "vcluster", Short: "Administer a Vertica cluster", - Long: `This CLI is used to manage a Vertica cluster with a REST API. The REST API endpoints are -exposed by the following services: + Long: `The vcluster CLI manages a Vertica cluster with a REST API. The REST API +endpoints are exposed by the following services: - Node Management Agent (NMA) - Embedded HTTPS service -This CLI tool combines REST calls to provide an interface so that you can -perform the following administrator operations: +vcluster combines REST calls to provide an interface so that you can perform +the following administrator operations: - Create a database - Scale a cluster up and down - Restart a database @@ -204,7 +204,7 @@ perform the following administrator operations: - Revive an Eon database - Add/Remove a subcluster - Sandbox/Unsandbox a subcluster -- Scrutinize a database +- Run scrutinize on a database - View the state of a database - Install packages on a database`, Version: CLIVersion, diff --git a/commands/cmd_add_node.go b/commands/cmd_add_node.go index c0111c1..d4ec5d9 100644 --- a/commands/cmd_add_node.go +++ b/commands/cmd_add_node.go @@ -50,13 +50,13 @@ func makeCmdAddNode() *cobra.Command { "Add host(s) to an existing database", `This subcommand adds one or more hosts to an existing database. -You need to provide the --add option followed by one or more hosts to add as a +You must provide the --add option followed by one or more hosts to add as a comma-separated list. You cannot add hosts to a sandbox subcluster in an Eon Mode database. -The --node-names option is utilized to address issues resulting from a failed -node addition attempt. It's crucial to include all expected nodes in the catalog +Use the --node-names option to address issues resulting from a failed node +addition attempt. It's crucial to include all expected nodes in the catalog when using this option. This subcommand removes any surplus nodes from the catalog, provided they are down, before commencing the node addition process. Omitting the option will skip this node trimming process. diff --git a/commands/cmd_add_subcluster.go b/commands/cmd_add_subcluster.go index 55838cb..e5ce9dd 100644 --- a/commands/cmd_add_subcluster.go +++ b/commands/cmd_add_subcluster.go @@ -54,7 +54,7 @@ func makeCmdAddSubcluster() *cobra.Command { You must provide a subcluster name with the --subcluster option. By default, the new subcluster is secondary. To add a primary subcluster, use -the --is-primary flag. +the --is-primary option. Examples: # Add a subcluster with config file diff --git a/commands/cmd_create_db.go b/commands/cmd_create_db.go index dde2969..608ca32 100644 --- a/commands/cmd_create_db.go +++ b/commands/cmd_create_db.go @@ -54,17 +54,15 @@ following locations, in order of precedence: - /opt/vertica/config/vertica_config.yaml if running vcluster from /opt/vertica/bin - $HOME/.config/vcluster/vertica_config.yaml -You can pass --config-param a comma-separated list of NAME=VALUE pairs to set -multiple configuration parameters when the database is created -(see Example below). +To set multiple configuration parameters when the database is created, pass +--config-param a comma-separated list of NAME=VALUE pairs. -To remove the local directories like catalog, depot, and data, you can use the +Remove the local directories like catalog, depot, and data, with the --force-cleanup-on-failure or --force-removal-at-creation options. The data deleted with these options is unrecoverable. -The password for the dbadmin user of this new database can be provided in a few ways. -It can be read from a file using --password-file, prompted by the CLI with ---read-password-from-prompt, or passed as plain text with --password as an option. +Provide the dbadmin password with the --password-file, --read-password-from-prompt, +or --password options. Examples: # Create a database and save the generated config file under custom directory diff --git a/commands/cmd_drop_db.go b/commands/cmd_drop_db.go index 76261f7..eef77d0 100644 --- a/commands/cmd_drop_db.go +++ b/commands/cmd_drop_db.go @@ -46,12 +46,16 @@ func makeCmdDropDB() *cobra.Command { For an Eon database, communal storage is not deleted. You can recover the dropped database with revive_db. -The config file must be specified to retrieve host information. If the config -file path is not specified via --config, the default path will be used (refer -to create_db subcommand for information about how the default config file path -is determined). When the command completes, the config file is removed. +The config file must be specified to retrieve host information. If --config +is not provided, a configuration file is created in one of the following +locations, in order of precedence: +- path set in VCLUSTER_CONFIG environment variable +- /opt/vertica/config/vertica_config.yaml if running vcluster from /opt/vertica/bin +- $HOME/.config/vcluster/vertica_config.yaml -To remove the local directories like catalog, depot, and data, you can use the +When the command completes, the config file is removed. + +To remove the local directories like catalog, depot, and data, use the --force-delete option. The data deleted with this option is unrecoverable. Examples: diff --git a/commands/cmd_install_packages.go b/commands/cmd_install_packages.go index a91897a..7047f74 100644 --- a/commands/cmd_install_packages.go +++ b/commands/cmd_install_packages.go @@ -48,15 +48,18 @@ func makeCmdInstallPackages() *cobra.Command { "Install default package(s) in database", `This subcommand installs default packages in the database. -The default packages are those under /opt/vertica/packages where Autoinstall -is marked true. Per package installation status will be returned. +You must provide the --hosts option followed by all hosts in the database as a +comma-separated list. + +Default packages are located in /opt/vertica/packages. During installation, the +status for each package is returned. Examples: - # Install default packages with user input. + # Install default packages with user input vcluster install_packages --db-name test_db \ --hosts 10.20.30.40,10.20.30.41,10.20.30.42 - # Force (re)install default packages with config file. + # Force (re)install default packages with config file vcluster install_packages --db-name test_db --force-reinstall \ --config /opt/vertica/config/vertica_cluster.yaml `, diff --git a/commands/cmd_list_all_nodes.go b/commands/cmd_list_all_nodes.go index 8b841af..61ca1da 100644 --- a/commands/cmd_list_all_nodes.go +++ b/commands/cmd_list_all_nodes.go @@ -44,17 +44,17 @@ func makeListAllNodes() *cobra.Command { newCmd, listAllNodesSubCmd, "List all nodes in the database", - `This subcommand queries the status of the nodes in the consensus and prints -whether they are currently up or down. + `This subcommand queries the status of the nodes in the database and prints +whether they are up or down. -The --hosts option specifies one or more hosts that the program -should communicate with. The program will return the first response it -receives from any of the specified hosts. +To provide its status, each host must run the spread daemon. -The --db-name and --catalog-path options are only needed when VCluster cannot -obtain node information from a running database and the config file is not provided. +You must provide the --hosts option one or more hosts as a comma-separated +list. list_allnodes returns the first response it receives from any host. -The only requirement for each host is that it is running the spread daemon. +The --db-name and --catalog-path options are required only when vcluster cannot +obtain node information from a running database and the config file is not +provided. Examples: # List the status of nodes with config file where password authentication is diff --git a/commands/cmd_manage_config.go b/commands/cmd_manage_config.go index 3ae268f..6e0c599 100644 --- a/commands/cmd_manage_config.go +++ b/commands/cmd_manage_config.go @@ -30,8 +30,8 @@ import ( func makeCmdManageConfig() *cobra.Command { cmd := makeSimpleCobraCmd( manageConfigSubCmd, - "Show or recover the content of the config file", - `This subcommand is used to print or recover the content of the config file.`) + "Display or recover the contents of the config file", + `This subcommand displays or recovers the contents of the config file.`) cmd.AddCommand(makeCmdConfigShow()) cmd.AddCommand(makeCmdConfigRecover()) diff --git a/commands/cmd_re_ip.go b/commands/cmd_re_ip.go index b3a1aa2..41209e1 100644 --- a/commands/cmd_re_ip.go +++ b/commands/cmd_re_ip.go @@ -43,19 +43,20 @@ func makeCmdReIP() *cobra.Command { newCmd, reIPSubCmd, "Re-ip database nodes", - `This subcommand alters the IP addresses of database nodes in the catalog. + `This subcommand changes the IP addresses of database nodes in the catalog. -The database must be offline when running this command. If an IP change -is required and the database is up, you can use restart_node to handle it. +The database must be down to change the IP addresses with re_ip. If +the database is up, you must run restart_node after re_ip for the +IP changes to take effect. -The file specified by the argument must be a JSON file in the following format: +The file specified by the re_ip-file option must be a JSON file in the +following format: [ {"from_address": "10.20.30.40", "to_address": "10.20.30.41"}, {"from_address": "10.20.30.42", "to_address": "10.20.30.43"} ] - -Only the nodes whose IP addresses you want to change need to be included in the -file. + +Include in the file only the nodes whose IP addresses you want to change. Examples: # Alter the IP address of database nodes with user input diff --git a/commands/cmd_remove_node.go b/commands/cmd_remove_node.go index d7e1815..1b105d4 100644 --- a/commands/cmd_remove_node.go +++ b/commands/cmd_remove_node.go @@ -46,7 +46,7 @@ func makeCmdRemoveNode() *cobra.Command { "Remove host(s) from an existing database", `This subcommand removes one or more nodes from an existing database. -You need to provide the --remove option followed by one or more hosts to +You must provide the --remove option followed by one or more hosts to remove as a comma-separated list. You cannot remove nodes from a sandboxed subcluster in an Eon Mode database. diff --git a/commands/cmd_replication.go b/commands/cmd_replication.go index a4daca5..06cef2a 100644 --- a/commands/cmd_replication.go +++ b/commands/cmd_replication.go @@ -23,7 +23,8 @@ func makeCmdReplication() *cobra.Command { cmd := makeSimpleCobraCmd( replicationSubCmd, "Handle database replication", - `This subcommand is used to start or show the status of database replication.`) + `This subcommand starts database replication or displays the status of an +in-progress replication operation.`) cmd.AddCommand(makeCmdStartReplication()) return cmd diff --git a/commands/cmd_revive_db.go b/commands/cmd_revive_db.go index b34c781..087c301 100644 --- a/commands/cmd_revive_db.go +++ b/commands/cmd_revive_db.go @@ -43,22 +43,21 @@ func makeCmdReviveDB() *cobra.Command { newCmd, reviveDBSubCmd, "Revive a database", - `This subcommand revives an Eon Mode database to a given set of hosts. -This could also restore an Eon Mode database to a given restore point. + `This subcommand revives an Eon Mode database on the specified hosts or restores +an Eon Mode database to the specified restore point. -The communal storage path must be provided and it cannot be empty. -If access to communal storage requires access keys, these can be provided -through the --config-param option. +The --communal-storage-location option is required. If access to communal +storage requires access keys, provide the keys with the --config-param option. -You must also specify a set of hosts that matches the number of hosts when the -database was running. You can omit the hosts only if --display-only -is specified. +The number of hosts that you provide to the --hosts option must match the +number of hosts in the existing database. You can omit the hosts only if +--display-only is specified. The name of the database must be provided. -When restoring to a restore point, the --restore-point-archive option must be -provided, and the targeted restore point within an archive must be specified -by either --restore-point-index or --restore-point-id (not both). +To restore a database to a restore point, you must provide the +--restore-point-archive option, and specify the restore point with either the +--restore-point-index or --restore-point-id option. Examples: # Revive a database with user input and save the generated config file diff --git a/commands/cmd_scrutinize.go b/commands/cmd_scrutinize.go index fb1c434..83bfbec 100644 --- a/commands/cmd_scrutinize.go +++ b/commands/cmd_scrutinize.go @@ -86,16 +86,17 @@ func makeCmdScrutinize() *cobra.Command { newCmd, scrutinizeSubCmd, "Scrutinize a database", - `This scrutinizes a database on a given set of hosts. + `This subcommand runs scrutinize to collect diagnostic information about a +database. -This subcommand is usually requested by Vertica support to collect -diagnostics from each host. +Vertica support might request that you run scrutinize when resolving a support +case. -If the --hosts option is specified, diagnostics will only be gathered from those -specific nodes. These nodes may be a subset of all the nodes in the database. +If you use the --hosts option, scrutinize gathers diagnostics from only the +specified hosts. -The diagnostics are bundled together in a tarball and stored at the following -directory: `+vclusterops.ScrutinizeOutputBasePath+`/VerticaScrutinize..tar. +The diagnostics are bundled together in a tar file and stored in +`+vclusterops.ScrutinizeOutputBasePath+`/VerticaScrutinize..tar. Examples: # Scrutinize all nodes in the database with config file diff --git a/commands/cmd_show_restore_points.go b/commands/cmd_show_restore_points.go index 58066ec..7b00537 100644 --- a/commands/cmd_show_restore_points.go +++ b/commands/cmd_show_restore_points.go @@ -40,13 +40,13 @@ func makeCmdShowRestorePoints() *cobra.Command { newCmd, showRestorePointsSubCmd, "Query and list restore point(s) in archive(s)", - `This subcommand queries and lists restore point(s) in archive(s). + `This subcommand queries and displays restore points in archives. + +The --start-timestamp and --end-timestamp options limit the restore points +query by creation timestamp. Both options accept UTC timestamps in date-time +and date-only format. For example: -Then --start-timestamp and --end-timestamp options both limit the scope of -creation timestamps of listed restore points. Both of them expect a timestamp -in date-time format or date-only format, for example: "2006-01-02 15:04:05", "2006-01-02", "2006-01-02 15:04:05.000000000". -Both of them expect a timestamp in UTC timezone. Examples: # List restore points without filters with user input diff --git a/commands/cmd_start_db.go b/commands/cmd_start_db.go index 7d2375e..6c384aa 100644 --- a/commands/cmd_start_db.go +++ b/commands/cmd_start_db.go @@ -129,7 +129,13 @@ func (c *CmdStartDB) setHiddenFlags(cmd *cobra.Command) { false, "", ) - hideLocalFlags(cmd, []string{"unsafe", "force", "allow_fallback_keygen", "ignore_cluster_lease", "fast", "trim-hosts"}) + cmd.Flags().BoolVar( + &c.startDBOptions.HostsInSandbox, + "hosts-in-sandbox", + false, + "", + ) + hideLocalFlags(cmd, []string{"unsafe", "force", "allow_fallback_keygen", "ignore_cluster_lease", "fast", "trim-hosts", "hosts-in-sandbox"}) } func (c *CmdStartDB) Parse(inputArgv []string, logger vlog.Printer) error { diff --git a/commands/cmd_stop_subcluster.go b/commands/cmd_stop_subcluster.go index f09120c..016c48e 100644 --- a/commands/cmd_stop_subcluster.go +++ b/commands/cmd_stop_subcluster.go @@ -51,8 +51,7 @@ func makeCmdStopSubcluster() *cobra.Command { You must provide the subcluster name with the --subcluster option. -All hosts in the subcluster will be stopped. You cannot stop a sandboxed -subcluster. +All hosts in the subcluster will be stopped. Examples: # Gracefully stop a subcluster with config file @@ -92,7 +91,7 @@ func (c *CmdStopSubcluster) setLocalFlags(cmd *cobra.Command) { &c.stopSCOptions.DrainSeconds, "drain-seconds", util.DefaultDrainSeconds, - util.GetEonFlagMsg("seconds to wait for user connections to close."+ + util.GetEonFlagMsg("seconds to wait for non-dbadmin user connections to close."+ " Default value is "+strconv.Itoa(util.DefaultDrainSeconds)+" seconds."+ " When the time expires, connections will be forcibly closed and the subcluster will shut down."+ " If the value is 0, VCluster closes all user connections immediately."+ diff --git a/commands/cmd_unsandbox.go b/commands/cmd_unsandbox.go index e0cb1fc..c82bd06 100644 --- a/commands/cmd_unsandbox.go +++ b/commands/cmd_unsandbox.go @@ -56,8 +56,6 @@ main cluster. When all subclusters are removed from a sandbox, the sandbox catalog and metadata are deleted. To reuse the sandbox name, you must manually clean the /metadata/ directory in your communal storage location. -To reuse the sandbox name, you must manually clean the /metadata/ -directory in your communal storage location. The comma-separated list of hosts passed to the --hosts option must include at least one up host in the main cluster. diff --git a/vclusterops/nma_vertica_version_op.go b/vclusterops/nma_vertica_version_op.go index b93577b..b42baf2 100644 --- a/vclusterops/nma_vertica_version_op.go +++ b/vclusterops/nma_vertica_version_op.go @@ -40,6 +40,7 @@ type nmaVerticaVersionOp struct { sandbox bool scName string readOnly bool + targetNodeIPs []string // used to filter desired nodes' info } func makeHostVersionMap() hostVersionMap { @@ -75,10 +76,13 @@ func makeNMAReadVerticaVersionOp(vdb *VCoordinationDatabase) nmaVerticaVersionOp return op } -// makeNMAVerticaVersionOpWithoutHosts is used when db is down -func makeNMAVerticaVersionOpWithoutHosts(sameVersion bool) nmaVerticaVersionOp { +// makeNMAVerticaVersionOpWithTargetHosts is used in start_db, VCluster will check Vertica +// version for the subclusters which contain target hosts +func makeNMAVerticaVersionOpWithTargetHosts(sameVersion bool, hosts []string) nmaVerticaVersionOp { // We set hosts to nil and isEon to false temporarily, and they will get the correct value from execute context in prepare() - return makeNMACheckVerticaVersionOp(nil /*hosts*/, sameVersion, false /*isEon*/) + op := makeNMACheckVerticaVersionOp(nil /*hosts*/, sameVersion, false /*isEon*/) + op.targetNodeIPs = hosts + return op } // makeNMAVerticaVersionOpAfterUnsandbox is used after unsandboxing @@ -175,12 +179,16 @@ func (op *nmaVerticaVersionOp) prepare(execContext *opEngineExecContext) error { op.SCToHostVersionMap[sc][host] = "" } } else { - // db is down + // start db op.HasIncomingSCNames = true if execContext.nmaVDatabase.CommunalStorageLocation != "" { op.IsEon = true } - for host, vnode := range execContext.nmaVDatabase.HostNodeMap { + hostNodeMap, err := op.prepareHostNodeMap(execContext) + if err != nil { + return err + } + for host, vnode := range hostNodeMap { op.hosts = append(op.hosts, host) // initialize the SCToHostVersionMap with empty versions sc := vnode.Subcluster.Name @@ -343,3 +351,39 @@ func (op *nmaVerticaVersionOp) readVersion() error { return nil } + +// prepareHostNodeMap is a helper to make a host-node map for nodes in target subclusters +func (op *nmaVerticaVersionOp) prepareHostNodeMap(execContext *opEngineExecContext) (map[string]*nmaVNode, error) { + hostNodeMap := execContext.nmaVDatabase.HostNodeMap + if len(op.targetNodeIPs) > 0 { + hostSCMap := make(map[string]string) + scHostsMap := make(map[string][]string) + for host, vnode := range execContext.nmaVDatabase.HostNodeMap { + hostSCMap[host] = vnode.Subcluster.Name + scHostsMap[vnode.Subcluster.Name] = append(scHostsMap[vnode.Subcluster.Name], host) + } + // find subclusters that hold the target hosts + targetSCs := []string{} + for _, host := range op.targetNodeIPs { + sc, ok := hostSCMap[host] + if ok { + targetSCs = append(targetSCs, sc) + } else { + return hostNodeMap, fmt.Errorf("[%s] host %s does not exist in the database", op.name, host) + } + } + // find all hosts that in target subclusters + allHostsInTargetSCs := []string{} + for _, sc := range targetSCs { + hosts, ok := scHostsMap[sc] + if ok { + allHostsInTargetSCs = append(allHostsInTargetSCs, hosts...) + } else { + return hostNodeMap, fmt.Errorf("[%s] internal error: subcluster %s was lost when preparing the hosts", op.name, sc) + } + } + // get host-node map for all hosts in target subclusters + hostNodeMap = util.FilterMapByKey(execContext.nmaVDatabase.HostNodeMap, allHostsInTargetSCs) + } + return hostNodeMap, nil +} diff --git a/vclusterops/start_db.go b/vclusterops/start_db.go index d39e96a..06364b9 100644 --- a/vclusterops/start_db.go +++ b/vclusterops/start_db.go @@ -36,6 +36,8 @@ type VStartDatabaseOptions struct { // you may not want to have both the NMA and Vertica server in the same container. // This feature requires version 24.2.0+. StartUpConf string + // whether the provided hosts are in a sandbox + HostsInSandbox bool } func VStartDatabaseOptionsFactory() VStartDatabaseOptions { @@ -114,8 +116,10 @@ func (vcc VClusterCommands) VStartDatabase(options *VStartDatabaseOptions) (vdbP // VER-93369 may improve this if the CLI knows which nodes are primary // from the config file var vdb VCoordinationDatabase - // retrieve database information from cluster_config.json for Eon databases - if options.IsEon { + // retrieve database information from cluster_config.json for Eon databases, + // skip this step for starting a sandbox because cluster_config.json does not + // contain accurate info of nodes in a sandbox + if !options.HostsInSandbox && options.IsEon { const warningMsg = " for an Eon database, start_db after revive_db could fail " + "because we cannot retrieve the correct database information" if options.CommunalStorageLocation != "" { @@ -276,13 +280,13 @@ func (vcc VClusterCommands) produceStartDBPreCheck(options *VStartDatabaseOption func (vcc VClusterCommands) produceStartDBInstructions(options *VStartDatabaseOptions, vdb *VCoordinationDatabase) ([]clusterOp, error) { var instructions []clusterOp - // vdb here should contains only primary nodes + // vdb here should contain only primary nodes nmaReadCatalogEditorOp, err := makeNMAReadCatalogEditorOp(vdb) if err != nil { return instructions, err } // require to have the same vertica version - nmaVerticaVersionOp := makeNMAVerticaVersionOpWithoutHosts(true) + nmaVerticaVersionOp := makeNMAVerticaVersionOpWithTargetHosts(true, options.Hosts) instructions = append(instructions, &nmaReadCatalogEditorOp, &nmaVerticaVersionOp,