Sync from server repo (e1bd36a236)

vclusterops/adapter_pool.go

@@ -16,8 +16,10 @@
 package vclusterops
 
 import (
+	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/vertica/vcluster/vclusterops/vlog"
 )
@@ -80,6 +82,10 @@ func (pool *AdapterPool) sendRequest(clusterHTTPRequest *ClusterHTTPRequest) err
 	// result channel to collect result from each host
 	resultChannel := make(chan HostHTTPResult, hostCount)
 
+	// use context to check whether a step has completed
+	ctx, cancel := context.WithCancel(context.Background())
+	go progressCheck(ctx, clusterHTTPRequest.Name, pool.log)
+
 	for i := 0; i < len(adapterToRequestCollection); i++ {
 		ar := adapterToRequestCollection[i]
 		// send request to the hosts
@@ -100,5 +106,32 @@ func (pool *AdapterPool) sendRequest(clusterHTTPRequest *ClusterHTTPRequest) err
 	}
 	close(resultChannel)
 
+	// cancel the progress check context when the result channel is closed
+	cancel()
+
 	return nil
 }
+
+// progressCheck checks whether a step (operation) has been completed.
+// Elapsed time of the step in seconds will be displayed.
+func progressCheck(ctx context.Context, name string, log vlog.Printer) {
+	const progressCheckInterval = 5
+	startTime := time.Now()
+
+	ticker := time.NewTicker(progressCheckInterval * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// context is canceled
+			// - when the requests to each host are completed, or
+			// - when the timeout is reached
+			return
+		case tickTime := <-ticker.C:
+			elapsedTime := tickTime.Sub(startTime)
+			log.PrintInfo("[%s] is still running. %.f seconds spent at this step.",
+				name, elapsedTime.Seconds())
+		}
+	}
+}

vclusterops/cluster_op.go

@@ -178,6 +178,7 @@ type ClusterOp interface {
 	logPrepare()
 	logExecute()
 	logFinalize()
+	setupBasicInfo()
 	loadCertsIfNeeded(certs *HTTPSCerts, findCertsInOptions bool) error
 	isSkipExecute() bool
 }
@@ -218,11 +219,20 @@ func (op *OpBase) parseAndCheckMapResponse(host, responseContent string) (OpResp
 	return responseObj, err
 }
 
+func (op *OpBase) setClusterHTTPRequestName() {
+	op.clusterHTTPRequest.Name = op.name
+}
+
 func (op *OpBase) setVersionToSemVar() {
 	op.clusterHTTPRequest.SemVar = SemVer{Ver: "1.0.0"}
 }
 
-// TODO: implement another parse function for list response
+func (op *OpBase) setupBasicInfo() {
+	op.clusterHTTPRequest = ClusterHTTPRequest{}
+	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
+	op.setClusterHTTPRequestName()
+	op.setVersionToSemVar()
+}
 
 func (op *OpBase) logResponse(host string, result HostHTTPResult) {
 	op.log.PrintInfo("[%s] result from host %s summary %s, details: %+v",
@@ -235,6 +245,7 @@ func (op *OpBase) logPrepare() {
 
 func (op *OpBase) logExecute() {
 	op.log.Info("Execute() called", "name", op.name)
+	op.log.PrintInfo("[%s] is running", op.name)
 }
 
 func (op *OpBase) logFinalize() {

vclusterops/cluster_op_engine.go

@@ -45,6 +45,7 @@ func (opEngine *VClusterOpEngine) Run(log vlog.Printer) error {
 	findCertsInOptions := opEngine.shouldGetCertsFromOptions()
 
 	for _, op := range opEngine.instructions {
+		op.setupBasicInfo()
 		op.logPrepare()
 		err := op.prepare(&execContext)
 		if err != nil {

vclusterops/http_request.go

@@ -52,4 +52,5 @@ type ClusterHTTPRequest struct {
 	RequestCollection map[string]HostHTTPRequest
 	ResultCollection  map[string]HostHTTPResult
 	SemVar            SemVer
+	Name              string
 }

vclusterops/https_add_subcluster_op.go

@@ -79,10 +79,6 @@ func (op *HTTPSAddSubclusterOp) setupRequestBody(hosts []string) error {
 }
 
 func (op *HTTPSAddSubclusterOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_check_db_running.go

@@ -76,10 +76,6 @@ func makeHTTPCheckRunningDBOp(log vlog.Printer, hosts []string,
 }
 
 func (op *HTTPCheckRunningDBOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_check_node_state_op.go

@@ -53,10 +53,6 @@ func makeHTTPCheckNodeStateOp(log vlog.Printer, hosts []string,
 }
 
 func (op *HTTPCheckNodeStateOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_check_subcluster_op.go

@@ -52,10 +52,6 @@ func makeHTTPSCheckSubclusterOp(log vlog.Printer, useHTTPPassword bool, userName
 }
 
 func (op *HTTPSCheckSubclusterOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_create_cluster_depot_op.go

@@ -63,10 +63,6 @@ func makeHTTPSCreateClusterDepotOp(log vlog.Printer, vdb *VCoordinationDatabase,
 }
 
 func (op *HTTPSCreateDepotOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_create_node_op.go

@@ -51,10 +51,6 @@ func makeHTTPSCreateNodeOp(log vlog.Printer, newNodeHosts []string, bootstrapHos
 }
 
 func (op *HTTPSCreateNodeOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_create_nodes_depot_op.go

@@ -53,10 +53,6 @@ func makeHTTPSCreateNodesDepotOp(log vlog.Printer, vdb *VCoordinationDatabase, n
 }
 
 func (op *HTTPSCreateNodesDepotOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_drop_node_op.go

@@ -57,10 +57,6 @@ func makeHTTPSDropNodeOp(log vlog.Printer, vnode string,
 }
 
 func (op *HTTPSDropNodeOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_drop_subcluster_op.go

@@ -50,10 +50,6 @@ func makeHTTPSDropSubclusterOp(hosts []string, scName string,
 }
 
 func (op *httpsDropSubclusterOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_find_subcluster_op.go

@@ -51,10 +51,6 @@ func makeHTTPSFindSubclusterOp(log vlog.Printer, hosts []string, useHTTPPassword
 }
 
 func (op *HTTPSFindSubclusterOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_get_cluster_info_op.go

@@ -54,10 +54,6 @@ func makeHTTPSGetClusterInfoOp(log vlog.Printer, dbName string, hosts []string,
 }
 
 func (op *httpsGetClusterInfoOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_get_nodes_info_op.go

@@ -48,10 +48,6 @@ func makeHTTPSGetNodesInfoOp(log vlog.Printer, dbName string, hosts []string,
 }
 
 func (op *httpsGetNodesInfoOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_get_up_nodes_op.go

@@ -52,10 +52,6 @@ func makeHTTPSGetUpNodesOp(log vlog.Printer, dbName string, hosts []string,
 }
 
 func (op *HTTPSGetUpNodesOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_install_packages.go → vclusterops/https_install_packages_op.go

@@ -47,10 +47,6 @@ func makeHTTPSInstallPackagesOp(log vlog.Printer, hosts []string, useHTTPPasswor
 }
 
 func (op *HTTPSInstallPackagesOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_mark_design_ksafe_op.go

@@ -64,10 +64,6 @@ func makeHTTPSMarkDesignKSafeOp(
 }
 
 func (op *HTTPSMarkDesignKSafeOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	// in practice, initiator only
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}

vclusterops/https_mark_nodes_ephemeral_op.go

@@ -49,10 +49,6 @@ func makeHTTPSMarkEphemeralNodeOp(log vlog.Printer, nodeName string,
 }
 
 func (op *HTTPSMarkEphemeralNodeOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_poll_node_state_op.go

@@ -112,10 +112,6 @@ func (op *HTTPSPollNodeStateOp) getPollingTimeout() int {
 }
 
 func (op *HTTPSPollNodeStateOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_poll_subscription_state_op.go

@@ -52,10 +52,6 @@ func (op *httpsPollSubscriptionStateOp) getPollingTimeout() int {
 }
 
 func (op *httpsPollSubscriptionStateOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_re_ip_op.go

@@ -53,10 +53,6 @@ func makeHTTPSReIPOp(nodeNamesList, hostToReIP []string,
 }
 
 func (op *httpsReIPOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for i, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PutMethod

vclusterops/https_rebalance_cluster_op.go

@@ -50,10 +50,6 @@ func makeHTTPSRebalanceClusterOp(log vlog.Printer, initiatorHost []string, useHT
 }
 
 func (op *HTTPSRebalanceClusterOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_rebalance_subcluster_shards_op.go

@@ -53,10 +53,6 @@ func makeHTTPSRebalanceSubclusterShardsOp(log vlog.Printer, bootstrapHost []stri
 }
 
 func (op *HTTPSRebalanceSubclusterShardsOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_reload_spread_op.go

@@ -52,10 +52,6 @@ func makeHTTPSReloadSpreadOp(log vlog.Printer, useHTTPPassword bool,
 }
 
 func (op *HTTPSReloadSpreadOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_spread_remove_node_op.go

@@ -53,10 +53,6 @@ func makeHTTPSSpreadRemoveNodeOp(log vlog.Printer, hostsToRemove []string, initi
 }
 
 func (op *HTTPSSpreadRemoveNodeOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_startup_command_op.go

@@ -51,10 +51,6 @@ func makeHTTPSStartUpCommandOp(log vlog.Printer, useHTTPPassword bool, userName
 }
 
 func (op *httpsStartUpCommandOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = GetMethod

vclusterops/https_stop_db_op.go

@@ -56,10 +56,6 @@ func makeHTTPSStopDBOp(log vlog.Printer, useHTTPPassword bool, userName string,
 }
 
 func (op *HTTPSStopDBOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod

vclusterops/https_sync_catalog_op.go

@@ -53,10 +53,6 @@ func makeHTTPSSyncCatalogOpWithoutHosts(log vlog.Printer, useHTTPPassword bool,
 }
 
 func (op *HTTPSSyncCatalogOp) setupClusterHTTPRequest(hosts []string) error {
-	op.clusterHTTPRequest = ClusterHTTPRequest{}
-	op.clusterHTTPRequest.RequestCollection = make(map[string]HostHTTPRequest)
-	op.setVersionToSemVar()
-
 	for _, host := range hosts {
 		httpRequest := HostHTTPRequest{}
 		httpRequest.Method = PostMethod