Merge pull request vmware#1357 from ksamoray/upgrade-run-9

Add support for v9.0 upgrade

nsxt/data_source_nsxt_edge_upgrade_group.go

@@ -13,9 +13,10 @@ import (
 )
 
 var (
-	edgeUpgradeGroup = "EDGE"
-	hostUpgradeGroup = "HOST"
-	mpUpgradeGroup   = "MP"
+	edgeUpgradeGroup     = "EDGE"
+	hostUpgradeGroup     = "HOST"
+	mpUpgradeGroup       = "MP"
+	finalizeUpgradeGroup = "FINALIZE_UPGRADE"
 )
 
 func dataSourceNsxtEdgeUpgradeGroup() *schema.Resource {

nsxt/resource_nsxt_upgrade_prepare.go

@@ -159,6 +159,37 @@ func prepareForUpgrade(d *schema.ResourceData, m interface{}) error {
 	return nil
 }
 
+func isVCF9HostUpgrade(m interface{}, targetVersion string) (bool, error) {
+	// In VCF9.0 and newer, overall upgrade status can be PAUSED when hosts are pre-upgraded. Pre-checks should still
+	// execute in this case
+	if util.VersionLower(targetVersion, "9.0.0") {
+		return false, nil
+	}
+	connector := getPolicyConnector(m)
+	client := upgrade.NewStatusSummaryClient(connector)
+	statusSummary, err := client.Get(nil, nil, nil)
+	if err != nil {
+		return false, err
+	}
+	if *statusSummary.OverallUpgradeStatus != nsxModel.UpgradeStatus_OVERALL_UPGRADE_STATUS_PAUSED {
+		return false, nil
+	}
+	// In this case, all components other than host will be NOT_STARTED, but hosts will be with status SUCCESS
+	for _, c := range statusSummary.ComponentStatus {
+		if *c.ComponentType == hostUpgradeGroup {
+			if *c.Status != nsxModel.ComponentUpgradeStatus_STATUS_SUCCESS {
+				return false, nil
+			}
+		} else {
+			// It's not a host - should be NOT_STARTED
+			if *c.Status != nsxModel.ComponentUpgradeStatus_STATUS_NOT_STARTED {
+				return false, nil
+			}
+		}
+	}
+	return true, nil
+}
+
 func getSummaryInfo(m interface{}) (string, bool, error) {
 	connector := getPolicyConnector(m)
 	summaryClient := upgrade.NewSummaryClient(connector)
@@ -175,8 +206,16 @@ func getSummaryInfo(m interface{}) (string, bool, error) {
 		return targetVersion, false, nil
 	}
 	if summary.UpgradeStatus == nil || (*summary.UpgradeStatus) != nsxModel.UpgradeSummary_UPGRADE_STATUS_NOT_STARTED {
-		log.Printf("Upgrade process has started, skip running precheck")
-		return targetVersion, false, nil
+		is9, err := isVCF9HostUpgrade(m, targetVersion)
+		if err != nil {
+			return "", false, err
+		}
+		if is9 {
+			log.Printf("Hosts have been pre-upgraded, prechecks should be running")
+		} else {
+			log.Printf("Upgrade process has started, skip running precheck")
+			return targetVersion, false, nil
+		}
 	}
 	return targetVersion, true, nil
 }

nsxt/resource_nsxt_upgrade_run.go

@@ -27,6 +27,18 @@ var upgradeComponentList = []string{
 	mpUpgradeGroup,
 }
 
+var upgradeComponentListPost9 = []string{
+	mpUpgradeGroup,
+	edgeUpgradeGroup,
+	hostUpgradeGroup,
+	finalizeUpgradeGroup,
+}
+
+var postCheckComponentList = []string{
+	edgeUpgradeGroup,
+	hostUpgradeGroup,
+}
+
 var componentToGroupKey = map[string]string{
 	edgeUpgradeGroup: "edge_group",
 	hostUpgradeGroup: "host_group",
@@ -71,6 +83,23 @@ type upgradeClientSet struct {
 	Interval int
 }
 
+func getTargetVersion(m interface{}) (string, error) {
+	connector := getPolicyConnector(m)
+	client := upgrade.NewSummaryClient(connector)
+	obj, err := client.Get()
+	if err != nil {
+		return "", err
+	}
+	return *obj.TargetVersion, nil
+}
+
+func getUpgradeComponentList(targetVersion string) []string {
+	if util.VersionHigherOrEqual(targetVersion, "9.0.0") {
+		return upgradeComponentListPost9
+	}
+	return upgradeComponentList
+}
+
 func newUpgradeClientSet(connector client.Connector, d *schema.ResourceData) *upgradeClientSet {
 	return &upgradeClientSet{
 		GroupClient:       upgrade.NewUpgradeUnitGroupsClient(connector),
@@ -104,6 +133,21 @@ func resourceNsxtUpgradeRun() *schema.Resource {
 			"host_group":           getUpgradeGroupSchema(true),
 			"edge_upgrade_setting": getUpgradeSettingSchema(true),
 			"host_upgrade_setting": getUpgradeSettingSchema(false),
+			"finalize_upgrade_setting": {
+				Type:     schema.TypeList,
+				Optional: true,
+				MaxItems: 1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"enabled": {
+							Type:        schema.TypeBool,
+							Description: "Finalize upgrade when complete",
+							Optional:    true,
+							Default:     true,
+						},
+					},
+				},
+			},
 			"timeout": {
 				Type:         schema.TypeInt,
 				Description:  "Upgrade status check timeout in seconds",
@@ -362,16 +406,28 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error {
 
 	connector := getPolicyConnectorWithHeaders(m, nil, false, false)
 	upgradeClientSet := newUpgradeClientSet(connector, d)
+	targetVersion, err := getTargetVersion(m)
+	if err != nil {
+		return err
+	}
 
 	log.Printf("[INFO] Updating UpgradeUnitGroup and UpgradePlanSetting.")
-	err := prepareUpgrade(upgradeClientSet, d, m)
+	err = prepareUpgrade(upgradeClientSet, d, targetVersion)
 	if err != nil {
 		return handleCreateError("NsxtUpgradeRun", id, err)
 	}
 
 	log.Printf("[INFO] Successfully update UpgradeUnitGroup and UpgradePlanSetting. Start Upgrade.")
+	finalizeUpgrade := true
+	if d.HasChange("finalize_upgrade_setting") {
+		finalizeSettings := d.Get("finalize_upgrade_setting").([]interface{})
+		if len(finalizeSettings) != 0 {
+			finalizeSettingsMap := finalizeSettings[0].(map[string]interface{})
+			finalizeUpgrade = finalizeSettingsMap["enabled"].(bool)
+		}
+	}
 
-	err = runUpgrade(upgradeClientSet, getPartialUpgradeMap(d))
+	err = runUpgrade(upgradeClientSet, getPartialUpgradeMap(d, targetVersion), targetVersion, finalizeUpgrade)
 	if err != nil {
 		return handleCreateError("NsxtUpgradeRun", id, err)
 	}
@@ -382,10 +438,10 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error {
 	return resourceNsxtUpgradeRunRead(d, m)
 }
 
-func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, m interface{}) error {
-	for _, component := range upgradeComponentList {
+func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, targetVersion string) error {
+	for _, component := range getUpgradeComponentList(targetVersion) {
 		// Customize MP upgrade is not allowed
-		if component == mpUpgradeGroup {
+		if component == mpUpgradeGroup || component == finalizeUpgradeGroup {
 			continue
 		}
 
@@ -438,13 +494,13 @@ func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData,
 	return nil
 }
 
-func getPartialUpgradeMap(d *schema.ResourceData) map[string]bool {
+func getPartialUpgradeMap(d *schema.ResourceData, targetVersion string) map[string]bool {
 	isPartialUpgradeMap := map[string]bool{
 		edgeUpgradeGroup: false,
 		hostUpgradeGroup: false,
 	}
-	for _, component := range upgradeComponentList {
-		if component == mpUpgradeGroup {
+	for _, component := range getUpgradeComponentList(targetVersion) {
+		if component == mpUpgradeGroup || component == finalizeUpgradeGroup {
 			continue
 		}
 		for _, groupI := range d.Get(componentToGroupKey[component]).([]interface{}) {
@@ -711,10 +767,14 @@ func updateComponentUpgradePlanSetting(settingClient plan.SettingsClient, d *sch
 	return err
 }
 
-func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string]bool) error {
+func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string]bool, targetVersion string, finalizeUpgrade bool) error {
 	partialUpgradeExist := false
 	prevComponent := ""
-	for _, component := range upgradeComponentList {
+	for _, c := range getUpgradeComponentList(targetVersion) {
+		component := c
+		if !finalizeUpgrade && component == finalizeUpgradeGroup {
+			continue
+		}
 		// After one component upgrade is completed, although the status of our next component is NOT_STARTED,
 		// there is a period that overall status is still IN_PROGRESS, which will prevent us to start the upgrade of next component.
 		// Wait here for the overall status become stable. Because there is potential upgrade triggered before, we wait here also
@@ -735,6 +795,16 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string
 				continue
 			}
 		}
+
+		// If component is already upgraded, resume
+		status, err := getUpgradeStatus(upgradeClientSet.StatusClient, &component)
+		if err != nil {
+			return err
+		}
+		if status.Status == model.ComponentUpgradeStatus_STATUS_SUCCESS {
+			log.Printf("Component %s already upgraded successfully, skipping", component)
+			continue
+		}
 		pendingStatus := []string{model.ComponentUpgradeStatus_STATUS_IN_PROGRESS}
 		targetStatus := []string{model.ComponentUpgradeStatus_STATUS_SUCCESS}
 		completeLog := fmt.Sprintf("[INFO] %s upgrade is completed.", component)
@@ -746,7 +816,10 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string
 			prevComponent = component
 			completeLog = fmt.Sprintf("[INFO] %s upgrade is partially completed.", component)
 		}
-		upgradeClientSet.PlanClient.Upgrade(&component)
+		err = upgradeClientSet.PlanClient.Upgrade(&component)
+		if err != nil {
+			return err
+		}
 		err = waitUpgradeForStatus(upgradeClientSet, &component, pendingStatus, targetStatus)
 		if err != nil {
 			return err
@@ -757,21 +830,18 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string
 }
 
 func runPostcheck(upgradeClient nsx.UpgradeClient, d *schema.ResourceData) {
-	for i := range upgradeComponentList {
-		component := upgradeComponentList[i]
-		if component != mpUpgradeGroup {
-			settingI := d.Get(componentToSettingKey[component]).([]interface{})
-			if len(settingI) == 0 {
-				continue
-			}
+	for _, component := range postCheckComponentList {
+		settingI := d.Get(componentToSettingKey[component]).([]interface{})
+		if len(settingI) == 0 {
+			continue
+		}
 
-			setting := settingI[0].(map[string]interface{})
-			postCheck := setting["post_upgrade_check"].(bool)
+		setting := settingI[0].(map[string]interface{})
+		postCheck := setting["post_upgrade_check"].(bool)
 
-			if postCheck {
-				log.Printf("[INFO] Start %s upgrade postcheck. Please use data source nsxt_upgrade_postcheck for results.", component)
-				upgradeClient.Executepostupgradechecks(component)
-			}
+		if postCheck {
+			log.Printf("[INFO] Start %s upgrade postcheck. Please use data source nsxt_upgrade_postcheck for results.", component)
+			upgradeClient.Executepostupgradechecks(component)
 		}
 	}
 }
@@ -838,6 +908,11 @@ func resourceNsxtUpgradeRunRead(d *schema.ResourceData, m interface{}) error {
 	if err != nil {
 		return handleReadError(d, "NsxtUpgradeRun", id, err)
 	}
+	targetVersion, err := getTargetVersion(m)
+	if err != nil {
+		return handleReadError(d, "NsxtUpgradeRun", id, err)
+	}
+	d.Set("target_version", targetVersion)
 	return nil
 }
 

nsxt/util/utils.go

@@ -13,9 +13,12 @@ import (
 var NsxVersion = ""
 
 func NsxVersionLower(ver string) bool {
+	return VersionLower(NsxVersion, ver)
+}
 
+func VersionLower(base, ver string) bool {
 	requestedVersion, err1 := version.NewVersion(ver)
-	currentVersion, err2 := version.NewVersion(NsxVersion)
+	currentVersion, err2 := version.NewVersion(base)
 	if err1 != nil || err2 != nil {
 		log.Printf("[ERROR] Failed perform version check for version %s", ver)
 		return true
@@ -24,9 +27,13 @@ func NsxVersionLower(ver string) bool {
 }
 
 func NsxVersionHigherOrEqual(ver string) bool {
+	return VersionHigherOrEqual(NsxVersion, ver)
+}
+
+func VersionHigherOrEqual(base, ver string) bool {
 
 	requestedVersion, err1 := version.NewVersion(ver)
-	currentVersion, err2 := version.NewVersion(NsxVersion)
+	currentVersion, err2 := version.NewVersion(base)
 	if err1 != nil || err2 != nil {
 		log.Printf("[ERROR] Failed perform version check for version %s", ver)
 		return false

website/docs/guides/upgrade.html.markdown

@@ -166,9 +166,20 @@ resource "nsxt_upgrade_run" "run" {
 }
 ```
 
+### Upgrade run stage with NSX v9.0 and above
+
+NSX v9.0 introduces a few changes to the flow of execution of the run resource: the upgrade begins with updating the NSX 
+management plane, then Edge appliances are upgraded, followed by ESXi hosts. Finally, the upgrade process is finalized, 
+when the entire process is complete, including the upgrade of the ESXi software.
+
+When the ESXi software is not upgraded, the finalization stage will fail. There is an additional component in the 
+`nsxt_upgrade_run` resource which allows execution without the finalization stage. This allows successful execution of 
+the NSX management plane, the Edge appliances and the NSX bits on the ESXi hosts, and postpone the ESXi OS upgrade to 
+a later time.
+
 ### Post upgrade checks
 
-Upgrade post check data sources can be used to examine the results of the edge and host upgrades, to cunclude if the 
+Upgrade post check data sources can be used to examine the results of the edge and host upgrades, to conclude if the 
 upgrade has completed successfully.
 
 The example below uses Terraform `check` to test the upgrade results.

website/docs/r/upgrade_run.html.markdown

@@ -88,6 +88,8 @@ The following arguments are supported:
     * `parallel` - (Optional) Upgrade Method to specify whether upgrades of UpgradeUnitGroups in this component are performed serially or in parallel. Default: True.
     * `post_upgrade_check` - (Optional) Flag to indicate whether run post upgrade check after upgrade. Default: True.
     * `stop_on_error` - (Optional) Flag to indicate whether to pause the upgrade plan execution when an error occurs. Default: False.
+* `finalize_upgrade_setting` - (Optional) FINALIZE_UPGRADE component upgrade plan setting.
+    * `enabled` - (Optional) Finalize upgrade after completion of all the components' upgrade is complete. Default: True.   
 
 ## Argument Reference