diff --git a/nsxt/data_source_nsxt_edge_upgrade_group.go b/nsxt/data_source_nsxt_edge_upgrade_group.go index b18b0acf7..16027463d 100644 --- a/nsxt/data_source_nsxt_edge_upgrade_group.go +++ b/nsxt/data_source_nsxt_edge_upgrade_group.go @@ -13,9 +13,10 @@ import ( ) var ( - edgeUpgradeGroup = "EDGE" - hostUpgradeGroup = "HOST" - mpUpgradeGroup = "MP" + edgeUpgradeGroup = "EDGE" + hostUpgradeGroup = "HOST" + mpUpgradeGroup = "MP" + finalizeUpgradeGroup = "FINALIZE_UPGRADE" ) func dataSourceNsxtEdgeUpgradeGroup() *schema.Resource { diff --git a/nsxt/resource_nsxt_upgrade_prepare.go b/nsxt/resource_nsxt_upgrade_prepare.go index dea9714ee..092c982e3 100644 --- a/nsxt/resource_nsxt_upgrade_prepare.go +++ b/nsxt/resource_nsxt_upgrade_prepare.go @@ -159,6 +159,37 @@ func prepareForUpgrade(d *schema.ResourceData, m interface{}) error { return nil } +func isVCF9HostUpgrade(m interface{}, targetVersion string) (bool, error) { + // In VCF9.0 and newer, overall upgrade status can be PAUSED when hosts are pre-upgraded. Pre-checks should still + // execute in this case + if util.VersionLower(targetVersion, "9.0.0") { + return false, nil + } + connector := getPolicyConnector(m) + client := upgrade.NewStatusSummaryClient(connector) + statusSummary, err := client.Get(nil, nil, nil) + if err != nil { + return false, err + } + if *statusSummary.OverallUpgradeStatus != nsxModel.UpgradeStatus_OVERALL_UPGRADE_STATUS_PAUSED { + return false, nil + } + // In this case, all components other than host will be NOT_STARTED, but hosts will be with status SUCCESS + for _, c := range statusSummary.ComponentStatus { + if *c.ComponentType == hostUpgradeGroup { + if *c.Status != nsxModel.ComponentUpgradeStatus_STATUS_SUCCESS { + return false, nil + } + } else { + // It's not a host - should be NOT_STARTED + if *c.Status != nsxModel.ComponentUpgradeStatus_STATUS_NOT_STARTED { + return false, nil + } + } + } + return true, nil +} + func getSummaryInfo(m interface{}) (string, bool, error) { connector := getPolicyConnector(m) summaryClient := upgrade.NewSummaryClient(connector) @@ -175,8 +206,16 @@ func getSummaryInfo(m interface{}) (string, bool, error) { return targetVersion, false, nil } if summary.UpgradeStatus == nil || (*summary.UpgradeStatus) != nsxModel.UpgradeSummary_UPGRADE_STATUS_NOT_STARTED { - log.Printf("Upgrade process has started, skip running precheck") - return targetVersion, false, nil + is9, err := isVCF9HostUpgrade(m, targetVersion) + if err != nil { + return "", false, err + } + if is9 { + log.Printf("Hosts have been pre-upgraded, prechecks should be running") + } else { + log.Printf("Upgrade process has started, skip running precheck") + return targetVersion, false, nil + } } return targetVersion, true, nil } diff --git a/nsxt/resource_nsxt_upgrade_run.go b/nsxt/resource_nsxt_upgrade_run.go index 622dea941..c48e05a08 100644 --- a/nsxt/resource_nsxt_upgrade_run.go +++ b/nsxt/resource_nsxt_upgrade_run.go @@ -27,6 +27,18 @@ var upgradeComponentList = []string{ mpUpgradeGroup, } +var upgradeComponentListPost9 = []string{ + mpUpgradeGroup, + edgeUpgradeGroup, + hostUpgradeGroup, + finalizeUpgradeGroup, +} + +var postCheckComponentList = []string{ + edgeUpgradeGroup, + hostUpgradeGroup, +} + var componentToGroupKey = map[string]string{ edgeUpgradeGroup: "edge_group", hostUpgradeGroup: "host_group", @@ -71,6 +83,23 @@ type upgradeClientSet struct { Interval int } +func getTargetVersion(m interface{}) (string, error) { + connector := getPolicyConnector(m) + client := upgrade.NewSummaryClient(connector) + obj, err := client.Get() + if err != nil { + return "", err + } + return *obj.TargetVersion, nil +} + +func getUpgradeComponentList(targetVersion string) []string { + if util.VersionHigherOrEqual(targetVersion, "9.0.0") { + return upgradeComponentListPost9 + } + return upgradeComponentList +} + func newUpgradeClientSet(connector client.Connector, d *schema.ResourceData) *upgradeClientSet { return &upgradeClientSet{ GroupClient: upgrade.NewUpgradeUnitGroupsClient(connector), @@ -104,6 +133,21 @@ func resourceNsxtUpgradeRun() *schema.Resource { "host_group": getUpgradeGroupSchema(true), "edge_upgrade_setting": getUpgradeSettingSchema(true), "host_upgrade_setting": getUpgradeSettingSchema(false), + "finalize_upgrade_setting": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeBool, + Description: "Finalize upgrade when complete", + Optional: true, + Default: true, + }, + }, + }, + }, "timeout": { Type: schema.TypeInt, Description: "Upgrade status check timeout in seconds", @@ -362,16 +406,28 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error { connector := getPolicyConnectorWithHeaders(m, nil, false, false) upgradeClientSet := newUpgradeClientSet(connector, d) + targetVersion, err := getTargetVersion(m) + if err != nil { + return err + } log.Printf("[INFO] Updating UpgradeUnitGroup and UpgradePlanSetting.") - err := prepareUpgrade(upgradeClientSet, d, m) + err = prepareUpgrade(upgradeClientSet, d, targetVersion) if err != nil { return handleCreateError("NsxtUpgradeRun", id, err) } log.Printf("[INFO] Successfully update UpgradeUnitGroup and UpgradePlanSetting. Start Upgrade.") + finalizeUpgrade := true + if d.HasChange("finalize_upgrade_setting") { + finalizeSettings := d.Get("finalize_upgrade_setting").([]interface{}) + if len(finalizeSettings) != 0 { + finalizeSettingsMap := finalizeSettings[0].(map[string]interface{}) + finalizeUpgrade = finalizeSettingsMap["enabled"].(bool) + } + } - err = runUpgrade(upgradeClientSet, getPartialUpgradeMap(d)) + err = runUpgrade(upgradeClientSet, getPartialUpgradeMap(d, targetVersion), targetVersion, finalizeUpgrade) if err != nil { return handleCreateError("NsxtUpgradeRun", id, err) } @@ -382,10 +438,10 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error { return resourceNsxtUpgradeRunRead(d, m) } -func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, m interface{}) error { - for _, component := range upgradeComponentList { +func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, targetVersion string) error { + for _, component := range getUpgradeComponentList(targetVersion) { // Customize MP upgrade is not allowed - if component == mpUpgradeGroup { + if component == mpUpgradeGroup || component == finalizeUpgradeGroup { continue } @@ -438,13 +494,13 @@ func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, return nil } -func getPartialUpgradeMap(d *schema.ResourceData) map[string]bool { +func getPartialUpgradeMap(d *schema.ResourceData, targetVersion string) map[string]bool { isPartialUpgradeMap := map[string]bool{ edgeUpgradeGroup: false, hostUpgradeGroup: false, } - for _, component := range upgradeComponentList { - if component == mpUpgradeGroup { + for _, component := range getUpgradeComponentList(targetVersion) { + if component == mpUpgradeGroup || component == finalizeUpgradeGroup { continue } for _, groupI := range d.Get(componentToGroupKey[component]).([]interface{}) { @@ -711,10 +767,14 @@ func updateComponentUpgradePlanSetting(settingClient plan.SettingsClient, d *sch return err } -func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string]bool) error { +func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string]bool, targetVersion string, finalizeUpgrade bool) error { partialUpgradeExist := false prevComponent := "" - for _, component := range upgradeComponentList { + for _, c := range getUpgradeComponentList(targetVersion) { + component := c + if !finalizeUpgrade && component == finalizeUpgradeGroup { + continue + } // After one component upgrade is completed, although the status of our next component is NOT_STARTED, // there is a period that overall status is still IN_PROGRESS, which will prevent us to start the upgrade of next component. // Wait here for the overall status become stable. Because there is potential upgrade triggered before, we wait here also @@ -735,6 +795,16 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string continue } } + + // If component is already upgraded, resume + status, err := getUpgradeStatus(upgradeClientSet.StatusClient, &component) + if err != nil { + return err + } + if status.Status == model.ComponentUpgradeStatus_STATUS_SUCCESS { + log.Printf("Component %s already upgraded successfully, skipping", component) + continue + } pendingStatus := []string{model.ComponentUpgradeStatus_STATUS_IN_PROGRESS} targetStatus := []string{model.ComponentUpgradeStatus_STATUS_SUCCESS} completeLog := fmt.Sprintf("[INFO] %s upgrade is completed.", component) @@ -746,7 +816,10 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string prevComponent = component completeLog = fmt.Sprintf("[INFO] %s upgrade is partially completed.", component) } - upgradeClientSet.PlanClient.Upgrade(&component) + err = upgradeClientSet.PlanClient.Upgrade(&component) + if err != nil { + return err + } err = waitUpgradeForStatus(upgradeClientSet, &component, pendingStatus, targetStatus) if err != nil { return err @@ -757,21 +830,18 @@ func runUpgrade(upgradeClientSet *upgradeClientSet, partialUpgradeMap map[string } func runPostcheck(upgradeClient nsx.UpgradeClient, d *schema.ResourceData) { - for i := range upgradeComponentList { - component := upgradeComponentList[i] - if component != mpUpgradeGroup { - settingI := d.Get(componentToSettingKey[component]).([]interface{}) - if len(settingI) == 0 { - continue - } + for _, component := range postCheckComponentList { + settingI := d.Get(componentToSettingKey[component]).([]interface{}) + if len(settingI) == 0 { + continue + } - setting := settingI[0].(map[string]interface{}) - postCheck := setting["post_upgrade_check"].(bool) + setting := settingI[0].(map[string]interface{}) + postCheck := setting["post_upgrade_check"].(bool) - if postCheck { - log.Printf("[INFO] Start %s upgrade postcheck. Please use data source nsxt_upgrade_postcheck for results.", component) - upgradeClient.Executepostupgradechecks(component) - } + if postCheck { + log.Printf("[INFO] Start %s upgrade postcheck. Please use data source nsxt_upgrade_postcheck for results.", component) + upgradeClient.Executepostupgradechecks(component) } } } @@ -838,6 +908,11 @@ func resourceNsxtUpgradeRunRead(d *schema.ResourceData, m interface{}) error { if err != nil { return handleReadError(d, "NsxtUpgradeRun", id, err) } + targetVersion, err := getTargetVersion(m) + if err != nil { + return handleReadError(d, "NsxtUpgradeRun", id, err) + } + d.Set("target_version", targetVersion) return nil } diff --git a/nsxt/util/utils.go b/nsxt/util/utils.go index 2e9920581..562d66402 100644 --- a/nsxt/util/utils.go +++ b/nsxt/util/utils.go @@ -13,9 +13,12 @@ import ( var NsxVersion = "" func NsxVersionLower(ver string) bool { + return VersionLower(NsxVersion, ver) +} +func VersionLower(base, ver string) bool { requestedVersion, err1 := version.NewVersion(ver) - currentVersion, err2 := version.NewVersion(NsxVersion) + currentVersion, err2 := version.NewVersion(base) if err1 != nil || err2 != nil { log.Printf("[ERROR] Failed perform version check for version %s", ver) return true @@ -24,9 +27,13 @@ func NsxVersionLower(ver string) bool { } func NsxVersionHigherOrEqual(ver string) bool { + return VersionHigherOrEqual(NsxVersion, ver) +} + +func VersionHigherOrEqual(base, ver string) bool { requestedVersion, err1 := version.NewVersion(ver) - currentVersion, err2 := version.NewVersion(NsxVersion) + currentVersion, err2 := version.NewVersion(base) if err1 != nil || err2 != nil { log.Printf("[ERROR] Failed perform version check for version %s", ver) return false diff --git a/website/docs/guides/upgrade.html.markdown b/website/docs/guides/upgrade.html.markdown index 387d76fa9..af90bc16a 100644 --- a/website/docs/guides/upgrade.html.markdown +++ b/website/docs/guides/upgrade.html.markdown @@ -166,9 +166,20 @@ resource "nsxt_upgrade_run" "run" { } ``` +### Upgrade run stage with NSX v9.0 and above + +NSX v9.0 introduces a few changes to the flow of execution of the run resource: the upgrade begins with updating the NSX +management plane, then Edge appliances are upgraded, followed by ESXi hosts. Finally, the upgrade process is finalized, +when the entire process is complete, including the upgrade of the ESXi software. + +When the ESXi software is not upgraded, the finalization stage will fail. There is an additional component in the +`nsxt_upgrade_run` resource which allows execution without the finalization stage. This allows successful execution of +the NSX management plane, the Edge appliances and the NSX bits on the ESXi hosts, and postpone the ESXi OS upgrade to +a later time. + ### Post upgrade checks -Upgrade post check data sources can be used to examine the results of the edge and host upgrades, to cunclude if the +Upgrade post check data sources can be used to examine the results of the edge and host upgrades, to conclude if the upgrade has completed successfully. The example below uses Terraform `check` to test the upgrade results. diff --git a/website/docs/r/upgrade_run.html.markdown b/website/docs/r/upgrade_run.html.markdown index a308055c4..02f74e07c 100644 --- a/website/docs/r/upgrade_run.html.markdown +++ b/website/docs/r/upgrade_run.html.markdown @@ -88,6 +88,8 @@ The following arguments are supported: * `parallel` - (Optional) Upgrade Method to specify whether upgrades of UpgradeUnitGroups in this component are performed serially or in parallel. Default: True. * `post_upgrade_check` - (Optional) Flag to indicate whether run post upgrade check after upgrade. Default: True. * `stop_on_error` - (Optional) Flag to indicate whether to pause the upgrade plan execution when an error occurs. Default: False. +* `finalize_upgrade_setting` - (Optional) FINALIZE_UPGRADE component upgrade plan setting. + * `enabled` - (Optional) Finalize upgrade after completion of all the components' upgrade is complete. Default: True. ## Argument Reference