Add drain logic for upgrade-controller SUC plans (#28)

* Add drain enable/disable in upgrade plan

* Add drain configuration in upgrade-controlelr SUC plans

* Add drain option parse logic to the controller

* Add description for the drain configuration

* Use singular naming for controlPlane and worker nodes drain

api/v1alpha1/upgradeplan_types.go

@@ -48,6 +48,17 @@ type UpgradePlanSpec struct {
 	// ReleaseVersion specifies the target version for platform upgrade.
 	// The version format is X.Y.Z, for example "3.0.2".
 	ReleaseVersion string `json:"releaseVersion"`
+	// Drain specifies whether control-plane and worker nodes should be drained.
+	// If left unspecified, drain is done on both control-plane and worker nodes by default.
+	// +optional
+	Drain *Drain `json:"drain"`
+}
+
+type Drain struct {
+	// +optional
+	ControlPlane *bool `json:"controlPlane"`
+	// +optional
+	Worker *bool `json:"worker"`
 }
 
 // UpgradePlanStatus defines the observed state of UpgradePlan

config/crd/bases/lifecycle.suse.com_upgradeplans.yaml

@@ -39,6 +39,16 @@ spec:
           spec:
             description: UpgradePlanSpec defines the desired state of UpgradePlan
             properties:
+              drain:
+                description: |-
+                  Drain specifies whether control-plane and worker nodes should be drained.
+                  If left unspecified, drain is done on both control-plane and worker nodes by default.
+                properties:
+                  controlPlane:
+                    type: boolean
+                  worker:
+                    type: boolean
+                type: object
               releaseVersion:
                 description: |-
                   ReleaseVersion specifies the target version for platform upgrade.

internal/controller/reconcile_kubernetes.go

@@ -27,7 +27,8 @@ func (r *UpgradePlanReconciler) reconcileKubernetes(ctx context.Context, upgrade
 		return ctrl.Result{}, fmt.Errorf("identifying target kubernetes version: %w", err)
 	}
 
-	controlPlanePlan := upgrade.KubernetesControlPlanePlan(kubernetesVersion)
+	drainControlPlane, drainWorker := parseDrainOptions(upgradePlan)
+	controlPlanePlan := upgrade.KubernetesControlPlanePlan(kubernetesVersion, drainControlPlane)
 	if err = r.Get(ctx, client.ObjectKeyFromObject(controlPlanePlan), controlPlanePlan); err != nil {
 		if !errors.IsNotFound(err) {
 			return ctrl.Result{}, err
@@ -50,7 +51,7 @@ func (r *UpgradePlanReconciler) reconcileKubernetes(ctx context.Context, upgrade
 		return ctrl.Result{Requeue: true}, nil
 	}
 
-	workerPlan := upgrade.KubernetesWorkerPlan(kubernetesVersion)
+	workerPlan := upgrade.KubernetesWorkerPlan(kubernetesVersion, drainWorker)
 	if err = r.Get(ctx, client.ObjectKeyFromObject(workerPlan), workerPlan); err != nil {
 		if !errors.IsNotFound(err) {
 			return ctrl.Result{}, err

internal/controller/reconcile_os.go

@@ -29,7 +29,8 @@ func (r *UpgradePlanReconciler) reconcileOS(ctx context.Context, upgradePlan *li
 		return ctrl.Result{}, r.createSecret(ctx, upgradePlan, secret)
 	}
 
-	controlPlanePlan := upgrade.OSControlPlanePlan(release.ReleaseVersion, secret.Name, &release.Components.OperatingSystem)
+	drainControlPlane, drainWorker := parseDrainOptions(upgradePlan)
+	controlPlanePlan := upgrade.OSControlPlanePlan(release.ReleaseVersion, secret.Name, &release.Components.OperatingSystem, drainControlPlane)
 	if err = r.Get(ctx, client.ObjectKeyFromObject(controlPlanePlan), controlPlanePlan); err != nil {
 		if !errors.IsNotFound(err) {
 			return ctrl.Result{}, err
@@ -57,7 +58,7 @@ func (r *UpgradePlanReconciler) reconcileOS(ctx context.Context, upgradePlan *li
 		return ctrl.Result{Requeue: true}, nil
 	}
 
-	workerPlan := upgrade.OSWorkerPlan(release.ReleaseVersion, secret.Name, &release.Components.OperatingSystem)
+	workerPlan := upgrade.OSWorkerPlan(release.ReleaseVersion, secret.Name, &release.Components.OperatingSystem, drainWorker)
 	if err = r.Get(ctx, client.ObjectKeyFromObject(workerPlan), workerPlan); err != nil {
 		if !errors.IsNotFound(err) {
 			return ctrl.Result{}, err

internal/controller/upgradeplan_controller.go

@@ -166,6 +166,23 @@ func isHelmUpgradeFinished(plan *lifecyclev1alpha1.UpgradePlan, conditionType st
 	return false
 }
 
+func parseDrainOptions(plan *lifecyclev1alpha1.UpgradePlan) (drainControlPlane bool, drainWorker bool) {
+	drainControlPlane = true
+	drainWorker = true
+
+	if plan.Spec.Drain != nil {
+		if plan.Spec.Drain.ControlPlane != nil {
+			drainControlPlane = *plan.Spec.Drain.ControlPlane
+		}
+
+		if plan.Spec.Drain.Worker != nil {
+			drainWorker = *plan.Spec.Drain.Worker
+		}
+	}
+
+	return drainControlPlane, drainWorker
+}
+
 func setPendingCondition(plan *lifecyclev1alpha1.UpgradePlan, conditionType, message string) {
 	condition := metav1.Condition{Type: conditionType, Status: metav1.ConditionUnknown, Reason: lifecyclev1alpha1.UpgradePending, Message: message}
 	meta.SetStatusCondition(&plan.Status.Conditions, condition)

internal/upgrade/base.go

@@ -1,6 +1,8 @@
 package upgrade
 
 import (
+	"time"
+
 	upgradecattlev1 "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -16,7 +18,7 @@ const (
 	ControlPlaneLabel = "node-role.kubernetes.io/control-plane"
 )
 
-func baseUpgradePlan(name string) *upgradecattlev1.Plan {
+func baseUpgradePlan(name string, drain bool) *upgradecattlev1.Plan {
 	const (
 		kind               = "Plan"
 		apiVersion         = "upgrade.cattle.io/v1"
@@ -37,6 +39,18 @@ func baseUpgradePlan(name string) *upgradecattlev1.Plan {
 		},
 	}
 
+	if drain {
+		timeout := 15 * time.Minute
+		deleteEmptyDirData := true
+		ignoreDaemonSets := true
+		plan.Spec.Drain = &upgradecattlev1.DrainSpec{
+			Timeout:            &timeout,
+			DeleteEmptydirData: &deleteEmptyDirData,
+			IgnoreDaemonSets:   &ignoreDaemonSets,
+			Force:              true,
+		}
+	}
+
 	return plan
 }
 

internal/upgrade/kubernetes.go

@@ -26,11 +26,11 @@ func kubernetesUpgradeImage(version string) string {
 	return rke2UpgradeImage
 }
 
-func KubernetesControlPlanePlan(version string) *upgradecattlev1.Plan {
+func KubernetesControlPlanePlan(version string, drain bool) *upgradecattlev1.Plan {
 	controlPlanePlanName := kubernetesPlanName(controlPlaneKey, version)
 	upgradeImage := kubernetesUpgradeImage(version)
 
-	controlPlanePlan := baseUpgradePlan(controlPlanePlanName)
+	controlPlanePlan := baseUpgradePlan(controlPlanePlanName, drain)
 	controlPlanePlan.Labels = map[string]string{
 		"k8s-upgrade": "control-plane",
 	}
@@ -75,12 +75,12 @@ func KubernetesControlPlanePlan(version string) *upgradecattlev1.Plan {
 	return controlPlanePlan
 }
 
-func KubernetesWorkerPlan(version string) *upgradecattlev1.Plan {
+func KubernetesWorkerPlan(version string, drain bool) *upgradecattlev1.Plan {
 	controlPlanePlanName := kubernetesPlanName(controlPlaneKey, version)
 	workerPlanName := kubernetesPlanName(workersKey, version)
 	upgradeImage := kubernetesUpgradeImage(version)
 
-	workerPlan := baseUpgradePlan(workerPlanName)
+	workerPlan := baseUpgradePlan(workerPlanName, drain)
 	workerPlan.Labels = map[string]string{
 		"k8s-upgrade": "worker",
 	}
@@ -108,9 +108,6 @@ func KubernetesWorkerPlan(version string) *upgradecattlev1.Plan {
 	}
 	workerPlan.Spec.Version = version
 	workerPlan.Spec.Cordon = true
-	workerPlan.Spec.Drain = &upgradecattlev1.DrainSpec{
-		Force: true,
-	}
 
 	return workerPlan
 }

internal/upgrade/os.go

@@ -64,9 +64,9 @@ func OSUpgradeSecret(releaseOS *release.OperatingSystem) (*corev1.Secret, error)
 	return secret, nil
 }
 
-func OSControlPlanePlan(releaseVersion, secretName string, releaseOS *release.OperatingSystem) *upgradecattlev1.Plan {
+func OSControlPlanePlan(releaseVersion, secretName string, releaseOS *release.OperatingSystem, drain bool) *upgradecattlev1.Plan {
 	controlPlanePlanName := osPlanName(controlPlaneKey, releaseOS.ZypperID, releaseOS.Version)
-	controlPlanePlan := baseOSPlan(controlPlanePlanName, releaseVersion, secretName)
+	controlPlanePlan := baseOSPlan(controlPlanePlanName, releaseVersion, secretName, drain)
 
 	controlPlanePlan.Labels = map[string]string{
 		"os-upgrade": "control-plane",
@@ -107,9 +107,9 @@ func OSControlPlanePlan(releaseVersion, secretName string, releaseOS *release.Op
 	return controlPlanePlan
 }
 
-func OSWorkerPlan(releaseVersion, secretName string, releaseOS *release.OperatingSystem) *upgradecattlev1.Plan {
+func OSWorkerPlan(releaseVersion, secretName string, releaseOS *release.OperatingSystem, drain bool) *upgradecattlev1.Plan {
 	workerPlanName := osPlanName(workersKey, releaseOS.ZypperID, releaseOS.Version)
-	workerPlan := baseOSPlan(workerPlanName, releaseVersion, secretName)
+	workerPlan := baseOSPlan(workerPlanName, releaseVersion, secretName, drain)
 
 	workerPlan.Labels = map[string]string{
 		"os-upgrade": "worker",
@@ -131,12 +131,12 @@ func OSWorkerPlan(releaseVersion, secretName string, releaseOS *release.Operatin
 	return workerPlan
 }
 
-func baseOSPlan(planName, releaseVersion, secretName string) *upgradecattlev1.Plan {
+func baseOSPlan(planName, releaseVersion, secretName string, drain bool) *upgradecattlev1.Plan {
 	const (
 		planImage = "registry.suse.com/bci/bci-base:15.5"
 	)
 
-	baseOSplan := baseUpgradePlan(planName)
+	baseOSplan := baseUpgradePlan(planName, drain)
 
 	secretPathRelativeToHost := fmt.Sprintf("/run/system-upgrade/secrets/%s", secretName)
 	mountPath := filepath.Join("/host", secretPathRelativeToHost)