controller: consider rollback nodes unavailable

Author: djoshy

pkg/controller/node/node_controller.go

@@ -1232,10 +1232,13 @@ func (ctrl *Controller) updateCandidateNode(mosc *mcfgv1.MachineOSConfig, mosb *
 func getAllCandidateMachines(layered bool, config *mcfgv1.MachineOSConfig, build *mcfgv1.MachineOSBuild, pool *mcfgv1.MachineConfigPool, nodesInPool []*corev1.Node, maxUnavailable int) ([]*corev1.Node, uint) {
 	unavail := getUnavailableMachines(nodesInPool, pool, layered, build)
 	if len(unavail) >= maxUnavailable {
-		klog.V(4).Infof("Pool %s: No nodes available for updates", pool.Name)
+		klog.V(4).Infof("getAllCandidateMachines: No capacity left for pool %s (unavail=%d >= maxUnavailable=%d)",
+			pool.Name, len(unavail), maxUnavailable)
 		return nil, 0
 	}
 	capacity := maxUnavailable - len(unavail)
+	klog.V(4).Infof("getAllCandidateMachines: Computed capacity=%d for pool %s", capacity, pool.Name)
+
 	failingThisConfig := 0
 	// We only look at nodes which aren't already targeting our desired config
 	var nodes []*corev1.Node

pkg/controller/node/status.go

@@ -373,13 +373,13 @@ func isNodeDone(node *corev1.Node, layered bool) bool {
 		return false
 	}
 
-	if layered {
-		// The MachineConfig annotations are loaded on boot-up by the daemon which
-		// isn't currently done for the image annotations, so the comparisons here
-		// are a bit more nuanced.
-		cimage, cok := node.Annotations[daemonconsts.CurrentImageAnnotationKey]
-		dimage, dok := node.Annotations[daemonconsts.DesiredImageAnnotationKey]
+	// The MachineConfig annotations are loaded on boot-up by the daemon which
+	// isn't currently done for the image annotations, so the comparisons here
+	// are a bit more nuanced.
+	cimage, cok := node.Annotations[daemonconsts.CurrentImageAnnotationKey]
+	dimage, dok := node.Annotations[daemonconsts.DesiredImageAnnotationKey]
 
+	if layered {
 		// If desired image is not set, but the pool is layered, this node can
 		// be considered ready for an update. This is the very first time node
 		// is being opted into layering.
@@ -398,6 +398,15 @@ func isNodeDone(node *corev1.Node, layered bool) bool {
 
 	}
 
+	// If not in layered mode, we also need to consider the case when the node is rolling back
+	// from layered to non-layered. In those cases, cconfig==dconfig, but the node
+	// will still need to do an update back to dconfig's OSImageURL. We can detect a
+	// rolling back node by checking if the cimage stills exists but the dimage does not exist.
+	if cok && !dok {
+		// The node is not "done" in this case, as the current image annotation still exists.
+		return false
+	}
+
 	return cconfig == dconfig && isNodeMCDState(node, daemonconsts.MachineConfigDaemonStateDone)
 }