checkup: Add guest-agent-ping probe replace wait cond to ready state

urrently the checkup setup only waits for the VMI to finish "booting",
i.e. the guest agent service to be ready.
However this is not enough in order to ensure that the VMI has been
configured, a procedure currently done on the cloud-init service.
When the configuration is complete, the script configuring the guest in
the cloud-init service adds a marker file.
Changing the wait condition to "ready" is stronger than waiting for
"guest-agent", since the guest-agent ping needs guest agent in order to
set the VMI to ready.

This commit:
- introduces a new waiting mechanism, using guest-agent-ping probe [0]
to poll-wait the guest until the file is present, and only then sets the
VMI ready condition to true.
- adds a wait for the VMI ready condition to be true.
- removed the now unnecessary wait for guest-agent-ready condition.

[0]
https://kubevirt.io/user-guide/virtual_machines/liveness_and_readiness_probes/#defining-guest-agent-ping-probes

Signed-off-by: Ram Lavi <[email protected]>

pkg/internal/checkup/checkup.go

@@ -98,7 +98,7 @@ func (c *Checkup) Setup(ctx context.Context) error {
 	c.vmi = createdVMI
 
 	var updatedVMIUnderTest *kvcorev1.VirtualMachineInstance
-	updatedVMIUnderTest, err = c.waitForVMIToBoot(setupCtx)
+	updatedVMIUnderTest, err = c.waitForVMIToBeReady(setupCtx)
 	if err != nil {
 		return err
 	}
@@ -159,11 +159,11 @@ func (c *Checkup) deleteVMUnderTestCM(ctx context.Context) error {
 	return c.client.DeleteConfigMap(ctx, c.namespace, c.vmUnderTestConfigMap.Name)
 }
 
-func (c *Checkup) waitForVMIToBoot(ctx context.Context) (*kvcorev1.VirtualMachineInstance, error) {
+func (c *Checkup) waitForVMIToBeReady(ctx context.Context) (*kvcorev1.VirtualMachineInstance, error) {
 	vmiFullName := ObjectFullName(c.vmi.Namespace, c.vmi.Name)
 	var updatedVMI *kvcorev1.VirtualMachineInstance
 
-	log.Printf("Waiting for VMI %q to boot...", vmiFullName)
+	log.Printf("Waiting for VMI %q to be ready...", vmiFullName)
 
 	conditionFn := func(ctx context.Context) (bool, error) {
 		var err error
@@ -173,7 +173,7 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context) (*kvcorev1.VirtualMachin
 		}
 
 		for _, condition := range updatedVMI.Status.Conditions {
-			if condition.Type == kvcorev1.VirtualMachineInstanceAgentConnected && condition.Status == corev1.ConditionTrue {
+			if condition.Type == kvcorev1.VirtualMachineInstanceReady && condition.Status == corev1.ConditionTrue {
 				return true, nil
 			}
 		}
@@ -182,10 +182,10 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context) (*kvcorev1.VirtualMachin
 	}
 	const pollInterval = 5 * time.Second
 	if err := wait.PollImmediateUntilWithContext(ctx, pollInterval, conditionFn); err != nil {
-		return nil, fmt.Errorf("failed to wait for VMI %q to boot: %w", vmiFullName, err)
+		return nil, fmt.Errorf("failed to wait for VMI %q be ready: %w", vmiFullName, err)
 	}
 
-	log.Printf("VMI %q had successfully booted", vmiFullName)
+	log.Printf("VMI %q has successfully reached ready condition", vmiFullName)
 
 	return updatedVMI, nil
 }
@@ -268,6 +268,7 @@ func newRealtimeVMI(checkupConfig config.Config) *kvcorev1.VirtualMachineInstanc
 		vmi.WithConfigMapDisk(configVolumeName, configDiskSerial),
 		vmi.WithCloudInitNoCloudVolume(cloudInitDiskName,
 			vmi.CloudInit(realtimeVMIBootCommands(configDiskSerial))),
+		vmi.WithReadinessFileProbe(config.BootScriptReadinessMarkerFileFullPath),
 	)
 }
 

pkg/internal/checkup/checkup_test.go

@@ -180,7 +180,7 @@ func (cs *clientStub) CreateVirtualMachineInstance(_ context.Context,
 	cs.createdVMIs[vmiFullName] = vmi
 
 	vmi.Status.Conditions = append(vmi.Status.Conditions, kvcorev1.VirtualMachineInstanceCondition{
-		Type:   kvcorev1.VirtualMachineInstanceAgentConnected,
+		Type:   kvcorev1.VirtualMachineInstanceReady,
 		Status: corev1.ConditionTrue,
 	})
 

pkg/internal/checkup/vmi/vmi.go

@@ -257,6 +257,23 @@ func CloudInit(bootCommands []string) string {
 	return sb.String()
 }
 
+func WithReadinessFileProbe(fileName string) Option {
+	return func(vmi *kvcorev1.VirtualMachineInstance) {
+		var readinessProbeCommand = []string{"cat", fileName}
+		vmi.Spec.ReadinessProbe = &kvcorev1.Probe{
+			Handler: kvcorev1.Handler{
+				Exec: &corev1.ExecAction{
+					Command: readinessProbeCommand,
+				},
+			},
+			FailureThreshold:    30,
+			InitialDelaySeconds: 90,
+			PeriodSeconds:       10,
+			TimeoutSeconds:      10,
+		}
+	}
+}
+
 func Pointer[T any](v T) *T {
 	return &v
 }