vms/vm-under-test: Move first boot to a separate service

The first-boot script running by the virt-builder command is run on a
service created by virt-builder. This service is guaranteed to run in
the final stages of the boot process [0].
This may create a race with the checkup, that is waiting on the
agentConnected condition [1] being added in order to run the checkup's
executor package. The race is happening since the guest-agent service
also runs during the final stages of the boot process.

In order to eliminate this race, removing the first-boot script in favor
of the new service, and moving the first boot script content
into a new service. This service is
- manually created on the customize-vm script.
- explicitly scheduled to run before the guest-agent service runs.
- set to run once, using ConditionPathExists unit option [2]. Before the
script reboots it creates an empty file, so that the service will not
run again.

Since the service runs and then reboots the log is not present on
journalctl, so it is manually saved to a log file on the guest.

[0] https://www.libguestfs.org/virt-builder.1.html
[1]
https://github.com/kiagnose/kubevirt-realtime-checkup/blob/97b5fbe47fc5df6513671db6a40e8e31b0815580/pkg/internal/checkup/checkup.go#L144
[2]
https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#ConditionPathExists=

Signed-off-by: Ram Lavi <[email protected]>

vms/vm-under-test/scripts/build-vm-image

@@ -24,6 +24,5 @@ virt-builder centosstream-8 \
   --root-password password:redhat \
   --install cloud-init,kernel-rt,tuned,rt-tests \
   --run /root/scripts/customize-vm \
-  --firstboot /root/scripts/first-boot \
   --selinux-relabel \
   --output /output/kubevirt-realtime-checkup-vm.qcow2

vms/vm-under-test/scripts/customize-vm

@@ -39,6 +39,79 @@ disable_swap() {
   sed -i '/swap/s/^/#/' /etc/fstab
 }
 
+# Create and enable the boot checkup service
+setup_boot_service() {
+  local service_name="realtime-checkup-boot.service"
+  local checkup_boot_script_full_path="/usr/bin/realtime-checkup-boot.sh"
+  local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name"
+  local checkup_boot_ready_marker_full_path="/var/realtime-checkup-image-ready-marker"
+  local checkup_boot_service_log_full_path="/var/realtime-checkup-boot.log"
+
+  setup_checkup_boot_script "$checkup_boot_script_full_path"
+
+  cat <<EOF > "$checkup_boot_service_full_path"
+[Unit]
+Description=Checkup Boot Script
+Before=qemu-guest-agent.service
+ConditionPathExists=!$checkup_boot_ready_marker_full_path
+[Service]
+Type=oneshot
+ExecStart=$checkup_boot_script_full_path $checkup_boot_ready_marker_full_path >$checkup_boot_service_log_full_path
+Restart=no
+User=root
+Group=root
+[Install]
+WantedBy=multi-user.target
+Wants=first-boot-complete.target
+EOF
+
+  systemctl enable "$checkup_boot_service_full_path"
+  systemctl start "$checkup_boot_service_full_path"
+}
+
+setup_checkup_boot_script() {
+  local checkup_boot_script_full_path=$1
+cat <<'EOF' > "$checkup_boot_script_full_path"
+#!/usr/bin/env bash
+#
+# This file is part of the kiagnose project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2024 Red Hat, Inc.
+#
+
+set -e
+
+checkup_boot_ready_marker_full_path=$1
+
+if systemctl --type swap list-units | grep -q '.swap'; then
+  systemctl mask "$(systemctl --type swap list-units | grep '.swap' | awk '{print $1}')"
+fi
+
+tuned_conf="/etc/tuned/realtime-virtual-guest-variables.conf"
+echo "isolated_cores=2-3" > "$tuned_conf"
+echo "isolate_managed_irq=Y" >> "$tuned_conf"
+tuned-adm profile realtime-virtual-guest
+
+touch $checkup_boot_ready_marker_full_path
+reboot
+EOF
+
+  chmod +x "$checkup_boot_script_full_path"
+}
+
 disable_services
 install_packages
 disable_swap
+setup_boot_service