Build live PXE and ISO images

src/cmd-buildextend-installer

@@ -31,32 +31,38 @@ if not args.build:
 
 print(f"Targeting build: {args.build}")
 
+# Hacky mode switch, until we can drop support for the installer images
+is_live = os.path.basename(sys.argv[0]).endswith('-live')
+image_type = 'live' if is_live else 'installer'
+meta_keys = {k: 'live-' + k if is_live else k for k in ('iso', 'kernel', 'initramfs')}
+
 workdir = os.path.abspath(os.getcwd())
 builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)
 
+img_qemu = os.path.join(builddir, buildmeta['images']['qemu']['path'])
 # Grab the commit hash for this build
 buildmeta_commit = buildmeta['ostree-commit']
 
 repo = os.path.join(workdir, 'tmp/repo')
 
 # Don't run if it's already been done, unless forced
-if 'iso' in buildmeta['images'] and not args.force:
-    print(f"Installer has already been built for {args.build}. Skipping.")
+if meta_keys['iso'] in buildmeta['images'] and not args.force:
+    print(f"Image has already been built for {args.build}. Skipping.")
     print("You can force a rebuild with '--force'.")
     sys.exit(0)
 
 base_name = buildmeta['name']
-iso_name = f'{base_name}-{args.build}-installer.iso'
+iso_name = f'{base_name}-{args.build}-{image_type}.iso'
 name_version = f'{base_name}-{args.build}'
 
-tmpdir = os.environ.get("FORCE_TMPDIR", f"{workdir}/tmp/buildpost-installer")
+tmpdir = os.environ.get("FORCE_TMPDIR", f"{workdir}/tmp/buildpost-{image_type}")
 if os.path.isdir(tmpdir):
     shutil.rmtree(tmpdir)
 
-tmpisoroot = os.path.join(tmpdir, 'installer')
+tmpisoroot = os.path.join(tmpdir, image_type)
 tmpisoimages = os.path.join(tmpisoroot, 'images')
 tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
 
@@ -85,9 +91,35 @@ def generate_iso():
         # initramfs isn't world readable by default so let's open up perms
         os.chmod(os.path.join(tmpisoimages, file), 0o755)
 
+    if is_live:
+        initramfs = os.path.join(tmpisoimages, 'initramfs.img')
+        tmp_squashfs = os.path.join(tmpdir, 'root.squashfs')
+        tmp_cpio = os.path.join(tmpdir, 'root.cpio')
+        tmp_initramfs = os.path.join(tmpdir, 'initramfs')
+
+        run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
+                     img_qemu, tmp_squashfs])
+        run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
+                '--quiet', '--reproducible', '--force-local',
+                '-D', os.path.dirname(tmp_squashfs), '-O', tmp_cpio],
+                input=os.path.basename(tmp_squashfs).encode())
+        # Compression is redundant but the kernel requires it
+        run_verbose(['gzip', '-1', tmp_cpio])
+
+        # Append the root cpio to the initramfs.
+        # The initramfs image is a hardlink to the uncompressed objects
+        # cache, so we can't modify it in place.
+        with open(tmp_initramfs, 'wb') as fdst:
+            with open(initramfs, 'rb') as fsrc:
+                shutil.copyfileobj(fsrc, fdst)
+            with open(tmp_cpio + '.gz', 'rb') as fsrc:
+                shutil.copyfileobj(fsrc, fdst)
+        os.rename(tmp_initramfs, initramfs)
+        os.unlink(tmp_squashfs)
+
     # TODO ignore EFI dir
     # Grab all the contents from the installer dir from the configs
-    run_verbose(["rsync", "-av", "src/config/installer/", f"{tmpisoroot}/"])
+    run_verbose(["rsync", "-av", f"src/config/{image_type}/", f"{tmpisoroot}/"])
 
     # These sections are based on lorax templates
     # see https://github.com/weldr/lorax/tree/master/share/templates.d/99-generic
@@ -181,7 +213,7 @@ def generate_iso():
             efitarfile = tempfile.NamedTemporaryFile(suffix=".tar")
             with tarfile.open(efitarfile.name, "w:", dereference=True) as tar:
                 tar.add(tmpimageefidir, arcname="/EFI", filter=strip)
-                tar.add('src/config/installer/EFI/', arcname='/EFI',
+                tar.add(f'src/config/{image_type}/EFI/', arcname='/EFI',
                         filter=strip)
 
             # Create the efiboot.img file (a fat filesystem) in the images/ dir
@@ -203,8 +235,8 @@ def generate_iso():
     if arch == "x86_64":
         run_verbose(['/usr/bin/isohybrid', tmpisofile])
 
-    kernel_name = f'{base_name}-{args.build}-installer-kernel'
-    initramfs_name = f'{base_name}-{args.build}-installer-initramfs.img'
+    kernel_name = f'{base_name}-{args.build}-{image_type}-kernel'
+    initramfs_name = f'{base_name}-{args.build}-{image_type}-initramfs.img'
     kernel_file = os.path.join(builddir, kernel_name)
     initramfs_file = os.path.join(builddir, initramfs_name)
     shutil.copyfile(os.path.join(tmpisoimages, "vmlinuz"), kernel_file)
@@ -215,15 +247,15 @@ def generate_iso():
     checksum = sha256sum_file(tmpisofile)
 
     buildmeta['images'].update({
-        'iso': {
+        meta_keys['iso']: {
             'path': iso_name,
             'sha256': checksum
         },
-        'kernel': {
+        meta_keys['kernel']: {
             'path': kernel_name,
             'sha256': kernel_checksum
         },
-        'initramfs': {
+        meta_keys['initramfs']: {
             'path': initramfs_name,
             'sha256': initramfs_checksum
         }

src/cmd-buildextend-live

@@ -0,0 +1 @@
+cmd-buildextend-installer

src/cmd-compress

@@ -41,7 +41,7 @@ else:
 print(f"Targeting build: {build}")
 
 # Don't compress certain images
-imgs_to_skip = ["iso", "vmware", "initramfs", "kernel"]
+imgs_to_skip = ["iso", "live-iso", "vmware", "initramfs", "live-initramfs", "kernel", "live-kernel"]
 
 
 def get_cpu_param(param):

src/gf-mksquashfs

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+dn=$(dirname "$0")
+# shellcheck source=src/cmdlib.sh
+. "${dn}"/cmdlib.sh
+# shellcheck source=src/libguestfish.sh
+. "${dn}"/libguestfish.sh
+
+# Usage: gf-mksquashfs <input image> <output image>
+# Example: gf-mksquashfs fedora-coreos.qcow2 fedora-coreos.squashfs
+#
+# This will generate a squashfs from the contents of the root partition.
+
+src="$1"
+dest="$2"
+
+if [[ $src == *.gz || $src == *.xz ]]; then
+    img="$(basename "$src")"
+    fatal "Cannot generate squashfs from $img; not an uncompressed image"
+fi
+
+set -x
+# Work in a tmpdir on the destination so that we don't inherit some MCS labeling
+# from the /tmp dir in the container. This also ensures that the final move is a
+# pure `rename()`.
+# See also:
+# https://github.com/coreos/coreos-assembler/issues/292
+# https://github.com/coreos/coreos-assembler/pull/394
+tmpd=$(mktemp -tdp "$(dirname "${dest}")" gf-mksquashfs.XXXXXX)
+tmp_dest=${tmpd}/image.squashfs
+
+coreos_gf_run "${src}" --ro
+
+root=$(coreos_gf findfs-label root)
+coreos_gf mount "${root}" /
+
+coreos_gf mksquashfs / "${tmp_dest}" compress:zstd
+
+coreos_gf_shutdown
+
+mv "${tmp_dest}" "${dest}"
+rm "${tmpd}" -rf

src/gf-platformid

@@ -18,7 +18,7 @@ src="$1"
 dest="$2"
 platformid="$3"
 
-if [[ $src == *.gz ]]; then
+if [[ $src == *.gz || $src == *.xz ]]; then
     img="$(basename "$src")"
     fatal "Cannot change ignition.platform.id on $img; not an uncompressed image"
 fi
@@ -27,7 +27,9 @@ set -x
 # Work in a tmpdir on the destination so that we don't inherit some MCS labeling
 # from the /tmp dir in the container. This also ensures that the final move is a
 # pure `rename()`.
-# See also: https://github.com/coreos/coreos-assembler/issues/292
+# See also:
+# https://github.com/coreos/coreos-assembler/issues/292
+# https://github.com/coreos/coreos-assembler/pull/394
 tmpd=$(mktemp -tdp "$(dirname "${dest}")" gf-platformid.XXXXXX)
 tmp_dest=${tmpd}/box.img
 

src/libguestfish.sh

@@ -12,15 +12,7 @@ coreos_gf_launch() {
     if [ -n "$GUESTFISH_PID" ]; then
         return
     fi
-    local src=$1
-    shift
-    local guestfish
-    guestfish[0]="guestfish"
-    guestfish[1]="--listen"
-    guestfish[3]="-a"
-    guestfish[4]="${src}"
-
-    eval "$("${guestfish[@]}")"
+    eval "$(guestfish --listen -a "$@")"
     if [ -z "$GUESTFISH_PID" ]; then
         fatal "guestfish didn't start up, see error messages above"
     fi