root_image

#!/usr/bin/env bash
#
# Create a VM image suitable for running automated tests
# Output: vm_image

set -o nounset
set -o errexit
set -o errtrace

ktest_dir=$(dirname "$(readlink -f "$0")")
debootstrap=$ktest_dir/debootstrap/debootstrap

. "$ktest_dir/lib/util.sh"
. "$ktest_dir/lib/common.sh"

if [[ $(id -u) != 0 ]] ; then
    echo this script must be run as root
    exit 1
fi

checkdep fallocate util-linux
checkdep mkfs.ext4 e2fsprogs
checkdep curl 

IMAGE_SIZE="10G"
#http is preferred over https here: on emulated CPUs, the encryption simply isn't worth it
MIRROR=http://deb.debian.org/debian/

usage()
{
    echo "root_image: create/update virtual machine root images for ktest"
    echo "Usage: root_image cmd [options]"
    echo "  create		Create a new image"
    echo "  update		Update an existing image"
    echo
    echo "options:"
    echo "  -h			Display this help and exit"
    echo "  -a <arch>		Architecture for vm (x86_64,armhf,i386,aarch64,sparc64,s390x,ppc64,riscv64)"
    echo "  -m <mirror>		Debian mirror"
    echo '  -i <image>		Image to create/update, defaults to /var/lib/ktest/root.$arch'
}

intervene()
{
	local yn="N";
	echo "Installing $2 failed: both binary install and $1";
	echo "You can start a shell to check whether any dependency issues can be fixed"
	echo "if you do (y), and exit the shell, it is assumed to be fixed and installing will continue"
	echo "if you don't, we give up."
        echo "you can also choose to skip package installation (s), though this is not recommended"
	read -p "Do you wish to start a shell to investigate? (y/s/N)" yn
	case $yn in
		[Yy]* )
			_chroot "$MNT" /tmpbuild/executer.sh /bin/bash;
		;;
		[sS]* )
			return 0;
		;;
		* ) return 1;;
	esac
}

install_fix_deps()
{
	trap 'echo "Could not install packages! consider switching to a supported architecture"; umount_image; rm "$ktest_image"' EXIT
	#debian has a separate dist for "experimental" / broken packages, in debian-ports it is called "unreleased".
	experimental_dist="experimental" && [[ ${MIRROR} == *"debian-ports"* ]] && experimental_dist=unreleased;
        SNAPSHOT_MIRROR="https://snapshot.debian.org/archive/debian" && [[ ${MIRROR} == *"debian-ports"* ]] && SNAPSHOT_MIRROR+="-ports";
	#get necessary certificates / pgp keyrings to work with unofficial and outdated snapshots
	_chroot "$MNT" apt-get -qq install debian-ports-archive-keyring ca-certificates
	#disable unwanted checks - we know the snapshots are not valid
	echo "Acquire::Check-Valid-Until false;" > $MNT/etc/apt/apt.conf.d/10-nocheckvalid
	#create a sources.list with more options to work around missing packages
	echo "deb [trusted=yes] $MIRROR sid main contrib" > $MNT/etc/apt/sources.list
	echo "deb-src [trusted=yes] $MIRROR sid main contrib" >> $MNT/etc/apt/sources.list
	echo "deb [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list
	echo "deb-src [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list
        #add snapshots specified in cross.conf
	for i in "${SID_SNAPSHOTS[@]}"; do
	echo "deb [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list
	echo "deb-src [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list
	done
        #when working with debian-ports, add the general sources as a last hope ... there *may* be a package compiling for this archwe need
	[[ ${MIRROR} == *"debian-ports"* ]] && echo "deb-src [trusted=yes] http://deb.debian.org/debian sid main contrib" >> $MNT/etc/apt/sources.list
	touch $MNT/etc/passwd
	touch $MNT/etc/shadow
	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
	_chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
	_chroot "$MNT" apt-get -qq install -f
	_chroot "$MNT" apt-get -qq install build-essential
	#if we can install them all at once, do so, otherwise, try one by one
	_chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}" || for i in "${PACKAGES[@]}"; do
		if [[ ! $(_chroot "$MNT" apt-cache show $i | grep Version) == "" ]]; then
		_chroot "$MNT" apt-get -qq install --allow-unauthenticated --no-install-recommends $i && continue;
		fi
		#installing binary failed.  Try to install from source.  Do it in a tmpfs folder so the image doesn't get overwhelmed:
		[[ ! -d ${MNT}/tmpbuild ]] && mkdir ${MNT}/tmpbuild;
		mount -t tmpfs none ${MNT}/tmpbuild;
		echo '#!/bin/bash' > "$MNT"/tmpbuild/executer.sh
		echo 'cd /tmpbuild/; $@' >> "$MNT"/tmpbuild/executer.sh
		chmod ago+x "$MNT"/tmpbuild/executer.sh
		_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq build-dep --allow-unauthenticated --no-install-recommends $i || intervene install-deps $i;
		#in case the intervention also installed it, continue:
		if [[ $(_chroot "$MNT" dpkg -l | grep "^ii  $i") == "" ]]; then
			_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq -b source --allow-unauthenticated $i || intervene build $i;
			[ -f "${MNT}/tmpbuild/${i}*.deb" ] && _chroot "$MNT" apt-get -qq install /tmpbuild/${i}*.deb;
		fi
		umount ${MNT}/tmpbuild;
		rmdir ${MNT}/tmpbuild;
	done
}

if [[ $# = 0 ]]; then
    usage
    exit 1
fi

ktest_image=""
ktest_arch="$(uname -m)"
CMD="cmd_$1"
shift

while getopts "ha:m:i:" arg; do
    case $arg in
	h)
	    usage
	    exit 0
	    ;;
	a)
	    ktest_arch=$OPTARG
	    ;;
	m)
	    MIRROR=$OPTARG
	    ;;
	i)
	    ktest_image=$OPTARG
	    ;;
    esac
done
shift $(( OPTIND - 1 ))

parse_arch "$ktest_arch"

[[ -z $ktest_image ]] && ktest_image=/var/lib/ktest/root.$DEBIAN_ARCH

mkdir -p "$(dirname "$ktest_image")"

PACKAGES=(kexec-tools less psmisc openssh-server curl		\
    pciutils							\
    build-essential make gcc g++				\
    autoconf automake autopoint bison				\
    pkg-config libtool-bin					\
    gdb strace linux-perf trace-cmd blktrace sysstat iotop htop	\
    hdparm mdadm lvm2						\
    btrfs-progs jfsutils nilfs-tools f2fs-tools			\
    bc attr gawk acl rsync git python3-docutils			\
    stress-ng)

# stress testing:
PACKAGES+=(fio dbench bonnie++ fsmark)

# bcachefs-tools build dependencies:
PACKAGES+=(libblkid-dev uuid-dev libscrypt-dev libsodium-dev)
PACKAGES+=(libkeyutils-dev liburcu-dev libudev-dev zlib1g-dev libattr1-dev)
PACKAGES+=(libaio-dev libzstd-dev liblz4-dev libfuse3-dev valgrind)
PACKAGES+=(llvm libclang-dev)

# quota tools:
PACKAGES+=(libudev-dev libldap2-dev)

# xfstests:
PACKAGES+=(acct bsdextrautils xfsprogs xfslibs-dev quota libcap2-bin)
PACKAGES+=(libattr1-dev libaio-dev libgdbm-dev libacl1-dev gettext)
PACKAGES+=(libssl-dev libgdbm-dev libgdbm-compat-dev liburing-dev)
PACKAGES+=(duperemove fsverity)

# xfsprogs:
PACKAGES+=(libinih-dev)

# nfs testing:
PACKAGES+=(nfs-kernel-server)

# nbd testing
PACKAGES+=(nbd-client nbd-server)

# dm testing:
PACKAGES+=(cryptsetup)

# weird block layer crap
PACKAGES+=(multipath-tools sg3-utils srptools)

# ZFS support
PACKAGES+=("linux-headers-generic")
# unless no other option when cross-compiling, ignore ZFS
# DKMS needs to cross-compile the module,
# against a different kernel on a different CPUarchitecture.
# this has to cause errors
[[ -z ${CROSS_COMPILE} ]] && PACKAGES+=(dkms zfsutils-linux zfs-dkms)

# suspend testing:
# [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)

EXCLUDE=(dmidecode nano rsyslog logrotate cron		\
    iptables nfacct vim-tiny				\
    debconf-i18n info gnupg libpam-systemd)

SYSTEMD_MASK=(dev-hvc0.device				\
    getty.target					\
    getty-static.service				\
    avahi-daemon.service				\
    crond.service					\
    exim4.service					\
    kdump.service					\
    hdparm.service					\
    cdrom.mount						\
    mdadm-raid.service					\
    lvm2-activation-early.service			\
    aoetools.service					\
    sysstat.service					\
    kexec-load.service					\
    kexec.service					\
    systemd-ask-password-console.path			\
    systemd-ask-password-wall.path			\
    systemd-update-utmp-runlevel.service		\
    systemd-update-utmp.service				\
    time-sync.target					\
    multipathd.service)

export DEBIAN_FRONTEND=noninteractive
export DEBCONF_NONINTERACTIVE_SEEN=true
export LC_ALL=C
export LANGUAGE=C
export LANG=C
# We compute it here to avoid on hosts systems (e.g. NixOS)
# that does not possess `chroot` in the isolated `PATH`
# to fail miserably.
export CHROOT=$(which chroot)

_chroot()
{
    PATH=/usr/sbin:/usr/bin:/sbin:/bin "$CHROOT" "$@"
}

update_files()
{
    install -m0644 "$ktest_dir/lib/fstab" "$MNT/etc/fstab"
    install -m0755 "$ktest_dir/lib/testrunner.wrapper" "$MNT/sbin/testrunner.wrapper"
    install -m0644 "$ktest_dir/lib/testrunner.service" "$MNT/lib/systemd/system/testrunner.service"

    ln -sf /lib/systemd/system/testrunner.service "$MNT/etc/systemd/system/multi-user.target.wants/testrunner.service"

    touch "$MNT/etc/resolv.conf"
    chmod 644 "$MNT/etc/resolv.conf"

    mkdir -p "$MNT/root/"
    install -m0644 "$MNT/etc/skel/.bashrc" "$MNT/root/"
    install -m0644 "$MNT/etc/skel/.profile" "$MNT/root/"

    mkdir -p "$MNT/var/log/core"
    chmod 777 "$MNT/var/log/core"

    # Disable systemd/udev stuff we don't need:

    # systemctl mask doesn't work for foreign archs
    #_chroot "$MNT" systemctl mask "${SYSTEMD_MASK[@]}"

    for i in "${SYSTEMD_MASK[@]}"; do
	(cd "$MNT/etc/systemd/system"; ln -sf /dev/null "$i")
    done

    cat > "$MNT/etc/systemd/journald.conf" <<-ZZ
[Journal]
Storage=none
ForwardToConsole=no
MaxLevelConsole=emerg
ZZ

    mkdir -p "$MNT/etc/network"
    cat > "$MNT/etc/network/interfaces" <<-ZZ
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet dhcp
ZZ

    # disable network interface renaming - it's unreliable
    mkdir -p "$MNT/etc/udev/rules.d/"
    ln -sf /dev/null "$MNT/etc/udev/rules.d/80-net-setup-link.rules"

    rm -f "$MNT/lib/udev/rules.d/*persistent*"
    rm -f "$MNT/lib/udev/rules.d/*lvm*"
    rm -f "$MNT/lib/udev/rules.d/*dm*"
    rm -f "$MNT/lib/udev/rules.d/*md-raid*"
    rm -f "$MNT/lib/udev/rules.d/*btrfs*"
    rm -f "$MNT/lib/udev/rules.d/*hdparm*"

    echo $(hostname)-kvm >"$MNT/etc/hostname"
}

update_packages()
{
    NO_RUSTUP=
    # systemd... !?
    mkdir -p "$MNT"/run/user/0
    cp /etc/resolv.conf "$MNT/etc/resolv.conf"
    #don't do rustup on unsupported architectures, just try to install from debian repositories
    [[ -z $(curl -qq -I "https://static.rust-lang.org/rustup/dist/${RUST_TRIPLE}/rustup-init" 2>/dev/null | grep "HTTP/2 200") ]] && NO_RUSTUP=1
    [[ -n $NO_RUSTUP ]] && PACKAGES+=(rustc rustc-dbgsym rustfmt rustfmt-dbgsym)

    _chroot "$MNT" mount -t proc none /proc
    _chroot "$MNT" mount -t devpts none /dev/pts
    if [[ $MIRROR == *"debian-ports"* || $1 == "sid" ]]; then
	install_fix_deps
    else
	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
    	_chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
    	_chroot "$MNT" apt-get -qq install -f
    	_chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}"
    fi
    rm -f "$MNT/var/cache/apt/archives/*.deb"

    if [[ -z $NO_RUSTUP ]]; then
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
	chmod 755 "$MNT"/tmp/rustup.sh

	_chroot "$MNT" /tmp/rustup.sh --default-host ${RUST_TRIPLE} -y
	echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
    fi;
}

trim_image()
{
    e2fsck -f "$1"
    resize2fs -M "$1"			# shrinks the file
    resize2fs "$1" "$IMAGE_SIZE"	# re-grows as sparse
}

umount_image()
{
    # Unmount everything under $MNT
    awk '{print $2}' /proc/mounts|
	grep "^$MNT"|
	sort -r|
	xargs umount

    rmdir "$MNT"
    trap '' EXIT
}

cmd_update()
{
    if [[ ! -e $ktest_image ]]; then
	echo "$ktest_image does not exist"
	exit 1
    fi

    MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)
    trap 'umount_image' EXIT

    cp "$ktest_image" "$ktest_image".new
    mount "$ktest_image".new "$MNT"

    update_packages none
    update_files

    umount_image
    trim_image "$ktest_image".new
    mv "$ktest_image".new "$ktest_image"
}

cmd_create()
{
    if [[ -e $ktest_image ]]; then
	echo "$ktest_image already exists"
	exit 1
    fi
    (cd "$ktest_dir"; git submodule update --init debootstrap)

    MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)
    trap '/bin/bash; umount_image; rm "$ktest_image"' EXIT

    fallocate -l "$IMAGE_SIZE" "$ktest_image"
    mkfs.ext4 -F "$ktest_image"
    mount "$ktest_image" "$MNT"

   local debian_release="trixie" #general release
   local keyring=""

    if [[ $MIRROR == *"debian-ports"* ]]; then
	debian_release="sid" #unofficial ports don't have named releases
	echo ""
	echo "WARNING: $ktest_arch is unsupported, using SID release for packages"
	echo "*******************************************************************"
	echo "PLEASE NOTE: this often has dependency problems between packages, "
	echo "and can prevent the install due to a dependency conflict "
	echo "If so, contact the debian maintainer for this architecture to fix it:"
	echo "${ktest_arch} : ${DEBIAN_ARCH}@buildd.debian.org"
	echo "*******************************************************************"
	echo ""
    fi

    #fallback: if the architecture can't be found, try official sid:
    if [[ -z $(curl -qq -I "${MIRROR}/dists/${debian_release}/main/binary-${DEBIAN_ARCH}/Release" 2>/dev/null | grep "HTTP/2 200") ]]; then
	echo "WARNING: $DEBIAN_ARCH $debian_release could not be found at ${MIRROR} (where it should be).  Falling back to standard sid"
        MIRROR="http://deb.debian.org/debian/"
	debian_release=sid
    fi

    DEBOOTSTRAP_DIR=$ktest_dir/debootstrap $debootstrap	\
	--no-check-gpg					\
	--arch="$DEBIAN_ARCH"				\
	--exclude=$(join_by , "${EXCLUDE[@]}")		\
	$keyring					\
	--foreign					\
	--components='main,contrib,non-free' \
	$debian_release "$MNT" "$MIRROR"

    if [[ -n ${CROSS_COMPILE} ]]; then
       statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static)
       cp ${statichelper} ${MNT}${statichelper}
    fi

    _chroot "$MNT" /debootstrap/debootstrap --second-stage
    _chroot "$MNT" dpkg --configure -a
    update_packages $debian_release
    update_files

    umount_image
    trim_image "$ktest_image"
}

if [[ $(type -t "$CMD") != function ]]; then
    usage
    exit 1
fi

$CMD "$@"