diff --git a/vmtest/__main__.py b/vmtest/__main__.py new file mode 100644 index 000000000..eca89575c --- /dev/null +++ b/vmtest/__main__.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 + +from collections import OrderedDict +import logging +import os +from pathlib import Path +import shlex +import subprocess +import sys + +from util import KernelVersion +from vmtest.config import ( + ARCHITECTURES, + HOST_ARCHITECTURE, + KERNEL_FLAVORS, + SUPPORTED_KERNEL_VERSIONS, + Kernel, +) +from vmtest.download import DownloadCompiler, DownloadKernel, download_in_thread +from vmtest.kmod import build_kmod +from vmtest.rootfsbuild import build_drgn_in_rootfs +from vmtest.vm import LostVMError, run_in_vm + +logger = logging.getLogger(__name__) + + +class _ProgressPrinter: + def __init__(self, file): + self._file = file + if hasattr(file, "fileno"): + try: + columns = os.get_terminal_size(file.fileno())[0] + self._color = True + except OSError: + columns = 80 + self._color = False + self._header = "#" * columns + self._passed = {} + self._failed = {} + + def _green(self, s: str) -> str: + if self._color: + return "\033[32m" + s + "\033[0m" + else: + return s + + def _red(self, s: str) -> str: + if self._color: + return "\033[31m" + s + "\033[0m" + else: + return s + + def update(self, category: str, name: str, passed: bool): + d = self._passed if passed else self._failed + d.setdefault(category, []).append(name) + + if self._failed: + header = self._red(self._header) + else: + header = self._green(self._header) + + print(header, file=self._file) + print(file=self._file) + + if self._passed: + first = True + for category, names in self._passed.items(): + if first: + first = False + print(self._green("Passed:"), end=" ") + else: + print(" ", end=" ") + print(f"{category}: {', '.join(names)}") + if self._failed: + first = True + for category, names in self._failed.items(): + if first: + first = False + print(self._red("Failed:"), end=" ") + else: + print(" ", end=" ") + print(f"{category}: {', '.join(names)}") + + print(file=self._file) + print(header, file=self._file, flush=True) + + +def _kdump_works(kernel: Kernel) -> bool: + if kernel.arch.name == "aarch64": + # kexec fails with "kexec: setup_2nd_dtb failed." on older versions. + # See + # http://lists.infradead.org/pipermail/kexec/2020-November/021740.html. + return KernelVersion(kernel.release) >= KernelVersion("5.10") + elif kernel.arch.name == "arm": + # Without virtual address translation, we can't debug vmcores. Besides, + # kexec fails with "Could not find a free area of memory of 0xXXX + # bytes...". + return False + elif kernel.arch.name == "ppc64": + # Without virtual address translation, we can't debug vmcores. + return False + # Before 6.1, sysrq-c hangs. + # return KernelVersion(kernel.release) >= KernelVersion("6.1") + elif kernel.arch.name == "s390x": + # Before 5.15, sysrq-c hangs. + return KernelVersion(kernel.release) >= KernelVersion("5.15") + elif kernel.arch.name == "x86_64": + return True + else: + assert False, kernel.arch.name + + +if __name__ == "__main__": + import argparse + + logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO + ) + parser = argparse.ArgumentParser( + description="test drgn in a virtual machine", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-d", + "--directory", + metavar="DIR", + type=Path, + default="build/vmtest", + help="directory for vmtest artifacts", + ) + parser.add_argument( + "-a", + "--architecture", + dest="architectures", + action="append", + choices=["all", "foreign", *sorted(ARCHITECTURES)], + default=argparse.SUPPRESS, + required=HOST_ARCHITECTURE is None, + help="architecture to test, " + '"all" to test all supported architectures, ' + 'or "foreign" to test all supported architectures other than the host architecture; ' + "may be given multiple times" + + ( + "" if HOST_ARCHITECTURE is None else f" (default: {HOST_ARCHITECTURE.name})" + ), + ) + parser.add_argument( + "-k", + "--kernel", + metavar="PATTERN|{all," + ",".join(KERNEL_FLAVORS) + "}", + dest="kernels", + action="append", + default=argparse.SUPPRESS, + help="kernel to test, " + '"all" to test all supported kernels, ' + "or flavor name to test all supported kernels of a specific flavor; " + "may be given multiple times (default: none)", + ) + parser.add_argument( + "-l", + "--local", + action="store_true", + help="run local tests", + ) + args = parser.parse_args() + + architecture_names = [] + if hasattr(args, "architectures"): + for name in args.architectures: + if name == "all": + architecture_names.extend(ARCHITECTURES) + elif name == "foreign": + architecture_names.extend( + [ + arch.name + for arch in ARCHITECTURES.values() + if arch is not HOST_ARCHITECTURE + ] + ) + else: + architecture_names.append(name) + architectures = [ + ARCHITECTURES[name] for name in OrderedDict.fromkeys(architecture_names) + ] + else: + architectures = [HOST_ARCHITECTURE] + + if hasattr(args, "kernels"): + kernels = [] + for pattern in args.kernels: + if pattern == "all": + kernels.extend( + [ + version + ".*" + flavor + for version in SUPPORTED_KERNEL_VERSIONS + for flavor in KERNEL_FLAVORS + ] + ) + elif pattern in KERNEL_FLAVORS: + kernels.extend( + [version + ".*" + pattern for version in SUPPORTED_KERNEL_VERSIONS] + ) + else: + kernels.append(pattern) + args.kernels = OrderedDict.fromkeys(kernels) + else: + args.kernels = [] + + if not args.kernels and not args.local: + parser.error("at least one of -k/--kernel or -l/--local is required") + + if args.kernels: + to_download = [DownloadCompiler(arch) for arch in architectures] + for pattern in args.kernels: + for arch in architectures: + to_download.append(DownloadKernel(arch, pattern)) + else: + to_download = [] + + progress = _ProgressPrinter(sys.stderr) + + with download_in_thread(args.directory, to_download) as downloads: + for arch in architectures: + if arch is HOST_ARCHITECTURE: + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "-i"], + env={ + **os.environ, + "CONFIGURE_FLAGS": "--enable-compiler-warnings=error", + }, + ) + if args.local: + logger.info("running local tests on %s", arch.name) + status = subprocess.call( + [ + sys.executable, + "-m", + "pytest", + "-v", + "--ignore=tests/linux_kernel", + ] + ) + progress.update(arch.name, "local", status == 0) + else: + rootfs = args.directory / arch.name / "rootfs" + build_drgn_in_rootfs(rootfs) + if args.local: + logger.info("running local tests on %s", arch.name) + status = subprocess.call( + [ + "unshare", + "--map-root-user", + "--map-users=auto", + "--map-groups=auto", + "--fork", + "--pid", + "--mount-proc=" + str(rootfs / "proc"), + "sh", + "-c", + r""" +set -e + +mount --bind . "$1/mnt" +chroot "$1" sh -c 'cd /mnt && pytest -v --ignore=tests/linux_kernel' +""", + "sh", + rootfs, + ] + ) + progress.update(arch.name, "local", status == 0) + for kernel in downloads: + if not isinstance(kernel, Kernel): + continue + kmod = build_kmod(args.directory, kernel) + if _kdump_works(kernel): + kdump_command = """\ + "$PYTHON" -Bm vmtest.enter_kdump + # We should crash and not reach this. + exit 1 +""" + else: + kdump_command = "" + test_command = rf""" +set -e + +export PYTHON={shlex.quote(sys.executable)} +export DRGN_TEST_KMOD={shlex.quote(str(kmod))} +export DRGN_RUN_LINUX_KERNEL_TESTS=1 +if [ -e /proc/vmcore ]; then + "$PYTHON" -Bm pytest -v tests/linux_kernel/vmcore +else + insmod "$DRGN_TEST_KMOD" + "$PYTHON" -Bm pytest -v tests/linux_kernel --ignore=tests/linux_kernel/vmcore +{kdump_command} +fi +""" + try: + status = run_in_vm( + test_command, + kernel, + args.directory / kernel.arch.name / "rootfs", + args.directory, + ) + except LostVMError as e: + print("error:", e, file=sys.stderr) + status = -1 + progress.update(kernel.arch.name, kernel.release, status == 0) diff --git a/vmtest/config.py b/vmtest/config.py index 2fc36d923..ca74bf251 100644 --- a/vmtest/config.py +++ b/vmtest/config.py @@ -204,6 +204,8 @@ class Architecture(NamedTuple): kernel_arch: str # Directory under arch/ in the Linux kernel source tree. kernel_srcarch: str + # Name of the architecture in Debian. + debian_arch: str # Linux kernel configuration options. kernel_config: str # Flavor-specific Linux kernel configuration options. @@ -224,6 +226,7 @@ class Architecture(NamedTuple): name="aarch64", kernel_arch="arm64", kernel_srcarch="arm64", + debian_arch="arm64", kernel_config=""" CONFIG_PCI_HOST_GENERIC=y CONFIG_RTC_CLASS=y @@ -253,6 +256,7 @@ class Architecture(NamedTuple): name="arm", kernel_arch="arm", kernel_srcarch="arm", + debian_arch="armhf", kernel_config=""" CONFIG_NR_CPUS=8 CONFIG_HIGHMEM=y @@ -286,6 +290,7 @@ class Architecture(NamedTuple): name="ppc64", kernel_arch="powerpc", kernel_srcarch="powerpc", + debian_arch="ppc64el", kernel_config=""" CONFIG_PPC64=y CONFIG_CPU_LITTLE_ENDIAN=y @@ -308,6 +313,7 @@ class Architecture(NamedTuple): name="s390x", kernel_arch="s390", kernel_srcarch="s390", + debian_arch="s390x", kernel_config=""" # Needed for CONFIG_KEXEC_FILE. CONFIG_CRYPTO_SHA256_S390=y @@ -321,6 +327,7 @@ class Architecture(NamedTuple): name="x86_64", kernel_arch="x86_64", kernel_srcarch="x86", + debian_arch="amd64", kernel_config=""" CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_CMOS=y diff --git a/vmtest/kbuild.py b/vmtest/kbuild.py index e0a15faa1..1b3279378 100644 --- a/vmtest/kbuild.py +++ b/vmtest/kbuild.py @@ -112,7 +112,7 @@ async def apply_patches(kernel_dir: Path) -> None: .decode() .strip() ) - logging.info("applying patches for kernel version %s", version) + logger.info("applying patches for kernel version %s", version) any_applied = False for patch in _PATCHES: for min_version, max_version in patch.versions: @@ -122,7 +122,7 @@ async def apply_patches(kernel_dir: Path) -> None: break else: continue - logging.info("applying %s", patch.name) + logger.info("applying %s", patch.name) any_applied = True proc = await asyncio.create_subprocess_exec( "git", @@ -147,9 +147,9 @@ async def apply_patches(kernel_dir: Path) -> None: sys.stderr.buffer.write(stderr) sys.stderr.buffer.flush() raise - logging.info("already applied") + logger.info("already applied") if not any_applied: - logging.info("no patches") + logger.info("no patches") class KBuild: diff --git a/vmtest/kmod.py b/vmtest/kmod.py index c35d63dfa..9e686917d 100644 --- a/vmtest/kmod.py +++ b/vmtest/kmod.py @@ -23,7 +23,7 @@ def build_kmod(download_dir: Path, kernel: Kernel) -> Path: kmod_source_dir = Path("tests/linux_kernel/kmod") source_files = ("drgn_test.c", "Makefile") if out_of_date(kmod, *[kmod_source_dir / filename for filename in source_files]): - logging.info("building %s", kmod) + logger.info("building %s", kmod) compiler = downloaded_compiler(download_dir, kernel.arch) kernel_build_dir = kernel.path / "build" @@ -55,7 +55,7 @@ def build_kmod(download_dir: Path, kernel: Kernel) -> Path: ) (tmp_dir / "drgn_test.ko").rename(kmod) else: - logging.info("%s is up to date", kmod) + logger.info("%s is up to date", kmod) return kmod diff --git a/vmtest/manage.py b/vmtest/manage.py index 9b9669eda..941b27109 100644 --- a/vmtest/manage.py +++ b/vmtest/manage.py @@ -360,7 +360,7 @@ async def main() -> None: if to_build: logger.info("kernel versions to build:") for tag, tag_arches_to_build in to_build: - logging.info( + logger.info( " %s (%s)", tag, ", ".join( diff --git a/vmtest/rootfsbuild.py b/vmtest/rootfsbuild.py new file mode 100644 index 000000000..42736d530 --- /dev/null +++ b/vmtest/rootfsbuild.py @@ -0,0 +1,198 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# SPDX-License-Identifier: LGPL-2.1-or-later + +import logging +from pathlib import Path +import subprocess +import sys +import tempfile +import typing + +if typing.TYPE_CHECKING: + if sys.version_info < (3, 8): + from typing_extensions import Literal + else: + from typing import Literal + +from vmtest.config import ARCHITECTURES, HOST_ARCHITECTURE, Architecture + +logger = logging.getLogger(__name__) + + +_ROOTFS_PACKAGES = [ + # drgn build dependencies. + "autoconf", + "automake", + "gcc", + "git", + "libdw-dev", + "libelf-dev", + "libkdumpfile-dev", + "libtool", + "make", + "pkgconf", + "python3", + "python3-dev", + "python3-pip", + "python3-setuptools", + # Test dependencies. + "iproute2", + "kexec-tools", + "kmod", + "python3-pyroute2", + "python3-pytest", + "zstd", +] + + +def build_rootfs( + arch: Architecture, + path: Path, + *, + btrfs: "Literal['never', 'always', 'auto']" = "auto", +) -> None: + if path.exists(): + logger.info("%s already exists", path) + return + + logger.info("creating debootstrap rootfs %s", path) + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=path.parent) as tmp_name: + tmp_dir = Path(tmp_name) + snapshot = False + + if btrfs != "never": + try: + import btrfsutil + + btrfsutil.create_subvolume(tmp_dir / path.name) + snapshot = True + except (ImportError, OSError): + if btrfs == "always": + raise + + subprocess.check_call( + [ + "unshare", + "--map-root-user", + "--map-users=auto", + "--map-groups=auto", + "sh", + "-c", + r""" +set -e + +arch="$1" +target="$2" +packages="$3" + +# We're not really an LXC container, but this convinces debootstrap to skip +# some operations that it can't do in a user namespace. +export container=lxc +debootstrap --variant=minbase --foreign --include="$packages" --arch="$arch" stable "$target" +chroot "$target" /debootstrap/debootstrap --second-stage +chroot "$target" apt clean +""", + "sh", + arch.debian_arch, + tmp_dir / path.name, + ",".join(_ROOTFS_PACKAGES), + ] + ) + (tmp_dir / path.name).rename(path) + logger.info("created debootstrap rootfs %s", path) + + if snapshot: + snapshot_dir = path.parent / (path.name + ".pristine") + btrfsutil.create_snapshot(path, snapshot_dir, read_only=True) + logger.info("created snapshot %s", snapshot_dir) + + +def build_drgn_in_rootfs(rootfs: Path) -> None: + logger.info("building drgn using %s", rootfs) + subprocess.check_call( + [ + "unshare", + "--mount", + "--map-root-user", + "--map-users=auto", + "--map-groups=auto", + "sh", + "-c", + r""" +set -e + +mount --bind . "$1/mnt" +chroot "$1" sh -c 'cd /mnt && CONFIGURE_FLAGS=--enable-compiler-warnings=error python3 setup.py build_ext -i' +""", + "sh", + rootfs, + ] + ) + + +if __name__ == "__main__": + import argparse + + logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO + ) + + parser = argparse.ArgumentParser( + description="build root filesystems for vmtest. " + "This requires debootstrap(8) and unprivileged user namespaces.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-d", + "--directory", + metavar="DIR", + type=Path, + default="build/vmtest", + help="directory for vmtest artifacts", + ) + parser.add_argument( + "--build-drgn", + action="store_true", + help="also build drgn in the current directory using the built rootfs", + ) + parser.add_argument( + "--btrfs", + choices=["never", "always", "auto"], + default="auto", + help="make the rootfs a Btrfs subvolume and create a read-only snapshot", + ) + parser.add_argument( + "architectures", + choices=["foreign", "all", *sorted(ARCHITECTURES)], + nargs="*", + # It would make more sense for this to be ["foreign"], but argparse + # checks that the default itself is in choices, not each item. We fix + # it up to a list below. + default="foreign", + help='architecture to build for, or "foreign" for all architectures other than the host architecture; may be given multiple times', + ) + args = parser.parse_args() + + if isinstance(args.architectures, str): + args.architectures = [args.architectures] + architectures = [] + for name in args.architectures: + if name == "foreign": + architectures.extend( + [ + arch + for arch in ARCHITECTURES.values() + if arch is not HOST_ARCHITECTURE + ] + ) + elif name == "all": + architectures.extend(ARCHITECTURES.values()) + else: + architectures.append(ARCHITECTURES[name]) + + for arch in architectures: + dir = args.directory / arch.name / "rootfs" + build_rootfs(arch, dir, btrfs=args.btrfs) + if args.build_drgn: + build_drgn_in_rootfs(dir) diff --git a/vmtest/vm.py b/vmtest/vm.py index 6ad6f32e7..c5bc1778a 100644 --- a/vmtest/vm.py +++ b/vmtest/vm.py @@ -9,6 +9,7 @@ import subprocess import sys import tempfile +from typing import Optional from util import nproc, out_of_date from vmtest.config import HOST_ARCHITECTURE, Kernel, local_kernel @@ -34,7 +35,11 @@ export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" {kdump_needs_nosmp} -trap 'poweroff -f' EXIT +# On exit, power off. We don't use the poweroff command because very minimal +# installations don't have it (e.g., the debootstrap minbase variant). The +# magic SysRq returns immediately without waiting for the poweroff, so we sleep +# for a while and panic if it takes longer than that. +trap 'echo o > /proc/sysrq-trigger && sleep 60' exit umask 022 @@ -183,7 +188,15 @@ class LostVMError(Exception): pass -def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> int: +def run_in_vm( + command: str, kernel: Kernel, root_dir: Optional[Path], build_dir: Path +) -> int: + if root_dir is None: + if kernel.arch is HOST_ARCHITECTURE: + root_dir = Path("/") + else: + root_dir = build_dir / kernel.arch.name / "rootfs" + qemu_exe = "qemu-system-" + kernel.arch.name match = re.search( r"QEMU emulator version ([0-9]+(?:\.[0-9]+)*)", @@ -226,11 +239,21 @@ def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> init_path = temp_path / "init" + unshare_args = [] if root_dir == Path("/"): host_virtfs_args = [] init = str(init_path.resolve()) host_dir_prefix = "" else: + # Try to detect if the rootfs was created without privileges (e.g., + # by vmtest.rootfsbuild) and remap the UIDs/GIDs if so. + if (root_dir / "bin" / "mount").stat().st_uid != 0: + unshare_args = [ + "unshare", + "--map-root-user", + "--map-users=auto", + "--map-groups=auto", + ] host_virtfs_args = [ "-virtfs", f"local,path=/,mount_tag=host,{virtfs_options}", @@ -254,6 +277,8 @@ def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> with subprocess.Popen( [ # fmt: off + *unshare_args, + qemu_exe, *kvm_args, # Limit the number of cores to 8, otherwise we can reach an OOM troubles. @@ -329,7 +354,7 @@ def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> metavar="DIR", type=Path, default="build/vmtest", - help="directory for build artifacts and downloaded kernels", + help="directory for vmtest artifacts", ) parser.add_argument( "--lost-status", @@ -352,9 +377,9 @@ def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> "-r", "--root-directory", metavar="DIR", - default=Path("/"), + default=argparse.SUPPRESS, type=Path, - help="directory to use as root directory in VM", + help="directory to use as root directory in VM (default: / for the host architecture, $directory/$arch/rootfs otherwise)", ) parser.add_argument( "command", @@ -371,6 +396,8 @@ def run_in_vm(command: str, kernel: Kernel, root_dir: Path, build_dir: Path) -> kernel = local_kernel(args.kernel.arch, Path(args.kernel.pattern)) else: kernel = next(download(args.directory, [args.kernel])) # type: ignore[assignment] + if not hasattr(args, "root_directory"): + args.root_directory = None try: command = " ".join(args.command) if args.command else "sh -i"