From 49a50f2e56a90bad47252dda0638e0e103cbf513 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Wed, 3 Jun 2020 19:11:04 +0200 Subject: [PATCH] Initial commit Signed-off-by: Benjamin Drung --- .gitignore | 5 + NEWS | 3 + README.md | 169 ++++++ bdebstrap | 750 ++++++++++++++++++++++++++ bdebstrap.1.md | 402 ++++++++++++++ bdebstrap.py | 1 + examples/Debian-buster-live.yaml | 28 + examples/Debian-unstable.yaml | 9 + examples/Ubuntu-20.04.yaml | 11 + setup.py | 84 +++ tests/__init__.py | 59 ++ tests/configs/commented-packages.yaml | 11 + tests/pylint.conf | 36 ++ tests/test_config.py | 364 +++++++++++++ tests/test_flake8.py | 58 ++ tests/test_helper.py | 110 ++++ tests/test_integration.py | 55 ++ tests/test_mmdebstrap.py | 149 +++++ tests/test_pylint.py | 76 +++ 19 files changed, 2380 insertions(+) create mode 100644 .gitignore create mode 100644 NEWS create mode 100644 README.md create mode 100755 bdebstrap create mode 100644 bdebstrap.1.md create mode 120000 bdebstrap.py create mode 100644 examples/Debian-buster-live.yaml create mode 100644 examples/Debian-unstable.yaml create mode 100644 examples/Ubuntu-20.04.yaml create mode 100755 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/configs/commented-packages.yaml create mode 100644 tests/pylint.conf create mode 100644 tests/test_config.py create mode 100644 tests/test_flake8.py create mode 100644 tests/test_helper.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_mmdebstrap.py create mode 100644 tests/test_pylint.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b433c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.1 +/builds/ +/*.egg-info/ +/.pybuild/ +__pycache__/ diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..94d234c --- /dev/null +++ b/NEWS @@ -0,0 +1,3 @@ +bdebstrap 0.1 (2020-06-03) + +* Initial release diff --git a/README.md b/README.md new file mode 100644 index 0000000..a17d3d0 --- /dev/null +++ b/README.md @@ -0,0 +1,169 @@ +bdebstrap +========= + +bdebstrap is an alternative to debootstrap and a wrapper around +[mmdebstrap](https://gitlab.mister-muffin.de/josch/mmdebstrap/) to support +YAML based configuration files. It inherits all benefits from mmdebstrap. The +support for configuration allows storing all customization in a YAML file +instead of having to use a very long one-liner call to mmdebstrap. It also +layering multiple customizations on top of each other, e.g. to support flavors +of an image. + +Usage examples +============== + +Minimal Debian unstable tarball +------------------------------- + +This example shows how to use a small YAML configuration to build a minimal +Debian unstable tarball. Assume following configuration is stored in +*examples/Debian-unstable.yaml*: + +```yaml +mmdebstrap: + architectures: + - amd64 + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + suite: unstable + target: root.tar.xz + variant: minbase +``` + +Then the tarball can be generated by running + +```sh +$ bdebstrap -c examples/Debian-unstable.yaml --name example1 +$ ls example1/ +config.yaml manifest root.tar.xz +``` + +Debian live system +------------------ + +This example shows how to use a YAML configuration to build a Debian 10 +(buster) live system. Assume following configuration is stored in +*examples/Debian-buster-live.yaml*: + +```yaml +mmdebstrap: + architectures: + - amd64 + cleanup-hooks: + - cp /dev/null "$1/etc/hostname" + - if test -f "$1/etc/resolv.conf"; then cp /dev/null "$1/etc/resolv.conf"; fi + customize-hooks: + - cp --preserve=timestamps -v "$1"/boot/vmlinuz* "$1${BDEBSTRAP_OUTPUT_DIR?}/vmlinuz" + - cp --preserve=timestamps -v "$1"/boot/initrd.img* "$1${BDEBSTRAP_OUTPUT_DIR?}/initrd.img" + - mkdir -p "$1/root/.ssh" + - upload ~/.ssh/id_rsa.pub /root/.ssh/authorized_keys + # Create a proper root password entry with "openssl passwd -6 $password" + - chroot "$1" usermod -p '$6$gxPiEmowud.yY/mT$SE1TTiHkw9mW3YtECxyluZtNPHN7IYPa.vRlWZZVtC8L6qG2PzGpwGIlgMDY79vucWD577fZm/EcA4LS3Koob0' root + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + packages: + - iproute2 + - less + - libpam-systemd # recommended by systemd and needed to not run into https://bugs.debian.org/751636 + - linux-image-cloud-amd64 + - live-boot + - locales + - openssh-server + - systemd-sysv # Use systemd as init system (otherwise /sbin/init would be missing) + suite: buster + target: root.squashfs + variant: minbase +``` + +This example assumes that *~/.ssh/id_rsa.pub* exists, because it will be +copied into the image to */root/.ssh/authorized_keys* to allow SSH access +using the user's SSH key. + +The squashfs image can be generated by running + +```sh +$ bdebstrap -c examples/Debian-buster-live.yaml --name example2 +$ ls example2/ +config.yaml initrd.img manifest root.squashfs vmlinuz +``` + +The kernel and initrd are copied out of the squashfs image using customize +hooks to allow them to be used directly by QEMU. To launch this image locally +with QEMU, the *root.squashfs* image needs to be provided by a HTTP server: + +```sh +$ python3 -m http.server -b localhost --directory example2 8080 +``` + +This command exposes the generated image via HTTP on localhost on port 8080. +QEMU can be started passing the TCP traffic on port 8080 to the webserver: + +```sh +$ cd example2 +$ qemu-system-x86_64 -machine accel=kvm -m 1G -device virtio-net-pci,netdev=net0 -monitor vc \ + -netdev user,id=net0,hostfwd=tcp::2222-:22,guestfwd=tcp:10.0.2.252:8080-tcp:localhost:8080,hostname=debian-live \ + -kernel ./vmlinuz -initrd ./initrd.img -append "boot=live fetch=http://10.0.2.252:8080/root.squashfs quiet" +``` + +To print the output on the launching terminal, add *-nographic -serial stdio* +to the QEMU command line and *console=ttyS0* to the *-append* parameter. Once +the virtual machine is started, it can be accessed via SSH: + +```sh +$ ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -p 2222 root@localhost +``` + +Prerequisites +============= + +* Python >= 3 +* Python modules: + * ruamel.yaml +* mmdebstrap (>= 0.6.0) +* pandoc (to generate the man page) +* squashfs-tools-ng (>= 0.8) for building squashfs images. Older versions of + squashfs-tools-ng throw errors + ([bug #31](https://github.com/AgentD/squashfs-tools-ng/issues/31)) and loose + the security capabilities + ([bug #32](https://github.com/AgentD/squashfs-tools-ng/issues/32)). + + +The test cases have additional Python module requirements: + +* flake8 +* pylint + +Thanks +====== + +I like to thank Johannes Schauer for developing +[mmdebstrap](https://gitlab.mister-muffin.de/josch/mmdebstrap/) and for quickly +responding to all my bug reports and feature requests. + +Contributing +============ + +Contributions are welcome. The source code has some test coverage, which should +be preserved. So please provide a test case for each bugfix and one or more +test cases for each new feature. Please follow +[How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/) +for writing good commit messages. + +Creating releases +================= + +To create a release, increase the version in `setup.py`, document the +noteworthy changes in `NEWS`, and commit the change. Tag the release: + +``` +git tag v$(./setup.py --version) +``` + +The xz-compressed release tarball can be generated by running: +``` +name="bdebstrap-$(./setup.py --version)" +git archive --prefix="$name/" HEAD | xz -c9 > "../$name.tar.xz" +gpg --output "../$name.tar.xz.asc" --armor --detach-sign "../$name.tar.xz" +``` diff --git a/bdebstrap b/bdebstrap new file mode 100755 index 0000000..2c3b270 --- /dev/null +++ b/bdebstrap @@ -0,0 +1,750 @@ +#!/usr/bin/python3 + +# Copyright (C) 2019-2020 Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Call mmdebstrap with parameters specified in a YAML file. +""" + +import argparse +import collections +import io +import logging +import os +import re +import shutil +import subprocess +import sys +import time + +import ruamel.yaml + +MANIFEST_FILENAME = "manifest" +OUTPUT_DIR = "/tmp/bdebstrap-output" +LOG_FORMAT = "%(asctime)s %(name)s %(levelname)s: %(message)s" +__script_name__ = os.path.basename(sys.argv[0]) if __name__ == "__main__" else __name__ + + +MMDEBSTRAP_OPTS = { + "aptopts": list, + "architectures": list, + "cleanup-hooks": list, + "components": list, + "customize-hooks": list, + "dpkgopts": list, + "essential-hooks": list, + "hostname": str, + "install-recommends": bool, + "keyrings": list, + "mirrors": list, + "mode": str, + "packages": list, + "setup-hooks": list, + "suite": str, + "target": str, + "variant": str, +} + + +class Mmdebstrap: + """Wrapper around calling mmdebstrap.""" + + def __init__(self, config): + self.config = config + self.logger = logging.getLogger(__script_name__) + + def construct_parameters(self, output_dir, simulate=False): + """Construct the parameter for mmdebstrap from a given dictionary.""" + # pylint: disable=too-many-branches + cmd = ["mmdebstrap", "-v"] + if simulate: + cmd += ["--simulate"] + mmdebstrap = self.config.get("mmdebstrap", {}) + if "variant" in mmdebstrap: + cmd.append("--variant={}".format(mmdebstrap["variant"])) + if "mode" in mmdebstrap: + cmd.append("--mode={}".format(mmdebstrap["mode"])) + if "aptopts" in mmdebstrap: + cmd += ["--aptopt={}".format(aptopt) for aptopt in mmdebstrap["aptopts"]] + if "keyrings" in mmdebstrap: + cmd += ["--keyring={}".format(keyring) for keyring in mmdebstrap["keyrings"]] + if "dpkgopts" in mmdebstrap: + cmd += ["--dpkgopt={}".format(dpkgopt) for dpkgopt in mmdebstrap["dpkgopts"]] + # For convenience use "packages" key as alias for "include" + if "packages" in mmdebstrap: + cmd.append("--include={}".format(",".join(mmdebstrap["packages"]))) + if "components" in mmdebstrap: + cmd.append("--components={}".format(",".join(mmdebstrap["components"]))) + if "architectures" in mmdebstrap: + cmd.append("--architectures={}".format(",".join(mmdebstrap["architectures"]))) + if "setup-hooks" in mmdebstrap: + cmd += ["--setup-hook={}".format(hook) for hook in mmdebstrap["setup-hooks"]] + cmd.append('--essential-hook=mkdir -p "$1{}"'.format(OUTPUT_DIR)) + if "essential-hooks" in mmdebstrap: + cmd += ["--essential-hook={}".format(hook) for hook in mmdebstrap["essential-hooks"]] + if "customize-hooks" in mmdebstrap: + cmd += ["--customize-hook={}".format(hook) for hook in mmdebstrap["customize-hooks"]] + # cleanup hooks are just hooks that run after all other customize hooks + if "cleanup-hooks" in mmdebstrap: + cmd += ["--customize-hook={}".format(hook) for hook in mmdebstrap["cleanup-hooks"]] + + # Special parameters not present in mmdebstrap + if "hostname" in mmdebstrap: + cmd.append( + '--customize-hook=echo "{}" > "$1/etc/hostname"'.format(mmdebstrap["hostname"]) + ) + if "install-recommends" in mmdebstrap and mmdebstrap["install-recommends"] is True: + cmd.append('--aptopt=Apt::Install-Recommends "true"') + cmd.append( + '--customize-hook=chroot "$1" dpkg-query ' + "-f='${Package}\\t${Version}\\n' -W > \"$1%s/manifest\"" % (OUTPUT_DIR) + ) + cmd.append('--customize-hook=sync-out "{}" "{}"'.format(OUTPUT_DIR, output_dir)) + cmd.append('--customize-hook=rm -rf "$1{}"'.format(OUTPUT_DIR)) + + # Positional arguments + cmd.append(mmdebstrap.get("suite", "-")) + cmd.append(mmdebstrap.get("target", "-")) + cmd += mmdebstrap.get("mirrors", []) + + return cmd + + def call(self, output_dir, simulate=False): + """Call mmdebstrap.""" + cmd = self.construct_parameters(output_dir, simulate) + self.logger.info("Calling %s", escape_cmd(cmd)) + subprocess.check_call(cmd) + + for file in (os.path.join(output_dir, "manifest"), + self.config.get("mmdebstrap", {}).get("target", "")): + if os.path.isfile(file): + clamp_mtime(file, self.config.source_date_epoch) + + clamp_mtime(output_dir, self.config.source_date_epoch) + + +def clamp_mtime(path, source_date_epoch): + """Clamp the modification time for the given path to SOURCE_DATE_EPOCH.""" + if not source_date_epoch: + return + stat = os.stat(path) + if stat.st_mtime > int(source_date_epoch): + os.utime(path, (int(source_date_epoch), int(source_date_epoch))) + + +def duration_str(duration): + """Return duration in the biggest useful time unit (hours, minutes, seconds).""" + if duration < 60: + return "%.3f seconds" % (duration) + if duration < 3600: + return "%i min %.3f s (= %.3f s)" % (duration // 60, duration % 60, duration) + + minutes = duration % 3600 + return "%i h %i min %.3f s (= %.3f s)" % ( + duration // 3600, + minutes // 60, + minutes % 60, + duration, + ) + + +def escape_cmd(cmd): + """Escape command line arguments for printing/logging.""" + unsafe_re = re.compile(r"[^\w@%+=:,./-]", re.ASCII) + + def quote(cmd_argv): + """Return a shell-escaped version of the string *cmd_argv*.""" + if unsafe_re.search(cmd_argv) is None: + return cmd_argv + parts = cmd_argv.split("'") + for i in range(0, len(parts), 2): + # Only escape parts that are not quoted with single quotes. + parts[i] = re.sub('(["$])', r"\\\1", parts[i]) + return '"' + "'".join(parts) + '"' + + return " ".join(quote(x) for x in cmd) + + +def sanitize_list(list_): + """Sanitize given list by removing all empty entries.""" + if list_ is None: + return None + return [x for x in list_ if x] + + +def parse_args(args): # pylint: disable=too-many-statements + """Parse the given command line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + # parser.add_argument("-t", "--target", help="path to a tarball filename or a directory") + # parser.add_argument("-m", "--manifest", help="Store packages manifest in given file") + parser.add_argument( + "-c", "--config", action="append", default=[], help="bdebstrap configuration YAML." + ) + parser.add_argument("-n", "--name", help="name of the generated golden image") + parser.add_argument( + "-e", "--env", action="append", default=[], help="add additional environment variable." + ) + parser.add_argument( + "-s", + "--simulate", + "--dry-run", + action="store_true", + help=( + "Run apt-get with --simulate. Only the package cache is initialized but no binary " + "packages are downloaded or installed. Use this option to quickly check whether a " + "package selection within a certain suite and variant can in principle be installed " + "as far as their dependencies go. If the output is a tarball, then no output is " + "produced. If the output is a directory, then the directory will be left populated " + "with the skeleton files and directories necessary for apt to run in it." + ), + ) + parser.add_argument("-b", "--output-base-dir", default=".", help="output base directory") + parser.add_argument("-o", "--output", help="output directory (default: output-base-dir/name)") + parser.add_argument( + "--debug", + dest="log_level", + help=( + "In addition to the output produced by --verbose, write detailed debugging " + "information to standard error." + ), + action="store_const", + const=logging.DEBUG, + default=logging.INFO, + ) + parser.add_argument( + "-q", + "--quiet", + dest="log_level", + help="Decrease output verbosity to warnings and errors", + action="store_const", + const=logging.WARNING, + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Remove existing output directory before creating a new one", + ) + parser.add_argument( + "-t", "--tmpdir", help="Temporary directory for building the image (default: /tmp)" + ) + + # Arguments from mmdebstrap + parser.add_argument( + "--variant", + choices=[ + "extract", + "custom", + "essential", + "apt", + "required", + "minbase", + "buildd", + "important", + "debootstrap", + "-", + "standard", + ], + help="Choose which package set to install.", + ) + parser.add_argument( + "--mode", + choices=[ + "auto", + "sudo", + "root", + "unshare", + "fakeroot", + "fakechroot", + "proot", + "chrootless", + ], + help=( + "Choose how to perform the chroot operation and create a filesystem with " + "ownership information different from the current user." + ), + ) + parser.add_argument( + "--aptopt", action="append", help="Pass arbitrary options or configuration files to apt." + ) + parser.add_argument( + "--keyring", action="append", help="Change the default keyring to use by apt." + ) + parser.add_argument( + "--dpkgopt", action="append", help="Pass arbitrary options or configuration files to dpkg." + ) + parser.add_argument( + "--hostname", help="Write the given HOSTNAME into /etc/hostname in the target chroot.", + ) + parser.add_argument( + "--install-recommends", + action="store_true", + help="Consider recommended packages as a dependency for installing.", + ) + parser.add_argument( + "--packages", + "--include", + action="append", + help=( + "Comma or whitespace separated list of packages which will be installed in " + "addition to the packages installed by the specified variant." + ), + ) + parser.add_argument( + "--components", + action="append", + help=( + "Comma or whitespace separated list of components like main, contrib and " + "non-free which will be used for all URI-only MIRROR arguments." + ), + ) + parser.add_argument( + "--architectures", + action="append", + help=( + "Comma or whitespace separated list of architectures. The first architecture " + "is the native architecture inside the chroot." + ), + ) + + parser.add_argument( + "--setup-hook", + metavar="COMMAND", + action="append", + help=( + "Execute arbitrary COMMAND right after initial setup (directory creation, " + "configuration of apt and dpkg, ...) but before any packages are downloaded or " + "installed. At that point, the chroot directory does not contain any executables and " + "thus cannot be chroot-ed into." + ), + ) + parser.add_argument( + "--essential-hook", + metavar="COMMAND", + action="append", + help=( + "Execute arbitrary COMMAND after the Essential:yes packages have been installed, " + "but before installing the remaining packages." + ), + ) + parser.add_argument( + "--customize-hook", + metavar="COMMAND", + action="append", + help=( + "Execute arbitrary COMMAND after the chroot is set up and all packages got installed " + "but before final cleanup actions are carried out." + ), + ) + parser.add_argument( + "--cleanup-hook", + metavar="COMMAND", + action="append", + help="Execute arbitrary COMMAND after all customize hooks have been executed.", + ) + + # Positional arguments from mmdebstrap + parser.add_argument( + "--suite", + help=( + "The suite may be a valid release code name (eg, sid, stretch, jessie) or a symbolic " + "name (eg, unstable, testing, stable, oldstable)." + ), + ) + parser.add_argument( + "--target", + help=( + "The optional target argument can either be the path to a directory, the path to a " + "tarball filename, the path to a squashfs image or '-'." + ), + ) + parser.add_argument( + "--mirrors", + action="append", + default=[], + help=( + "Comma separated list of mirrors. If no mirror option is provided, " + "http://deb.debian.org/debian is used." + ), + ) + + parser.add_argument( + metavar="suite", + dest="suite_positional", + nargs="?", + help=( + "The suite may be a valid release code name (eg, sid, stretch, jessie) or a symbolic " + "name (eg, unstable, testing, stable, oldstable)." + ), + ) + parser.add_argument( + metavar="target", + dest="target_positional", + nargs="?", + help=( + "The optional target argument can either be the path to a directory, the path to a " + "tarball filename, the path to a squashfs image or '-'." + ), + ) + parser.add_argument( + metavar="mirrors", + dest="mirrors_positional", + nargs="*", + help=( + "APT mirror to use. If no mirror option is provided, " + "http://deb.debian.org/debian is used." + ), + ) + + args = parser.parse_args(args) + + env_dict = {} + for env in args.env: + if "=" not in env: + parser.error( + "Failed to parse --env '%s'. It needs to be in the format KEY=value." % env + ) + key, value = env.split("=", 1) + env_dict[key] = value + args.env = env_dict + + if args.packages: + args.packages = [p for l in args.packages for p in re.split(",| ", l) if p] + if args.components: + args.components = [c for l in args.components for c in re.split(",| ", l) if c] + if args.architectures: + args.architectures = [a for l in args.architectures for a in re.split(",| ", l) if a] + args.mirrors = [m for mirror_list in args.mirrors for m in mirror_list.split(",") if m] + + # Positional arguments override optional arguments (or extent them in case of "mirrors") + if args.suite_positional: + args.suite = args.suite_positional + if args.target_positional: + args.target = args.target_positional + args.mirrors += [m for m in args.mirrors_positional if m] + del args.suite_positional + del args.target_positional + del args.mirrors_positional + + # Sanitize (clear empty entries in lists) + args.aptopt = sanitize_list(args.aptopt) + args.config = sanitize_list(args.config) + args.dpkgopt = sanitize_list(args.dpkgopt) + args.keyring = sanitize_list(args.keyring) + args.setup_hook = sanitize_list(args.setup_hook) + args.essential_hook = sanitize_list(args.essential_hook) + args.customize_hook = sanitize_list(args.customize_hook) + args.cleanup_hook = sanitize_list(args.cleanup_hook) + + return args + + +def dict_merge(this, other): + """ + Update this dictionary with the key/value pairs from other, merging existing keys. + Return ``None``. + + Inspired by ``dict.update()``, instead of updating only top-level keys, + dict_merge recurses down into nested dicts. Dictionaries are updated + recursively and lists are appended. The ``other`` dict is merged into + ``this``. + + :param this: dictionary onto which the merge is executed + :param other: dictionary merged into ``this`` + :return: None + """ + for key in other.keys(): + if (key in this and isinstance(this[key], collections.abc.MutableMapping) + and isinstance(other[key], collections.abc.Mapping)): + dict_merge(this[key], other[key]) + elif (key in this and isinstance(this[key], collections.abc.MutableSequence) + and isinstance(other[key], collections.abc.Sequence)): + this[key] += other[key] + else: + this[key] = other[key] + + +class Config(dict): + """YAML configuration for bdebstrap.""" + + _ENV_PREFIX = "BDEBSTRAP_" + _KEYS = {"env", "mmdebstrap", "name"} + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + self.logger = logging.getLogger(__script_name__) + self.yaml = ruamel.yaml.YAML() + + def _set_mmdebstrap_option(self, option, value): + """Set the given mmdebstrap option (overwriting existing values).""" + if "mmdebstrap" not in self: + self["mmdebstrap"] = {} + self["mmdebstrap"][option] = value + + def _append_mmdebstrap_option(self, option, value): + """Append the given mmdebstrap option to the list of values.""" + if "mmdebstrap" not in self: + self["mmdebstrap"] = {} + if option in self["mmdebstrap"]: + self["mmdebstrap"][option] += value + else: + self["mmdebstrap"][option] = value + + def add_command_line_arguments(self, args): # pylint: disable=too-many-branches + """Add/Override configs from the given command line arguments.""" + for config_filename in args.config: + self.load(config_filename) + + if args.env: + if "env" not in self: + self["env"] = {} + for key, value in args.env.items(): + self["env"][key] = value + if args.name: + self["name"] = args.name + + if args.variant: + self._set_mmdebstrap_option("variant", args.variant) + if args.mode: + self._set_mmdebstrap_option("mode", args.mode) + if args.aptopt: + self._append_mmdebstrap_option("aptopts", args.aptopt) + if args.keyring: + self._append_mmdebstrap_option("keyrings", args.keyring) + if args.dpkgopt: + self._append_mmdebstrap_option("dpkgopts", args.dpkgopt) + if args.hostname: + self._set_mmdebstrap_option("hostname", args.hostname) + if args.install_recommends: + self._set_mmdebstrap_option("install-recommends", args.install_recommends) + if args.packages: + self._append_mmdebstrap_option("packages", args.packages) + if args.components: + self._append_mmdebstrap_option("components", args.components) + if args.architectures: + self._append_mmdebstrap_option("architectures", args.architectures) + if args.setup_hook: + self._append_mmdebstrap_option("setup-hooks", args.setup_hook) + if args.essential_hook: + self._append_mmdebstrap_option("essential-hooks", args.essential_hook) + if args.customize_hook: + self._append_mmdebstrap_option("customize-hooks", args.customize_hook) + if args.cleanup_hook: + self._append_mmdebstrap_option("cleanup-hooks", args.cleanup_hook) + if args.suite: + self._set_mmdebstrap_option("suite", args.suite) + if args.target: + self._set_mmdebstrap_option("target", args.target) + if args.mirrors: + self._append_mmdebstrap_option("mirrors", args.mirrors) + + def env_items(self): + """Return key-value pair of environment variables.""" + return sorted( + list(self.get("env", {}).items()) + + [ + (self._ENV_PREFIX + "NAME", self["name"]), + (self._ENV_PREFIX + "OUTPUT_DIR", OUTPUT_DIR), + ] + ) + + def check(self): + """Check the format of the configuration.""" + unknown_top_level_keys = sorted(k for k in self.keys() if k not in self._KEYS) + if unknown_top_level_keys: + self.logger.warning( + "Ignoring unknown top level keys: %s", ", ".join(unknown_top_level_keys) + ) + + if "mmdebstrap" in self: + for key, value in self["mmdebstrap"].items(): + if key not in MMDEBSTRAP_OPTS: + self.logger.warning("Ignoring unknown mmdebstrap option '%s'.", key) + continue + if not isinstance(value, MMDEBSTRAP_OPTS[key]): + raise ValueError( + "Unexpected type '%s' for mmdebstrap option '%s'. " + "Excepted: %s." % (type(value), key, MMDEBSTRAP_OPTS[key]) + ) + else: + self.logger.warning("The configuration does not contain a 'mmdebstrap' entry.") + + if "name" not in self: + raise ValueError("The configuration does not contain a 'name' entry.") + + def load(self, config_filename): + """Loading configuration from given config file.""" + self.logger.info("Loading configuration from '%s'...", config_filename) + try: + with open(config_filename) as config_file: + config = self.yaml.load(config_file) + except OSError as error: + self.logger.error( + "Failed to open configuration '%s': %s", config_filename, error.args[1] + ) + raise + + if "mmdebstrap" in config and "include" in config["mmdebstrap"]: + mmdebstrap = config["mmdebstrap"] + mmdebstrap["packages"] = mmdebstrap["include"] + mmdebstrap.get("packages", []) + del mmdebstrap["include"] + + dict_merge(self, config) + + def sanitize_packages(self): + """Sanitize packages list by removing duplicates (keeping the latest one).""" + if "mmdebstrap" not in self or "packages" not in self["mmdebstrap"]: + return + + packages = collections.OrderedDict() + for package in self["mmdebstrap"]["packages"]: + if not package: + # Cover commented out and empty entries + continue + if "=" in package: + name, version = package.split("=", 1) + packages[name] = "=" + version + elif "/" in package: + name, version = package.split("/", 1) + packages[name] = "/" + version + else: + packages[package] = "" + + self["mmdebstrap"]["packages"] = [k + v for k, v in packages.items()] + + def save(self, config_filename, simulate=False): + """Save configuration to given config filename.""" + self.logger.info( + "%s configuration to '%s'.", + "Simulate saving" if simulate else "Saving", + config_filename, + ) + if simulate: + self.yaml.dump(dict(self), io.StringIO()) + else: + with open(config_filename, "w") as config_file: + self.yaml.dump(dict(self), config_file) + clamp_mtime(config_filename, self.source_date_epoch) + + @property + def source_date_epoch(self): + """Return SOURCE_DATE_EPOCH (for reproducible builds).""" + return self.get("env", {}).get("SOURCE_DATE_EPOCH") + + def set_source_date_epoch(self): + """Set SOURCE_DATE_EPOCH (for reproducible builds) if not already set.""" + + if "env" not in self: + self["env"] = {} + if self["env"].get("SOURCE_DATE_EPOCH") is None: + self["env"]["SOURCE_DATE_EPOCH"] = int(time.time()) + + +def prepare_output_dir(output_dir, force, simulate=False): + """Ensure that the output directory exists and is empty.""" + logger = logging.getLogger(__script_name__) + + if os.path.exists(output_dir) and os.listdir(output_dir): + if force: + logger.info( + "%s existing output directory '%s'.", + "Simulate removing" if simulate else "Removing", + output_dir, + ) + if not simulate: + for content in os.listdir(output_dir): + path = os.path.join(output_dir, content) + try: + shutil.rmtree(path) + except NotADirectoryError: + os.remove(path) + else: + logger.error( + "The output directory '%s' already exist and is not empty. " + "Use --force to remove it.", + output_dir, + ) + return False + + if not os.path.isdir(output_dir): + logger.info( + "%s output directory '%s'...", + "Simulate creating" if simulate else "Creating", + output_dir, + ) + if not simulate: + os.makedirs(output_dir) + + return True + + +def main(argv): + """Call mmdebstrap with parameters specified in a YAML file.""" + start_time = time.time() + args = parse_args(argv) + logging.basicConfig(level=args.log_level, format=LOG_FORMAT) + logger = logging.getLogger(__script_name__) + + config = Config() + try: + config.add_command_line_arguments(args) + config.sanitize_packages() + config.set_source_date_epoch() + config.check() + except OSError: + sys.exit(1) + except ValueError as error: + logger.error("%s", error) + sys.exit(1) + + if not args.output: + args.output = os.path.join(args.output_base_dir, config["name"]) + + if not prepare_output_dir(args.output, args.force, args.simulate): + sys.exit(1) + config.save(os.path.join(args.output, "config.yaml"), args.simulate) + + # Set environment variables + for env, value in config.env_items(): + if os.environ.get(env) != str(value): + logger.info("Setting environment variable %s=%s", env, value) + os.environ[env] = str(value) + if args.tmpdir: + os.environ["TMPDIR"] = args.tmpdir + + mmdebstrap = config.get("mmdebstrap", {}) + + if "target" in mmdebstrap and "/" not in mmdebstrap["target"]: + mmdebstrap["target"] = os.path.join(args.output, mmdebstrap["target"]) + + if "LD_PRELOAD" in os.environ: + # gtk3-nocsd preloads libgtk3-nocsd.so.0 which fails on cross-builds + del os.environ["LD_PRELOAD"] + + try: + Mmdebstrap(config).call(args.output, args.simulate) + except subprocess.CalledProcessError as error: + logger.info("Execution time: %s", duration_str(time.time() - start_time)) + logger.error( + "mmdebstrap failed with exit code %i. See above for details.", error.returncode + ) + sys.exit(1) + + logger.info("Build successful in '%s'.", mmdebstrap["target"]) + logger.info("Execution time: %s", duration_str(time.time() - start_time)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/bdebstrap.1.md b/bdebstrap.1.md new file mode 100644 index 0000000..e4deade --- /dev/null +++ b/bdebstrap.1.md @@ -0,0 +1,402 @@ +--- +date: 2020-05-26 +footer: bdebstrap +header: "bdebstrap's Manual" +layout: page +license: 'Licensed under the MIT license' +section: 1 +title: BDEBSTRAP +--- + +# NAME + +bdebstrap - YAML config based multi-mirror Debian chroot creation tool + +# SYNOPSIS + +**bdebstrap** [**-h**|**\--help**] [**-c**|**\--config** *CONFIG*] +[**-n**|**\--name** *NAME*] [**-e**|**\--env** *ENV*] +[**-s**|**\--simulate**|**\--dry-run**] +[**-b**|**\--output-base-dir** *OUTPUT_BASE_DIR*] +[**-o**|**\--output** *OUTPUT*] [**\--debug**] [**-q**|**\--quiet**] +[**-f**|**\--force**] [**-t**|**\--tmpdir** *TMPDIR*] +[**\--variant** {*extract*,*custom*,*essential*,*apt*,*required*,*minbase*,*buildd*,*important*,*debootstrap*,*-*,*standard*}] +[**\--mode** {*auto*,*sudo*,*root*,*unshare*,*fakeroot*,*fakechroot*,*proot*,*chrootless*}] +[**\--aptopt** *APTOPT*] [**\--keyring** *KEYRING*] [**\--dpkgopt** *DPKGOPT*] +[**\--hostname** *HOSTNAME*] [**\--install-recommends**] +[**\--packages**|**\--include** *PACKAGES*] [**\--components** *COMPONENTS*] +[**\--architectures** *ARCHITECTURES*] +[**\--setup-hook** *COMMAND*] [**\--essential-hook** *COMMAND*] +[**\--customize-hook** *COMMAND*] [**\--cleanup-hook** *COMMAND*] +[**\--suite** *SUITE*] [**\--target** *TARGET*] [**\--mirrors** *MIRRORS*] +[*SUITE* [*TARGET* [*MIRROR*...]]] + +# DESCRIPTION + +**bdebstrap** creates a Debian chroot of *SUITE* into *TARGET* from one or more +*MIRROR*s and places meta-data in the *OUTPUT* directory: A *config.yaml* +containing the configuration for the build (useful for rebuilds) and a +*manifest* listing all installed packages and versions. If *TARGET* is just a +filename (and not include the path), it will be placed in the *OUTPUT* +directory as well. **bdebstrap** extents mmdebtrap to make it configurable via +YAML configuration files for more complex builds. + +The configuration parameters can be passed to **bdebstrap** as command line +arguments or in one or more configuration YAML files. The content of YAML files +will be merged by appending lists and recursively merging maps. Arguments +specified on the command line will take precedence over values provided in the +YAML configuration file. The final merged parameters will be stored in the +output directory as *config.yaml*. + +# OPTIONS + +**-h**, **\--help** +: Show a help message and exit + +**-c** *CONFIG*, **\--config** *CONFIG* +: Configuration YAML file. See YAML CONFIGURATION below for the expected + structure of this file. This parameter can be specified multiple times. The + content of YAML files will be merged by appending lists and recursively + merging maps. + +**-n** *NAME*, **\--name** *NAME* +: name of the generated golden image. If no output directory is specified, + the golden image will be placed in *OUTPUT_BASE_DIR*/*NAME*. + +**-e** *ENV*, **\--env** *ENV* +: Add an additional environment variable. These environment variable will be + set when calling the hooks. + +**-s**, **\--simulate**, **\--dry-run** +: Run apt-get with **\--simulate**. Only the package cache is initialized + but no binary packages are downloaded or installed. Use this option to + quickly check whether a package selection within a certain suite and + variant can in principle be installed as far as their dependencies go. If + the output is a tarball, then no output is produced. If the output is a + directory, then the directory will be left populated with the skeleton + files and directories necessary for apt to run in it. + +**-b** *OUTPUT_BASE_DIR*, **\--output-base-dir** *OUTPUT_BASE_DIR* +: output base directory. By default it is the current directory. + +**-o** *OUTPUT*, **\--output** *OUTPUT* +: output directory (default: output-base-dir/name) + +**\--debug** +: In addition to the output produced by **\--verbose**, write detailed + debugging information to standard error. + +**-q**, **\--quiet** +: Decrease output verbosity to warnings and errors + +**-f**, **\--force** +: Remove existing output directory before creating a new one + +**-t** *TMPDIR*, **\--tmpdir** *TMPDIR* +: Temporary directory for building the image (default: /tmp) + +**\--variant** {*extract*,*custom*,*essential*,*apt*,*required*,*minbase*,*buildd*,*important*,*debootstrap*,*-*,*standard*} +: Choose which package set to install. + +**\--mode** {*auto*,*sudo*,*root*,*unshare*,*fakeroot*,*fakechroot*,*proot*,*chrootless*} +: Choose how to perform the chroot operation and create a filesystem with + ownership information different from the current user. + +**\--aptopt** *APTOPT* +: Pass arbitrary options or configuration files to apt. + +**\--keyring** *KEYRING* +: Change the default keyring to use by apt. + +**\--dpkgopt** *DPKGOPT* +: Pass arbitrary options or configuration files to dpkg. + +**\--hostname** *HOSTNAME* +: Write the given *HOSTNAME* into */etc/hostname* in the target chroot. + +**\--install-recommends** +: Consider recommended packages as a dependency for installing. + +**\--packages** *PACKAGES*, **\--include** *PACKAGES* +: Comma or whitespace separated list of packages which will be installed in + addition to the packages installed by the specified variant. + +**\--components** *COMPONENTS* +: Comma or whitespace separated list of components like main, contrib and + non-free which will be used for all URI-only *MIRROR* arguments. + +**\--architectures** *ARCHITECTURES* +: Comma or whitespace separated list of architectures. The first + architecture is the native architecture inside the chroot. + +**\--setup-hook** *COMMAND* +: Execute arbitrary *COMMAND* right after initial setup (directory creation, + configuration of apt and dpkg, ...) but before any packages are downloaded + or installed. At that point, the chroot directory does not contain any + executables and thus cannot be chroot-ed into. This option can be specified + multiple times. + +**\--essential-hook** *COMMAND* +: Execute arbitrary *COMMAND* after the Essential:yes packages have been + installed, but before installing the remaining packages. This option can be + specified multiple times. + +**\--customize-hook** *COMMAND* +: Execute arbitrary *COMMAND* after the chroot is set up and all packages got + installed but before final cleanup actions are carried out. This option can + be specified multiple times. + +**\--cleanup-hook** *COMMAND* +: Execute arbitrary *COMMAND* after all customize hooks have been executed. + This option can be specified multiple times. + +**\--suite** *SUITE*, *SUITE* +: The suite may be a valid release code name (eg, sid, stretch, jessie) or + a symbolic name (eg, unstable, testing, stable, oldstable). + +**\--target** *TARGET*, *TARGET* +: The optional target argument can either be the path to a directory, the + path to a tarball filename, the path to a squashfs image or *-*. + +**\--mirrors** *MIRRORS*, *MIRRORS* +: Comma separated list of mirrors. If no mirror option is provided, + http://deb.debian.org/debian is used. + +# YAML CONFIGURATION + +This section describes the expected data-structure hierarchy of the YAML +configuration file(s). The top-level structure is expected to be a mapping. +The top-level mapping may contain following keys: + +### env + +mapping of environment variables names to their values. Environment variables +can be overridden by specifying them with **\--env** using the same name. These +environment variable are set before calling the hooks. + +### name + +String. Name of the generated golden image. Can be overridden by **\--name**. + +### mmdebstrap + +mapping. The values here are passed to mmdebstrap(1). Following keys might +be specified: + +**aptopts** +: list of arbitrary options or configuration files (string) to apt. + Additional apt options can be specified with **\--aptopt**. + +**architectures** +: list of architectures (string). The first architecture is the native + architecture inside the chroot. Additional architectures can be specified + with **\--architectures**. + +**components** +: list of components (string) like main, contrib and non-free which will be + used for all URI-only *MIRROR* arguments. Additional components can be + specified with **\--components**. + +**dpkgopts** +: list of arbitrary options or configuration files (string) to dpkg. + Additional dpkg options can be specified with **\--dpkgopt**. + +**hostname** +: String. If specified, write the given *hostname* into */etc/hostname* in + the target chroot. This parameter does not exist in **mmdebstrap** and is + implemented as customize hook for **mmdebstrap**. Can be overridden by + **\--hostname**. + +**install-recommends** +: Boolean. If set to *True*, the APT option *Apt::Install-Recommends "true"* + is passed to **mmdebstrap** via **\--aptopt**. Can be overridden by + **\--install-recommends**. + +**keyrings** +: list of default keyring to use by apt. Additional keyring files can be + specified with **\--keyring**. + +**mirrors** +: list of mirrors (string). Additional mirrors can be specified with + **\--mirrors**. + +**mode** +: Choose how to perform the chroot operation and create a filesystem with + ownership information different from the current user. It needs to be one + of *auto*, *sudo*, *root*, *unshare*, *fakeroot*, *fakechroot*, *proot*, or + *chrootless*. See mmdebstrap(1) for details. Can be overridden by + **\--mode**. + +**packages** +: list of packages (string) which will be installed in addition to the + packages installed by the specified variant. Additional packages can be + specified with **\--packages** or **\--include**. This setting is passed to + **mmdebstrap** using the **\--include** parameter. + +**setup-hooks** +: list of setup hooks (string). Execute arbitrary commands right after + initial setup (directory creation, configuration of apt and dpkg, ...) but + before any packages are downloaded or installed. At that point, the chroot + directory does not contain any executables and thus cannot be chroot-ed + into. See **HOOKS** in mmdebstrap(1) for more information and examples. + Additional setup hooks can be specified with **\--setup-hook**. + +**essential-hooks** +: list of essential hooks (string). Execute arbitrary commands after the + Essential:yes packages have been installed, but before installing the + remaining packages. See **HOOKS** in mmdebstrap(1) for more information and + examples. Additional essential hooks can be specified with + **\--essential-hook**. + +**customize-hooks** +: list of customize hooks (string). Execute arbitrary commands after the + chroot is set up and all packages got installed but before final cleanup + actions are carried out. See **HOOKS** in mmdebstrap(1) for more + information and examples. Additional customize hooks can be specified with + **\--customize-hook**. + +**cleanup-hooks** +: list of cleanup hooks (string). Cleanup hooks are just hooks that are run + directly after all other customize hooks. See **customize-hooks** above. + Additional cleanup hooks can be specified with **\--cleanup-hook**. + +**suite** +: String. The suite may be a valid release code name (eg, sid, stretch, + jessie) or a symbolic name (eg, unstable, testing, stable, oldstable). Can + be overridden by **\--suite**. + +**target** +: String. The target argument can either be the path to a directory, the path + to a tarball filename, the path to a squashfs image or *-*. Can be + overridden by **\--target**. + +**variant** +: Choose which package set to install. It needs to be one of *extract*, + *custom*, *essential*, *apt*, *required*, *minbase*, *buildd*, *important*, + *debootstrap*, *-*, *standard*. See mmdebstrap(1) for details. Can be + overridden by **\--variant**. + +# HOOKS + +**bdebstrap** enhances the hooks provided by **mmdebstrap**. Hooks can use the +environment variables specified via the *env* configuration option or the +**\--env** parameter. **bdebstrap** sets following environment variables by +default to be consumed by the hooks: + +**BDEBSTRAP_NAME** +: name of the generated golden image which is set via the **name** + configuration option of the **\--name** parameter. + +**BDEBSTRAP_OUTPUT_DIR** +: Path of a temporary directory inside the chroot. Files and directories + that are placed inside this directory will be copied out of the image into + the output directory. This temporary directory will be removed in a final + cleanup hook. + +# EXAMPLES + +### Minimal Debian unstable tarball + +This example shows how to use a small YAML configuration to build a minimal +Debian unstable tarball. Assume following configuration is stored in +*unstable.yaml*: + +```yaml +mmdebstrap: + architectures: + - amd64 + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + suite: unstable + target: root.tar.xz + variant: minbase +``` + +Then the tarball can be generated by running + +``` +$ bdebstrap -c unstable.yaml --name example1 +$ ls example1/ +config.yaml manifest root.tar.xz +``` + +### Debian live system + +This example shows how to use a YAML configuration to build a Debian 10 +(buster) live system. Assume following configuration is stored in *live.yaml*: + +```yaml +mmdebstrap: + architectures: + - amd64 + cleanup-hooks: + - cp /dev/null "$1/etc/hostname" + - if test -f "$1/etc/resolv.conf"; then cp /dev/null "$1/etc/resolv.conf"; fi + customize-hooks: + - cp --preserve=timestamps -v "$1"/boot/vmlinuz* "$1${BDEBSTRAP_OUTPUT_DIR?}/vmlinuz" + - cp --preserve=timestamps -v "$1"/boot/initrd.img* "$1${BDEBSTRAP_OUTPUT_DIR?}/initrd.img" + - mkdir -p "$1/root/.ssh" + - upload ~/.ssh/id_rsa.pub /root/.ssh/authorized_keys + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + packages: + - iproute2 + - less + - libpam-systemd + - linux-image-cloud-amd64 + - live-boot + - locales + - openssh-server + - systemd-sysv + suite: buster + target: root.squashfs + variant: minbase +``` + +This example assumes that *~/.ssh/id_rsa.pub* exists, because it will be +copied into the image to */root/.ssh/authorized_keys* to allow SSH access +using the user's SSH key. + +The squashfs image can be generated by running + +``` +$ bdebstrap -c live.yaml --name example2 +$ ls example2/ +config.yaml initrd.img manifest root.squashfs vmlinuz +``` + +The kernel and initrd are copied out of the squashfs image using customize +hooks to allow them to be used directly by QEMU. To launch this image locally +with QEMU, the *root.squashfs* image needs to be provided by a HTTP server: + +``` +$ python3 -m http.server -b localhost --directory example2 8080 +``` + +This command exposes the generated image via HTTP on localhost on port 8080. +QEMU can be started passing the TCP traffic on port 8080 to the webserver: + +``` +$ cd example2 +$ qemu-system-x86_64 -machine accel=kvm -m 1G -device virtio-net-pci,netdev=net0 -monitor vc \ + -netdev user,id=net0,hostfwd=tcp::2222-:22,guestfwd=tcp:10.0.2.252:8080-tcp:localhost:8080,hostname=debian-live \ + -kernel ./vmlinuz -initrd ./initrd.img -append "boot=live fetch=http://10.0.2.252:8080/root.squashfs quiet" +``` + +To print the output on the launching terminal, add *-nographic -serial stdio* +to the QEMU command line and *console=ttyS0* to the *-append* parameter. Once +the virtual machine is started, it can be accessed via SSH: + +``` +$ ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -p 2222 root@localhost +``` + +# SEE ALSO + +mmdebstrap(1), debootstrap(8) + +# AUTHOR + +Benjamin Drung diff --git a/bdebstrap.py b/bdebstrap.py new file mode 120000 index 0000000..09ffadb --- /dev/null +++ b/bdebstrap.py @@ -0,0 +1 @@ +bdebstrap \ No newline at end of file diff --git a/examples/Debian-buster-live.yaml b/examples/Debian-buster-live.yaml new file mode 100644 index 0000000..261fa1c --- /dev/null +++ b/examples/Debian-buster-live.yaml @@ -0,0 +1,28 @@ +mmdebstrap: + architectures: + - amd64 + cleanup-hooks: + - cp /dev/null "$1/etc/hostname" + - if test -f "$1/etc/resolv.conf"; then cp /dev/null "$1/etc/resolv.conf"; fi + customize-hooks: + - cp --preserve=timestamps -v "$1"/boot/vmlinuz* "$1${BDEBSTRAP_OUTPUT_DIR?}/vmlinuz" + - cp --preserve=timestamps -v "$1"/boot/initrd.img* "$1${BDEBSTRAP_OUTPUT_DIR?}/initrd.img" + - mkdir -p "$1/root/.ssh" + - upload ~/.ssh/id_rsa.pub /root/.ssh/authorized_keys + # Create a proper root password entry with "openssl passwd -6 $password" + - chroot "$1" usermod -p '$6$gxPiEmowud.yY/mT$SE1TTiHkw9mW3YtECxyluZtNPHN7IYPa.vRlWZZVtC8L6qG2PzGpwGIlgMDY79vucWD577fZm/EcA4LS3Koob0' root + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + packages: + - iproute2 + - less + - libpam-systemd # recommended by systemd and needed to not run into https://bugs.debian.org/751636 + - linux-image-cloud-amd64 + - live-boot + - locales + - openssh-server + - systemd-sysv # Use systemd as init system (otherwise /sbin/init would be missing) + suite: buster + target: root.squashfs + variant: minbase diff --git a/examples/Debian-unstable.yaml b/examples/Debian-unstable.yaml new file mode 100644 index 0000000..08e64d8 --- /dev/null +++ b/examples/Debian-unstable.yaml @@ -0,0 +1,9 @@ +mmdebstrap: + architectures: + - amd64 + keyrings: + - /usr/share/keyrings/debian-archive-keyring.gpg + mode: unshare + suite: unstable + target: root.tar.xz + variant: minbase diff --git a/examples/Ubuntu-20.04.yaml b/examples/Ubuntu-20.04.yaml new file mode 100644 index 0000000..35c7bc9 --- /dev/null +++ b/examples/Ubuntu-20.04.yaml @@ -0,0 +1,11 @@ +mmdebstrap: + architectures: + - amd64 + keyrings: + - /usr/share/keyrings/ubuntu-archive-keyring.gpg + mirrors: + - http://archive.ubuntu.com/ubuntu + mode: unshare + suite: focal + target: root.tar.xz + variant: minbase diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..2285f9b --- /dev/null +++ b/setup.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 + +# Copyright (C) 2019, Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Setup for bdebstrap""" + +import distutils.cmd +import distutils.command.build +import distutils.command.clean +import os +import subprocess + +from setuptools import setup + +MAN_PAGES = ["bdebstrap.1"] + + +class DocCommand(distutils.cmd.Command): + """A custom command to build the documentation using pandoc.""" + + description = "run pandoc to generate man pages" + user_options = [] + + def initialize_options(self): + """Set default values for options.""" + + def finalize_options(self): + """Post-process options.""" + + def run(self): + """Run pandoc.""" + for man_page in MAN_PAGES: + command = ["pandoc", "-s", "-t", "man", man_page + ".md", "-o", man_page] + self.announce("running command: %s" % " ".join(command), level=distutils.log.INFO) + subprocess.check_call(command) + + +class BuildCommand(distutils.command.build.build): + """Custom build command (calling doc beforehand).""" + + def run(self): + self.run_command("doc") + distutils.command.build.build.run(self) + + +class CleanCommand(distutils.command.clean.clean): + """Custom clean command (removing generated man pages).""" + + def run(self): + for man_page in MAN_PAGES: + if os.path.exists(man_page): + self.announce("removing %s" % (man_page), level=distutils.log.INFO) + os.remove(man_page) + distutils.command.clean.clean.run(self) + + +if __name__ == "__main__": + setup( + name="bdebstrap", + version="0.1", + description="Benjamin's multi-mirror Debian chroot creation tool", + long_description=("TODO"), + author="Benjamin Drung", + author_email="bdrung@posteo.de", + url="https://github.com/bdrung/bdebstrap", + license="MIT", + cmdclass={"doc": DocCommand, "build": BuildCommand, "clean": CleanCommand}, + install_requires=["ruamel.yaml"], + scripts=["bdebstrap"], + py_modules=[], + data_files=[("/usr/share/man/man1", MAN_PAGES)], + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b6e8589 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2017, Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Helper functions for testing.""" + +import inspect +import os +import sys +import unittest + + +def get_source_files(): + """Return a list of sources files/directories (to check with flake8/pylint).""" + scripts = ["bdebstrap"] + modules = ["tests"] + py_files = ["setup.py"] + + files = [] + for code_file in scripts + modules + py_files: + is_script = code_file in scripts + if not os.path.exists(code_file): # pragma: no cover + # The alternative path is needed for Debian's pybuild + alternative = os.path.join(os.environ.get("OLDPWD", ""), code_file) + code_file = alternative if os.path.exists(alternative) else code_file + if is_script: + with open(code_file, "rb") as script_file: + shebang = script_file.readline().decode("utf-8") + if ((sys.version_info[0] == 3 and "python3" in shebang) + or ("python" in shebang and "python3" not in shebang)): + files.append(code_file) + else: + files.append(code_file) + return files + + +def unittest_verbosity(): + """ + Return the verbosity setting of the currently running unittest. + + If no test is running, return None. + """ + frame = inspect.currentframe() + while frame: + self = frame.f_locals.get("self") + if isinstance(self, unittest.TestProgram): + return self.verbosity + frame = frame.f_back + return None # pragma: no cover diff --git a/tests/configs/commented-packages.yaml b/tests/configs/commented-packages.yaml new file mode 100644 index 0000000..ee390a4 --- /dev/null +++ b/tests/configs/commented-packages.yaml @@ -0,0 +1,11 @@ +name: commented-packages +mmdebstrap: + architectures: + - amd64 + mode: unshare + suite: unstable + target: root.tar.xz + variant: minbase + packages: + - # empty + - netconsole diff --git a/tests/pylint.conf b/tests/pylint.conf new file mode 100644 index 0000000..3615f74 --- /dev/null +++ b/tests/pylint.conf @@ -0,0 +1,36 @@ +[MASTER] + +# Pickle collected data for later comparisons. +persistent=no + + +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=locally-disabled + + +[REPORTS] + +# Tells whether to display a full report or only the messages +reports=no + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=99 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..faab5b7 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,364 @@ +# Copyright (C) 2019 Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Test configuration handling of bdebstrap.""" + +import logging +import os +import unittest + +from bdebstrap import Config, dict_merge, parse_args + +EXAMPLE_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "..", "examples") +TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "configs") + + +class TestArguments(unittest.TestCase): + """ + This unittest class tests the argument parsing. + """ + + def test_empty_args(self): + """Test setting arguments to empty strings.""" + args = parse_args( + [ + "--aptopt=", + "--architectures=", + "--cleanup-hook=", + "--components=", + "--config=", + "--customize-hook=", + "--dpkgopt=", + "--essential-hook=", + "--keyring=", + "--mirrors=", + "--packages=", + "--setup-hook=", + ] + ) + self.assertDictContainsSubset( + { + "aptopt": [], + "architectures": [], + "cleanup_hook": [], + "components": [], + "config": [], + "customize_hook": [], + "dpkgopt": [], + "essential_hook": [], + "keyring": [], + "mirrors": [], + "packages": [], + "setup_hook": [], + }, + args.__dict__, + ) + + def test_no_args(self): + """Test calling bdebstrap without arguments.""" + args = parse_args([]) + self.assertEqual( + args.__dict__, + { + "aptopt": None, + "architectures": None, + "cleanup_hook": None, + "components": None, + "config": [], + "customize_hook": None, + "dpkgopt": None, + "env": {}, + "essential_hook": None, + "force": False, + "hostname": None, + "install_recommends": False, + "keyring": None, + "log_level": logging.INFO, + "mirrors": [], + "mode": None, + "name": None, + "output_base_dir": ".", + "output": None, + "packages": None, + "setup_hook": None, + "simulate": False, + "suite": None, + "target": None, + "tmpdir": None, + "variant": None, + }, + ) + + def test_parse_env(self): + """Test parsing --env parameters.""" + args = parse_args(["-e", "KEY=VALUE", "--env", "FOO=bar"]) + self.assertEqual(args.env, {"FOO": "bar", "KEY": "VALUE"}) + + def test_malformed_env(self): + """Test malformed --env parameter (missing equal sign).""" + with self.assertRaises(SystemExit): + parse_args(["--env", "invalid"]) + + def test_optional_args(self): + """Test optional arguments (which also have positional ones).""" + args = parse_args( + [ + "--suite", + "unstable", + "--target", + "unstable.tar", + "--mirrors", + "deb http://deb.debian.org/debian unstable main," + "deb http://deb.debian.org/debian unstable non-free", + "--mirrors", + "deb http://deb.debian.org/debian unstable contrib", + ] + ) + self.assertDictContainsSubset( + { + "mirrors": [ + "deb http://deb.debian.org/debian unstable main", + "deb http://deb.debian.org/debian unstable non-free", + "deb http://deb.debian.org/debian unstable contrib", + ], + "suite": "unstable", + "target": "unstable.tar", + }, + args.__dict__, + ) + + def test_positional_args(self): + """Test positional arguments (overwriting optional ones).""" + args = parse_args( + [ + "--suite", + "bullseye", + "--target", + "bullseye.tar", + "--mirrors", + "deb http://deb.debian.org/debian unstable main," + "deb http://deb.debian.org/debian unstable non-free", + "unstable", + "unstable.tar", + "deb http://deb.debian.org/debian unstable contrib", + ] + ) + self.assertDictContainsSubset( + { + "mirrors": [ + "deb http://deb.debian.org/debian unstable main", + "deb http://deb.debian.org/debian unstable non-free", + "deb http://deb.debian.org/debian unstable contrib", + ], + "suite": "unstable", + "target": "unstable.tar", + }, + args.__dict__, + ) + + def test_split(self): + """Test splitting comma and space separated values.""" + args = parse_args( + [ + "--packages", + "distro-info ionit,netconsole", + "--include", + "openssh-server,restricted-ssh-commands", + "--components", + "main,non-free contrib", + "--architectures", + "amd64,i386", + ] + ) + self.assertEqual( + { + k: v + for k, v in args.__dict__.items() + if k in {"architectures", "components", "packages"} + }, + { + "architectures": ["amd64", "i386"], + "components": ["main", "non-free", "contrib"], + "packages": [ + "distro-info", + "ionit", + "netconsole", + "openssh-server", + "restricted-ssh-commands", + ], + }, + ) + + +class TestConfig(unittest.TestCase): + """ + This unittest class tests the Config object. + """ + + maxDiff = None + + def test_add_command_line_arguments(self): + """Test Config.add_command_line_arguments().""" + args = parse_args( + [ + "-c", + os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml"), + "--name", + "Debian-unstable", + ] + ) + config = Config() + config.add_command_line_arguments(args) + self.assertEqual( + config, + { + "mmdebstrap": { + "architectures": ["amd64"], + "keyrings": ["/usr/share/keyrings/debian-archive-keyring.gpg"], + "mode": "unshare", + "suite": "unstable", + "target": "root.tar.xz", + "variant": "minbase", + }, + "name": "Debian-unstable", + }, + ) + + def test_config_and_arguments(self): + """Test Config.add_command_line_arguments() with config file and arguments.""" + args = parse_args( + [ + "-c", + os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml"), + "--name", + "Debian-unstable", + "--variant", + "standard", + "--mode", + "root", + "--aptopt", + 'Apt::Install-Recommends "0"', + "--keyring", + "/usr/share/keyrings", + "--dpkgopt", + "force-confdef", + "--dpkgopt", + "force-confold", + "--include", + "ionit,netconsole", + "--components", + "main,non-free", + "--architectures", + "i386", + "--mirrors", + "http://deb.debian.org/debian", + "unstable", + "unstable.tar", + ] + ) + config = Config() + config.add_command_line_arguments(args) + self.assertDictEqual( + config, + { + "mmdebstrap": { + "aptopts": ['Apt::Install-Recommends "0"'], + "architectures": ["amd64", "i386"], + "components": ["main", "non-free"], + "dpkgopts": ["force-confdef", "force-confold"], + "keyrings": [ + "/usr/share/keyrings/debian-archive-keyring.gpg", + "/usr/share/keyrings", + ], + "mirrors": ["http://deb.debian.org/debian"], + "mode": "root", + "packages": ["ionit", "netconsole"], + "suite": "unstable", + "target": "unstable.tar", + "variant": "standard", + }, + "name": "Debian-unstable", + }, + ) + + @staticmethod + def test_check_example(): + """Test example unstable.yaml file.""" + config = Config() + config.load(os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml")) + config["name"] = "Debian-unstable" + config.check() + + @staticmethod + def test_commented_packages(): + """Test commented-packages.yaml file.""" + config = Config() + config.load(os.path.join(TEST_CONFIG_DIR, "commented-packages.yaml")) + config.sanitize_packages() + config.check() + + def test_env_items(self): + """Test environment variables for example unstable.yaml.""" + config = Config() + config.load(os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml")) + config["name"] = "Debian-unstable" + self.assertEqual( + config.env_items(), + [ + ("BDEBSTRAP_NAME", "Debian-unstable"), + ("BDEBSTRAP_OUTPUT_DIR", "/tmp/bdebstrap-output"), + ], + ) + + def test_loading(self): + """Test loading a YAML configuration file.""" + config = Config() + config.load(os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml")) + self.assertEqual( + config, + { + "mmdebstrap": { + "architectures": ["amd64"], + "keyrings": ["/usr/share/keyrings/debian-archive-keyring.gpg"], + "mode": "unshare", + "suite": "unstable", + "target": "root.tar.xz", + "variant": "minbase", + } + }, + ) + + def test_source_date_epoch(self): + """Test getting and setting SOURCE_DATE_EPOCH.""" + config = Config() + self.assertIsNone(config.source_date_epoch) + with unittest.mock.patch("time.time", return_value=1581694618.0388665): + config.set_source_date_epoch() + self.assertEqual(config.source_date_epoch, 1581694618) + + +class TestDictMerge(unittest.TestCase): + """Unittests for dict_merge function.""" + + def test_merge_lists(self): + """Test merging nested dicts.""" + items = {"A": ["A1", "A2", "A3"], "C": 4} + dict_merge(items, {"A": ["A4", "A5"]}) + self.assertEqual(items, {"A": ["A1", "A2", "A3", "A4", "A5"], "C": 4}) + + def test_merge_nested_dicts(self): + """Test merging nested dicts.""" + items = {"A": {"A1": 0, "A4": 4}, "C": 4} + dict_merge(items, {"A": {"A1": 1, "A5": 5}}) + self.assertEqual(items, {"A": {"A1": 1, "A4": 4, "A5": 5}, "C": 4}) diff --git a/tests/test_flake8.py b/tests/test_flake8.py new file mode 100644 index 0000000..6133cb6 --- /dev/null +++ b/tests/test_flake8.py @@ -0,0 +1,58 @@ +# Copyright (C) 2017-2018, Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Run flake8 check.""" + +import subprocess +import sys +import unittest + +from . import get_source_files, unittest_verbosity + + +class Flake8TestCase(unittest.TestCase): + """ + This unittest class provides a test that runs the flake8 code + checker (which combines pycodestyle and pyflakes) on the Python + source code. The list of source files is provided by the + get_source_files() function. + """ + + def test_flake8(self): + """Test: Run flake8 on Python source code.""" + cmd = [sys.executable, "-m", "flake8", "--max-line-length=99"] + get_source_files() + if unittest_verbosity() >= 2: + sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd))) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True + ) + + out, err = process.communicate() + if process.returncode != 0: # pragma: no cover + msgs = [] + if err: + msgs.append( + "flake8 exited with code {} and has unexpected output on stderr:\n{}".format( + process.returncode, err.decode().rstrip() + ) + ) + if out: + msgs.append("flake8 found issues:\n{}".format(out.decode().rstrip())) + if not msgs: + msgs.append( + "flake8 exited with code {} and has no output on stdout or stderr.".format( + process.returncode + ) + ) + self.fail("\n".join(msgs)) diff --git a/tests/test_helper.py b/tests/test_helper.py new file mode 100644 index 0000000..c382e71 --- /dev/null +++ b/tests/test_helper.py @@ -0,0 +1,110 @@ +# Copyright (C) 2020 Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Test helper functions of bdebstrap.""" + +import os +import unittest +import unittest.mock + +from bdebstrap import clamp_mtime, duration_str, escape_cmd + + +class TestClampMtime(unittest.TestCase): + """ + This unittest class tests the clamp_mtime function. + """ + + @staticmethod + @unittest.mock.patch("os.stat") + @unittest.mock.patch("os.utime") + def test_clamp(utime_mock, stat_mock): + """Test clamping the modification time.""" + stat_mock.return_value = os.stat_result( + (33261, 16535979, 64769, 1, 1000, 1000, 17081, 1581451059, 1581451059, 1581451059) + ) + clamp_mtime("/example", 1581433737) + utime_mock.assert_called_once_with("/example", (1581433737, 1581433737)) + + @staticmethod + @unittest.mock.patch("os.stat") + @unittest.mock.patch("os.utime") + def test_not_clamping(utime_mock, stat_mock): + """Test not clamping the modification time.""" + stat_mock.return_value = os.stat_result( + (33261, 16535979, 64769, 1, 1000, 1000, 17081, 1581451059, 1581451059, 1581451059) + ) + clamp_mtime("/example", 1581506399) + utime_mock.assert_not_called() + + @staticmethod + @unittest.mock.patch("os.stat") + def test_no_source_date_epoch(stat_mock): + """Test doing nothing if SOURCE_DATE_EPOCH is not set.""" + clamp_mtime("/example", None) + stat_mock.assert_not_called() + + +class TestDuration(unittest.TestCase): + """ + This unittest class tests the duration_str function. + """ + + def test_seconds(self): + """Test calling duration_str(3.606104612350464).""" + self.assertEqual(duration_str(3.606104612350464), "3.606 seconds") + + def test_minutes(self): + """Test calling duration_str(421.88086652755737).""" + self.assertEqual(duration_str(421.88086652755737), "7 min 1.881 s (= 421.881 s)") + + def test_hours(self): + """Test calling duration_str(7397.447488069534).""" + self.assertEqual(duration_str(7397.447488069534), "2 h 3 min 17.447 s (= 7397.447 s)") + + +class TestEscapeCmd(unittest.TestCase): + """ + This unittest class tests the escape_cmd function. + """ + + def test_simple(self): + """Test calling escape_cmd(["free"]).""" + self.assertEqual(escape_cmd(["free"]), "free") + + def test_spaces(self): + """Test calling escape_cmd(["scp", "source", "a space"]).""" + self.assertEqual(escape_cmd(["scp", "source", "a space"]), 'scp source "a space"') + + def test_escape(self): + """Test calling escape_cmd(["dpkg-query", r"-f=${Package}\t${Version}\n", "-W"]).""" + self.assertEqual( + escape_cmd(["dpkg-query", r"-f=${Package}\t${Version}\n", "-W"]), + r'dpkg-query "-f=\${Package}\t\${Version}\n" -W', + ) + + def test_customize_hook(self): + """Test calling escape_cmd on mmdebstrap customize hook.""" + self.assertEqual( + escape_cmd( + [ + "mmdebstrap", + '--customize-hook=chroot "$1" dpkg-query ' + "-f='${Package}\\t${Version}\\n' -W > \"$1/tmp/bdebstrap/manifest\"", + ] + ), + r'mmdebstrap "--customize-hook=chroot \"\$1\" dpkg-query ' + r"-f='${Package}\t${Version}\n' -W " + r'> \"\$1/tmp/bdebstrap/manifest\""', + ) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..3c23a00 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,55 @@ +# Copyright (C) 2020 Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Test Mmdebstrap class of bdebstrap.""" + +import os +import shutil +import subprocess +import unittest + +from bdebstrap import main + +BUILDS_DIR = os.path.join(os.path.dirname(__file__), "builds") +EXAMPLE_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "..", "examples") + + +@unittest.skipIf( + not os.environ.get("USE_INTERNET", False), + "Needs Internet access. Set environment variable USE_INTERNET to run it.", +) +class TestIntegration(unittest.TestCase): + """ + This unittest class implements integration tests that need Internet connection. + """ + + def tearDown(self): + shutil.rmtree(BUILDS_DIR) + + @staticmethod + def test_reproducible(): + """Test building Debian unstable reproducible.""" + config = os.path.join(EXAMPLE_CONFIG_DIR, "Debian-unstable.yaml") + # Build Debian unstable once. + main(["-c", config, "-b", BUILDS_DIR, "-n", "unstable"]) + config = os.path.join(BUILDS_DIR, "unstable", "config.yaml") + # Rebuild Debian unstable again. + main(["-c", config, "-o", os.path.join(BUILDS_DIR, "unstable-rebuild")]) + subprocess.check_call( + [ + "diffoscope", + os.path.join(BUILDS_DIR, "unstable"), + os.path.join(BUILDS_DIR, "unstable-rebuild"), + ] + ) diff --git a/tests/test_mmdebstrap.py b/tests/test_mmdebstrap.py new file mode 100644 index 0000000..d022d0f --- /dev/null +++ b/tests/test_mmdebstrap.py @@ -0,0 +1,149 @@ +# Copyright (C) 2020 Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Test Mmdebstrap class of bdebstrap.""" + +import unittest + +from bdebstrap import Mmdebstrap + + +class TestMmdebstrap(unittest.TestCase): + """ + This unittest class tests the Mmdebstrap object. + """ + + def test_debian_example(self): + """Test Mmdebstrap with Debian unstable config.""" + mmdebstrap = Mmdebstrap( + { + "mmdebstrap": { + "architectures": ["i386"], + "install-recommends": True, + "keyrings": ["/usr/share/keyrings/debian-archive-keyring.gpg"], + "mode": "unshare", + "suite": "unstable", + "target": "example.tar.xz", + "variant": "minbase", + } + } + ) + self.assertEqual( + mmdebstrap.construct_parameters("/output"), + [ + "mmdebstrap", + "-v", + "--variant=minbase", + "--mode=unshare", + "--keyring=/usr/share/keyrings/debian-archive-keyring.gpg", + "--architectures=i386", + '--essential-hook=mkdir -p "$1/tmp/bdebstrap-output"', + '--aptopt=Apt::Install-Recommends "true"', + "--customize-hook=chroot \"$1\" dpkg-query -f='${Package}\\t${Version}\\n' -W " + '> "$1/tmp/bdebstrap-output/manifest"', + '--customize-hook=sync-out "/tmp/bdebstrap-output" "/output"', + '--customize-hook=rm -rf "$1/tmp/bdebstrap-output"', + "unstable", + "example.tar.xz", + ], + ) + + def test_dry_run(self): + """Test Mmdebstrap with dry run set.""" + mmdebstrap = Mmdebstrap({"mmdebstrap": {"suite": "unstable", "target": "example.tar.xz"}}) + self.assertEqual( + mmdebstrap.construct_parameters("/output", True), + [ + "mmdebstrap", + "-v", + "--simulate", + '--essential-hook=mkdir -p "$1/tmp/bdebstrap-output"', + "--customize-hook=chroot \"$1\" dpkg-query -f='${Package}\\t${Version}\\n' -W " + '> "$1/tmp/bdebstrap-output/manifest"', + '--customize-hook=sync-out "/tmp/bdebstrap-output" "/output"', + '--customize-hook=rm -rf "$1/tmp/bdebstrap-output"', + "unstable", + "example.tar.xz", + ], + ) + + def test_hooks(self): + """Test Mmdebstrap with custom hooks.""" + mmdebstrap = Mmdebstrap( + { + "mmdebstrap": { + "cleanup-hooks": ['rm -f "$0/etc/udev/rules.d/70-persistent-net.rules"'], + "customize-hooks": [ + 'chroot "$0" update-alternatives --set editor /usr/bin/vim.basic' + ], + "essential-hooks": ["copy-in /etc/bash.bashrc /etc"], + "hostname": "example", + "setup-hooks": [], + "suite": "buster", + "target": "buster.tar.xz", + } + } + ) + self.assertEqual( + mmdebstrap.construct_parameters("/output"), + [ + "mmdebstrap", + "-v", + '--essential-hook=mkdir -p "$1/tmp/bdebstrap-output"', + "--essential-hook=copy-in /etc/bash.bashrc /etc", + '--customize-hook=chroot "$0" update-alternatives --set editor /usr/bin/vim.basic', + '--customize-hook=rm -f "$0/etc/udev/rules.d/70-persistent-net.rules"', + '--customize-hook=echo "example" > "$1/etc/hostname"', + "--customize-hook=chroot \"$1\" dpkg-query -f='${Package}\\t${Version}\\n' -W " + '> "$1/tmp/bdebstrap-output/manifest"', + '--customize-hook=sync-out "/tmp/bdebstrap-output" "/output"', + '--customize-hook=rm -rf "$1/tmp/bdebstrap-output"', + "buster", + "buster.tar.xz", + ], + ) + + def test_extra_opts(self): + """Test Mmdebstrap with extra options.""" + mmdebstrap = Mmdebstrap( + { + "mmdebstrap": { + "aptopts": ['Acquire::http { Proxy "http://proxy:3128/"; }'], + "components": ["main", "non-free", "contrib"], + "dpkgopts": ["force-confdef", "force-confold"], + "packages": ["bash-completions", "vim"], + "suite": "unstable", + "target": "example.tar.xz", + } + } + ) + self.assertEqual( + mmdebstrap.construct_parameters("/output"), + [ + "mmdebstrap", + "-v", + '--aptopt=Acquire::http { Proxy "http://proxy:3128/"; }', + "--dpkgopt=force-confdef", + "--dpkgopt=force-confold", + "--include=bash-completions,vim", + "--components=main,non-free,contrib", + '--essential-hook=mkdir -p "$1/tmp/bdebstrap-output"', + "--customize-hook=chroot \"$1\" dpkg-query -f='${Package}\\t${Version}\\n' -W " + '> "$1/tmp/bdebstrap-output/manifest"', + '--customize-hook=sync-out "/tmp/bdebstrap-output" "/output"', + '--customize-hook=rm -rf "$1/tmp/bdebstrap-output"', + "unstable", + "example.tar.xz", + ], + ) diff --git a/tests/test_pylint.py b/tests/test_pylint.py new file mode 100644 index 0000000..2f475b3 --- /dev/null +++ b/tests/test_pylint.py @@ -0,0 +1,76 @@ +# Copyright (C) 2010, Stefano Rivera +# Copyright (C) 2017-2018, Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +"""Run pylint.""" + +import os +import re +import subprocess +import sys +import unittest + +from . import get_source_files, unittest_verbosity + +CONFIG = os.path.join(os.path.dirname(__file__), "pylint.conf") + + +class PylintTestCase(unittest.TestCase): + """ + This unittest class provides a test that runs the pylint code check + on the Python source code. The list of source files is provided by + the get_source_files() function and pylint is purely configured via + a config file. + """ + + def test_pylint(self): + """Test: Run pylint on Python source code.""" + + cmd = [sys.executable, "-m", "pylint", "--rcfile=" + CONFIG, "--"] + get_source_files() + if unittest_verbosity() >= 2: + sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd))) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True + ) + out, err = process.communicate() + + if process.returncode != 0: # pragma: no cover + # Strip trailing summary (introduced in pylint 1.7). This summary might look like: + # + # ------------------------------------ + # Your code has been rated at 10.00/10 + # + out = re.sub( + "^(-+|Your code has been rated at .*)$", "", out.decode(), flags=re.MULTILINE + ).rstrip() + + # Strip logging of used config file (introduced in pylint 1.8) + err = re.sub("^Using config file .*\n", "", err.decode()).rstrip() + + msgs = [] + if err: + msgs.append( + "pylint exited with code {} and has unexpected output on stderr:\n{}".format( + process.returncode, err + ) + ) + if out: + msgs.append("pylint found issues:\n{}".format(out)) + if not msgs: + msgs.append( + "pylint exited with code {} and has no output on stdout or stderr.".format( + process.returncode + ) + ) + self.fail("\n".join(msgs))