From eddc96a1dc892298fd49f68aab18cca0d6e57434 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Mon, 15 Mar 2021 13:49:18 +0100 Subject: [PATCH 1/6] ci(TEST-60-BONDBRIDGEVLANIFCFG): use toplevel Makefile Signed-off-by: David Disseldorp --- test/TEST-60-BONDBRIDGEVLANIFCFG/Makefile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/TEST-60-BONDBRIDGEVLANIFCFG/Makefile b/test/TEST-60-BONDBRIDGEVLANIFCFG/Makefile index aad2705947..2dcab8164b 100644 --- a/test/TEST-60-BONDBRIDGEVLANIFCFG/Makefile +++ b/test/TEST-60-BONDBRIDGEVLANIFCFG/Makefile @@ -1,10 +1 @@ -all: - @$(MAKE) -s --no-print-directory -C ../.. all - @V=$(V) basedir=../.. testdir=../ ./test.sh --all -setup: - @$(MAKE) --no-print-directory -C ../.. all - @basedir=../.. testdir=../ ./test.sh --setup -clean: - @basedir=../.. testdir=../ ./test.sh --clean -run: - @basedir=../.. testdir=../ ./test.sh --run +-include ../Makefile.testdir From cabe94c262e535a41c91c20ab1d7f4eb82b44267 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Wed, 3 Mar 2021 19:09:07 +0100 Subject: [PATCH 2/6] ci(test): export basedir and testdir as absolute paths Individual test scripts may change working directory, so relative paths should be avoided. Signed-off-by: David Disseldorp --- test/Makefile.testdir | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Makefile.testdir b/test/Makefile.testdir index 33c2a9a4c1..e06c68b059 100644 --- a/test/Makefile.testdir +++ b/test/Makefile.testdir @@ -2,11 +2,11 @@ all: @$(MAKE) -s --no-print-directory -C ../.. all - @V=$(V) basedir=../.. testdir=../ ./test.sh --all + @V=$(V) basedir=$(realpath ../..) testdir="$(realpath ../)" ./test.sh --all setup: @$(MAKE) --no-print-directory -C ../.. all - @basedir=../.. testdir=../ ./test.sh --setup + @basedir="$(realpath ../..)" testdir="$(realpath ../)" ./test.sh --setup clean: - @basedir=../.. testdir=../ ./test.sh --clean + @basedir="$(realpath ../..)" testdir="$(realpath ../)" ./test.sh --clean run: - @basedir=../.. testdir=../ ./test.sh --run + @basedir="$(realpath ../..)" testdir="$(realpath ../)" ./test.sh --run From 6d0c86702d857828be46430d2265e2034213fffc Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Fri, 26 Feb 2021 17:42:45 +0100 Subject: [PATCH 3/6] feat(skipcpio): move cpio on-disk format into separate header Preparation for reusing cpio archive definitions. skipcpio's PROGRAM_VERSION_STRING is currently unused so drop it. Signed-off-by: David Disseldorp --- skipcpio/skipcpio.c | 28 +--------------------------- skipcpio/skipcpio.h | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 27 deletions(-) create mode 100644 skipcpio/skipcpio.h diff --git a/skipcpio/skipcpio.c b/skipcpio/skipcpio.c index fd9b4e46ad..6a08cffe6e 100644 --- a/skipcpio/skipcpio.c +++ b/skipcpio/skipcpio.c @@ -17,8 +17,6 @@ along with this program; If not, see . */ -#define PROGRAM_VERSION_STRING "1" - #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif @@ -26,29 +24,7 @@ #include #include #include - -#define CPIO_MAGIC "070701" -#define CPIO_END "TRAILER!!!" -#define CPIO_ENDLEN (sizeof(CPIO_END) - 1) - -#define CPIO_ALIGNMENT 4 - -struct cpio_header { - char c_magic[6]; - char c_ino[8]; - char c_mode[8]; - char c_uid[8]; - char c_gid[8]; - char c_nlink[8]; - char c_mtime[8]; - char c_filesize[8]; - char c_dev_maj[8]; - char c_dev_min[8]; - char c_rdev_maj[8]; - char c_rdev_min[8]; - char c_namesize[8]; - char c_chksum[8]; -} __attribute__((packed)); +#include "skipcpio.h" struct buf_struct { struct cpio_header h; @@ -62,8 +38,6 @@ union buf_union { static union buf_union buf; -#define ALIGN_UP(n, a) (((n) + (a) - 1) & (~((a) - 1))) - int main(int argc, char **argv) { FILE *f; diff --git a/skipcpio/skipcpio.h b/skipcpio/skipcpio.h new file mode 100644 index 0000000000..f7b6505ee2 --- /dev/null +++ b/skipcpio/skipcpio.h @@ -0,0 +1,42 @@ +/* + Copyright (C) 2021 Harald Hoyer + Copyright (C) 2021 Red Hat, Inc. All rights reserved. + + This program is free software: you can redistribute it and/or modify + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; If not, see . +*/ + +#define CPIO_MAGIC "070701" +#define CPIO_END "TRAILER!!!" +#define CPIO_ENDLEN (sizeof(CPIO_END) - 1) + +#define CPIO_ALIGNMENT 4 + +struct cpio_header { + char c_magic[6]; + char c_ino[8]; + char c_mode[8]; + char c_uid[8]; + char c_gid[8]; + char c_nlink[8]; + char c_mtime[8]; + char c_filesize[8]; + char c_dev_maj[8]; + char c_dev_min[8]; + char c_rdev_maj[8]; + char c_rdev_min[8]; + char c_namesize[8]; + char c_chksum[8]; +} __attribute__((packed)); + +#define ALIGN_UP(n, a) (((n) + (a) - 1) & (~((a) - 1))) From 7df678e71ce25fdcb70ef5555c7a4bb3c302d617 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Fri, 26 Feb 2021 17:37:43 +0100 Subject: [PATCH 4/6] feat(padcpio): new helper binary for cpio data alignment 1. Purpose Improve cpio archive creation performance and space efficiency by ensuring that file data is aligned within the archive to the filesystem block size. For filesystems supporting reflinks (e.g. XFS and Btrfs) this ensures that cpio archive data can share the same copy-on-write extents as the archive source files. A GNU cpio binary capable of copy-on-write cloning data via copy_file_range is required to make proper use of this. Padding can't be added to cpio archives arbitrarily, so we need to inject extra files into the archive to provide filesystem alignment. 2. Behaviour Read a zero terminated file list from stdin. If an input file, when cpio serialized, is aligned to then print it to stdout. If the cpio archived file data would be unaligned then create a padding file in which, when cpio archived before the input file, will provide cpio alignment and print the pad file path before the input file path. GNU cpio reorders hardlinks, so avoid extra complexity by deferring them (unpadded) to the end of the archive. Signed-off-by: David Disseldorp --- Makefile | 10 +- dracut.spec | 1 + skipcpio/padcpio.c | 380 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 skipcpio/padcpio.c diff --git a/Makefile b/Makefile index c3fbd95b87..56ea08067c 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ manpages = $(man1pages) $(man5pages) $(man7pages) $(man8pages) .PHONY: install clean archive rpm srpm testimage test all check AUTHORS CONTRIBUTORS doc dracut-version.sh -all: dracut-version.sh dracut.pc dracut-install skipcpio/skipcpio dracut-util +all: dracut-version.sh dracut.pc dracut-install skipcpio/skipcpio skipcpio/padcpio dracut-util %.o : %.c $(CC) -c $(CFLAGS) $(CPPFLAGS) $(KMOD_CFLAGS) $< -o $@ @@ -85,6 +85,9 @@ SKIPCPIO_OBJECTS = skipcpio/skipcpio.o skipcpio/skipcpio.o: skipcpio/skipcpio.c skipcpio/skipcpio: $(SKIPCPIO_OBJECTS) +skipcpio/padcpio.o: skipcpio/padcpio.c +skipcpio/padcpio: skipcpio/padcpio.o + UTIL_OBJECTS = util/util.o util/util.o: util/util.c util/util: $(UTIL_OBJECTS) @@ -193,6 +196,9 @@ endif if [ -f skipcpio/skipcpio ]; then \ install -m 0755 skipcpio/skipcpio $(DESTDIR)$(pkglibdir)/skipcpio; \ fi + if [ -f skipcpio/padcpio ]; then \ + install -m 0755 skipcpio/padcpio $(DESTDIR)$(pkglibdir)/padcpio; \ + fi if [ -f dracut-util ]; then \ install -m 0755 dracut-util $(DESTDIR)$(pkglibdir)/dracut-util; \ fi @@ -218,7 +224,7 @@ clean: $(RM) dracut-*.rpm dracut-*.tar.bz2 dracut-*.tar.xz $(RM) dracut-version.sh $(RM) dracut-install install/dracut-install $(DRACUT_INSTALL_OBJECTS) - $(RM) skipcpio/skipcpio $(SKIPCPIO_OBJECTS) + $(RM) skipcpio/skipcpio $(SKIPCPIO_OBJECTS) skipcpio/padcpio skipcpio/padcpio.o $(RM) dracut-util util/util $(UTIL_OBJECTS) $(RM) $(manpages) dracut.html $(RM) dracut.pc diff --git a/dracut.spec b/dracut.spec index 1ca7bde0e0..e6f7f4f21d 100644 --- a/dracut.spec +++ b/dracut.spec @@ -298,6 +298,7 @@ echo 'dracut_rescue_image="yes"' > $RPM_BUILD_ROOT%{dracutlibdir}/dracut.conf.d/ %{dracutlibdir}/dracut-install %{dracutlibdir}/dracut-util %{dracutlibdir}/skipcpio +%{dracutlibdir}/padcpio %config(noreplace) %{_sysconfdir}/dracut.conf %if 0%{?fedora} || 0%{?suse_version} || 0%{?rhel} %{dracutlibdir}/dracut.conf.d/01-dist.conf diff --git a/skipcpio/padcpio.c b/skipcpio/padcpio.c new file mode 100644 index 0000000000..58e0287998 --- /dev/null +++ b/skipcpio/padcpio.c @@ -0,0 +1,380 @@ +/* + Copyright (C) 2021 SUSE LLC + + This program is free software: you can redistribute it and/or modify + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; If not, see . +*/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#define _LARGEFILE64_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "skipcpio.h" + +#ifdef DEBUG +/* make CFLAGS="-DDEBUG" */ +#define dout(...) fprintf(stderr, __VA_ARGS__) +#else +#define dout(...) +#endif + +static void usage(int status) +{ + fprintf(stdout, + "Usage: padcpio -a ALIGNMENT [-d PADDIR] [-m SIZE] [-o OFFSET]\n\n" + " -a --align Pad input file data to ALIGNMENT (required)\n" + " -d --paddir Create PADDIR for padding files (default=pad)\n" + " -m --min Don't pad for files under SIZE (default=1)\n" + " -o --offset Calculate padding from OFFSET (default=0)\n" + " -h --help Show this help\n\n" + "Example:\n" + " echo \"This data will be 4K aligned within out.cpio\" > file\n" + " printf \"file\\0\" | padcpio -a 4k | cpio -o --null -H newc -O out.cpio\n\n"); + + exit(status); +} + +static int unit_multiply(const char *unit, unsigned long *_val) +{ + unsigned long val = *_val; + + switch (unit[0]) { + case 'T': + case 't': + val *= 1024; /* fall through */ + case 'G': + case 'g': + val *= 1024; /* fall through */ + case 'M': + case 'm': + val *= 1024; /* fall through */ + case 'K': + case 'k': + val *= 1024; + if (unit[1] != '\0') + return -1; + /* fall through */ + case '\0': + break; + default: + return -1; + } + if (val < *_val) + return -1; /* overflow */ + + *_val = val; + return 0; +} + +const char *skip_relative_dot_slash(const char *path) +{ + const char *p = path; + if (p[0] == '/') { + fprintf(stderr, "error: %s is an absolute path\n", path); + return NULL; + } + while (p[0] == '.' && p[1] == '/') { + p++; + while (p[0] == '/') + p++; + } + if (p[0] == '\0') { + fprintf(stderr, "error: %s is invalid\n", path); + return NULL; + } + return p; +} + +static void parse_args(int argc, char *argv[], + unsigned long *_pad_align, + const char **_pad_dir, unsigned long *_min_file_size, off_t *_archive_off) +{ + unsigned long pad_align = 0; + const char *pad_dir = "pad"; + unsigned long min_file_size = 1; + unsigned long archive_off = 0; + int c; + int sret; + struct stat sb; + static struct option const options[] = { + {"alignment", required_argument, NULL, 'a'}, + {"paddir", required_argument, NULL, 'd'}, + {"min", required_argument, NULL, 'm'}, + {"offset", required_argument, NULL, 'o'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + while ((c = getopt_long(argc, argv, "a:d:m:h", options, NULL)) != -1) { + char *eptr; + switch (c) { + case 'a': + eptr = NULL; + pad_align = strtoul(optarg, &eptr, 10); + if (eptr == optarg || unit_multiply(eptr, &pad_align) < 0) { + fprintf(stderr, "invalid alignment value \'%s\'\n", optarg); + usage(EXIT_FAILURE); + } + break; + case 'd': + pad_dir = skip_relative_dot_slash(optarg); + if (pad_dir == NULL) { + usage(EXIT_FAILURE); + } + /* no support for nested pad paths or trailing slashes */ + if (strchr(pad_dir, '/') != NULL) { + fprintf(stderr, "paddir \'%s\' invalid: " "nested path or trailing slashes\n", pad_dir); + usage(EXIT_FAILURE); + } + break; + case 'm': + eptr = NULL; + min_file_size = strtoul(optarg, &eptr, 10); + if (eptr == optarg || unit_multiply(eptr, &min_file_size) < 0) { + fprintf(stderr, "invalid minimum size \'%s\'\n", optarg); + usage(EXIT_FAILURE); + } + break; + case 'o': + eptr = NULL; + archive_off = strtoul(optarg, &eptr, 10); + if (eptr == optarg || unit_multiply(eptr, &min_file_size) < 0) { + fprintf(stderr, "invalid offset \'%s\'\n", optarg); + usage(EXIT_FAILURE); + } + break; + case 'h': + usage(EXIT_SUCCESS); + break; + default: + usage(EXIT_FAILURE); + } + } + + if (optind == 1 || optind != argc) { + usage(EXIT_FAILURE); + } + + if (pad_align == 0 || (pad_align & (pad_align - 1)) != 0) { + fprintf(stderr, "invalid alignment \'%lu\': must be a power of two\n", pad_align); + usage(EXIT_FAILURE); + } + + sret = stat(pad_dir, &sb); + if (sret == 0 || errno != ENOENT) { + fprintf(stderr, "error: paddir \'%s\' path must not exist\n", pad_dir); + exit(1); + } + + *_pad_align = pad_align; + *_pad_dir = pad_dir; + *_min_file_size = min_file_size; + *_archive_off = archive_off; +} + +int pad_file_prepend(const char *pad_dirname, int pad_num, const char *in_path, + size_t in_data_len, unsigned long pad_align, off_t *archive_off) +{ + char pad_path[PATH_MAX]; + int ret; + int fd; + off_t real_data_off; + off_t aligned_data_off; + off_t next_off; + off_t pad_data_len; + + ret = snprintf(pad_path, sizeof(pad_path), "%s/%d", pad_dirname, pad_num); + if (ret >= sizeof(pad_path)) + return -E2BIG; + + fd = open(pad_path, O_CREAT | O_EXCL | O_WRONLY, 0600); + if (fd < 0) { + fprintf(stderr, "failed to create padding at %s: %s\n", pad_path, strerror(errno)); + return -errno; + } + + real_data_off = ALIGN_UP(*archive_off + sizeof(struct cpio_header) + strlen(pad_path) + 1, CPIO_ALIGNMENT) + /* pad data will go here */ + + ALIGN_UP(sizeof(struct cpio_header) + strlen(in_path) + 1, CPIO_ALIGNMENT); + + aligned_data_off = ALIGN_UP(real_data_off, pad_align); + pad_data_len = aligned_data_off - real_data_off; + + /* take into account size of header and pad_path for next entry */ + ret = ftruncate(fd, pad_data_len); + if (ret < 0) { + fprintf(stderr, "failed to truncate %s to %lu: %s\n", pad_path, pad_data_len, strerror(errno)); + ret = -errno; + close(fd); + return ret; + } + ret = close(fd); + if (ret < 0) + return -errno; + + dout("pad file %s size %lu inserted before %s size %lu\n", pad_path, pad_data_len, in_path, in_data_len); + + fprintf(stdout, "%s%c", pad_path, '\0'); + next_off = ALIGN_UP(*archive_off + sizeof(struct cpio_header) + strlen(pad_path) + 1, CPIO_ALIGNMENT) + + ALIGN_UP(pad_data_len, CPIO_ALIGNMENT); + dout("%s: archive offset: [%lu, %lu)\n", pad_path, *archive_off, next_off); + *archive_off = next_off; + + fprintf(stdout, "%s%c", in_path, '\0'); + next_off = ALIGN_UP(*archive_off + sizeof(struct cpio_header) + strlen(in_path) + 1, CPIO_ALIGNMENT) + + ALIGN_UP(in_data_len, CPIO_ALIGNMENT); + dout("%s: archive offset: [%lu, %lu)\n", in_path, *archive_off, next_off); + *archive_off = next_off; + + return 0; +} + +char *hardlink_names = NULL; +size_t hardlink_names_size = 0; +off_t hardlink_names_off = 0; + +int main(int argc, char **argv) +{ + unsigned long pad_align = 0; + const char *pad_dirname = NULL; + unsigned long min_file_size = 1; + int pad_num = 0; + int sret; + struct stat sb; + char *line = NULL; + size_t len = 0; + ssize_t nread; + int ret = EXIT_FAILURE; + off_t archive_off = 0; + + parse_args(argc, argv, &pad_align, &pad_dirname, &min_file_size, &archive_off); + + while ((nread = getdelim(&line, &len, '\0', stdin)) != -1) { + off_t this_doff; + const char *in_path; + + if (nread >= PATH_MAX) { + fprintf(stderr, "input path too large\n"); + goto err_line_free; + } + + in_path = skip_relative_dot_slash(line); + if (in_path == NULL) { + goto err_line_free; + } + + sret = lstat(in_path, &sb); + if (sret < 0) { + fprintf(stderr, "stat failed for %s\n", line); + goto err_line_free; + } + + this_doff = ALIGN_UP(archive_off + sizeof(struct cpio_header) + + strlen(in_path) + 1, CPIO_ALIGNMENT); + + if (S_ISLNK(sb.st_mode)) { + ssize_t bytes; + char lnk_tgt[PATH_MAX]; + bytes = readlink(in_path, lnk_tgt, sizeof(lnk_tgt)); + if (bytes <= 0 || bytes >= sizeof(lnk_tgt)) { + fprintf(stderr, "readlink failed for %s\n", line); + goto err_line_free; + } + fprintf(stdout, "%s%c", in_path, '\0'); + dout("%s: archive offset: [%lu, %lu)\n", in_path, archive_off, + ALIGN_UP(this_doff + bytes, CPIO_ALIGNMENT)); + archive_off = ALIGN_UP(this_doff + bytes, CPIO_ALIGNMENT); + continue; + } + + if (!S_ISREG(sb.st_mode)) { + /* non-file or size under minimum for padding */ + fprintf(stdout, "%s%c", in_path, '\0'); + dout("%s: archive offset: [%lu, %lu)\n", in_path, archive_off, this_doff); + archive_off = this_doff; + continue; + } + + if (sb.st_nlink > 1) { + /* + * GNU cpio deferrs hardlink processing until last link. + * Avoid the complexity of determining when they appear + * by just deferring them all to the end of the archive + * without any padding. + */ + size_t len = strlen(in_path) + 1; + if (hardlink_names_off + len > hardlink_names_size) { + char *hardlink_names_n = realloc(hardlink_names, hardlink_names_off + len); + if (hardlink_names_n == NULL) + goto err_line_free; + hardlink_names = hardlink_names_n; + hardlink_names_size = hardlink_names_off + len; + } + memcpy(&hardlink_names[hardlink_names_off], in_path, len); + hardlink_names_off += len; + dout("%s: hardlink deferred to end\n", in_path); + continue; + } + + if (sb.st_size < min_file_size || this_doff == ALIGN_UP(this_doff, pad_align)) { + /* data segment under min-size for padding or already aligned */ + fprintf(stdout, "%s%c", in_path, '\0'); + dout("%s: archive offset: [%lu, %lu)\n", in_path, archive_off, + ALIGN_UP(this_doff + sb.st_size, CPIO_ALIGNMENT)); + archive_off = ALIGN_UP(this_doff + sb.st_size, CPIO_ALIGNMENT); + continue; + } + + if (pad_num == 0) { + ret = mkdir(pad_dirname, 0700); + if (ret < 0) { + fprintf(stderr, "Cannot create '%s'\n", pad_dirname); + goto err_line_free; + } + + fprintf(stdout, "%s%c", pad_dirname, '\0'); + dout("%s: archive offset: [%lu, %lu)\n", pad_dirname, archive_off, + ALIGN_UP(archive_off + sizeof(struct cpio_header) + strlen(pad_dirname) + 1, + CPIO_ALIGNMENT)); + archive_off = ALIGN_UP(archive_off + sizeof(struct cpio_header) + + strlen(pad_dirname) + 1, CPIO_ALIGNMENT); + } + ret = pad_file_prepend(pad_dirname, pad_num, in_path, sb.st_size, pad_align, &archive_off); + if (ret < 0) { + fprintf(stderr, "Cannot create '%s'\n", pad_dirname); + goto err_line_free; + } + pad_num++; + } + + if (hardlink_names_off != 0 && fwrite(hardlink_names, hardlink_names_off, 1, stdout) != 1) + goto err_line_free; + free(line); + free(hardlink_names); + return EXIT_SUCCESS; +err_line_free: + free(line); + free(hardlink_names); + return EXIT_FAILURE; +} From 0ee332b6cf8e26ffbb80240838ea7e2301b0e96d Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Mon, 1 Mar 2021 01:51:42 +0100 Subject: [PATCH 5/6] ci(TEST-62-CPIO): add CPIO test Provides some coverage for the skipcpio and padcpio binaries. Signed-off-by: David Disseldorp --- test/TEST-62-CPIO/Makefile | 1 + test/TEST-62-CPIO/test.sh | 163 +++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 test/TEST-62-CPIO/Makefile create mode 100755 test/TEST-62-CPIO/test.sh diff --git a/test/TEST-62-CPIO/Makefile b/test/TEST-62-CPIO/Makefile new file mode 100644 index 0000000000..2dcab8164b --- /dev/null +++ b/test/TEST-62-CPIO/Makefile @@ -0,0 +1 @@ +-include ../Makefile.testdir diff --git a/test/TEST-62-CPIO/test.sh b/test/TEST-62-CPIO/test.sh new file mode 100755 index 0000000000..eef9d608e6 --- /dev/null +++ b/test/TEST-62-CPIO/test.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# This file is part of dracut. +# SPDX-License-Identifier: GPL-2.0-or-later + +TEST_DESCRIPTION="test skipcpio and padcpio utilities" + +test_check() { + which cpio dd truncate find sort diff &>/dev/null +} + +skipcpio_simple() { + mkdir -p "$CPIO_TESTDIR/skipcpio_simple/first_archive" + pushd "$CPIO_TESTDIR/skipcpio_simple/first_archive" + + for ((i=0; i < 3; i++)); do + echo "first archive file $i" >> ./$i + done + find . -print0 | sort -z \ + | cpio -o --null -H newc --file "$CPIO_TESTDIR/skipcpio_simple.cpio" + popd + + mkdir -p "$CPIO_TESTDIR/skipcpio_simple/second_archive" + pushd "$CPIO_TESTDIR/skipcpio_simple/second_archive" + + for ((i=10; i < 13; i++)); do + echo "second archive file $i" >> ./$i + done + # could also use the new cpio --chain option here... + find . -print0 | sort -z \ + | cpio -o --null -H newc >> "$CPIO_TESTDIR/skipcpio_simple.cpio" + popd + + cat "$CPIO_TESTDIR/skipcpio_simple.cpio" | cpio -i --list \ + > "$CPIO_TESTDIR/skipcpio_simple.list" + cat < "$CPIO_TESTDIR/skipcpio_simple.list" + cat < "$CPIO_TESTDIR/padcpio_simple/f1" + echo "$f2_content" > "$CPIO_TESTDIR/padcpio_simple/f2" + truncate --size 6K "$CPIO_TESTDIR/padcpio_simple/f2" + echo "$f3_content" > "$CPIO_TESTDIR/padcpio_simple/f3" + + pushd "$CPIO_TESTDIR/padcpio_simple" + echo -n -e "f1\0f2\0f3\0" \ + | $basedir/skipcpio/padcpio --align 4096 --paddir pad \ + | cpio -o --null -H newc --file "$CPIO_TESTDIR/padcpio_simple.cpio" + popd + + dd status=none bs=1 skip=4096 count="${#f1_content}" \ + if="$CPIO_TESTDIR/padcpio_simple.cpio" | grep "$f1_content" + dd status=none bs=1 skip=8192 count="${#f2_content}" \ + if="$CPIO_TESTDIR/padcpio_simple.cpio" | grep "$f2_content" + dd status=none bs=1 skip=16384 count="${#f3_content}" \ + if="$CPIO_TESTDIR/padcpio_simple.cpio" | grep "$f3_content" + cpio -i --list --file "$CPIO_TESTDIR/padcpio_simple.cpio" \ + > "$CPIO_TESTDIR/padcpio_simple.list" + cat < "$CPIO_TESTDIR/padcpio_min/f1" + echo "$f2_content" > "$CPIO_TESTDIR/padcpio_min/f2" + echo "$f3_content" > "$CPIO_TESTDIR/padcpio_min/f3" + truncate --size 4K "$CPIO_TESTDIR/padcpio_min/f3" + + pushd "$CPIO_TESTDIR/padcpio_min" + find . -print0 | sort -z \ + | $basedir/skipcpio/padcpio --min 4K --align 4K --paddir pad \ + | cpio -o --null -H newc --file "$CPIO_TESTDIR/padcpio_min.cpio" + popd + + dd status=none bs=1 skip=4096 count="${#f3_content}" \ + if="$CPIO_TESTDIR/padcpio_min.cpio" | grep "$f3_content" +} + +# GNU cpio defers hardlink processing until the last link is encountered. To +# avoid this tracking padcpio just puts them (unaligned) at the end of the +# archive. +padcpio_links() { + mkdir -p "$CPIO_TESTDIR/padcpio_links" + echo "this is hardlinked" > "$CPIO_TESTDIR/padcpio_links/f1" + ln "$CPIO_TESTDIR/padcpio_links/f1" "$CPIO_TESTDIR/padcpio_links/f2" + echo "this is a symlink target" > "$CPIO_TESTDIR/padcpio_links/f3" + truncate --size 6K "$CPIO_TESTDIR/padcpio_links/f3" + ln -s "f3" "$CPIO_TESTDIR/padcpio_links/f4" + + pushd "$CPIO_TESTDIR/padcpio_links" + echo -n -e "f1\0f2\0f3\0f4\0" \ + | $basedir/skipcpio/padcpio --align 4096 --paddir paddy \ + | cpio -o --null -H newc --file "$CPIO_TESTDIR/padcpio_links.cpio" + popd + + cpio -i --list --file "$CPIO_TESTDIR/padcpio_links.cpio" \ + > "$CPIO_TESTDIR/padcpio_links.list" + cat < Date: Wed, 17 Feb 2021 01:05:37 +0100 Subject: [PATCH 6/6] feat(dracut.sh): add "--cpio-reflink" option for requesting GNU cpio reflinks The new GNU cpio "--reflink" parameter sees it use copy_file_range() when copying between source and destination, allowing for copy-on-write optimization if supported by the underlying filesystem (e.g. Btrfs or XFS). When calling cpio with --reflink the file list is piped through padcpio to ensure optimal alignment for extent sharing. Microcode and initramfs proper archives are chained together using the new cpio "--chain" parameter, which ensures that subsequent archives are appended in a reflink friendly fashion. RFC: This shouldn't be merged until GNU cpio --reflink --chain patches have made it upstream. Signed-off-by: David Disseldorp --- dracut-bash-completion.sh | 3 +- dracut.8.asc | 9 ++++ dracut.sh | 94 +++++++++++++++++++++++++++++++-------- 3 files changed, 87 insertions(+), 19 deletions(-) diff --git a/dracut-bash-completion.sh b/dracut-bash-completion.sh index 286fd1ceae..0ccb8a98b3 100644 --- a/dracut-bash-completion.sh +++ b/dracut-bash-completion.sh @@ -32,7 +32,8 @@ _dracut() { --local --hostonly --no-hostonly --fstab --help --bzip2 --lzma --xz --zstd --no-compress --gzip --list-modules --show-modules --keep --printsize --regenerate-all --noimageifnotneeded --early-microcode - --no-early-microcode --print-cmdline --reproducible --uefi' + --no-early-microcode --print-cmdline --reproducible --uefi + --cpio-reflink' [ARG]='-a -m -o -d -I -k -c -L --kver --add --force-add --add-drivers --omit-drivers --modules --omit --drivers --filesystems --install --fwdir --libdirs --fscks --add-fstab --mount --device --nofscks diff --git a/dracut.8.asc b/dracut.8.asc index dcbfe5976e..8a99919042 100644 --- a/dracut.8.asc +++ b/dracut.8.asc @@ -531,6 +531,15 @@ will not be able to boot. Specifies the kernel image, which to include in the UEFI executable. The default is _/lib/modules//vmlinuz_ or _/boot/vmlinuz-_ +**--cpio-reflink**:: + Attempt to use the GNU cpio --reflink option, which optimizes archive creation for + copy-on-write filesystems by using the copy_file_range(2) syscall. When specified, + initramfs archives are also padded to ensure optimal data alignment for extent + sharing. To retain reflink data deduplication benefits, this should be used + alongside the **--no-compress** and **--no-strip** parameters, with initramfs + source files, **--tmpdir** staging area and destination all on the same + copy-on-write capable filesystem. + ENVIRONMENT ----------- diff --git a/dracut.sh b/dracut.sh index 8890203a6d..e96f176db2 100755 --- a/dracut.sh +++ b/dracut.sh @@ -228,6 +228,7 @@ Creates initial ramdisk images for preloading modules otherwise you will not be able to boot. --no-compress Do not compress the generated initramfs. This will override any other compression options. + --cpio-reflink Request that cpio perform reflinks for file data. --list-modules List all available dracut modules. -M, --show-modules Print included module's name to standard output during build. @@ -414,6 +415,7 @@ rearrange_params() { --long zstd \ --long no-compress \ --long gzip \ + --long cpio-reflink \ --long list-modules \ --long show-modules \ --long keep \ @@ -772,6 +774,7 @@ while :; do --zstd) compress_l="zstd" ;; --no-compress) _no_compress_l="cat" ;; --gzip) compress_l="gzip" ;; + --cpio-reflink) cpio_reflink="yes" ;; --list-modules) do_list="yes" ;; -M | --show-modules) show_modules_l="yes" @@ -1147,6 +1150,28 @@ trap 'exit 1;' SIGINT readonly initdir="${DRACUT_TMPDIR}/initramfs" mkdir -p "$initdir" +if [[ $cpio_reflink == "yes" ]]; then + if [[ "$(cpio --help)" == *--reflink* ]]; then + # both XFS and Btrfs require data to be FS block-size aligned for proper + # extent sharing / reflinks. padcpio ensures this. + if [[ -d "$dracutbasedir/skipcpio" ]]; then + padcpio="$dracutbasedir/skipcpio/padcpio" + else + padcpio="$dracutbasedir/padcpio" + fi + if [[ -x "$padcpio" ]]; then + # align based on statfs optimal transfer size + padcpio_align=$(stat --file-system -c "%s" -- "$initdir") + else + dinfo "cpio-reflink ignored due to lack of padcpio" + unset cpio_reflink + fi + else + dinfo "cpio-reflink ignored due to lack of support in $(which cpio)" + unset cpio_reflink + fi +fi + # shellcheck disable=SC2154 if [[ $early_microcode == yes ]] || { [[ $acpi_override == yes ]] && [[ -d $acpi_table_dir ]]; }; then readonly early_cpio_dir="${DRACUT_TMPDIR}/earlycpio" @@ -2196,6 +2221,8 @@ if dracut_module_included "squash"; then fi if [[ $do_strip == yes ]] && ! [[ $DRACUT_FIPS_MODE ]]; then + # warn that stripping files negates (dedup) benefits of using reflink + [ -n "$cpio_reflink" ] && dinfo "inefficient: strip is enabled alongside cpio reflink" dinfo "*** Stripping files ***" find "$initdir" -type f \ -executable -not -path '*/lib/modules/*.ko' -print0 \ @@ -2266,15 +2293,28 @@ if [[ $create_early_cpio == yes ]]; then fi # The microcode blob is _before_ the initramfs blob, not after - if ! ( - umask 077 - cd "$early_cpio_dir/d" - find . -print0 | sort -z \ - | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null \ - ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet > "${DRACUT_TMPDIR}/initramfs.img" - ); then - dfatal "dracut: creation of $outfile failed" - exit 1 + if [[ -n "$cpio_reflink" ]]; then + if ! ( + umask 077 + cd "$early_cpio_dir/d" + find . -print0 | sort -z | "$padcpio" --min "$padcpio_align" --align "$padcpio_align" \ + | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null \ + ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet --reflink -O "${DRACUT_TMPDIR}/initramfs.img" + ); then + dfatal "dracut: creation of reflinked $outfile failed" + exit 1 + fi + else + if ! ( + umask 077 + cd "$early_cpio_dir/d" + find . -print0 | sort -z \ + | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null \ + ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet > "${DRACUT_TMPDIR}/initramfs.img" + ); then + dfatal "dracut: creation of $outfile failed" + exit 1 + fi fi fi @@ -2325,15 +2365,33 @@ case $compress in ;; esac -if ! ( - umask 077 - cd "$initdir" - find . -print0 | sort -z \ - | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet \ - | $compress >> "${DRACUT_TMPDIR}/initramfs.img" -); then - dfatal "dracut: creation of $outfile failed" - exit 1 +if [[ -n "$cpio_reflink" && "$compress" == "cat" ]]; then + # determine padding offset if we're appending to microcode + i=$(stat -c "%s" -- "${DRACUT_TMPDIR}/initramfs.img" 2>/dev/null) + if ! ( + umask 077 + cd "$initdir" + find . -print0 | sort -z \ + | "$padcpio" --min "$padcpio_align" --align "$padcpio_align" ${i:+--offset "$i"} \ + | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null \ + ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet --reflink ${i:+--chain} \ + -O "${DRACUT_TMPDIR}/initramfs.img" + ); then + dfatal "dracut: creation of reflinked $outfile failed" + exit 1 + fi +else + [ -n "$cpio_reflink" ] && dinfo "cpio-reflink ignored due to compression" + if ! ( + umask 077 + cd "$initdir" + find . -print0 | sort -z \ + | cpio ${CPIO_REPRODUCIBLE:+--reproducible} --null ${cpio_owner:+-R "$cpio_owner"} -H newc -o --quiet \ + | $compress >> "${DRACUT_TMPDIR}/initramfs.img" + ); then + dfatal "dracut: creation of $outfile failed" + exit 1 + fi fi # shellcheck disable=SC2154