repart: Add compression support

Now that mkfs.btrfs is adding support for compressing the generated filesystem (kdave/btrfs-progs#882), let's add general support for specifying the compression algorithm and compression level to use. We opt to not parse the specified compression algorithm and instead pass it on as is to the mkfs tool. This has a few benefits: - We support every compression algorithm supported by every tool automatically. - Users don't need to modify systemd-repart if a mkfs tool learns a new compression algorithm in the future - We don't need to maintain a bunch of tables for filesystem to map from our generic compression algorithm enum to the filesystem specific names.
DaanDeMeyer · Aug 30, 2024 · 5686efd · 5686efd
1 parent c573084
commit 5686efd
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 5 deletions.
diff --git a/man/repart.d.xml b/man/repart.d.xml
@@ -789,6 +789,32 @@
 
         <xi:include href="version-info.xml" xpointer="v256"/></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>Compression=</varname></term>
+
+        <listitem><para>Specify the compression algorithm to use for the filesystem configured with
+        <varname>Format=</varname>. Takes a single argument specifying the compression algorithm. </para>
+
+        <para>Note that this setting is only taken into account when the filesystem configured with
+        <varname>Format=</varname> supports compression (btrfs, squashfs, erofs).</para>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><varname>CompressionLevel=</varname></term>
+
+        <listitem><para>Specify the compression level to use for the filesystem configured with
+        <varname>Format=</varname>. Takes a single argument specifying the compression level to use for the
+        configured compression algorithm.</para>
+
+        <para>Note that this setting is only taken into account when the filesystem configured with
+        <varname>Format=</varname> supports compression and the <varname>Compression=</varname> setting is
+        configured explicitly.</para>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

diff --git a/src/home/homework-luks.c b/src/home/homework-luks.c
@@ -2378,6 +2378,8 @@ int home_create_luks(
                             user_record_luks_discard(h),
                             /* quiet = */ true,
                             /* sector_size = */ 0,
+                            /* compression = */ NULL,
+                            /* compression_level= */ 0,
                             extra_mkfs_options);
         if (r < 0)
                 return r;

diff --git a/src/partition/makefs.c b/src/partition/makefs.c
@@ -78,6 +78,8 @@ static int run(int argc, char *argv[]) {
                                /* discard = */ true,
                                /* quiet = */ true,
                                /* sector_size = */ 0,
+                               /* compression = */ NULL,
+                               /* compression_level = */ 0,
                                /* extra_mkfs_options = */ NULL);
 }
 

diff --git a/src/partition/repart.c b/src/partition/repart.c
@@ -312,6 +312,8 @@ typedef struct Partition {
         MinimizeMode minimize;
         uint64_t verity_data_block_size;
         uint64_t verity_hash_block_size;
+        char *compression;
+        uint64_t compression_level;
 
         uint64_t gpt_flags;
         int no_auto;
@@ -471,6 +473,7 @@ static Partition* partition_free(Partition *p) {
         strv_free(p->make_directories);
         strv_free(p->subvolumes);
         free(p->verity_match_key);
+        free(p->compression);
 
         iovec_done(&p->roothash);
 
@@ -506,6 +509,7 @@ static void partition_foreignize(Partition *p) {
         p->make_directories = strv_free(p->make_directories);
         p->subvolumes = strv_free(p->subvolumes);
         p->verity_match_key = mfree(p->verity_match_key);
+        p->compression = mfree(p->compression);
 
         p->priority = 0;
         p->weight = 1000;
@@ -1965,6 +1969,8 @@ static int partition_read_definition(Partition *p, const char *path, const char
                 { "Partition", "VerityHashBlockSizeBytes", config_parse_block_size,        0, &p->verity_hash_block_size  },
                 { "Partition", "MountPoint",               config_parse_mountpoint,        0, p                           },
                 { "Partition", "EncryptedVolume",          config_parse_encrypted_volume,  0, p                           },
+                { "Partition", "Compression",              config_parse_string,            0, &p->compression             },
+                { "Partition", "CompressionLevel",         config_parse_uint64,            0, &p->compression_level       },
                 {}
         };
         int r;
@@ -5256,7 +5262,8 @@ static int context_mkfs(Context *context) {
 
                 r = make_filesystem(partition_target_path(t), p->format, strempty(p->new_label), root,
                                     p->fs_uuid, arg_discard, /* quiet = */ false,
-                                    context->fs_sector_size, extra_mkfs_options);
+                                    context->fs_sector_size, p->compression, p->compression_level,
+                                    extra_mkfs_options);
                 if (r < 0)
                         return r;
 
@@ -6820,6 +6827,8 @@ static int context_minimize(Context *context) {
                                     fs_uuid,
                                     arg_discard, /* quiet = */ false,
                                     context->fs_sector_size,
+                                    p->compression,
+                                    p->compression_level,
                                     extra_mkfs_options);
                 if (r < 0)
                         return r;
@@ -6901,6 +6910,8 @@ static int context_minimize(Context *context) {
                                     arg_discard,
                                     /* quiet = */ false,
                                     context->fs_sector_size,
+                                    p->compression,
+                                    p->compression_level,
                                     extra_mkfs_options);
                 if (r < 0)
                         return r;

diff --git a/src/shared/mkfs-util.c b/src/shared/mkfs-util.c
@@ -323,6 +323,8 @@ int make_filesystem(
                 bool discard,
                 bool quiet,
                 uint64_t sector_size,
+                char *compression,
+                uint64_t compression_level,
                 char * const *extra_mkfs_args) {
 
         _cleanup_free_ char *mkfs = NULL, *mangled_label = NULL;
@@ -456,6 +458,20 @@ int make_filesystem(
                 if (quiet && strv_extend(&argv, "-q") < 0)
                         return log_oom();
 
+                if (compression) {
+                        _cleanup_free_ char *c = NULL;
+
+                        c = strdup(compression);
+                        if (!c)
+                                return log_oom();
+
+                        if (compression_level > 0 && strextendf(&c, ":%" PRIu64, compression_level) < 0)
+                                return log_oom();
+
+                        if (strv_extend_many(&argv, "--compression", c) < 0)
+                                return log_oom();
+                }
+
                 /* mkfs.btrfs unconditionally warns about several settings changing from v5.15 onwards which
                  * isn't silenced by "-q", so let's redirect stdout to /dev/null as well. */
                 if (quiet)
@@ -570,11 +586,25 @@ int make_filesystem(
                                 root, node,
                                 "-noappend");
 
+                if (compression) {
+                        if (strv_extend_many(&argv, "-comp", compression) < 0)
+                                return log_oom();
+
+                        if (compression_level > 0) {
+                                if (strv_extend(&argv, "-Xcompression-level") < 0)
+                                        return log_oom();
+
+                                if (strv_extendf(&argv, "%"PRIu64, compression_level) < 0)
+                                        return log_oom();
+                        }
+                }
+
                 /* mksquashfs -quiet option is pretty new so let's redirect stdout to /dev/null instead. */
                 if (quiet)
                         stdio_fds[1] = -EBADF;
 
         } else if (streq(fstype, "erofs")) {
+                _cleanup_free_ char *c = NULL;
 
                 argv = strv_new(mkfs,
                                 "-U", vol_id,
@@ -583,6 +613,18 @@ int make_filesystem(
                 if (quiet && strv_extend(&argv, "--quiet") < 0)
                         return log_oom();
 
+                if (compression) {
+                        c = strjoin("-z", compression);
+                        if (!c)
+                                return log_oom();
+
+                        if (compression_level > 0 && strextendf(&c, ",level=%"PRIu64, compression_level) < 0)
+                                return log_oom();
+
+                        if (strv_extend(&argv, c) < 0)
+                                return log_oom();
+                }
+
         } else
                 /* Generic fallback for all other file systems */
                 argv = strv_new(mkfs, node);

diff --git a/src/shared/mkfs-util.h b/src/shared/mkfs-util.h
@@ -20,6 +20,8 @@ int make_filesystem(
                 bool discard,
                 bool quiet,
                 uint64_t sector_size,
+                char *compression,
+                uint64_t compression_level,
                 char * const *extra_mkfs_args);
 
 int mkfs_options_from_env(const char *component, const char *fstype, char ***ret);
diff --git a/src/test/test-loop-block.c b/src/test/test-loop-block.c
@@ -251,16 +251,16 @@ static int run(int argc, char *argv[]) {
         assert_se(r >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL, 0, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL, 0, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL, 0, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL, 0, NULL) >= 0);
 
         dissected = dissected_image_unref(dissected);
 

diff --git a/test/units/TEST-58-REPART.sh b/test/units/TEST-58-REPART.sh
@@ -1312,6 +1312,39 @@ testcase_list_devices() {
     systemd-repart --list-devices
 }
 
+testcase_compression() {
+    local workdir image defs
+
+    workdir="$(mktemp --directory "/tmp/test-repart.compression.XXXXXXXXXX")"
+    # shellcheck disable=SC2064
+    trap "rm -rf '${workdir:?}'" RETURN
+
+    image="$workdir/image.img"
+    defs="$workdir/defs"
+    mkdir "$defs"
+
+    # TODO: add btrfs once btrfs-progs v6.11 is available in distributions.
+    for format in squashfs erofs; do
+        if ! command -v "mkfs.$format" >/dev/null && ! command -v "mk$format" >/dev/null; then
+            continue
+        fi
+
+        rm -f "$image"
+
+        tee "$defs/10-root.conf" <<EOF
+[Partition]
+Type=root
+Format=$format
+Compression=zstd
+CompressionLevel=3
+CopyFiles=$defs:/def
+SizeMinBytes=48M
+EOF
+
+        systemd-repart --empty=create --size=auto --pretty=yes --dry-run=no --definitions="$defs" "$image"
+    done
+}
+
 OFFLINE="yes"
 run_testcases