Skip to content

Commit

Permalink
repart: Add compression support
Browse files Browse the repository at this point in the history 
Now that mkfs.btrfs is adding support for compressing the generated
filesystem (kdave/btrfs-progs#882), let's
add general support for specifying the compression algorithm and
compression level to use.

We opt to not parse the specified compression algorithm and instead
pass it on as is to the mkfs tool. This has a few benefits:

- We support every compression algorithm supported by every tool
  automatically.
- Users don't need to modify systemd-repart if a mkfs tool learns a
  new compression algorithm in the future
- We don't need to maintain a bunch of tables for filesystem to map
  from our generic compression algorithm enum to the filesystem specific
  names.

We don't add support for btrfs just yet until the corresponding PR
in btrfs-progs is merged.
  • Loading branch information
@DaanDeMeyer
DaanDeMeyer committed Sep 3, 2024
1 parent 6b5d3d2 commit 27cacec
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 38 deletions.
62 changes: 62 additions & 0 deletions man/repart.d.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,68 @@

<xi:include href="version-info.xml" xpointer="v256"/></listitem>
</varlistentry>

<varlistentry>
<term><varname>Compression=</varname></term>

<listitem><para>Specify the compression algorithm to use for the filesystem configured with
<varname>Format=</varname>. Takes a single argument specifying the compression algorithm.</para>

<para>Note that this setting is only taken into account when the filesystem configured with
<varname>Format=</varname> supports compression (btrfs, squashfs, erofs). Here's an incomplete list
of compression algorithms supported by the filesystems known to
<command>systemd-repart</command>:</para>

<table>
<title>File System Compression Algorithms</title>

<tgroup cols='3' align='left' colsep='1' rowsep='1'>
<colspec colname="filesystem" />
<colspec colname="algorithms" />
<colspec colname="manpage" />

<thead>
<row>
<entry>File System</entry>
<entry>Compression Algorithms</entry>
<entry>Documentation</entry>
</row>
</thead>

<tbody>
<row>
<entry><constant>squashfs</constant></entry>
<entry>gzip, lzo, lz4, xz, zstd, lzma</entry>
<entry><member><citerefentry project='man-pages'><refentrytitle>mksquashfs</refentrytitle><manvolnum>1</manvolnum></citerefentry></member></entry>
</row>

<row>
<entry><constant>erofs</constant></entry>
<entry>lz4, lz4hc, lzma, deflate, libdeflate, zstd</entry>
<entry><member><citerefentry project='man-pages'><refentrytitle>mkfs.erofs</refentrytitle><manvolnum>1</manvolnum></citerefentry></member></entry>
</row>
</tbody>
</tgroup>
</table>

<xi:include href="version-info.xml" xpointer="v257"/></listitem>
</varlistentry>

<varlistentry>
<term><varname>CompressionLevel=</varname></term>

<listitem><para>Specify the compression level to use for the filesystem configured with
<varname>Format=</varname>. Takes a single argument specifying the compression level to use for the
configured compression algorithm. The possible compression levels and their meaning are filesystem
specific (refer to the filesystem's documentation for the exact meaning of a particular compression
level).</para>

<para>Note that this setting is only taken into account when the filesystem configured with
<varname>Format=</varname> supports compression and the <varname>Compression=</varname> setting is
configured explicitly.</para>

<xi:include href="version-info.xml" xpointer="v257"/></listitem>
</varlistentry>
</variablelist>
</refsect1>

Expand Down
2 changes: 2 additions & 0 deletions src/home/homework-luks.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2378,6 +2378,8 @@ int home_create_luks(
user_record_luks_discard(h),
/* quiet = */ true,
/* sector_size = */ 0,
/* compression = */ NULL,
/* compression_level= */ NULL,
extra_mkfs_options);
if (r < 0)
return r;
Expand Down
2 changes: 2 additions & 0 deletions src/partition/makefs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ static int run(int argc, char *argv[]) {
/* discard = */ true,
/* quiet = */ true,
/* sector_size = */ 0,
/* compression = */ NULL,
/* compression_level = */ NULL,
/* extra_mkfs_options = */ NULL);
}

Expand Down
79 changes: 46 additions & 33 deletions src/partition/repart.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,8 @@ typedef struct Partition {
MinimizeMode minimize;
uint64_t verity_data_block_size;
uint64_t verity_hash_block_size;
char *compression;
char *compression_level;

uint64_t gpt_flags;
int no_auto;
Expand Down Expand Up @@ -545,6 +547,8 @@ static Partition* partition_free(Partition *p) {
ordered_hashmap_free(p->subvolumes);
free(p->default_subvolume);
free(p->verity_match_key);
free(p->compression);
free(p->compression_level);

iovec_done(&p->roothash);

Expand Down Expand Up @@ -581,6 +585,8 @@ static void partition_foreignize(Partition *p) {
p->subvolumes = ordered_hashmap_free(p->subvolumes);
p->default_subvolume = mfree(p->default_subvolume);
p->verity_match_key = mfree(p->verity_match_key);
p->compression = mfree(p->compression);
p->compression_level = mfree(p->compression_level);

p->priority = 0;
p->weight = 1000;
Expand Down Expand Up @@ -2088,38 +2094,40 @@ static int partition_finalize_fstype(Partition *p, const char *path) {
static int partition_read_definition(Partition *p, const char *path, const char *const *conf_file_dirs) {

ConfigTableItem table[] = {
{ "Partition", "Type", config_parse_type, 0, &p->type },
{ "Partition", "Label", config_parse_label, 0, &p->new_label },
{ "Partition", "UUID", config_parse_uuid, 0, p },
{ "Partition", "Priority", config_parse_int32, 0, &p->priority },
{ "Partition", "Weight", config_parse_weight, 0, &p->weight },
{ "Partition", "PaddingWeight", config_parse_weight, 0, &p->padding_weight },
{ "Partition", "SizeMinBytes", config_parse_size4096, -1, &p->size_min },
{ "Partition", "SizeMaxBytes", config_parse_size4096, 1, &p->size_max },
{ "Partition", "PaddingMinBytes", config_parse_size4096, -1, &p->padding_min },
{ "Partition", "PaddingMaxBytes", config_parse_size4096, 1, &p->padding_max },
{ "Partition", "FactoryReset", config_parse_bool, 0, &p->factory_reset },
{ "Partition", "CopyBlocks", config_parse_copy_blocks, 0, p },
{ "Partition", "Format", config_parse_fstype, 0, &p->format },
{ "Partition", "CopyFiles", config_parse_copy_files, 0, &p->copy_files },
{ "Partition", "ExcludeFiles", config_parse_exclude_files, 0, &p->exclude_files_source },
{ "Partition", "ExcludeFilesTarget", config_parse_exclude_files, 0, &p->exclude_files_target },
{ "Partition", "MakeDirectories", config_parse_make_dirs, 0, &p->make_directories },
{ "Partition", "Encrypt", config_parse_encrypt, 0, &p->encrypt },
{ "Partition", "Verity", config_parse_verity, 0, &p->verity },
{ "Partition", "VerityMatchKey", config_parse_string, 0, &p->verity_match_key },
{ "Partition", "Flags", config_parse_gpt_flags, 0, &p->gpt_flags },
{ "Partition", "ReadOnly", config_parse_tristate, 0, &p->read_only },
{ "Partition", "NoAuto", config_parse_tristate, 0, &p->no_auto },
{ "Partition", "GrowFileSystem", config_parse_tristate, 0, &p->growfs },
{ "Partition", "SplitName", config_parse_string, 0, &p->split_name_format },
{ "Partition", "Minimize", config_parse_minimize, 0, &p->minimize },
{ "Partition", "Subvolumes", config_parse_subvolumes, 0, &p->subvolumes },
{ "Partition", "DefaultSubvolume", config_parse_default_subvolume, 0, &p->default_subvolume },
{ "Partition", "VerityDataBlockSizeBytes", config_parse_block_size, 0, &p->verity_data_block_size },
{ "Partition", "VerityHashBlockSizeBytes", config_parse_block_size, 0, &p->verity_hash_block_size },
{ "Partition", "MountPoint", config_parse_mountpoint, 0, p },
{ "Partition", "EncryptedVolume", config_parse_encrypted_volume, 0, p },
{ "Partition", "Type", config_parse_type, 0, &p->type },
{ "Partition", "Label", config_parse_label, 0, &p->new_label },
{ "Partition", "UUID", config_parse_uuid, 0, p },
{ "Partition", "Priority", config_parse_int32, 0, &p->priority },
{ "Partition", "Weight", config_parse_weight, 0, &p->weight },
{ "Partition", "PaddingWeight", config_parse_weight, 0, &p->padding_weight },
{ "Partition", "SizeMinBytes", config_parse_size4096, -1, &p->size_min },
{ "Partition", "SizeMaxBytes", config_parse_size4096, 1, &p->size_max },
{ "Partition", "PaddingMinBytes", config_parse_size4096, -1, &p->padding_min },
{ "Partition", "PaddingMaxBytes", config_parse_size4096, 1, &p->padding_max },
{ "Partition", "FactoryReset", config_parse_bool, 0, &p->factory_reset },
{ "Partition", "CopyBlocks", config_parse_copy_blocks, 0, p },
{ "Partition", "Format", config_parse_fstype, 0, &p->format },
{ "Partition", "CopyFiles", config_parse_copy_files, 0, &p->copy_files },
{ "Partition", "ExcludeFiles", config_parse_exclude_files, 0, &p->exclude_files_source },
{ "Partition", "ExcludeFilesTarget", config_parse_exclude_files, 0, &p->exclude_files_target },
{ "Partition", "MakeDirectories", config_parse_make_dirs, 0, &p->make_directories },
{ "Partition", "Encrypt", config_parse_encrypt, 0, &p->encrypt },
{ "Partition", "Verity", config_parse_verity, 0, &p->verity },
{ "Partition", "VerityMatchKey", config_parse_string, 0, &p->verity_match_key },
{ "Partition", "Flags", config_parse_gpt_flags, 0, &p->gpt_flags },
{ "Partition", "ReadOnly", config_parse_tristate, 0, &p->read_only },
{ "Partition", "NoAuto", config_parse_tristate, 0, &p->no_auto },
{ "Partition", "GrowFileSystem", config_parse_tristate, 0, &p->growfs },
{ "Partition", "SplitName", config_parse_string, 0, &p->split_name_format },
{ "Partition", "Minimize", config_parse_minimize, 0, &p->minimize },
{ "Partition", "Subvolumes", config_parse_subvolumes, 0, &p->subvolumes },
{ "Partition", "DefaultSubvolume", config_parse_default_subvolume, 0, &p->default_subvolume },
{ "Partition", "VerityDataBlockSizeBytes", config_parse_block_size, 0, &p->verity_data_block_size },
{ "Partition", "VerityHashBlockSizeBytes", config_parse_block_size, 0, &p->verity_hash_block_size },
{ "Partition", "MountPoint", config_parse_mountpoint, 0, p },
{ "Partition", "EncryptedVolume", config_parse_encrypted_volume, 0, p },
{ "Partition", "Compression", config_parse_string, CONFIG_PARSE_STRING_SAFE_AND_ASCII, &p->compression },
{ "Partition", "CompressionLevel", config_parse_string, CONFIG_PARSE_STRING_SAFE_AND_ASCII, &p->compression_level },
{}
};
_cleanup_free_ char *filename = NULL;
Expand Down Expand Up @@ -5469,7 +5477,8 @@ static int context_mkfs(Context *context) {

r = make_filesystem(partition_target_path(t), p->format, strempty(p->new_label), root,
p->fs_uuid, arg_discard, /* quiet = */ false,
context->fs_sector_size, extra_mkfs_options);
context->fs_sector_size, p->compression, p->compression_level,
extra_mkfs_options);
if (r < 0)
return r;

Expand Down Expand Up @@ -7033,6 +7042,8 @@ static int context_minimize(Context *context) {
fs_uuid,
arg_discard, /* quiet = */ false,
context->fs_sector_size,
p->compression,
p->compression_level,
extra_mkfs_options);
if (r < 0)
return r;
Expand Down Expand Up @@ -7114,6 +7125,8 @@ static int context_minimize(Context *context) {
arg_discard,
/* quiet = */ false,
context->fs_sector_size,
p->compression,
p->compression_level,
extra_mkfs_options);
if (r < 0)
return r;
Expand Down
25 changes: 24 additions & 1 deletion src/shared/mkfs-util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ int make_filesystem(
bool discard,
bool quiet,
uint64_t sector_size,
char *compression,
char *compression_level,
char * const *extra_mkfs_args) {

_cleanup_free_ char *mkfs = NULL, *mangled_label = NULL;
Expand Down Expand Up @@ -570,19 +572,40 @@ int make_filesystem(
root, node,
"-noappend");

if (compression) {
if (strv_extend_many(&argv, "-comp", compression) < 0)
return log_oom();

if (compression_level && strv_extend_many(&argv, "-Xcompression-level", compression_level) < 0)
return log_oom();
}

/* mksquashfs -quiet option is pretty new so let's redirect stdout to /dev/null instead. */
if (quiet)
stdio_fds[1] = -EBADF;

} else if (streq(fstype, "erofs")) {

argv = strv_new(mkfs,
"-U", vol_id,
node, root);

if (quiet && strv_extend(&argv, "--quiet") < 0)
return log_oom();

if (compression) {
_cleanup_free_ char *c = NULL;

c = strjoin("-z", compression);
if (!c)
return log_oom();

if (compression_level && !strextend(&c, ",level=", compression_level))
return log_oom();

if (strv_extend(&argv, c) < 0)
return log_oom();
}

} else
/* Generic fallback for all other file systems */
argv = strv_new(mkfs, node);
Expand Down
2 changes: 2 additions & 0 deletions src/shared/mkfs-util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ int make_filesystem(
bool discard,
bool quiet,
uint64_t sector_size,
char *compression,
char *compression_level,
char * const *extra_mkfs_args);

int mkfs_options_from_env(const char *component, const char *fstype, char ***ret);
8 changes: 4 additions & 4 deletions src/test/test-loop-block.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,16 +251,16 @@ static int run(int argc, char *argv[]) {
assert_se(r >= 0);

assert_se(sd_id128_randomize(&id) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);

assert_se(sd_id128_randomize(&id) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);

assert_se(sd_id128_randomize(&id) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);

assert_se(sd_id128_randomize(&id) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL) >= 0);
assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);

dissected = dissected_image_unref(dissected);

Expand Down
Loading

0 comments on commit 27cacec

Please sign in to comment.