btrfs-progs: mkfs: add --compression option

maharmstone · maharmstone · commit 802c51b64973 · 2024-09-03T11:18:31.000+01:00
Adds an option --compression to mkfs.btrfs, to allow creating files
using zstd when using --rootdir.

Signed-off-by: Mark Harmstone &lt;maharmstone@fb.com&gt;
diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst
@@ -188,6 +188,12 @@ OPTIONS
 
                 $ mkfs.btrfs -O list-all
 
+--compression <algo>
+        Try to compress files when using *--rootdir*.  Supported values for *algo* are
+        *none* (the default) and *zstd*.  As with the kernel, :command:`mkfs.btrfs` won't
+        write compressed extents when they would be larger than the uncompressed versions,
+        and will mark a file as `nocompress` if its beginning is found to be incompressible.
+
 -f|--force
         Forcibly overwrite the block devices when an existing filesystem is detected.
         By default, :command:`mkfs.btrfs` will utilize *libblkid* to check for any known
diff --git a/Makefile b/Makefile
@@ -740,11 +740,11 @@ btrfsck.static: btrfs.static
 
 mkfs.btrfs: $(mkfs_objects) $(objects) libbtrfsutil.a
 	@echo "    [LD]     $@"
-	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBS)
+	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_COMP)
 
 mkfs.btrfs.static: $(static_mkfs_objects) $(static_objects) $(static_libbtrfs_objects)
 	@echo "    [LD]     $@"
-	$(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS)
+	$(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS) $(STATIC_LIBS_COMP)
 
 btrfstune: $(tune_objects) $(objects) libbtrfsutil.a
 	@echo "    [LD]     $@"
diff --git a/common/extent-tree-utils.c b/common/extent-tree-utils.c
@@ -285,3 +285,93 @@ int btrfs_record_file_extent(struct btrfs_trans_handle *trans,
 	}
 	return ret;
 }
+
+int btrfs_record_file_extent_comp(struct btrfs_trans_handle *trans,
+				  struct btrfs_root *root, u64 objectid,
+				  struct btrfs_inode_item *inode,
+				  u64 file_pos, u64 disk_bytenr,
+				  u64 extent_num_bytes, u64 ram_bytes,
+				  enum btrfs_compression_type comp)
+{
+	struct btrfs_fs_info *info = root->fs_info;
+	struct btrfs_root *extent_root = btrfs_extent_root(info, disk_bytenr);
+	struct extent_buffer *leaf;
+	struct btrfs_file_extent_item *fi;
+	struct btrfs_key ins_key;
+	struct btrfs_path *path;
+	struct btrfs_extent_item *ei;
+	u64 nbytes;
+	u64 extent_bytenr;
+	u64 extent_offset;
+	int ret = 0;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	ins_key.objectid = disk_bytenr;
+	ins_key.type = BTRFS_EXTENT_ITEM_KEY;
+	ins_key.offset = extent_num_bytes;
+
+	ret = btrfs_insert_empty_item(trans, extent_root, path,
+					&ins_key, sizeof(*ei));
+	if (ret == 0) {
+		leaf = path->nodes[0];
+		ei = btrfs_item_ptr(leaf, path->slots[0],
+					struct btrfs_extent_item);
+
+		btrfs_set_extent_refs(leaf, ei, 0);
+		btrfs_set_extent_generation(leaf, ei, trans->transid);
+		btrfs_set_extent_flags(leaf, ei,
+					BTRFS_EXTENT_FLAG_DATA);
+		btrfs_mark_buffer_dirty(leaf);
+
+		ret = btrfs_update_block_group(trans, disk_bytenr,
+					       extent_num_bytes, 1, 0);
+		if (ret)
+			goto fail;
+	} else if (ret != -EEXIST) {
+		goto fail;
+	}
+
+	ret = remove_from_free_space_tree(trans, disk_bytenr, extent_num_bytes);
+	if (ret)
+		goto fail;
+
+	btrfs_run_delayed_refs(trans, -1);
+	extent_bytenr = disk_bytenr;
+	extent_offset = 0;
+
+	btrfs_release_path(path);
+	ins_key.objectid = objectid;
+	ins_key.type = BTRFS_EXTENT_DATA_KEY;
+	ins_key.offset = file_pos;
+	ret = btrfs_insert_empty_item(trans, root, path, &ins_key, sizeof(*fi));
+	if (ret)
+		goto fail;
+	leaf = path->nodes[0];
+	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
+	btrfs_set_file_extent_generation(leaf, fi, trans->transid);
+	btrfs_set_file_extent_type(leaf, fi, BTRFS_FILE_EXTENT_REG);
+	btrfs_set_file_extent_disk_bytenr(leaf, fi, extent_bytenr);
+	btrfs_set_file_extent_disk_num_bytes(leaf, fi, extent_num_bytes);
+	btrfs_set_file_extent_offset(leaf, fi, extent_offset);
+	btrfs_set_file_extent_num_bytes(leaf, fi, ram_bytes);
+	btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
+	btrfs_set_file_extent_compression(leaf, fi, comp);
+	btrfs_set_file_extent_encryption(leaf, fi, 0);
+	btrfs_set_file_extent_other_encoding(leaf, fi, 0);
+	btrfs_mark_buffer_dirty(leaf);
+
+	nbytes = btrfs_stack_inode_nbytes(inode) + ram_bytes;
+	btrfs_set_stack_inode_nbytes(inode, nbytes);
+	btrfs_release_path(path);
+
+	ret = btrfs_inc_extent_ref(trans, extent_bytenr, extent_num_bytes,
+				   0, root->root_key.objectid, objectid,
+				   file_pos - extent_offset);
+
+fail:
+	btrfs_free_path(path);
+	return ret;
+}
diff --git a/common/extent-tree-utils.h b/common/extent-tree-utils.h
@@ -19,6 +19,7 @@
 
 #include "kerncompat.h"
 #include "kernel-lib/bitops.h"
+#include "kernel-shared/compression.h"
 
 struct btrfs_inode_item;
 struct btrfs_path;
@@ -32,5 +33,11 @@ int btrfs_record_file_extent(struct btrfs_trans_handle *trans,
 			     struct btrfs_inode_item *inode,
 			     u64 file_pos, u64 disk_bytenr,
 			     u64 num_bytes);
+int btrfs_record_file_extent_comp(struct btrfs_trans_handle *trans,
+				  struct btrfs_root *root, u64 objectid,
+				  struct btrfs_inode_item *inode,
+				  u64 file_pos, u64 disk_bytenr,
+				  u64 num_bytes, u64 ram_bytes,
+				  enum btrfs_compression_type comp);
 
 #endif
diff --git a/convert/source-ext2.c b/convert/source-ext2.c
@@ -445,7 +445,8 @@ static int ext2_create_file_extents(struct btrfs_trans_handle *trans,
 		if (num_bytes > inode_size)
 			num_bytes = inode_size;
 		ret = btrfs_insert_inline_extent(trans, root, objectid,
-						 0, buffer, num_bytes);
+						 0, buffer, num_bytes,
+						 0, num_bytes);
 		if (ret)
 			goto fail;
 		nbytes = btrfs_stack_inode_nbytes(btrfs_inode) + num_bytes;
@@ -489,7 +490,8 @@ static int ext2_create_symlink(struct btrfs_trans_handle *trans,
 	pathname = (char *)&(ext2_inode->i_block[0]);
 	BUG_ON(pathname[inode_size] != 0);
 	ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
-					 pathname, inode_size + 1);
+					 pathname, inode_size + 1,
+					 0, inode_size + 1);
 	btrfs_set_stack_inode_nbytes(btrfs_inode, inode_size + 1);
 	return ret;
 }
diff --git a/convert/source-reiserfs.c b/convert/source-reiserfs.c
@@ -384,7 +384,8 @@ static int reiserfs_convert_tail(struct btrfs_trans_handle *trans,
 				      length, offset, convert_flags);
 
 	ret = btrfs_insert_inline_extent(trans, root, objectid,
-					 offset, body, length);
+					 offset, body, length,
+					 0, length);
 	if (ret)
 		return ret;
 
@@ -538,7 +539,7 @@ static int reiserfs_copy_symlink(struct btrfs_trans_handle *trans,
 	len = get_ih_item_len(tp_item_head(&path));
 
 	ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
-					 symlink, len + 1);
+					 symlink, len + 1, 0, len + 1);
 	btrfs_set_stack_inode_nbytes(btrfs_inode, len + 1);
 fail:
 	pathrelse(&path);
diff --git a/kernel-shared/file-item.c b/kernel-shared/file-item.c
@@ -26,6 +26,7 @@
 #include "kernel-shared/extent_io.h"
 #include "kernel-shared/uapi/btrfs.h"
 #include "common/internal.h"
+#include "common/messages.h"
 
 #define MAX_CSUM_ITEMS(r, size) ((((BTRFS_LEAF_DATA_SIZE(r->fs_info) - \
 			       sizeof(struct btrfs_item) * 2) / \
@@ -86,7 +87,9 @@ int btrfs_insert_file_extent(struct btrfs_trans_handle *trans,
 
 int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			       struct btrfs_root *root, u64 objectid,
-			       u64 offset, const char *buffer, size_t size)
+			       u64 offset, const char *buffer, size_t size,
+			       enum btrfs_compression_type comp,
+			       u64 ram_bytes)
 {
 	struct btrfs_key key;
 	struct btrfs_path *path;
@@ -117,8 +120,8 @@ int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			    struct btrfs_file_extent_item);
 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
 	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
-	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
-	btrfs_set_file_extent_compression(leaf, ei, 0);
+	btrfs_set_file_extent_ram_bytes(leaf, ei, ram_bytes);
+	btrfs_set_file_extent_compression(leaf, ei, comp);
 	btrfs_set_file_extent_encryption(leaf, ei, 0);
 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
 
diff --git a/kernel-shared/file-item.h b/kernel-shared/file-item.h
@@ -10,6 +10,7 @@
 #include "kernel-shared/ctree.h"
 #include "kernel-shared/uapi/btrfs_tree.h"
 #include "kernel-shared/accessors.h"
+#include "kernel-shared/compression.h"
 
 struct bio;
 struct inode;
@@ -91,6 +92,7 @@ int btrfs_csum_file_block(struct btrfs_trans_handle *trans, u64 logical,
 			  u64 csum_objectid, u32 csum_type, const char *data);
 int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			       struct btrfs_root *root, u64 objectid,
-			       u64 offset, const char *buffer, size_t size);
+			       u64 offset, const char *buffer, size_t size,
+			       enum btrfs_compression_type comp, u64 ram_bytes);
 
 #endif
diff --git a/mkfs/main.c b/mkfs/main.c
@@ -443,6 +443,7 @@ static const char * const mkfs_usage[] = {
 	OPTLINE("-u|--subvol SUBDIR:FLAGS", "create SUBDIR as subvolume rather than normal directory, can be specified multiple times"),
 	OPTLINE("--shrink", "(with --rootdir) shrink the filled filesystem to minimal size"),
 	OPTLINE("-K|--nodiscard", "do not perform whole device TRIM"),
+	OPTLINE("--compression ALGO", "compression algorithm to use, none (default), zstd"),
 	OPTLINE("-f|--force", "force overwrite of existing filesystem"),
 	"General:",
 	OPTLINE("-q|--quiet", "no messages except errors"),
@@ -1101,6 +1102,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 	size_t source_dir_len = 0;
 	struct rootdir_subvol *rds;
 	bool has_default_subvol = false;
+	bool compression = false;
 	LIST_HEAD(subvols);
 
 	cpu_detect_flags();
@@ -1115,6 +1117,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			GETOPT_VAL_CHECKSUM,
 			GETOPT_VAL_GLOBAL_ROOTS,
 			GETOPT_VAL_DEVICE_UUID,
+			GETOPT_VAL_COMPRESSION,
 		};
 		static const struct option long_options[] = {
 			{ "byte-count", required_argument, NULL, 'b' },
@@ -1142,6 +1145,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			{ "quiet", 0, NULL, 'q' },
 			{ "verbose", 0, NULL, 'v' },
 			{ "shrink", no_argument, NULL, GETOPT_VAL_SHRINK },
+			{ "compression", required_argument, NULL,
+				GETOPT_VAL_COMPRESSION },
 #if EXPERIMENTAL
 			{ "param", required_argument, NULL, GETOPT_VAL_PARAM },
 			{ "num-global-roots", required_argument, NULL, GETOPT_VAL_GLOBAL_ROOTS },
@@ -1314,6 +1319,23 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			case 'q':
 				bconf_be_quiet();
 				break;
+			case GETOPT_VAL_COMPRESSION:
+				if (!strcmp(optarg, "none")) {
+					compression = false;
+				} else if (!strcmp(optarg, "zstd")) {
+#if COMPRESSION_ZSTD
+					compression = true;
+#else
+					error("zstd support not compiled in");
+					ret = 1;
+					goto error;
+#endif
+				} else {
+					error("unrecognized compression type %s", optarg);
+					ret = 1;
+					goto error;
+				}
+				break;
 			case GETOPT_VAL_DEVICE_UUID:
 				strncpy_null(dev_uuid, optarg, BTRFS_UUID_UNPARSED_SIZE);
 				break;
@@ -2014,7 +2036,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		}
 
 		ret = btrfs_mkfs_fill_dir(trans, source_dir, root,
-					  &subvols);
+					  &subvols, compression);
 		if (ret) {
 			error("error while filling filesystem: %d", ret);
 			btrfs_abort_transaction(trans, ret);
diff --git a/mkfs/rootdir.c b/mkfs/rootdir.c
diff --git a/mkfs/rootdir.h b/mkfs/rootdir.h
