btrfs-progs: mkfs: add --compression

Documentation/mkfs.btrfs.rst

@@ -212,6 +212,15 @@ OPTIONS
 
                 $ mkfs.btrfs -O list-all
 
+--compress <algo>:<level>
+        Try to compress files when using *--rootdir*.  Supported values for *algo* are
+        *no* (the default), *zlib*, *lzo*, and *zstd*.  The optional value *level* is a
+        compression level, from 1 to 9 for ZLIB and from 1 to 15 for ZSTD.
+
+        As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when
+        they would be larger than the uncompressed versions, and will mark a file as
+        `nocompress` if its beginning is found to be incompressible.
+
 -f|--force
         Forcibly overwrite the block devices when an existing filesystem is detected.
         By default, :command:`mkfs.btrfs` will utilize *libblkid* to check for any known

Makefile

@@ -740,11 +740,11 @@ btrfsck.static: btrfs.static
 
 mkfs.btrfs: $(mkfs_objects) $(objects) libbtrfsutil.a
 	@echo "    [LD]     $@"
-	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBS)
+	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_COMP)
 
 mkfs.btrfs.static: $(static_mkfs_objects) $(static_objects) $(static_libbtrfs_objects)
 	@echo "    [LD]     $@"
-	$(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS)
+	$(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS) $(STATIC_LIBS_COMP)
 
 btrfstune: $(tune_objects) $(objects) libbtrfsutil.a
 	@echo "    [LD]     $@"

common/extent-tree-utils.c

@@ -285,3 +285,93 @@ int btrfs_record_file_extent(struct btrfs_trans_handle *trans,
 	}
 	return ret;
 }
+
+int btrfs_record_file_extent_comp(struct btrfs_trans_handle *trans,
+				  struct btrfs_root *root, u64 objectid,
+				  struct btrfs_inode_item *inode,
+				  u64 file_pos, u64 disk_bytenr,
+				  u64 extent_num_bytes, u64 ram_bytes,
+				  enum btrfs_compression_type comp)
+{
+	struct btrfs_fs_info *info = root->fs_info;
+	struct btrfs_root *extent_root = btrfs_extent_root(info, disk_bytenr);
+	struct extent_buffer *leaf;
+	struct btrfs_file_extent_item *fi;
+	struct btrfs_key ins_key;
+	struct btrfs_path *path;
+	struct btrfs_extent_item *ei;
+	u64 nbytes;
+	u64 extent_bytenr;
+	u64 extent_offset;
+	int ret = 0;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	ins_key.objectid = disk_bytenr;
+	ins_key.type = BTRFS_EXTENT_ITEM_KEY;
+	ins_key.offset = extent_num_bytes;
+
+	ret = btrfs_insert_empty_item(trans, extent_root, path,
+					&ins_key, sizeof(*ei));
+	if (ret == 0) {
+		leaf = path->nodes[0];
+		ei = btrfs_item_ptr(leaf, path->slots[0],
+					struct btrfs_extent_item);
+
+		btrfs_set_extent_refs(leaf, ei, 0);
+		btrfs_set_extent_generation(leaf, ei, trans->transid);
+		btrfs_set_extent_flags(leaf, ei,
+					BTRFS_EXTENT_FLAG_DATA);
+		btrfs_mark_buffer_dirty(leaf);
+
+		ret = btrfs_update_block_group(trans, disk_bytenr,
+					       extent_num_bytes, 1, 0);
+		if (ret)
+			goto fail;
+	} else if (ret != -EEXIST) {
+		goto fail;
+	}
+
+	ret = remove_from_free_space_tree(trans, disk_bytenr, extent_num_bytes);
+	if (ret)
+		goto fail;
+
+	btrfs_run_delayed_refs(trans, -1);
+	extent_bytenr = disk_bytenr;
+	extent_offset = 0;
+
+	btrfs_release_path(path);
+	ins_key.objectid = objectid;
+	ins_key.type = BTRFS_EXTENT_DATA_KEY;
+	ins_key.offset = file_pos;
+	ret = btrfs_insert_empty_item(trans, root, path, &ins_key, sizeof(*fi));
+	if (ret)
+		goto fail;
+	leaf = path->nodes[0];
+	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
+	btrfs_set_file_extent_generation(leaf, fi, trans->transid);
+	btrfs_set_file_extent_type(leaf, fi, BTRFS_FILE_EXTENT_REG);
+	btrfs_set_file_extent_disk_bytenr(leaf, fi, extent_bytenr);
+	btrfs_set_file_extent_disk_num_bytes(leaf, fi, extent_num_bytes);
+	btrfs_set_file_extent_offset(leaf, fi, extent_offset);
+	btrfs_set_file_extent_num_bytes(leaf, fi, ram_bytes);
+	btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
+	btrfs_set_file_extent_compression(leaf, fi, comp);
+	btrfs_set_file_extent_encryption(leaf, fi, 0);
+	btrfs_set_file_extent_other_encoding(leaf, fi, 0);
+	btrfs_mark_buffer_dirty(leaf);
+
+	nbytes = btrfs_stack_inode_nbytes(inode) + ram_bytes;
+	btrfs_set_stack_inode_nbytes(inode, nbytes);
+	btrfs_release_path(path);
+
+	ret = btrfs_inc_extent_ref(trans, extent_bytenr, extent_num_bytes,
+				   0, root->root_key.objectid, objectid,
+				   file_pos - extent_offset);
+
+fail:
+	btrfs_free_path(path);
+	return ret;
+}

common/extent-tree-utils.h

@@ -19,6 +19,7 @@
 
 #include "kerncompat.h"
 #include "kernel-lib/bitops.h"
+#include "kernel-shared/compression.h"
 
 struct btrfs_inode_item;
 struct btrfs_path;
@@ -32,5 +33,11 @@ int btrfs_record_file_extent(struct btrfs_trans_handle *trans,
 			     struct btrfs_inode_item *inode,
 			     u64 file_pos, u64 disk_bytenr,
 			     u64 num_bytes);
+int btrfs_record_file_extent_comp(struct btrfs_trans_handle *trans,
+				  struct btrfs_root *root, u64 objectid,
+				  struct btrfs_inode_item *inode,
+				  u64 file_pos, u64 disk_bytenr,
+				  u64 num_bytes, u64 ram_bytes,
+				  enum btrfs_compression_type comp);
 
 #endif

convert/source-ext2.c

@@ -457,7 +457,8 @@ static int ext2_create_file_extents(struct btrfs_trans_handle *trans,
 		if (num_bytes > inode_size)
 			num_bytes = inode_size;
 		ret = btrfs_insert_inline_extent(trans, root, objectid,
-						 0, buffer, num_bytes);
+						 0, buffer, num_bytes,
+						 0, num_bytes);
 		if (ret)
 			goto fail;
 		nbytes = btrfs_stack_inode_nbytes(btrfs_inode) + num_bytes;
@@ -506,7 +507,8 @@ static int ext2_create_symlink(struct btrfs_trans_handle *trans,
 	pathname = (char *)&(ext2_inode->i_block[0]);
 	BUG_ON(pathname[inode_size] != 0);
 	ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
-					 pathname, inode_size);
+					 pathname, inode_size,
+					 0, inode_size);
 	btrfs_set_stack_inode_nbytes(btrfs_inode, inode_size);
 	return ret;
 }

convert/source-reiserfs.c

@@ -384,7 +384,8 @@ static int reiserfs_convert_tail(struct btrfs_trans_handle *trans,
 				      length, offset, convert_flags);
 
 	ret = btrfs_insert_inline_extent(trans, root, objectid,
-					 offset, body, length);
+					 offset, body, length,
+					 0, length);
 	if (ret)
 		return ret;
 
@@ -544,7 +545,7 @@ static int reiserfs_copy_symlink(struct btrfs_trans_handle *trans,
 		goto fail;
 	}
 	ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
-					 symlink, len);
+					 symlink, len, 0, len);
 	btrfs_set_stack_inode_nbytes(btrfs_inode, len);
 fail:
 	pathrelse(&path);

kernel-shared/file-item.c

@@ -87,7 +87,9 @@ int btrfs_insert_file_extent(struct btrfs_trans_handle *trans,
 
 int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			       struct btrfs_root *root, u64 objectid,
-			       u64 offset, const char *buffer, size_t size)
+			       u64 offset, const char *buffer, size_t size,
+			       enum btrfs_compression_type comp,
+			       u64 ram_bytes)
 {
 	struct btrfs_fs_info *fs_info = trans->fs_info;
 	struct btrfs_key key;
@@ -123,8 +125,8 @@ int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			    struct btrfs_file_extent_item);
 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
 	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
-	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
-	btrfs_set_file_extent_compression(leaf, ei, 0);
+	btrfs_set_file_extent_ram_bytes(leaf, ei, ram_bytes);
+	btrfs_set_file_extent_compression(leaf, ei, comp);
 	btrfs_set_file_extent_encryption(leaf, ei, 0);
 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
 

kernel-shared/file-item.h

@@ -10,6 +10,7 @@
 #include "kernel-shared/ctree.h"
 #include "kernel-shared/uapi/btrfs_tree.h"
 #include "kernel-shared/accessors.h"
+#include "kernel-shared/compression.h"
 
 struct bio;
 struct inode;
@@ -91,7 +92,8 @@ int btrfs_csum_file_block(struct btrfs_trans_handle *trans, u64 logical,
 			  u64 csum_objectid, u32 csum_type, const char *data);
 int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans,
 			       struct btrfs_root *root, u64 objectid,
-			       u64 offset, const char *buffer, size_t size);
+			       u64 offset, const char *buffer, size_t size,
+			       enum btrfs_compression_type comp, u64 ram_bytes);
 /*
  * For symlink we allow up to PATH_MAX - 1 (PATH_MAX includes the terminating NUL,
  * but fs doesn't store that terminating NUL).

mkfs/main.c

@@ -443,6 +443,7 @@ static const char * const mkfs_usage[] = {
 	OPTLINE("-u|--subvol TYPE:SUBDIR", "create SUBDIR as subvolume rather than normal directory, can be specified multiple times"),
 	OPTLINE("--shrink", "(with --rootdir) shrink the filled filesystem to minimal size"),
 	OPTLINE("-K|--nodiscard", "do not perform whole device TRIM"),
+	OPTLINE("--compress ALGO:LEVEL", "compression algorithm and level to use; ALGO can be no (default), zlib, lzo, zstd"),
 	OPTLINE("-f|--force", "force overwrite of existing filesystem"),
 	"General:",
 	OPTLINE("-q|--quiet", "no messages except errors"),
@@ -1058,6 +1059,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 	char *source_dir = NULL;
 	struct rootdir_subvol *rds;
 	bool has_default_subvol = false;
+	enum btrfs_compression_type compression = BTRFS_COMPRESS_NONE;
+	u64 compression_level = 0;
 	LIST_HEAD(subvols);
 
 	cpu_detect_flags();
@@ -1072,6 +1075,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			GETOPT_VAL_CHECKSUM,
 			GETOPT_VAL_GLOBAL_ROOTS,
 			GETOPT_VAL_DEVICE_UUID,
+			GETOPT_VAL_COMPRESS,
 		};
 		static const struct option long_options[] = {
 			{ "byte-count", required_argument, NULL, 'b' },
@@ -1099,6 +1103,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			{ "quiet", 0, NULL, 'q' },
 			{ "verbose", 0, NULL, 'v' },
 			{ "shrink", no_argument, NULL, GETOPT_VAL_SHRINK },
+			{ "compress", required_argument, NULL,
+				GETOPT_VAL_COMPRESS },
 #if EXPERIMENTAL
 			{ "param", required_argument, NULL, GETOPT_VAL_PARAM },
 			{ "num-global-roots", required_argument, NULL, GETOPT_VAL_GLOBAL_ROOTS },
@@ -1272,6 +1278,42 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			case 'q':
 				bconf_be_quiet();
 				break;
+			case GETOPT_VAL_COMPRESS: {
+				char *colon;
+				size_t type_size;
+
+				if (!strcmp(optarg, "no")) {
+					compression = BTRFS_COMPRESS_NONE;
+					break;
+				}
+
+				colon = strstr(optarg, ":");
+
+				if (colon)
+					type_size = colon - optarg;
+				else
+					type_size = strlen(optarg);
+
+				if (!strncmp(optarg, "zlib", type_size)) {
+					compression = BTRFS_COMPRESS_ZLIB;
+				} else if (!strncmp(optarg, "lzo", type_size)) {
+					compression = BTRFS_COMPRESS_LZO;
+				} else if (!strncmp(optarg, "zstd", type_size)) {
+					compression = BTRFS_COMPRESS_ZSTD;
+				} else {
+					error("unrecognized compression type %s",
+					      optarg);
+					ret = 1;
+					goto error;
+				}
+
+				if (colon)
+					compression_level = arg_strtou64(colon + 1);
+				else
+					compression_level = 0;
+
+				break;
+			}
 			case GETOPT_VAL_DEVICE_UUID:
 				strncpy_null(dev_uuid, optarg, BTRFS_UUID_UNPARSED_SIZE);
 				break;
@@ -1953,7 +1995,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		}
 
 		ret = btrfs_mkfs_fill_dir(trans, source_dir, root,
-					  &subvols);
+					  &subvols, compression,
+					  compression_level);
 		if (ret) {
 			error("error while filling filesystem: %d", ret);
 			btrfs_abort_transaction(trans, ret);