diff --git a/Cargo.toml b/Cargo.toml index 2671d1c..d9420f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,8 +33,10 @@ zip = { version = "0.6.4", optional = true } flate2 = { version = "1.0.25", optional = true } xz2 = { version = "0.1.7", optional = true } toml_edit = { version = "0.19.9", optional = true } +walkdir = "2.3.3" [dev-dependencies] assert_fs = "1" tokio = {version = "1.24", features = ["macros"]} wiremock = "0.5" +clap = { version = "4.3.19", features = ["derive"] } diff --git a/examples/compress.rs b/examples/compress.rs new file mode 100644 index 0000000..e406c57 --- /dev/null +++ b/examples/compress.rs @@ -0,0 +1,57 @@ +//! Example that makes it easy to mess around with the compression backend +//! +//! ```ignore +//! cargo run --example compress --features=compression -- src src.tar.gz --with-root=some/dir +//! ``` +//! +//! ```ignore +//! cargo run --example compress --features=compression -- src src.zip --with-root=some/dir +//! ``` +#![allow(unused_imports)] +#![allow(unused_variables)] + +use axoasset::{AxoassetError, LocalAsset}; +use camino::Utf8PathBuf; +use clap::Parser; + +#[derive(Parser)] +struct Cli { + src_path: Utf8PathBuf, + dest_path: Utf8PathBuf, + #[clap(long)] + with_root: Option, +} + +fn main() { + let args = Cli::parse(); + + doit(args).unwrap() +} + +fn doit(args: Cli) -> Result<(), AxoassetError> { + #[cfg(feature = "compression-tar")] + if args.dest_path.as_str().ends_with("tar.zstd") { + return LocalAsset::tar_zstd_dir(args.src_path, args.dest_path, args.with_root); + } + #[cfg(feature = "compression-tar")] + if args.dest_path.as_str().ends_with("tar.xz") { + return LocalAsset::tar_xz_dir(args.src_path, args.dest_path, args.with_root); + } + #[cfg(feature = "compression-tar")] + if args.dest_path.as_str().ends_with("tar.gz") { + return LocalAsset::tar_gz_dir(args.src_path, args.dest_path, args.with_root); + } + #[cfg(feature = "compression-zip")] + if args.dest_path.as_str().ends_with("zip") { + return LocalAsset::zip_dir(args.src_path, args.dest_path, args.with_root); + } + + if !cfg!(any( + feature = "compression-tar", + feature = "compression-zip" + )) { + panic!("this example must be built with --features=compression") + } else { + panic!("unsupported dest_path extension") + } +} diff --git a/src/compression.rs b/src/compression.rs index 8914fa2..d6cc782 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1,8 +1,6 @@ //! Compression-related methods, all used in `axoasset::Local` -use crate::error::*; use camino::Utf8Path; -use std::fs; /// Internal tar-file compression algorithms #[cfg(feature = "compression-tar")] @@ -20,15 +18,20 @@ pub(crate) enum CompressionImpl { pub(crate) fn tar_dir( src_path: &Utf8Path, dest_path: &Utf8Path, + with_root: Option<&Utf8Path>, compression: &CompressionImpl, -) -> Result<()> { +) -> crate::error::Result<()> { + use crate::error::*; use flate2::{write::ZlibEncoder, Compression, GzBuilder}; + use std::fs; use xz2::write::XzEncoder; // Set up the archive/compression - // The contents of the zip (e.g. a tar) - let dir_name = src_path.file_name().unwrap(); - let zip_contents_name = format!("{dir_name}.tar"); + // dir_name here is a prefix directory/path that the src dir's contents will be stored + // under when being tarred. Having it be empty means the contents + // will be placed in the root of the tarball. + let dir_name = with_root.unwrap_or_else(|| Utf8Path::new("")); + let zip_contents_name = format!("{}.tar", dest_path.file_name().unwrap()); let final_zip_file = match fs::File::create(dest_path) { Ok(file) => file, Err(details) => { @@ -154,78 +157,70 @@ pub(crate) fn tar_dir( } #[cfg(feature = "compression-zip")] -pub(crate) fn zip_dir(src_path: &Utf8Path, dest_path: &Utf8Path) -> Result<()> { - use zip::ZipWriter; - - // Set up the archive/compression - let final_zip_file = match fs::File::create(dest_path) { - Ok(file) => file, - Err(details) => { - return Err(AxoassetError::LocalAssetWriteNewFailed { - dest_path: dest_path.to_string(), - details, - }) - } +pub(crate) fn zip_dir( + src_path: &Utf8Path, + dest_path: &Utf8Path, + with_root: Option<&Utf8Path>, +) -> zip::result::ZipResult<()> { + use std::{ + fs::File, + io::{Read, Write}, }; + use zip::{result::ZipError, write::FileOptions, CompressionMethod}; - // Wrap our file in compression - let mut zip = ZipWriter::new(final_zip_file); + let file = File::create(dest_path)?; - let dir = match std::fs::read_dir(src_path) { - Ok(dir) => dir, - Err(details) => { - return Err(AxoassetError::LocalAssetReadFailed { - origin_path: src_path.to_string(), - details, - }) - } - }; + // The `zip` crate lacks the conveniences of the `tar` crate so we need to manually + // walk through all the subdirs of `src_path` and copy each entry. walkdir streamlines + // that process for us. + let walkdir = walkdir::WalkDir::new(src_path); + let it = walkdir.into_iter(); - for entry in dir { - if let Err(details) = copy_into_zip(entry, &mut zip) { - return Err(AxoassetError::LocalAssetArchive { - reason: format!("failed to create file in zip: {dest_path}"), - details, - }); + let mut zip = zip::ZipWriter::new(file); + let options = FileOptions::default().compression_method(CompressionMethod::STORE); + + // If there's a root prefix, add entries for all of its components + if let Some(root) = with_root { + for path in root.ancestors() { + if !path.as_str().is_empty() { + zip.add_directory(path.as_str(), options)?; + } } } - // Finish up the compression - let _zip_file = match zip.finish() { - Ok(file) => file, - Err(details) => { - return Err(AxoassetError::LocalAssetArchive { - reason: format!("failed to write archive: {dest_path}"), - details: details.into(), - }) + let mut buffer = Vec::new(); + for entry in it.filter_map(|e| e.ok()) { + let path = entry.path(); + // Get the relative path of this file/dir that will be used in the zip + let Some(name) = path + .strip_prefix(src_path) + .ok() + .and_then(Utf8Path::from_path) + else { + return Err(ZipError::UnsupportedArchive("unsupported path format")); + }; + // Optionally apply the root prefix + let name = if let Some(root) = with_root { + root.join(name) + } else { + name.to_owned() + }; + + // Write file or directory explicitly + // Some unzip tools unzip files with directory paths correctly, some do not! + if path.is_file() { + zip.start_file(name, options)?; + let mut f = File::open(path)?; + + f.read_to_end(&mut buffer)?; + zip.write_all(&buffer)?; + buffer.clear(); + } else if !name.as_str().is_empty() { + // Only if not root! Avoids path spec / warning + // and mapname conversion failed error on unzip + zip.add_directory(name, options)?; } - }; - // Drop the file to close it - Ok(()) -} - -/// Copies a file into a provided `ZipWriter`. Mostly factored out so that we can bunch up -/// a bunch of `std::io::Error`s without having to individually handle them. -#[cfg(feature = "compression-zip")] -fn copy_into_zip( - entry: std::result::Result, - zip: &mut zip::ZipWriter, -) -> std::result::Result<(), std::io::Error> { - use std::io::{self, BufReader}; - use zip::{write::FileOptions, CompressionMethod}; - - let entry = entry?; - if entry.file_type()?.is_file() { - let options = FileOptions::default().compression_method(CompressionMethod::Stored); - let file = fs::File::open(entry.path())?; - let mut buf = BufReader::new(file); - let file_name = entry.file_name(); - // FIXME: ...don't do this lossy conversion? - let utf8_file_name = file_name.to_string_lossy(); - zip.start_file(utf8_file_name.clone(), options)?; - io::copy(&mut buf, zip)?; - } else { - todo!("implement zip subdirs! (or was this a symlink?)"); } + zip.finish()?; Ok(()) } diff --git a/src/local.rs b/src/local.rs index d518465..ca064e4 100644 --- a/src/local.rs +++ b/src/local.rs @@ -277,50 +277,85 @@ impl LocalAsset { } /// Creates a new .tar.gz file from a provided directory + /// + /// The with_root argument specifies that all contents of dest_dir should be placed + /// under the given path within the archive. If None then the contents of the dir will + /// be placed directly in the root. root_dir can be a proper path with subdirs + /// (e.g. `root_dir = "some/dir/prefix"` is valid). #[cfg(any(feature = "compression", feature = "compression-tar"))] pub fn tar_gz_dir( origin_dir: impl AsRef, dest_dir: impl AsRef, + with_root: Option>, ) -> Result<()> { crate::compression::tar_dir( Utf8Path::new(origin_dir.as_ref()), Utf8Path::new(dest_dir.as_ref()), + with_root.as_ref().map(|p| p.as_ref()), &crate::compression::CompressionImpl::Gzip, ) } /// Creates a new .tar.xz file from a provided directory + /// + /// The with_root argument specifies that all contents of dest_dir should be placed + /// under the given path within the archive. If None then the contents of the dir will + /// be placed directly in the root. root_dir can be a proper path with subdirs + /// (e.g. `root_dir = "some/dir/prefix"` is valid). #[cfg(any(feature = "compression", feature = "compression-tar"))] pub fn tar_xz_dir( origin_dir: impl AsRef, dest_dir: impl AsRef, + with_root: Option>, ) -> Result<()> { crate::compression::tar_dir( Utf8Path::new(origin_dir.as_ref()), Utf8Path::new(dest_dir.as_ref()), + with_root.as_ref().map(|p| p.as_ref()), &crate::compression::CompressionImpl::Xzip, ) } /// Creates a new .tar.zstd file from a provided directory + /// + /// The with_root argument specifies that all contents of dest_dir should be placed + /// under the given path within the archive. If None then the contents of the dir will + /// be placed directly in the root. root_dir can be a proper path with subdirs + /// (e.g. `root_dir = "some/dir/prefix"` is valid). #[cfg(any(feature = "compression", feature = "compression-tar"))] pub fn tar_zstd_dir( origin_dir: impl AsRef, dest_dir: impl AsRef, + with_root: Option>, ) -> Result<()> { crate::compression::tar_dir( Utf8Path::new(origin_dir.as_ref()), Utf8Path::new(dest_dir.as_ref()), + with_root.as_ref().map(|p| p.as_ref()), &crate::compression::CompressionImpl::Zstd, ) } /// Creates a new .zip file from a provided directory + /// + /// The with_root argument specifies that all contents of dest_dir should be placed + /// under the given path within the archive. If None then the contents of the dir will + /// be placed directly in the root. root_dir can be a proper path with subdirs + /// (e.g. `root_dir = "some/dir/prefix"` is valid). #[cfg(any(feature = "compression", feature = "compression-zip"))] - pub fn zip_dir(origin_dir: impl AsRef, dest_dir: impl AsRef) -> Result<()> { + pub fn zip_dir( + origin_dir: impl AsRef, + dest_dir: impl AsRef, + with_root: Option>, + ) -> Result<()> { crate::compression::zip_dir( Utf8Path::new(origin_dir.as_ref()), Utf8Path::new(dest_dir.as_ref()), + with_root.as_ref().map(|p| p.as_ref()), ) + .map_err(|e| AxoassetError::LocalAssetArchive { + reason: format!("failed to write tar: {}", dest_dir.as_ref()), + details: e.into(), + }) } } diff --git a/tests/source.rs b/tests/source.rs index 1470f1e..a18f798 100644 --- a/tests/source.rs +++ b/tests/source.rs @@ -72,7 +72,7 @@ fn json_invalid() { // Get the span for a non-substring (string literal isn't pointing into the String) let res = source.deserialize_json::(); assert!(res.is_err()); - let Err(AxoassetError::Json{ span: Some(_), .. }) = res else { + let Err(AxoassetError::Json { span: Some(_), .. }) = res else { panic!("span was invalid"); }; } @@ -129,7 +129,7 @@ goodbye = // Get the span for a non-substring (string literal isn't pointing into the String) let res = source.deserialize_toml::(); assert!(res.is_err()); - let Err(AxoassetError::Toml{ span: Some(_), .. }) = res else { + let Err(AxoassetError::Toml { span: Some(_), .. }) = res else { panic!("span was invalid"); }; } @@ -169,7 +169,7 @@ goodbye = // Get the span for a non-substring (string literal isn't pointing into the String) let res = source.deserialize_toml_edit(); assert!(res.is_err()); - let Err(AxoassetError::TomlEdit{ span: Some(_), .. }) = res else { + let Err(AxoassetError::TomlEdit { span: Some(_), .. }) = res else { panic!("span was invalid"); }; }