diff --git a/UtocEmulator/UTOC.Stream.Emulator/ModConfig.json b/UtocEmulator/UTOC.Stream.Emulator/ModConfig.json index 870bac5..a0ec376 100644 --- a/UtocEmulator/UTOC.Stream.Emulator/ModConfig.json +++ b/UtocEmulator/UTOC.Stream.Emulator/ModConfig.json @@ -2,7 +2,7 @@ "ModId": "UTOC.Stream.Emulator", "ModName": "IO Store Emulator", "ModAuthor": "Rirurin", - "ModVersion": "1.0.4", + "ModVersion": "1.0.5", "ModDescription": "Simulates Unreal Engine IO Store files (.utoc \u002B .ucas). Allows mods to add or replace existing files using files on disk", "ModDll": "UTOC.Stream.Emulator.dll", "ModIcon": "Preview.png", diff --git a/UtocEmulator/fileemu-utoc-stream-emulator/Cargo.toml b/UtocEmulator/fileemu-utoc-stream-emulator/Cargo.toml index 7eefede..477b8d9 100644 --- a/UtocEmulator/fileemu-utoc-stream-emulator/Cargo.toml +++ b/UtocEmulator/fileemu-utoc-stream-emulator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fileemu-utoc-stream-emulator" -version = "1.0.4" +version = "1.0.5" edition = "2021" authors = [ "Rirurin" ] description = "Simulates Unreal Engine IO Store files. Allows mods to add or replace existing files using files on disk." diff --git a/UtocEmulator/fileemu-utoc-stream-emulator/src/asset_collector.rs b/UtocEmulator/fileemu-utoc-stream-emulator/src/asset_collector.rs index 551d1f7..60432ad 100644 --- a/UtocEmulator/fileemu-utoc-stream-emulator/src/asset_collector.rs +++ b/UtocEmulator/fileemu-utoc-stream-emulator/src/asset_collector.rs @@ -295,7 +295,7 @@ pub fn add_from_folders_inner(parent: TocDirectorySyncRef, os_path: &PathBuf, pr if *utoc_meta_lock == None { *utoc_meta_lock = Some(UtocMetadata::new()); } - utoc_meta_lock.as_mut().unwrap().add_entries(fs::read(fs_obj.path()).unwrap()); + utoc_meta_lock.as_mut().unwrap().add_entries::(fs::read(fs_obj.path()).unwrap()); } else { profiler.add_skipped_file(fs_obj.path().to_str().unwrap(), format!("No file extension"), file_size); } diff --git a/UtocEmulator/fileemu-utoc-stream-emulator/src/io_package.rs b/UtocEmulator/fileemu-utoc-stream-emulator/src/io_package.rs index 5025185..af3a15c 100644 --- a/UtocEmulator/fileemu-utoc-stream-emulator/src/io_package.rs +++ b/UtocEmulator/fileemu-utoc-stream-emulator/src/io_package.rs @@ -11,6 +11,7 @@ type ExportFilterFlags = u8; // and this one too... use byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt}; use crate::{ + metadata::{UtocMetadata, UtocMetaImportType}, pak_package::{FObjectImport, FObjectExport, GameName, NameMap}, string::{ FMappedName, FStringDeserializerText, FString16, Hasher16 }, toc_factory::{TocResolverCommon, TocResolverType2} @@ -19,7 +20,8 @@ use std::{ error::Error, fs::File, fmt, - io::{BufReader, Cursor, ErrorKind, Read, Seek, SeekFrom, Write} + io::{BufReader, Cursor, ErrorKind, Read, Seek, SeekFrom, Write}, + sync::MutexGuard }; // IoStoreObjectIndex is a 64 bit value consisting of a hash of a target string for the lower 62 bits and an object type for the highest 2 // expect for Empty which represents a null value and Export which contains an index to another item on the export tree @@ -423,13 +425,53 @@ pub struct ContainerHeaderPackage { } impl ContainerHeaderPackage { + // Set all graph package imports as container summary imports. This behavior will likely be kept to + // maintain compatibility with older P3RE mods since validation does break some other assets + fn imports_from_graph_packges_unvalidated(graph_packages: &Vec) -> Vec { + let mut import_ids = vec![]; + for i in graph_packages { + import_ids.push(i.imported_package_id); + } + import_ids + } + // From graph package imports, but with validating the name entries for matching file names + // Not all files actually have the same graph package imports as container header imports (Unreal shenanigans) + // However, there's an edge case with files ending in an underscore + number, where Unreal won't serialize that + // last portion of the string. I'd need an external tool to fix this + fn import_from_graph_packages_validated + (reader: &mut TReader, summary: &PackageSummaryExports, graph_packages: &Vec) -> Vec { + let mut import_ids = vec![]; + reader.seek(SeekFrom::Start(summary.name_offset as u64)).unwrap(); + let mut names = vec![]; + for i in 0..summary.name_count { + names.push(FString16::from_buffer_text::(reader).unwrap().unwrap()); + } + let mut path_name_hashes = vec![]; + loop { // we only want to hash file paths, which Unreal always serializes at the beginning + if path_name_hashes.len() == names.len() || !names[path_name_hashes.len()].starts_with("/") { + break; + } + path_name_hashes.push(Hasher16::get_cityhash64(&names[path_name_hashes.len()])); + } + for i in graph_packages { + if path_name_hashes.contains(&i.imported_package_id) { + import_ids.push(i.imported_package_id); + } + } + import_ids + } + // If required, import ids can be manually specified from the metadata file. Trying to generate a + // UCAS file with no external metadata was always going to be a challenge + fn import_from_metadata_file(meta_guard: &mut MutexGuard>) -> Vec { + vec![] + } // Parse the package file to extract the values needed to build a store entry in the container header pub fn from_package_summary< TExportBundle: ExportBundle, TSummary: PackageIoSummaryDeserialize, TReader: Read + Seek, TByteOrder: byteorder::ByteOrder - >(file_reader: &mut TReader, hash: u64, size: u64, path: &str) -> Self { // consume the file object, we're only going to need it in here + >(file_reader: &mut TReader, hash: u64, size: u64, path: &str, meta_guard: &mut MutexGuard>) -> Self { // consume the file object, we're only going to need it in here let package_summary = TSummary::to_package_summary::(file_reader).unwrap(); let export_count = package_summary.get_export_count() as u32; file_reader.seek(SeekFrom::Start(package_summary.export_bundle_offset as u64)).unwrap(); // jump to FExportBundleHeader start @@ -438,35 +480,21 @@ impl ContainerHeaderPackage { ); // Go through each export bundle to look for the highest index file_reader.seek(SeekFrom::Start(package_summary.graph_offset as u64)).unwrap(); // go to FGraphPackage (imported_packages_count) let graph_packages = FGraphPackage::list_from_buffer::(file_reader); - // previously, utoc emulator only relied on obtaining it's container header import ids from the package's graph package ids (which in itself was a bit of a hack) - // however, this causes issues in regards to localized data, since graph package also includes ids for localization data not included in the container header - // this causes a lot of weird behaviour. This hack involves reading the first file entries (Unreal always serializes asset file paths first, followed by script paths) - // and then verifying that it's hash is within the graph package hashes. if both conditions are met, then it's allowed to be added as an import - let mut import_ids = Vec::with_capacity(graph_packages.len()); - if export_bundle_count == 1 { - for i in &graph_packages { - import_ids.push(i.imported_package_id); - } - } else { - file_reader.seek(SeekFrom::Start(package_summary.name_offset as u64)).unwrap(); - let mut names = vec![]; - for i in 0..package_summary.name_count { - names.push(FString16::from_buffer_text::(file_reader).unwrap().unwrap()); - } - let mut path_name_hashes = vec![]; - loop { // we only want to hash file paths - if path_name_hashes.len() == names.len() || !names[path_name_hashes.len()].starts_with("/") { - break; + let import_ids = match meta_guard.as_ref() { + Some(m) => { + match m.get_import_type(hash) { + UtocMetaImportType::GraphPackageValidated => { + //println!("VALIDATE PACKAGE {} ({:X})", path, hash); + ContainerHeaderPackage::import_from_graph_packages_validated::(file_reader, &package_summary, &graph_packages) + }, + UtocMetaImportType::Manual => ContainerHeaderPackage::import_from_metadata_file(meta_guard), + UtocMetaImportType::GraphPackageUnvalidated => ContainerHeaderPackage::imports_from_graph_packges_unvalidated(&graph_packages) } - path_name_hashes.push(Hasher16::get_cityhash64(&names[path_name_hashes.len()])); - } - for i in &graph_packages { - if path_name_hashes.contains(&i.imported_package_id) { - import_ids.push(i.imported_package_id); - } - } - } - //println!("ASSET {}, {} imports, {} export bundles", path, import_ids.len(), export_bundle_count); + }, + None => ContainerHeaderPackage::imports_from_graph_packges_unvalidated(&graph_packages) + }; + //println!("ASSET {} ({:X}), {} imports, {} export bundles, {} exports", path, hash, import_ids.len(), export_bundle_count, export_count); + let load_order = 0; // This doesn't seem to matter? Self { hash, @@ -611,9 +639,11 @@ mod tests { let os_file = File::open(path).unwrap(); let file_size = Metadata::get_file_size(&os_file); let mut os_reader = BufReader::new(os_file); + /* ContainerHeaderPackage::from_package_summary::< ExportBundleHeader4, PackageSummary2, BufReader, NativeEndian >(&mut os_reader, 0, file_size, &format!("aa")); + */ } #[test] diff --git a/UtocEmulator/fileemu-utoc-stream-emulator/src/metadata.rs b/UtocEmulator/fileemu-utoc-stream-emulator/src/metadata.rs index 0d9c920..9d6c192 100644 --- a/UtocEmulator/fileemu-utoc-stream-emulator/src/metadata.rs +++ b/UtocEmulator/fileemu-utoc-stream-emulator/src/metadata.rs @@ -1,7 +1,7 @@ use bitflags::bitflags; use byteorder::{LittleEndian, ReadBytesExt}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, io::{Cursor, Seek, SeekFrom}, mem, sync::Mutex @@ -21,28 +21,81 @@ bitflags! { pub static UTOC_METADATA: Mutex> = Mutex::new(None); // .utocmeta structure: -// count: u64 -// chunk_hashes: [u64; length] -// flags: [u8; length] +// version: u32 @ 0x0 +// alt_auto_import_count: u32 @ 0x4 +// manual_import_count: u32 @ 0x8 +// compressed_package_count: u32 @ 0xc +// alt_import_assets: [u64; length] @ alt_auto_imports_offset +// manual_import_assets @ manual_imports_offset +// asset_hash: u64, +// count: u64, +// imports: [u64; count] +// compressed_assets: [u64; length] @ compressed_package_offset +// compressed_asset_flags: [u8; length] @ compressed_package_offset + compressed_assets #[derive(Debug, PartialEq)] pub struct UtocMetadata { - chunk_flags: HashMap + // version: u32, + //alt_auto_imports_offset: u32, + //manual_imports_offset: u32, + //compressed_package_offset: u32, + alt_import_assets: HashSet, + manual_import_assets: HashMap>, + compressed_assets: HashMap +} + +pub enum UtocMetaImportType { + GraphPackageUnvalidated, + GraphPackageValidated, + Manual } impl UtocMetadata { pub fn new() -> Self { - Self { chunk_flags: HashMap::new() } + Self { + alt_import_assets: HashSet::new(), + manual_import_assets: HashMap::new(), + compressed_assets: HashMap::new() + } } - pub fn add_entries(&mut self, data: Vec) { + pub fn add_entries(&mut self, data: Vec) { let mut metadata_reader = Cursor::new(data); - let val_count = metadata_reader.read_u64::().unwrap(); - let flags_offset = mem::size_of::() as u64 + mem::size_of::() as u64 * val_count; - for i in 0..val_count { - metadata_reader.seek(SeekFrom::Start(mem::size_of::() as u64 + 0xc * i)); - let curr_hash = metadata_reader.read_u64::().unwrap(); - metadata_reader.seek(SeekFrom::Start(flags_offset + i)); - let chunk_flag = UtocMetadataFlags::from_bits_truncate(metadata_reader.read_u8().unwrap()); - self.chunk_flags.insert(curr_hash, chunk_flag); + // read header + let version = metadata_reader.read_u32::().unwrap(); + let alt_auto_import_count = metadata_reader.read_u32::().unwrap(); + let manual_import_count = metadata_reader.read_u32::().unwrap(); + let compressed_package_count = metadata_reader.read_u32::().unwrap(); + println!("{}, {}, {}, {}", version, alt_auto_import_count, manual_import_count, compressed_package_count); + // read alt auto imports + for i in 0..alt_auto_import_count { + let curr_import = metadata_reader.read_u64::().unwrap(); + //println!("adding import {:X}", &curr_import); + self.alt_import_assets.insert(curr_import); + } + // read manual imports + for i in 0..manual_import_count { + let mut manual_imports = vec![]; + let manual_asset = metadata_reader.read_u64::().unwrap(); + let import_count = metadata_reader.read_u64::().unwrap(); + for j in 0..import_count { + manual_imports.push(metadata_reader.read_u64::().unwrap()); + } + self.manual_import_assets.insert(manual_asset, manual_imports); + } + // read compressed packages + let mut compressed_assets = Vec::with_capacity(compressed_package_count as usize); + for i in 0..compressed_package_count { + compressed_assets.push(metadata_reader.read_u64::().unwrap()); + } + for i in 0..compressed_package_count { + self.compressed_assets.insert(compressed_assets[i as usize], UtocMetadataFlags::from_bits(metadata_reader.read_u8().unwrap()).unwrap()); + } + } + pub fn get_import_type(&self, asset: u64) -> UtocMetaImportType { + if self.alt_import_assets.contains(&asset) { + return UtocMetaImportType::GraphPackageValidated; + } else if self.manual_import_assets.contains_key(&asset) { + return UtocMetaImportType::Manual; } + return UtocMetaImportType::GraphPackageUnvalidated; } } \ No newline at end of file diff --git a/UtocEmulator/fileemu-utoc-stream-emulator/src/toc_factory.rs b/UtocEmulator/fileemu-utoc-stream-emulator/src/toc_factory.rs index 1cc9950..b20538e 100644 --- a/UtocEmulator/fileemu-utoc-stream-emulator/src/toc_factory.rs +++ b/UtocEmulator/fileemu-utoc-stream-emulator/src/toc_factory.rs @@ -373,8 +373,8 @@ impl TocResolverType2 { container_header.packages.push(ContainerHeaderPackage::from_package_summary::< ExportBundleHeader4, TSummary, BufReader, byteorder::NativeEndian >( - &mut file_reader, - self.chunk_ids[index].get_raw_hash(), curr_file.file_size, &self.files[index].os_path + &mut file_reader, self.chunk_ids[index].get_raw_hash(), + curr_file.file_size, &self.files[index].os_path, meta_guard )); } // write into container data