revised metadata format

UtocEmulator/UTOC.Stream.Emulator/ModConfig.json

@@ -2,7 +2,7 @@
   "ModId": "UTOC.Stream.Emulator",
   "ModName": "IO Store Emulator",
   "ModAuthor": "Rirurin",
-  "ModVersion": "1.0.4",
+  "ModVersion": "1.0.5",
   "ModDescription": "Simulates Unreal Engine IO Store files (.utoc \u002B .ucas). Allows mods to add or replace existing files using files on disk",
   "ModDll": "UTOC.Stream.Emulator.dll",
   "ModIcon": "Preview.png",

UtocEmulator/fileemu-utoc-stream-emulator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fileemu-utoc-stream-emulator"
-version = "1.0.4"
+version = "1.0.5"
 edition = "2021"
 authors = [ "Rirurin" ]
 description = "Simulates Unreal Engine IO Store files. Allows mods to add or replace existing files using files on disk."

UtocEmulator/fileemu-utoc-stream-emulator/src/asset_collector.rs

@@ -295,7 +295,7 @@ pub fn add_from_folders_inner(parent: TocDirectorySyncRef, os_path: &PathBuf, pr
                                 if *utoc_meta_lock == None {
                                     *utoc_meta_lock = Some(UtocMetadata::new());
                                 }
-                                utoc_meta_lock.as_mut().unwrap().add_entries(fs::read(fs_obj.path()).unwrap());
+                                utoc_meta_lock.as_mut().unwrap().add_entries::<byteorder::NativeEndian>(fs::read(fs_obj.path()).unwrap());
                             } else {
                                 profiler.add_skipped_file(fs_obj.path().to_str().unwrap(), format!("No file extension"), file_size);
                             }

UtocEmulator/fileemu-utoc-stream-emulator/src/io_package.rs

@@ -11,6 +11,7 @@ type ExportFilterFlags = u8; // and this one too...
 
 use byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt};
 use crate::{
+    metadata::{UtocMetadata, UtocMetaImportType},
     pak_package::{FObjectImport, FObjectExport, GameName, NameMap},
     string::{ FMappedName, FStringDeserializerText, FString16, Hasher16 },
     toc_factory::{TocResolverCommon, TocResolverType2}
@@ -19,7 +20,8 @@ use std::{
     error::Error,
     fs::File,
     fmt,
-    io::{BufReader, Cursor, ErrorKind, Read, Seek, SeekFrom, Write}
+    io::{BufReader, Cursor, ErrorKind, Read, Seek, SeekFrom, Write},
+    sync::MutexGuard
 };
 // IoStoreObjectIndex is a 64 bit value consisting of a hash of a target string for the lower 62 bits and an object type for the highest 2
 // expect for Empty which represents a null value and Export which contains an index to another item on the export tree
@@ -423,13 +425,53 @@ pub struct ContainerHeaderPackage {
 }
 
 impl ContainerHeaderPackage {
+    // Set all graph package imports as container summary imports. This behavior will likely be kept to 
+    // maintain compatibility with older P3RE mods since validation does break some other assets
+    fn imports_from_graph_packges_unvalidated(graph_packages: &Vec<FGraphPackage>) -> Vec<u64> {
+        let mut import_ids = vec![];
+        for i in graph_packages {
+            import_ids.push(i.imported_package_id);
+        }
+        import_ids
+    }
+    // From graph package imports, but with validating the name entries for matching file names
+    // Not all files actually have the same graph package imports as container header imports (Unreal shenanigans)
+    // However, there's an edge case with files ending in an underscore + number, where Unreal won't serialize that
+    // last portion of the string. I'd need an external tool to fix this
+    fn import_from_graph_packages_validated<TReader: Read + Seek, TByteOrder: byteorder::ByteOrder>
+    (reader: &mut TReader, summary: &PackageSummaryExports, graph_packages: &Vec<FGraphPackage>) -> Vec<u64> {
+        let mut import_ids = vec![];
+        reader.seek(SeekFrom::Start(summary.name_offset as u64)).unwrap();
+        let mut names = vec![];
+        for i in 0..summary.name_count {
+            names.push(FString16::from_buffer_text::<TReader, TByteOrder>(reader).unwrap().unwrap());
+        }
+        let mut path_name_hashes = vec![];
+        loop { // we only want to hash file paths, which Unreal always serializes at the beginning
+            if path_name_hashes.len() == names.len() || !names[path_name_hashes.len()].starts_with("/") {
+                break;
+            }
+            path_name_hashes.push(Hasher16::get_cityhash64(&names[path_name_hashes.len()]));
+        }
+        for i in graph_packages {
+            if path_name_hashes.contains(&i.imported_package_id) {
+                import_ids.push(i.imported_package_id);
+            }
+        }
+        import_ids
+    }
+    // If required, import ids can be manually specified from the metadata file. Trying to generate a
+    // UCAS file with no external metadata was always going to be a challenge
+    fn import_from_metadata_file(meta_guard: &mut MutexGuard<Option<UtocMetadata>>) -> Vec<u64> {
+        vec![]
+    }
     // Parse the package file to extract the values needed to build a store entry in the container header
     pub fn from_package_summary<
         TExportBundle: ExportBundle,
         TSummary: PackageIoSummaryDeserialize,
         TReader: Read + Seek,
         TByteOrder: byteorder::ByteOrder
-    >(file_reader: &mut TReader, hash: u64, size: u64, path: &str) -> Self { // consume the file object, we're only going to need it in here
+    >(file_reader: &mut TReader, hash: u64, size: u64, path: &str, meta_guard: &mut MutexGuard<Option<UtocMetadata>>) -> Self { // consume the file object, we're only going to need it in here
         let package_summary = TSummary::to_package_summary::<TReader, TByteOrder>(file_reader).unwrap();
         let export_count = package_summary.get_export_count() as u32;
         file_reader.seek(SeekFrom::Start(package_summary.export_bundle_offset as u64)).unwrap(); // jump to FExportBundleHeader start
@@ -438,35 +480,21 @@ impl ContainerHeaderPackage {
         ); // Go through each export bundle to look for the highest index
         file_reader.seek(SeekFrom::Start(package_summary.graph_offset as u64)).unwrap(); // go to FGraphPackage (imported_packages_count)
         let graph_packages = FGraphPackage::list_from_buffer::<TReader, TByteOrder>(file_reader);
-        // previously, utoc emulator only relied on obtaining it's container header import ids from the package's graph package ids (which in itself was a bit of a hack)
-        // however, this causes issues in regards to localized data, since graph package also includes ids for localization data not included in the container header
-        // this causes a lot of weird behaviour. This hack involves reading the first file entries (Unreal always serializes asset file paths first, followed by script paths)
-        // and then verifying that it's hash is within the graph package hashes. if both conditions are met, then it's allowed to be added as an import
-        let mut import_ids = Vec::with_capacity(graph_packages.len());
-        if export_bundle_count == 1 {
-            for i in &graph_packages {
-                import_ids.push(i.imported_package_id);
-            }
-        } else {
-            file_reader.seek(SeekFrom::Start(package_summary.name_offset as u64)).unwrap();
-            let mut names = vec![];
-            for i in 0..package_summary.name_count {
-                names.push(FString16::from_buffer_text::<TReader, TByteOrder>(file_reader).unwrap().unwrap());
-            }
-            let mut path_name_hashes = vec![];
-            loop { // we only want to hash file paths
-                if path_name_hashes.len() == names.len() || !names[path_name_hashes.len()].starts_with("/") {
-                    break;
+        let import_ids = match meta_guard.as_ref() {
+            Some(m) => {
+                match m.get_import_type(hash) {
+                    UtocMetaImportType::GraphPackageValidated => {
+                        //println!("VALIDATE PACKAGE {} ({:X})", path, hash);
+                        ContainerHeaderPackage::import_from_graph_packages_validated::<TReader, TByteOrder>(file_reader, &package_summary, &graph_packages)
+                    },
+                    UtocMetaImportType::Manual => ContainerHeaderPackage::import_from_metadata_file(meta_guard),
+                    UtocMetaImportType::GraphPackageUnvalidated => ContainerHeaderPackage::imports_from_graph_packges_unvalidated(&graph_packages)
                 }
-                path_name_hashes.push(Hasher16::get_cityhash64(&names[path_name_hashes.len()]));
-            }
-            for i in &graph_packages {
-                if path_name_hashes.contains(&i.imported_package_id) {
-                    import_ids.push(i.imported_package_id);
-                }
-            }
-        }
-        //println!("ASSET {}, {} imports, {} export bundles", path, import_ids.len(), export_bundle_count);
+            },
+            None => ContainerHeaderPackage::imports_from_graph_packges_unvalidated(&graph_packages)
+        };
+        //println!("ASSET {} ({:X}), {} imports, {} export bundles, {} exports", path, hash, import_ids.len(), export_bundle_count, export_count);
+
         let load_order = 0; // This doesn't seem to matter?
         Self {
             hash,
@@ -611,9 +639,11 @@ mod tests {
         let os_file = File::open(path).unwrap();
         let file_size = Metadata::get_file_size(&os_file);
         let mut os_reader = BufReader::new(os_file);
+        /* 
         ContainerHeaderPackage::from_package_summary::<
             ExportBundleHeader4, PackageSummary2, BufReader<File>, NativeEndian
         >(&mut os_reader, 0, file_size, &format!("aa"));
+        */
     }
 
     #[test]

UtocEmulator/fileemu-utoc-stream-emulator/src/metadata.rs

@@ -1,7 +1,7 @@
 use bitflags::bitflags;
 use byteorder::{LittleEndian, ReadBytesExt};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     io::{Cursor, Seek, SeekFrom},
     mem,
     sync::Mutex
@@ -21,28 +21,81 @@ bitflags! {
 pub static UTOC_METADATA: Mutex<Option<UtocMetadata>> = Mutex::new(None);
 
 // .utocmeta structure:
-// count: u64
-// chunk_hashes: [u64; length]
-// flags: [u8; length]
+// version: u32 @ 0x0
+// alt_auto_import_count: u32 @ 0x4
+// manual_import_count: u32 @ 0x8
+// compressed_package_count: u32 @ 0xc
+// alt_import_assets: [u64; length] @ alt_auto_imports_offset
+// manual_import_assets @ manual_imports_offset
+//      asset_hash: u64,
+//      count: u64,
+//      imports: [u64; count]
+// compressed_assets: [u64; length] @ compressed_package_offset
+// compressed_asset_flags: [u8; length] @ compressed_package_offset + compressed_assets
 #[derive(Debug, PartialEq)]
 pub struct UtocMetadata {
-    chunk_flags: HashMap<u64, UtocMetadataFlags>
+    // version: u32,
+    //alt_auto_imports_offset: u32,
+    //manual_imports_offset: u32,
+    //compressed_package_offset: u32,
+    alt_import_assets: HashSet<u64>,
+    manual_import_assets: HashMap<u64, Vec<u64>>,
+    compressed_assets: HashMap<u64, UtocMetadataFlags>
+}
+
+pub enum UtocMetaImportType {
+    GraphPackageUnvalidated,
+    GraphPackageValidated,
+    Manual
 }
 
 impl UtocMetadata {
     pub fn new() -> Self {
-        Self { chunk_flags: HashMap::new() }
+        Self { 
+            alt_import_assets: HashSet::new(),
+            manual_import_assets: HashMap::new(),
+            compressed_assets: HashMap::new()
+        }
     }
-    pub fn add_entries(&mut self, data: Vec<u8>) {
+    pub fn add_entries<TByteOrder: byteorder::ByteOrder>(&mut self, data: Vec<u8>) {
         let mut metadata_reader = Cursor::new(data);
-        let val_count = metadata_reader.read_u64::<byteorder::LittleEndian>().unwrap();
-        let flags_offset = mem::size_of::<u64>() as u64 + mem::size_of::<u64>() as u64 * val_count;
-        for i in 0..val_count {
-            metadata_reader.seek(SeekFrom::Start(mem::size_of::<u64>() as u64 + 0xc * i));
-            let curr_hash = metadata_reader.read_u64::<byteorder::LittleEndian>().unwrap();
-            metadata_reader.seek(SeekFrom::Start(flags_offset + i));
-            let chunk_flag = UtocMetadataFlags::from_bits_truncate(metadata_reader.read_u8().unwrap());
-            self.chunk_flags.insert(curr_hash, chunk_flag);
+        // read header
+        let version = metadata_reader.read_u32::<TByteOrder>().unwrap();
+        let alt_auto_import_count = metadata_reader.read_u32::<TByteOrder>().unwrap();
+        let manual_import_count = metadata_reader.read_u32::<TByteOrder>().unwrap();
+        let compressed_package_count = metadata_reader.read_u32::<TByteOrder>().unwrap();
+        println!("{}, {}, {}, {}", version, alt_auto_import_count, manual_import_count, compressed_package_count);
+        // read alt auto imports
+        for i in 0..alt_auto_import_count {
+            let curr_import = metadata_reader.read_u64::<TByteOrder>().unwrap();
+            //println!("adding import {:X}", &curr_import);
+            self.alt_import_assets.insert(curr_import);
+        }
+        // read manual imports
+        for i in 0..manual_import_count {
+            let mut manual_imports = vec![];
+            let manual_asset = metadata_reader.read_u64::<TByteOrder>().unwrap();
+            let import_count = metadata_reader.read_u64::<TByteOrder>().unwrap();
+            for j in 0..import_count {
+                manual_imports.push(metadata_reader.read_u64::<TByteOrder>().unwrap());
+            }
+            self.manual_import_assets.insert(manual_asset, manual_imports);
+        }
+        // read compressed packages
+        let mut compressed_assets = Vec::with_capacity(compressed_package_count as usize);
+        for i in 0..compressed_package_count {
+            compressed_assets.push(metadata_reader.read_u64::<TByteOrder>().unwrap());
+        }
+        for i in 0..compressed_package_count {
+            self.compressed_assets.insert(compressed_assets[i as usize], UtocMetadataFlags::from_bits(metadata_reader.read_u8().unwrap()).unwrap());
+        }
+    }
+    pub fn get_import_type(&self, asset: u64) -> UtocMetaImportType {
+        if self.alt_import_assets.contains(&asset) {
+            return UtocMetaImportType::GraphPackageValidated;
+        } else if self.manual_import_assets.contains_key(&asset) {
+            return UtocMetaImportType::Manual;
         }
+        return UtocMetaImportType::GraphPackageUnvalidated;
     }
 }

UtocEmulator/fileemu-utoc-stream-emulator/src/toc_factory.rs

@@ -373,8 +373,8 @@ impl TocResolverType2 {
             container_header.packages.push(ContainerHeaderPackage::from_package_summary::<
                 ExportBundleHeader4, TSummary, BufReader<File>, byteorder::NativeEndian
             >(
-                &mut file_reader, 
-                self.chunk_ids[index].get_raw_hash(), curr_file.file_size, &self.files[index].os_path
+                &mut file_reader, self.chunk_ids[index].get_raw_hash(), 
+                curr_file.file_size, &self.files[index].os_path, meta_guard
             ));
         }
         // write into container data