From 89ae0d757523b68f6c70026f8ae6dc8b2d78a0ca Mon Sep 17 00:00:00 2001 From: Kizari Date: Mon, 13 Feb 2023 18:08:12 +1300 Subject: [PATCH] Flagrum 1.4 --- Flagrum.Blender/__init__.py | 2 +- .../import_export/gfxbin/gfxbinheader.py | 17 + .../import_export/gfxbin/gmdl/gmdl.py | 90 + .../import_export/gfxbin/gmdl/gmdlbone.py | 17 + .../import_export/gfxbin/gmdl/gmdlmesh.py | 134 ++ .../gfxbin/gmdl/gmdlmeshobject.py | 29 + .../import_export/gfxbin/gmdl/gmdlmeshpart.py | 13 + .../gfxbin/gmdl/gmdlmodelpart.py | 15 + .../import_export/gfxbin/gmdl/gmdlnode.py | 31 + .../gfxbin/gmdl/gmdlsubgeometry.py | 17 + .../gfxbin/gmdl/gmdlvertexelement.py | 15 + .../gfxbin/gmdl/gmdlvertexelementformat.py | 124 ++ .../gfxbin/gmdl/gmdlvertexstream.py | 23 + .../import_export/gfxbin/gmtl/buffer.py | 31 + .../import_export/gfxbin/gmtl/colour.py | 15 + .../import_export/gfxbin/gmtl/gmtl.py | 142 ++ .../import_export/gfxbin/gmtl/sampler.py | 45 + .../import_export/gfxbin/gmtl/shaderbinary.py | 11 + .../gfxbin/gmtl/shaderprogram.py | 23 + .../import_export/gfxbin/gmtl/texture.py | 33 + .../import_export/gfxbin/gmtl/uniform.py | 25 + .../import_export/gfxbin/msgpack_reader.py | 136 ++ .../import_export/gfxbin/type_format.py | 46 + Flagrum.Blender/import_export/gmdlimporter.py | 434 ++++ Flagrum.Blender/import_export/gmtlimporter.py | 318 +++ .../import_export/import_context.py | 135 +- Flagrum.Blender/import_export/interop.py | 7 +- Flagrum.Blender/import_export/menu.py | 40 +- Flagrum.Blender/utilities/timer.py | 14 + Flagrum.Console/GfxbinTests.cs | 1007 ---------- Flagrum.Console/Program.cs | 60 +- Flagrum.Console/Program2.cs | 223 --- .../Scripts/Archive/IndexingScripts.cs | 69 +- .../Scripts/Forspoken/ModScripts.cs | 85 + Flagrum.Console/Scripts/Terrain/HebScripts.cs | 146 +- .../Scripts/Terrain/TerrainPaletteScripts.cs | 2 +- .../Texture/TextureConversionScripts.cs | 85 +- Flagrum.Console/Utilities/FileFinder.cs | 50 +- Flagrum.Console/Utilities/MaterialFinder.cs | 563 +++--- Flagrum.Core/Archive/ArchiveFile.cs | 67 +- Flagrum.Core/Archive/ArchiveHelper.cs | 378 ++-- .../DataSource/IArchiveFileDataSource.cs | 7 + .../Archive/DataSource/MemoryDataSource.cs | 18 + .../Archive/DataSource/OpenFileDataSource.cs | 32 + Flagrum.Core/Archive/EbonyArchive.cs | 162 ++ ...ArchiveHeader.cs => EbonyArchiveHeader.cs} | 26 +- Flagrum.Core/Archive/EbonyArchiveManager.cs | 40 + Flagrum.Core/Archive/EbonyArchiveReader.cs | 89 + Flagrum.Core/Archive/EbonyArchiveWriter.cs | 389 ++++ Flagrum.Core/Archive/EbonyReplace.cs | 43 + Flagrum.Core/Archive/Packer.cs | 431 ---- Flagrum.Core/Archive/Unpacker.cs | 287 --- Flagrum.Core/Flagrum.Core.csproj | 2 + Flagrum.Core/Gfxbin/Btex/BtexConverter.cs | 323 +-- Flagrum.Core/Gfxbin/Btex/ImageFormat.cs | 22 + Flagrum.Core/Gfxbin/Gmdl/Components/Mesh.cs | 3 +- .../Components/VertexElementDescription.cs | 42 +- .../Gfxbin/Gmdl/MessagePack/Gfxbin.cs | 321 +++ .../Gfxbin/Gmdl/MessagePack/GfxbinAABB.cs | 9 + .../Gfxbin/Gmdl/MessagePack/GfxbinBone.cs | 23 + .../Gfxbin/Gmdl/MessagePack/GfxbinHeader.cs | 17 + .../Gfxbin/Gmdl/MessagePack/GfxbinMesh.cs | 148 ++ .../Gmdl/MessagePack/GfxbinMeshObject.cs | 24 + .../Gfxbin/Gmdl/MessagePack/GfxbinMeshPart.cs | 15 + .../Gmdl/MessagePack/GfxbinModelPart.cs | 17 + .../Gfxbin/Gmdl/MessagePack/GfxbinNode.cs | 42 + .../Gmdl/MessagePack/GfxbinSubgeometry.cs | 19 + .../Gmdl/MessagePack/GfxbinVertexElement.cs | 17 + .../Gmdl/MessagePack/GfxbinVertexStream.cs | 22 + .../Gfxbin/Gmdl/MessagePack/Gpubin.cs | 321 +++ .../IMessagePackDifferentFirstItem.cs | 6 + .../Gmdl/MessagePack/IMessagePackItem.cs | 6 + .../Gmdl/MessagePack/MessagePackReader.cs | 210 ++ .../Gmdl/MessagePack/MessagePackType.cs | 47 + Flagrum.Core/Gfxbin/Gmtl/MaterialReader.cs | 13 +- Flagrum.Core/Utilities/BoyerMoore.cs | 112 ++ Flagrum.Core/Utilities/Cryptography.cs | 147 +- Flagrum.Core/Utilities/Extensions.cs | 79 + Flagrum.Core/Utilities/Serialization.cs | 13 +- Flagrum.Core/Utilities/Types/Enum.cs | 50 + Flagrum.Core/Utilities/Types/LuminousGame.cs | 7 + Flagrum.Desktop/App.xaml.cs | 61 +- Flagrum.Desktop/Architecture/RelayCommand.cs | 29 + .../Architecture/ViewportHelper.cs | 87 +- Flagrum.Desktop/Flagrum.Desktop.csproj | 81 +- Flagrum.Desktop/MainViewModel.cs | 11 +- Flagrum.Desktop/MainWindow.xaml | 19 +- Flagrum.Desktop/MainWindow.xaml.cs | 24 +- Flagrum.Desktop/Services/WpfService.cs | 5 +- Flagrum.Desktop/patreon.png | Bin 0 -> 11424 bytes Flagrum.Desktop/wwwroot/app.css | 9 + .../wwwroot/branding/mws_logo_white.svg | 47 + .../wwwroot/branding/patreon-logo.png | Bin 0 -> 3980 bytes Flagrum.Web/App.ja-JP.resx | 3 + Flagrum.Web/App.resx | 6 + Flagrum.Web/App.zh-hans.resx | 3 + Flagrum.Web/App.zh-hant.resx | 3 + .../Components/Controls/DonateButton.razor | 6 +- .../Components/Controls/EnumSelect.razor | 6 +- Flagrum.Web/Components/Graph/DiagramData.cs | 50 - .../Components/Graph/DiagramPersistence.cs | 99 - Flagrum.Web/Components/Graph/StandardGroup.cs | 18 - .../Graph/StandardGroupRenderer.razor | 46 - Flagrum.Web/Components/Graph/StandardNode.cs | 86 - .../Graph/StandardNodeRenderer.razor | 61 - Flagrum.Web/Components/Graph/StandardPort.cs | 48 - .../Graph/StandardPortRenderer.razor | 31 - .../AssetExplorer/Base/AddressBar.razor.cs | 2 +- .../AssetExplorer/Base/AssetExplorer.razor | 4 +- .../AssetExplorer/Base/AssetExplorer.razor.cs | 4 +- .../AssetExplorer/Base/ExplorerListView.razor | 3 +- .../AssetExplorer/Base/ExplorerRow.razor | 1 - .../AssetExplorer/Base/ExplorerTreeView.razor | 4 +- .../Features/AssetExplorer/Base/Preview.razor | 86 +- .../AssetExplorer/Data/ExportContext.cs | 9 +- .../Export/ExportContextMenu.razor | 28 +- .../Export/ExportFolderModal.razor | 40 +- .../Export/ExportWithDependenciesModal.razor | 93 +- .../GameView/GameViewAddressBar.cs | 70 +- .../GameView/GameViewFileList.cs | 15 +- .../Features/AssetExplorer/Index.razor | 35 +- .../AssetExplorer/Previews/ModelPreview.razor | 25 +- .../Previews/TexturePreview.razor | 58 +- Flagrum.Web/Features/Information/About.razor | 10 +- Flagrum.Web/Features/Information/About.resx | 16 +- .../Features/ModLibrary/Data/ModListing.cs | 10 - .../ModLibrary/Editor/ModelBrowserModal.razor | 146 -- .../ModLibrary/Editor/ModelExplorerRow.razor | 63 - .../Features/ModLibrary/_Imports.razor | 2 - .../{EarcMods => ModManager}/Data/Fmod.cs | 6 +- .../{EarcMods => ModManager}/Data/FmodEarc.cs | 6 +- .../Data/FmodEarcFile.cs | 2 +- .../{EarcMods => ModManager}/Data/FmodFile.cs | 2 +- .../Data/FmodFragment.cs | 2 +- .../Data/Legacy.cs} | 6 +- .../Editor.ja-JP.resx | 0 .../{EarcMods => ModManager}/Editor.razor | 88 +- .../{EarcMods => ModManager}/Editor.resx | 0 .../Editor.zh-hans.resx | 174 +- .../Editor.zh-hant.resx | 174 +- .../{EarcMods => ModManager}/Index.ja-JP.resx | 0 .../{EarcMods => ModManager}/Index.razor | 20 +- .../{EarcMods => ModManager}/Index.razor.cs | 46 +- .../{EarcMods => ModManager}/Index.resx | 3 + .../Index.zh-hans.resx | 276 +-- .../Index.zh-hant.resx | 276 +-- .../ModCard.razor} | 50 +- .../ModComponentBase.cs | 18 +- .../Modals/ModCardModal.ja-jp.resx | 0 .../Modals/ModCardModal.razor | 17 +- .../Modals/ModCardModal.resx | 0 .../Modals/ModCardModal.zh-hans.resx | 0 .../Modals/ModCardModal.zh-hant.resx | 0 .../Modals/UriSelectModal.ja-JP.resx | 0 .../Modals/UriSelectModal.razor | 1 - .../Modals/UriSelectModal.resx | 0 .../Modals/UriSelectModal.zh-hans.resx | 48 +- .../Modals/UriSelectModal.zh-hant.resx | 48 +- .../ModManager/Services/AssetConverter.cs | 125 ++ .../ModManager/Services/FFXVModManager.cs | 285 +++ .../Services/ForspokenModManager.cs | 131 ++ .../Services/ModManagerServiceBase.cs | 400 ++++ .../SequenceEditor/SequenceEditor.razor | 10 - .../SequenceEditor/SequenceEditor.razor.cs | 134 -- .../Features/Settings/Data/FlagrumProfile.cs | 54 + .../Settings/{ => Data}/LanguageTable.cs | 2 +- .../Settings/Modals/ProfileModal.razor | 78 + .../Features/Settings/PatreonCallback.razor | 55 + .../Features/Settings/ProfilesPage.razor | 88 + .../Features/Settings/ProfilesPage.resx | 36 + .../Features/Settings/SettingsPage.ja-JP.resx | 6 - .../Features/Settings/SettingsPage.razor | 235 ++- .../Features/Settings/SettingsPage.resx | 13 +- .../Settings/SettingsPage.zh-hans.resx | 6 - .../Settings/SettingsPage.zh-hant.resx | 6 - .../Settings/Utilities/ForspokenPatcher.cs | 58 + .../Components/NewWorkshopModCard.ja-JP.resx} | 0 .../Components/NewWorkshopModCard.razor} | 2 +- .../Components/NewWorkshopModCard.resx} | 0 .../NewWorkshopModCard.zh-hans.resx} | 42 +- .../NewWorkshopModCard.zh-hant.resx} | 42 +- .../Components/SteamWorkshopLink.razor | 6 +- .../Components/SteamWorkshopLinkGroup.razor | 0 .../Components/WorkshopModCard.razor} | 4 +- .../Components/WorkshopModCardGroup.razor} | 11 +- .../Data/ModelReplacementPresetMetadata.cs | 2 +- .../Data/WorkshopModBuildContext.cs} | 36 +- .../Data/WorkshopModStats.cs} | 74 +- .../Editor/Index.ja-JP.resx | 0 .../Editor/Index.razor | 1 + .../Editor/Index.razor.cs | 76 +- .../Editor/Index.resx | 0 .../Editor/Index.zh-hans.resx | 330 +-- .../Editor/Index.zh-hant.resx | 330 +-- .../Editor/ModelBrowserModal.ja-JP.resx | 0 .../Editor/ModelBrowserModal.razor | 116 ++ .../Editor/ModelBrowserModal.resx | 0 .../Editor/ModelBrowserModal.zh-hans.resx | 48 +- .../Editor/ModelBrowserModal.zh-hant.resx | 48 +- .../Editor/ModelReplacement.ja-JP.resx | 0 .../Editor/ModelReplacement.razor | 2 +- .../Editor/ModelReplacement.resx | 0 .../Editor/ModelReplacement.zh-hans.resx | 54 +- .../Editor/ModelReplacement.zh-hant.resx | 54 +- .../Editor/ModelReplacementRow.ja-JP.resx | 0 .../Editor/ModelReplacementRow.razor | 4 +- .../Editor/ModelReplacementRow.resx | 0 .../Editor/ModelReplacementRow.zh-hans.resx | 48 +- .../Editor/ModelReplacementRow.zh-hant.resx | 48 +- .../Editor/ModelReplacements.ja-JP.resx | 0 .../Editor/ModelReplacements.razor | 4 +- .../Editor/ModelReplacements.resx | 0 .../Editor/ModelReplacements.zh-hans.resx | 54 +- .../Editor/ModelReplacements.zh-hant.resx | 54 +- .../Index.ja-JP.resx | 0 .../{ModLibrary => WorkshopMods}/Index.razor | 5 +- .../{ModLibrary => WorkshopMods}/Index.resx | 0 .../Index.zh-hans.resx | 372 ++-- .../Index.zh-hant.resx | 372 ++-- .../Stats/Outfit.razor | 56 +- .../Stats/Replacement.razor | 2 +- .../Stats/StyleEdit.razor | 2 +- .../Stats/Weapon.razor | 58 +- .../{ModLibrary => WorkshopMods}/Upload.razor | 4 +- Flagrum.Web/Flagrum.Web.csproj | 10 +- Flagrum.Web/Layout/Bootstrapper.cs | 90 +- Flagrum.Web/Layout/MainLayout.razor | 47 +- Flagrum.Web/Layout/MainMenu.ja-JP.resx | 2 +- Flagrum.Web/Layout/MainMenu.razor | 37 +- Flagrum.Web/Layout/MainMenu.resx | 2 +- Flagrum.Web/Layout/MainMenu.zh-hans.resx | 2 +- Flagrum.Web/Layout/MainMenu.zh-hant.resx | 2 +- Flagrum.Web/Layout/PatchNotes.ja-JP.resx | 11 +- Flagrum.Web/Layout/PatchNotes.razor | 11 +- Flagrum.Web/Layout/PatchNotes.resx | 21 +- Flagrum.Web/Layout/PatchNotes.zh-hans.resx | 8 +- Flagrum.Web/Layout/PatchNotes.zh-hant.resx | 8 +- Flagrum.Web/Layout/TypeTreeComponent.razor | 70 - .../Persistence/Entities/AssetExplorerNode.cs | 44 +- Flagrum.Web/Persistence/Entities/AssetUri.cs | 12 +- .../Entities/ModManager/EarcMod.cs | 1778 ++++++++--------- .../Entities/ModManager/EarcModEarc.cs | 2 +- Flagrum.Web/Persistence/Entities/StatePair.cs | 15 +- Flagrum.Web/Persistence/FlagrumDbContext.cs | 14 +- Flagrum.Web/Services/AppStateService.cs | 83 +- Flagrum.Web/Services/Binmod.cs | 12 +- Flagrum.Web/Services/BinmodBuilder.cs | 34 +- Flagrum.Web/Services/DependencyInjection.cs | 5 +- Flagrum.Web/Services/EnvironmentPacker.cs | 19 +- Flagrum.Web/Services/IWpfService.cs | 3 +- Flagrum.Web/Services/ProfileService.cs | 482 +++++ Flagrum.Web/Services/SettingsService.cs | 507 ++--- Flagrum.Web/Services/SteamWorkshopService.cs | 16 +- Flagrum.Web/Services/TerrainPacker.cs | 20 +- Flagrum.Web/Services/TextureConverter.cs | 129 +- Flagrum.Web/Services/UriMapper.cs | 46 +- .../Components/AutoHideEvent.cs | 0 .../Components/ContextMenu.razor | 0 .../Components/ContextMenuBase.cs | 0 .../Components/ContextMenuDirection.cs | 0 .../Components/ContextMenuTrigger.cs | 0 .../Components/Item.razor | 0 .../Components/MenuRenderingContext.cs | 0 .../Components/MenuTreeComponent.cs | 0 .../Components/MouseButtonTrigger.cs | 0 .../Components/Seperator.razor | 0 .../Components/SubMenu.razor | 0 .../Components/_Imports.razor | 0 .../EventArgs/ItemAppearingEventArgs.cs | 0 .../EventArgs/ItemClickEventArgs.cs | 0 .../EventArgs/MenuAppearingEventArgs.cs | 0 .../EventArgs/MenuHidingEventArgs.cs | 0 .../BlazorContextMenuSettingsBuilder.cs | 0 .../Helpers/Helpers.cs | 0 .../Helpers/ServiceCollectionExtensions.cs | 0 .../Services/BlazorContextMenuService.cs | 0 .../Services/BlazorContextMenuSettings.cs | 0 .../Services/ContextMenuStorage.cs | 0 .../Services/InternalContextMenuHandler.cs | 0 .../Services/MenuTreeTraverser.cs | 0 .../Vendor/Patreon/IdentityResponse.cs | 25 + .../Vendor/Patreon/PatreonConstants.cs | 10 + .../Services/Vendor/Patreon/PatreonStatus.cs | 9 + .../Vendor/Patreon/PatreonTokenResponse.cs | 16 + Flagrum.Web/Styles/app.css | 9 + Flagrum.Web/Utilities/Extensions.cs | 16 + Flagrum.Web/_Imports.razor | 5 +- 287 files changed, 11627 insertions(+), 7232 deletions(-) create mode 100644 Flagrum.Blender/import_export/gfxbin/gfxbinheader.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdl.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlbone.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmesh.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshobject.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshpart.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmodelpart.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlnode.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlsubgeometry.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelement.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelementformat.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexstream.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/buffer.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/colour.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/gmtl.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/sampler.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/shaderbinary.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/shaderprogram.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/texture.py create mode 100644 Flagrum.Blender/import_export/gfxbin/gmtl/uniform.py create mode 100644 Flagrum.Blender/import_export/gfxbin/msgpack_reader.py create mode 100644 Flagrum.Blender/import_export/gfxbin/type_format.py create mode 100644 Flagrum.Blender/import_export/gmdlimporter.py create mode 100644 Flagrum.Blender/import_export/gmtlimporter.py create mode 100644 Flagrum.Blender/utilities/timer.py delete mode 100644 Flagrum.Console/GfxbinTests.cs delete mode 100644 Flagrum.Console/Program2.cs create mode 100644 Flagrum.Console/Scripts/Forspoken/ModScripts.cs create mode 100644 Flagrum.Core/Archive/DataSource/IArchiveFileDataSource.cs create mode 100644 Flagrum.Core/Archive/DataSource/MemoryDataSource.cs create mode 100644 Flagrum.Core/Archive/DataSource/OpenFileDataSource.cs create mode 100644 Flagrum.Core/Archive/EbonyArchive.cs rename Flagrum.Core/Archive/{ArchiveHeader.cs => EbonyArchiveHeader.cs} (64%) create mode 100644 Flagrum.Core/Archive/EbonyArchiveManager.cs create mode 100644 Flagrum.Core/Archive/EbonyArchiveReader.cs create mode 100644 Flagrum.Core/Archive/EbonyArchiveWriter.cs create mode 100644 Flagrum.Core/Archive/EbonyReplace.cs delete mode 100644 Flagrum.Core/Archive/Packer.cs delete mode 100644 Flagrum.Core/Archive/Unpacker.cs create mode 100644 Flagrum.Core/Gfxbin/Btex/ImageFormat.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gfxbin.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinAABB.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinBone.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinHeader.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMesh.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshObject.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshPart.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinModelPart.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinNode.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinSubgeometry.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexElement.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexStream.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gpubin.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackDifferentFirstItem.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackItem.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackReader.cs create mode 100644 Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackType.cs create mode 100644 Flagrum.Core/Utilities/BoyerMoore.cs create mode 100644 Flagrum.Core/Utilities/Types/Enum.cs create mode 100644 Flagrum.Core/Utilities/Types/LuminousGame.cs create mode 100644 Flagrum.Desktop/Architecture/RelayCommand.cs create mode 100644 Flagrum.Desktop/patreon.png create mode 100644 Flagrum.Desktop/wwwroot/branding/mws_logo_white.svg create mode 100644 Flagrum.Desktop/wwwroot/branding/patreon-logo.png delete mode 100644 Flagrum.Web/Components/Graph/DiagramData.cs delete mode 100644 Flagrum.Web/Components/Graph/DiagramPersistence.cs delete mode 100644 Flagrum.Web/Components/Graph/StandardGroup.cs delete mode 100644 Flagrum.Web/Components/Graph/StandardGroupRenderer.razor delete mode 100644 Flagrum.Web/Components/Graph/StandardNode.cs delete mode 100644 Flagrum.Web/Components/Graph/StandardNodeRenderer.razor delete mode 100644 Flagrum.Web/Components/Graph/StandardPort.cs delete mode 100644 Flagrum.Web/Components/Graph/StandardPortRenderer.razor delete mode 100644 Flagrum.Web/Features/ModLibrary/Data/ModListing.cs delete mode 100644 Flagrum.Web/Features/ModLibrary/Editor/ModelBrowserModal.razor delete mode 100644 Flagrum.Web/Features/ModLibrary/Editor/ModelExplorerRow.razor delete mode 100644 Flagrum.Web/Features/ModLibrary/_Imports.razor rename Flagrum.Web/Features/{EarcMods => ModManager}/Data/Fmod.cs (98%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Data/FmodEarc.cs (87%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Data/FmodEarcFile.cs (95%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Data/FmodFile.cs (95%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Data/FmodFragment.cs (97%) rename Flagrum.Web/Features/{EarcMods/Data/EarcModMetadata.cs => ModManager/Data/Legacy.cs} (78%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Editor.ja-JP.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Editor.razor (82%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Editor.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Editor.zh-hans.resx (97%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Editor.zh-hant.resx (97%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.ja-JP.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.razor (86%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.razor.cs (90%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.resx (98%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.zh-hans.resx (97%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Index.zh-hant.resx (97%) rename Flagrum.Web/Features/{EarcMods/EarcModCard.razor => ModManager/ModCard.razor} (82%) rename Flagrum.Web/Features/{EarcMods => ModManager}/ModComponentBase.cs (84%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/ModCardModal.ja-jp.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/ModCardModal.razor (91%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/ModCardModal.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/ModCardModal.zh-hans.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/ModCardModal.zh-hant.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/UriSelectModal.ja-JP.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/UriSelectModal.razor (98%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/UriSelectModal.resx (100%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/UriSelectModal.zh-hans.resx (97%) rename Flagrum.Web/Features/{EarcMods => ModManager}/Modals/UriSelectModal.zh-hant.resx (97%) create mode 100644 Flagrum.Web/Features/ModManager/Services/AssetConverter.cs create mode 100644 Flagrum.Web/Features/ModManager/Services/FFXVModManager.cs create mode 100644 Flagrum.Web/Features/ModManager/Services/ForspokenModManager.cs create mode 100644 Flagrum.Web/Features/ModManager/Services/ModManagerServiceBase.cs delete mode 100644 Flagrum.Web/Features/SequenceEditor/SequenceEditor.razor delete mode 100644 Flagrum.Web/Features/SequenceEditor/SequenceEditor.razor.cs create mode 100644 Flagrum.Web/Features/Settings/Data/FlagrumProfile.cs rename Flagrum.Web/Features/Settings/{ => Data}/LanguageTable.cs (91%) create mode 100644 Flagrum.Web/Features/Settings/Modals/ProfileModal.razor create mode 100644 Flagrum.Web/Features/Settings/PatreonCallback.razor create mode 100644 Flagrum.Web/Features/Settings/ProfilesPage.razor create mode 100644 Flagrum.Web/Features/Settings/ProfilesPage.resx create mode 100644 Flagrum.Web/Features/Settings/Utilities/ForspokenPatcher.cs rename Flagrum.Web/Features/{ModLibrary/Components/NewCard.ja-JP.resx => WorkshopMods/Components/NewWorkshopModCard.ja-JP.resx} (100%) rename Flagrum.Web/Features/{ModLibrary/Components/NewCard.razor => WorkshopMods/Components/NewWorkshopModCard.razor} (95%) rename Flagrum.Web/Features/{ModLibrary/Components/NewCard.resx => WorkshopMods/Components/NewWorkshopModCard.resx} (100%) rename Flagrum.Web/Features/{ModLibrary/Components/NewCard.zh-hans.resx => WorkshopMods/Components/NewWorkshopModCard.zh-hans.resx} (97%) rename Flagrum.Web/Features/{ModLibrary/Components/NewCard.zh-hant.resx => WorkshopMods/Components/NewWorkshopModCard.zh-hant.resx} (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Components/SteamWorkshopLink.razor (90%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Components/SteamWorkshopLinkGroup.razor (100%) rename Flagrum.Web/Features/{ModLibrary/Components/ModListingCard.razor => WorkshopMods/Components/WorkshopModCard.razor} (96%) rename Flagrum.Web/Features/{ModLibrary/Components/ModListingGroup.razor => WorkshopMods/Components/WorkshopModCardGroup.razor} (58%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Data/ModelReplacementPresetMetadata.cs (85%) rename Flagrum.Web/Features/{ModLibrary/Data/BuildContext.cs => WorkshopMods/Data/WorkshopModBuildContext.cs} (88%) rename Flagrum.Web/Features/{ModLibrary/Data/StatsModel.cs => WorkshopMods/Data/WorkshopModStats.cs} (75%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.ja-JP.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.razor (99%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.razor.cs (78%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/Index.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelBrowserModal.ja-JP.resx (100%) create mode 100644 Flagrum.Web/Features/WorkshopMods/Editor/ModelBrowserModal.razor rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelBrowserModal.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelBrowserModal.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelBrowserModal.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacement.ja-JP.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacement.razor (99%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacement.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacement.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacement.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacementRow.ja-JP.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacementRow.razor (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacementRow.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacementRow.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacementRow.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacements.ja-JP.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacements.razor (96%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacements.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacements.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Editor/ModelReplacements.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Index.ja-JP.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Index.razor (93%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Index.resx (100%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Index.zh-hans.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Index.zh-hant.resx (97%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Stats/Outfit.razor (70%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Stats/Replacement.razor (93%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Stats/StyleEdit.razor (95%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Stats/Weapon.razor (69%) rename Flagrum.Web/Features/{ModLibrary => WorkshopMods}/Upload.razor (97%) delete mode 100644 Flagrum.Web/Layout/TypeTreeComponent.razor create mode 100644 Flagrum.Web/Services/ProfileService.cs rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/AutoHideEvent.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/ContextMenu.razor (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/ContextMenuBase.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/ContextMenuDirection.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/ContextMenuTrigger.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/Item.razor (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/MenuRenderingContext.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/MenuTreeComponent.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/MouseButtonTrigger.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/Seperator.razor (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/SubMenu.razor (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Components/_Imports.razor (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/EventArgs/ItemAppearingEventArgs.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/EventArgs/ItemClickEventArgs.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/EventArgs/MenuAppearingEventArgs.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/EventArgs/MenuHidingEventArgs.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Helpers/BlazorContextMenuSettingsBuilder.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Helpers/Helpers.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Helpers/ServiceCollectionExtensions.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Services/BlazorContextMenuService.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Services/BlazorContextMenuSettings.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Services/ContextMenuStorage.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Services/InternalContextMenuHandler.cs (100%) rename Flagrum.Web/Services/Vendor/{ => BlazorContextMenuLibrary}/Services/MenuTreeTraverser.cs (100%) create mode 100644 Flagrum.Web/Services/Vendor/Patreon/IdentityResponse.cs create mode 100644 Flagrum.Web/Services/Vendor/Patreon/PatreonConstants.cs create mode 100644 Flagrum.Web/Services/Vendor/Patreon/PatreonStatus.cs create mode 100644 Flagrum.Web/Services/Vendor/Patreon/PatreonTokenResponse.cs create mode 100644 Flagrum.Web/Utilities/Extensions.cs diff --git a/Flagrum.Blender/__init__.py b/Flagrum.Blender/__init__.py index db8d3007..b4e99365 100644 --- a/Flagrum.Blender/__init__.py +++ b/Flagrum.Blender/__init__.py @@ -20,7 +20,7 @@ bl_info = { "name": "Flagrum", - "version": (1, 2, 5), + "version": (1, 3, 0), "blender": (3, 0, 0), "location": "File > Import-Export", "description": "Blender add-on for Flagrum", diff --git a/Flagrum.Blender/import_export/gfxbin/gfxbinheader.py b/Flagrum.Blender/import_export/gfxbin/gfxbinheader.py new file mode 100644 index 00000000..b22332db --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gfxbinheader.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + + +@dataclass +class GfxbinHeader: + version: int + dependencies: dict[str, str] + hashes: list[int] + + def __init__(self, reader): + self.version = reader.read() + self.dependencies = reader.read() + + hash_count = reader.read() + self.hashes = [] + for i in range(hash_count): + self.hashes.append(reader.read()) diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdl.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdl.py new file mode 100644 index 00000000..55234139 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdl.py @@ -0,0 +1,90 @@ +from dataclasses import dataclass + +from .gmdlbone import GmdlBone +from .gmdlmeshobject import GmdlMeshObject +from .gmdlmodelpart import GmdlModelPart +from .gmdlnode import GmdlNode +from ..gfxbinheader import GfxbinHeader + + +@dataclass +class Gmdl: + header: GfxbinHeader + aabb: list[list[float]] + instance_name_format: int + shader_class_format: int + shader_sampler_description_format: int + shader_parameter_list_format: int + child_class_format: int + bones: list[GmdlBone] + nodes: list[GmdlNode] + unknown: float + gpubin_count: int + gpubin_hashes: list[int] + mesh_objects: list[GmdlMeshObject] + unknown2: bool + name: str + parts: list[GmdlModelPart] + unknown3: float + unknown4: float + unknown5: float + unknown6: float + unknown7: float + gpubin_size_count: int + gpubin_sizes: list[int] + + def __init__(self, reader): + self.header = GfxbinHeader(reader) + self.aabb = [[reader.read(), reader.read(), reader.read()], [reader.read(), reader.read(), reader.read()]] + + if self.header.version < 20220707: + self.instance_name_format = reader.read() + self.shader_class_format = reader.read() + self.shader_sampler_description_format = reader.read() + self.shader_parameter_list_format = reader.read() + self.child_class_format = reader.read() + + bone_count = reader.read() + self.bones = [] + for i in range(bone_count): + self.bones.append(GmdlBone(reader, self.header.version)) + + node_count = reader.read() + self.nodes = [] + for i in range(node_count): + self.nodes.append(GmdlNode(reader, i == 0, self.header.version)) + + if self.header.version >= 20220707: + self.unknown = reader.read() + self.gpubin_count = reader.read() + else: + self.gpubin_count = 1 + + self.gpubin_hashes = [] + for i in range(self.gpubin_count): + self.gpubin_hashes.append(reader.read()) + + mesh_object_count = reader.read() + self.mesh_objects = [] + for i in range(mesh_object_count): + self.mesh_objects.append(GmdlMeshObject(reader, i == 0, self.header.version)) + + self.unknown2 = reader.read() + self.name = reader.read() + + parts_count = reader.read() + self.parts = [] + for i in range(parts_count): + self.parts.append(GmdlModelPart(reader)) + + if self.header.version >= 20220707: + self.unknown3 = reader.read() + self.unknown4 = reader.read() + self.unknown5 = reader.read() + self.unknown6 = reader.read() + self.unknown7 = reader.read() + + self.gpubin_size_count = reader.read() + self.gpubin_sizes = [] + for i in range(self.gpubin_size_count): + self.gpubin_sizes.append(reader.read()) diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlbone.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlbone.py new file mode 100644 index 00000000..7c806180 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlbone.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + + +@dataclass +class GmdlBone: + name: str + lod_index: int + unique_index: int + + def __init__(self, reader, version: int): + self.name = reader.read() + self.lod_index = reader.read() + + if version >= 20220707: + self.unique_index = reader.read() + else: + self.unique_index = self.lod_index >> 16 diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmesh.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmesh.py new file mode 100644 index 00000000..4e5357c8 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmesh.py @@ -0,0 +1,134 @@ +from dataclasses import dataclass + +from .gmdlmeshpart import GmdlMeshPart +from .gmdlsubgeometry import GmdlSubgeometry +from .gmdlvertexstream import GmdlVertexStream + + +@dataclass +class GmdlMesh: + name: str + unknown: int + bone_ids: list[int] + vertex_layout_type: int + unknown2: bool + AABB: list[list[float]] + is_oriented_bb: bool + orientedBB: list[list[float]] + primitive_type: int + face_index_count: int + face_index_type: int + gpubin_index: int + face_index_offset: int + face_index_size: int + vertex_count: int + vertex_streams: list[GmdlVertexStream] + vertex_buffer_offset: int + vertex_buffer_size: int + instance_number: int + subgeometry_count: int + subgeometries: list[GmdlSubgeometry] + unknown6: int + unknown_offset: int + unknown_size: int + material_hash: int + draw_priority_offset: int + unknown7: bool + unknown8: bool + lod_near: float + lod_far: float + lod_fade: float + unknown11: bool + unknown12: bool + parts_id: int + parts: list[GmdlMeshPart] + unknown9: bool + flags: int + unknown10: bool + breakable_bone_index: int + low_lod_shadow_cascade_no: int + + def __init__(self, reader, version: int): + self.name = reader.read() + + if version < 20220707: + self.unknown = reader.read() + bone_id_count = reader.read() + self.bone_ids = [] + for i in range(bone_id_count): + self.bone_ids.append(reader.read()) + + self.vertex_layout_type = reader.read() + self.unknown2 = reader.read() + + self.AABB = [ + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()] + ] + + self.is_oriented_bb = reader.read() + + self.orientedBB = [ + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()] + ] + + self.primitive_type = reader.read() + self.face_index_count = reader.read() + self.face_index_type = reader.read() + + if version >= 20220707: + self.gpubin_index = reader.read() + else: + self.gpubin_index = 0 + + self.face_index_offset = reader.read() + self.face_index_size = reader.read() + self.vertex_count = reader.read() + + vertex_stream_count = reader.read() + self.vertex_streams = [] + for i in range(vertex_stream_count): + self.vertex_streams.append(GmdlVertexStream(reader)) + + self.vertex_buffer_offset = reader.read() + self.vertex_buffer_size = reader.read() + + if version < 20220707: + self.instance_number = reader.read() + + self.subgeometry_count = reader.read() + self.subgeometries = [] + for i in range(self.subgeometry_count): + self.subgeometries.append(GmdlSubgeometry(reader)) + + if version >= 20220707: + self.unknown6 = reader.read() + self.unknown_offset = reader.read() + self.unknown_size = reader.read() + + self.material_hash = reader.read() + self.draw_priority_offset = reader.read() + self.unknown7 = reader.read() + self.unknown8 = reader.read() + self.lod_near = reader.read() + self.lod_far = reader.read() + self.lod_fade = reader.read() + + if version < 20220707: + self.unknown11 = reader.read() + self.unknown12 = reader.read() + + self.parts_id = reader.read() + parts_count = reader.read() + self.parts = [] + for i in range(parts_count): + self.parts.append(GmdlMeshPart(reader)) + + self.unknown9 = reader.read() + self.flags = reader.read() + self.unknown10 = reader.read() + self.breakable_bone_index = reader.read() + self.low_lod_shadow_cascade_no = reader.read() diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshobject.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshobject.py new file mode 100644 index 00000000..f1e3de8e --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshobject.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass + +from .gmdlmesh import GmdlMesh + + +@dataclass +class GmdlMeshObject: + unknown: bool + name: str + clusters: list[str] + meshes: list[GmdlMesh] + + def __init__(self, reader, is_first, version: int): + if is_first: + self.unknown = False + else: + self.unknown = reader.read() + + self.name = reader.read() + + cluster_count = reader.read() + self.clusters = [] + for i in range(cluster_count): + self.clusters.append(reader.read()) + + mesh_count = reader.read() + self.meshes = [] + for i in range(mesh_count): + self.meshes.append(GmdlMesh(reader, version)) diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshpart.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshpart.py new file mode 100644 index 00000000..311ced16 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmeshpart.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + + +@dataclass +class GmdlMeshPart: + parts_id: int + start_index: int + index_count: int + + def __init__(self, reader): + self.parts_id = reader.read() + self.start_index = reader.read() + self.index_count = reader.read() diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmodelpart.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmodelpart.py new file mode 100644 index 00000000..3ee65ff4 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlmodelpart.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass +class GmdlModelPart: + name: str + id: int + unknown: str + flags: bool + + def __init__(self, reader): + self.name = reader.read() + self.id = reader.read() + self.unknown = reader.read() + self.flags = reader.read() diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlnode.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlnode.py new file mode 100644 index 00000000..02ecd2dd --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlnode.py @@ -0,0 +1,31 @@ +from dataclasses import dataclass + + +@dataclass +class GmdlNode: + matrix: list[list[float]] + unknown: float + name: str + unknown2: int + unknown3: int + unknown4: int + + def __init__(self, reader, is_first: bool, version: int): + self.matrix = [ + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()], + [reader.read(), reader.read(), reader.read()] + ] + + if is_first: + self.unknown = 0.0 + elif version >= 20220707: + self.unknown = reader.read() + + self.name = reader.read() + + if version >= 20220707: + self.unknown2 = reader.read() + self.unknown3 = reader.read() + self.unknown4 = reader.read() diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlsubgeometry.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlsubgeometry.py new file mode 100644 index 00000000..d89d55bf --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlsubgeometry.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + + +@dataclass +class GmdlSubgeometry: + AABB: list[list[float]] + start_index: int + primitive_count: int + cluster_index_bitflag: int + draw_order: int + + def __init__(self, reader): + self.AABB = [[reader.read(), reader.read(), reader.read()], [reader.read(), reader.read(), reader.read()]] + self.start_index = reader.read() + self.primitive_count = reader.read() + self.cluster_index_bitflag = reader.read() + self.draw_order = reader.read() diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelement.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelement.py new file mode 100644 index 00000000..c6e14031 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelement.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + +from .gmdlvertexelementformat import ElementFormat + + +@dataclass +class GmdlVertexElement: + offset: int + semantic: str + format: ElementFormat + + def __init__(self, reader): + self.offset = reader.read() + self.semantic = reader.read() + self.format = ElementFormat(reader.read()) diff --git a/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelementformat.py b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelementformat.py new file mode 100644 index 00000000..c147b557 --- /dev/null +++ b/Flagrum.Blender/import_export/gfxbin/gmdl/gmdlvertexelementformat.py @@ -0,0 +1,124 @@ +from enum import IntEnum + + +class ElementFormat(IntEnum): + Disable = 0x0, + XYZW32_Float = 0x1, + XYZW32_UintN = 0x2, + XYZW32_Uint = 0x3, + XYZW32_SintN = 0x4, + XYZW32_Sint = 0x5, + XYZW16_Float = 0x6, + XYZW16_UintN = 0x7, + XYZW16_Uint = 0x8, + XYZW16_SintN = 0x9, + XYZW16_Sint = 0xA, + XYZW8_Float = 0xB, + XYZW8_UintN = 0xC, + XYZW8_Uint = 0xD, + XYZW8_SintN = 0xE, + XYZW8_Sint = 0xF, + XYZ32_Float = 0x10, + XYZ32_UintN = 0x11, + XYZ32_Uint = 0x12, + XYZ32_SintN = 0x13, + XYZ32_Sint = 0x14, + XY32_Float = 0x15, + XY32_UintN = 0x16, + XY32_Uint = 0x17, + XY32_SintN = 0x18, + XY32_Sint = 0x19, + XY16_Float = 0x1A, + XY16_UintN = 0x1B, + XY16_Uint = 0x1C, + XY16_SintN = 0x1D, + XY16_Sint = 0x1E, + X32_Float = 0x1F, + X32_UintN = 0x20, + X32_Uint = 0x21, + X32_SintN = 0x22, + X32_Sint = 0x23, + X16_Float = 0x24, + X16_UintN = 0x25, + X16_Uint = 0x26, + X16_SintN = 0x27, + X16_Sint = 0x28, + X8_Float = 0x29, + X8_UintN = 0x2A, + X8_Uint = 0x2B, + X8_SintN = 0x2C, + X8_Sint = 0x2D + + @staticmethod + def get_size(element): + if element.format == ElementFormat.XYZ32_Float: + return 12 + elif element.format == ElementFormat.XY16_SintN: + return 4 + elif element.format == ElementFormat.XY16_UintN: + return 4 + elif element.format == ElementFormat.XY16_Float: + return 4 + elif element.format == ElementFormat.XYZW8_UintN: + return 4 + elif element.format == ElementFormat.XYZW8_SintN: + return 4 + elif element.format == ElementFormat.XYZW16_Uint: + return 8 + elif element.format == ElementFormat.XYZW32_Uint: + return 16 + else: + print(f"[ERROR] Unsupported element format {str(element.format)} on {element.semantic}") + return None + + @staticmethod + def get_count(element): + if element.format == ElementFormat.XYZ32_Float: + return 3 + elif element.format == ElementFormat.XY16_SintN: + return 2 + elif element.format == ElementFormat.XY16_UintN: + return 2 + elif element.format == ElementFormat.XY16_Float: + return 2 + elif element.format == ElementFormat.XY32_Float: + return 2 + elif element.format == ElementFormat.XYZW8_UintN or element.format == ElementFormat.XYZW8_Uint: + return 4 + elif element.format == ElementFormat.XYZW8_SintN or element.format == ElementFormat.XYZW8_Sint: + return 4 + elif element.format == ElementFormat.XYZW16_Uint: + return 4 + elif element.format == ElementFormat.XYZW32_Uint: + return 4 + else: + print(f"[ERROR] Unsupported element format {str(element.format)} on {element.semantic}") + return None + + @staticmethod + def get_data_type(element): + if element.format == ElementFormat.XYZ32_Float: + return "= 20220707: + self.unknown = reader.read() + + self.uniform_count = reader.read() + self.buffer_count = reader.read() + self.texture_count = reader.read() + self.sampler_count = reader.read() + self.total_uniform_count = reader.read() + self.total_buffer_count = reader.read() + self.total_texture_count = reader.read() + self.total_sampler_count = reader.read() + self.shader_binary_count = reader.read() + self.shader_program_count = reader.read() + self.gpubin_size = reader.read() + self.stringbin_size = reader.read() + self.name_hash = reader.read() + + if self.header.version >= 20220707: + self.unknown_hash = reader.read() + + self.blend_type = reader.read() + self.blend_factor = reader.read() + self.render_state_bits = reader.read() + self.skin_vs_max_bone_count = reader.read() + self.brdf_type = reader.read() + self.htpk_uri_offset = reader.read() + + self.uniforms = [] + for i in range(self.total_uniform_count): + self.uniforms.append(Uniform(reader)) + + self.buffers = [] + for i in range(self.total_buffer_count): + self.buffers.append(Buffer(reader)) + + self.textures = [] + for i in range(self.total_texture_count): + self.textures.append(Texture(reader)) + + self.samplers = [] + for i in range(self.total_sampler_count): + self.samplers.append(Sampler(reader)) + + self.shader_binaries = [] + for i in range(self.shader_binary_count): + self.shader_binaries.append(ShaderBinary(reader)) + + self.shader_programs = [] + for i in range(self.shader_program_count): + self.shader_programs.append(ShaderProgram(reader)) + + self.gpubin = reader.read() + self.stringbin = reader.read() + + self.name = self._unpack_string(self.name_offset) + + for i in range(len(self.buffers)): + self.buffers[i].shader_gen_name = self._unpack_string(self.buffers[i].shader_gen_name_offset) + self.buffers[i].values = self._unpack_values(self.buffers[i].offset, self.buffers[i].size) + + for i in range(len(self.textures)): + self.textures[i].name = self._unpack_string(self.textures[i].name_offset) + self.textures[i].shader_gen_name = self._unpack_string(self.textures[i].shader_gen_name_offset) + self.textures[i].uri = self._unpack_string(self.textures[i].uri_offset) + + def get_buffer(self, shader_gen_name: str): + for i in range(len(self.buffers)): + if self.buffers[i].shader_gen_name == shader_gen_name: + return self.buffers[i] + return None + + def _unpack_values(self, offset: int, size: int): + result = [] + for i in range(int(size / 4)): + result.append(struct.unpack_from(" 0: + result = struct.unpack_from("<" + str(size - 1) + "s", self.buffer, self.offset)[0].decode('utf-8') + self.offset += size + return result + else: + return "" + elif format_type == Format.BooleanFalse: + return False + elif format_type == Format.BooleanTrue: + return True + elif format_type == Format.Bin8: + size = struct.unpack_from("= 20220707: + for bone in self.game_model.bones: + self.bone_table[counter] = bone.name + counter += 1 + else: + for bone in self.game_model.bones: + self.bone_table[bone.unique_index] = bone.name + + def _import_meshes(self): + if self.context.import_lods: + self.lods = {} + for mesh_object in self.game_model.mesh_objects: + for mesh in mesh_object.meshes: + self.lods[mesh.lod_near] = None + + counter = 0 + for key in self.lods: + self.lods[key] = bpy.data.collections.new("LOD" + str(counter)) + self.context.collection.children.link(self.lods[key]) + counter += 1 + + if self.context.import_vems: + self.has_vems = False + for mesh_object in self.game_model.mesh_objects: + for mesh in mesh_object.meshes: + if (mesh.flags & 67108864) > 0: + self.has_vems = True + + if self.has_vems: + self.vems = bpy.data.collections.new("VEMs") + self.context.collection.children.link(self.vems) + + for mesh_object in self.game_model.mesh_objects: + for mesh in mesh_object.meshes: + self._import_mesh(mesh) + + layer = bpy.context.view_layer + layer.update() + + def _import_mesh(self, mesh_data: GmdlMesh): + context = self.context + game_model = self.game_model + + # Skip LODs if setting is not checked + if not context.import_lods and mesh_data.lod_near != 0: + return + + # Skip VEMs if setting is not checked + if not context.import_vems and (mesh_data.flags & 67108864) > 0: + return + + print("") + print(f"Importing {mesh_data.name}...") + buffer = self._get_gpubin_buffer(mesh_data.gpubin_index) + + # Get face indices + if mesh_data.face_index_type == 0: + data_type = "= 20220707: + u = uv_data[c][0] + v = uv_data[c][1] + if v >= 0: + v_tile = int(v) + coords[c][0] = u + (v_tile % 10) + coords[c][1] = (v_tile + int(v_tile / 10) + 1) - v + else: + coords[c][0] = u + coords[c][1] = 1 - v + else: + # The V coordinate is set as 1-V to flip from FBX coordinate system + coords[c][0] = uv_data[c][0] + coords[c][1] = 1 - uv_data[c][1] + + uv_dictionary = {i: uv for i, uv in enumerate(coords)} + per_loop_list = [0.0] * len(mesh.loops) + + for loop in mesh.loops: + per_loop_list[loop.index] = uv_dictionary.get(loop.vertex_index) + + per_loop_list = [uv for pair in per_loop_list for uv in pair] + + mesh.uv_layers[uv_map_index].data.foreach_set("uv", per_loop_list) + uv_map_index += 1 + + self.timer.print("Generating UV maps") + + # Generate each of the color maps + for i in range(4): + key = "COLOR" + str(i) + if key not in semantics: + continue + + colors = semantics[key] + + per_loop_list = [0.0] * len(mesh.loops) + + for loop in mesh.loops: + if loop.vertex_index < len(colors): + per_loop_list[loop.index] = colors[loop.vertex_index] + + per_loop_list = [colors for pair in per_loop_list for colors in pair] + new_name = "colorSet" + if i > 0: + new_name += str(i) + mesh.vertex_colors.new(name=new_name) + mesh.vertex_colors[i].data.foreach_set("color", per_loop_list) + + self.timer.print("Generating vertex colours") + + mesh.validate() + mesh.update() + + self.timer.print("Validating and updating mesh") + + mesh_object = bpy.data.objects.new(mesh_data.name, mesh) + + if (mesh_data.flags & 67108864) > 0: + self.vems.objects.link(mesh_object) + elif context.import_lods: + self.lods[mesh_data.lod_near].objects.link(mesh_object) + else: + context.collection.objects.link(mesh_object) + + self.timer.print("Linking mesh object") + + # Add the parts system + model_parts = {} + for model_part in game_model.parts: + model_parts[model_part.id] = model_part.name + + for parts_group in mesh_data.parts: + parts_layer = mesh.attributes.new(name=model_parts[parts_group.parts_id], + type='BOOLEAN', + domain='FACE') + + sequence = [] + start_index = int(parts_group.start_index / 3) + index_count = int(parts_group.index_count / 3) + end_index = start_index + index_count + for i in range(len(mesh.polygons)): + sequence.append(start_index <= i < end_index) + + parts_layer.data.foreach_set("value", sequence) + + self.timer.print("Generating parts data") + + # Import custom normals + mesh.update(calc_edges=True) + + clnors = array('f', [0.0] * (len(mesh.loops) * 3)) + mesh.loops.foreach_get("normal", clnors) + mesh.polygons.foreach_set("use_smooth", [True] * len(mesh.polygons)) + + normals = [] + for normal in semantics["NORMAL0"]: + result = self.correction_matrix @ Vector([normal[0], normal[1], normal[2]]) + result.normalize() + normals.append(result) + + mesh.normals_split_custom_set_from_vertices(normals) + mesh.use_auto_smooth = True + + self.timer.print("Generating custom normals") + + # Add the vertex weights from each weight map + weights = {} + for key in self.bone_table: + weights[key] = [] + + if len(self.bone_table) > 0: + for i in range(2): + if "BLENDWEIGHT" + str(i) not in semantics: + continue + + blend_weight = semantics["BLENDWEIGHT" + str(i)] + blend_indices = semantics["BLENDINDICES" + str(i)] + for j in range(len(blend_weight)): + for k in range(4): + # No need to import zero weights + if blend_weight[j][k] == 0: + continue + weights[blend_indices[j][k]].append(([j], blend_weight[j][k])) + + for key in weights: + bone_name = self.bone_table[key] + vertex_group = mesh_object.vertex_groups.new(name=bone_name) + for item in weights[key]: + vertex_group.add(item[0], item[1], 'ADD') + + self.timer.print("Generating weight data") + + # Link the mesh to the armature + if len(self.bone_table) > 0: + mod = mesh_object.modifiers.new( + type="ARMATURE", name=context.collection.name) + mod.use_vertex_groups = True + + armature = bpy.data.objects[context.collection.name] + mod.object = armature + + mesh_object.parent = armature + else: + # Collection wasn't linked on armature set, so do it now + if context.collection.name not in bpy.context.scene.collection.children: + bpy.context.scene.collection.children.link(context.collection) + + self.timer.print("Linking mesh to armature") + + # Process the material + material_uri = self._get_material_uri(mesh_data.material_hash) + + try: + material_importer = GmtlImporter(self.context, material_uri) + material = material_importer.generate_material(has_light_map) + + if material is not None: + mesh_object.data.materials.append(material) + except: + print(f"[ERROR] Failed to import GMTL data from {material_uri}") + + self.timer.print("Generating automatic materials") + + def _get_gpubin_buffer(self, index: int) -> bytes: + if self.game_model.header.version < 20220707: + file_path = self.context.path_without_extension + ".gpubin" + else: + file_path = self.context.path_without_extension + "_" + str(index) + ".gpubin" + + if index not in self.gpubins: + with open(file_path, mode="rb") as file: + self.gpubins[index] = file.read() + + return self.gpubins[index] + + def _get_material_uri(self, uri_hash: int): + return self.game_model.header.dependencies[str(uri_hash)] diff --git a/Flagrum.Blender/import_export/gmtlimporter.py b/Flagrum.Blender/import_export/gmtlimporter.py new file mode 100644 index 00000000..87ec6b23 --- /dev/null +++ b/Flagrum.Blender/import_export/gmtlimporter.py @@ -0,0 +1,318 @@ +import os +from dataclasses import dataclass + +import bpy +from bpy.types import Material, NodeTree + +from .gfxbin.gmtl.gmtl import Gmtl +from .gfxbin.msgpack_reader import MessagePackReader +from .import_context import ImportContext + + +@dataclass +class GmtlImporter: + context: ImportContext + gmtl: Gmtl + + def __init__(self, context: ImportContext, uri: str): + self.context = context + + material_path = self.context.get_absolute_path_from_uri(uri) + if material_path is not None: + with open(material_path, mode="rb") as file: + reader = MessagePackReader(file.read()) + + self.gmtl = Gmtl(reader) + else: + self.gmtl = None + + def generate_material(self, has_light_map: bool) -> Material: + if self.gmtl is None: + return None + + gmtl = self.gmtl + material = bpy.data.materials.new(name=gmtl.name) + + material.use_nodes = True + material.use_backface_culling = True + bsdf = material.node_tree.nodes["Principled BSDF"] + multiply = material.node_tree.nodes.new('ShaderNodeMixRGB') + multiply.blend_type = 'MULTIPLY' + multiply.inputs[0].default_value = 1.0 + material.node_tree.links.new(bsdf.inputs['Base Color'], multiply.outputs['Color']) + + uvscale_buffer = gmtl.get_buffer("UVScale") + + needs_scaling = uvscale_buffer is not None and ( + uvscale_buffer.values[0] != 1.0 or (len(uvscale_buffer.values) > 1 and uvscale_buffer.values[1] != 1.0)) + + uv_scale = None + if needs_scaling: + map1 = material.node_tree.nodes.new('ShaderNodeUVMap') + map1.uv_map = "map1" + uv_scale = material.node_tree.nodes.new('ShaderNodeMapping') + uv_scale.inputs[3].default_value[0] = uvscale_buffer[0] + uv_scale.inputs[3].default_value[1] = uvscale_buffer[ + len(uvscale_buffer) - 1] + material.node_tree.links.new(uv_scale.inputs['Vector'], map1.outputs['UV']) + + co_texture = None + aoto_separate_rgb = None + noto_texture = None + + has_normal1 = False + has_basecolor = False + for texture_metadata in gmtl.textures: + if texture_metadata.uri.endswith(".sb"): + continue + + texture_slot = texture_metadata.shader_gen_name.upper() + + if "NORMAL1" in texture_slot or "NORMALTEXTURE1" in texture_slot: + has_normal1 = True + if "BASECOLOR0" in texture_slot or "BASECOLORTEXTURE0" in texture_slot: + has_basecolor = True + + bump = None + if has_normal1: + bump = material.node_tree.nodes.new('ShaderNodeBump') + bump.inputs[1].default_value = 0.001 + material.node_tree.links.new(bsdf.inputs['Normal'], bump.outputs['Normal']) + + for texture_metadata in gmtl.textures: + if texture_metadata.uri.endswith(".sb"): + continue + + texture_path = self.context.get_absolute_path_from_uri(texture_metadata.uri) + if texture_path is None: + continue + + texture = material.node_tree.nodes.new('ShaderNodeTexImage') + texture.image = bpy.data.images.load(texture_path, check_existing=True) + if texture_path.split('\\')[-1].split('.')[-2] == "1001": + texture.image.source = 'TILED' + + if needs_scaling: + material.node_tree.links.new(texture.inputs['Vector'], uv_scale.outputs['Vector']) + + texture_slot = texture_metadata.shader_gen_name.upper() + + if "BASECOLOR0" in texture_slot or "BASECOLORTEXTURE0" in texture_slot: + material.node_tree.links.new(multiply.inputs['Color1'], texture.outputs['Color']) + if texture_metadata.name.endswith("_ba") or texture_metadata.name.endswith("_ba_$h"): + material.node_tree.links.new(bsdf.inputs['Alpha'], texture.outputs['Alpha']) + material.blend_method = 'CLIP' + elif "NORMAL0" in texture_slot or "NRT0" in texture_slot or "NORMALTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + if texture_metadata.name.endswith("_nrt") or texture_metadata.name.endswith( + "_nrt_$h"): + norm_map = material.node_tree.nodes.new('ShaderNodeNormalMap') + material.node_tree.links.new(bsdf.inputs['Normal'], norm_map.outputs['Normal']) + rgb = material.node_tree.nodes.new('ShaderNodeRGB') + rgb.outputs[0].default_value = (1.0, 1.0, 1.0, 1.0) + invert = material.node_tree.nodes.new('ShaderNodeInvert') + separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + combine_rgb = material.node_tree.nodes.new('ShaderNodeCombineRGB') + material.node_tree.links.new(separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(combine_rgb.inputs['R'], separate_rgb.outputs['R']) + material.node_tree.links.new(combine_rgb.inputs['G'], separate_rgb.outputs['G']) + material.node_tree.links.new(norm_map.inputs['Color'], combine_rgb.outputs['Image']) + material.node_tree.links.new(invert.inputs['Color'], texture.outputs['Alpha']) + material.node_tree.links.new(bsdf.inputs['Transmission'], invert.outputs['Color']) + material.node_tree.links.new(bsdf.inputs['Roughness'], separate_rgb.outputs['B']) + material.node_tree.links.new(combine_rgb.inputs['B'], rgb.outputs['Color']) + elif texture_metadata.name.endswith("_nro") or texture_metadata.name.endswith( + "_nro_$h"): + norm_map = material.node_tree.nodes.new('ShaderNodeNormalMap') + material.node_tree.links.new(bsdf.inputs['Normal'], norm_map.outputs['Normal']) + rgb = material.node_tree.nodes.new('ShaderNodeRGB') + rgb.outputs[0].default_value = (1.0, 1.0, 1.0, 1.0) + separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + combine_rgb = material.node_tree.nodes.new('ShaderNodeCombineRGB') + material.node_tree.links.new(separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(combine_rgb.inputs['R'], separate_rgb.outputs['R']) + material.node_tree.links.new(combine_rgb.inputs['G'], separate_rgb.outputs['G']) + material.node_tree.links.new(norm_map.inputs['Color'], combine_rgb.outputs['Image']) + material.node_tree.links.new(multiply.inputs['Color2'], texture.outputs['Alpha']) + material.node_tree.links.new(bsdf.inputs['Roughness'], separate_rgb.outputs['B']) + material.node_tree.links.new(combine_rgb.inputs['B'], rgb.outputs['Color']) + else: + normalise_group = self._setup_normalise_group() + normalise = material.node_tree.nodes.new('ShaderNodeGroup') + normalise.node_tree = normalise_group + material.node_tree.links.new(normalise.inputs['Color'], texture.outputs['Color']) + + if has_normal1: + material.node_tree.links.new(bump.inputs['Normal'], normalise.outputs['Normal']) + else: + material.node_tree.links.new(bsdf.inputs['Normal'], normalise.outputs['Normal']) + elif "METALLICTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + material.node_tree.links.new(bsdf.inputs['Metallic'], texture.outputs['Color']) + elif "ROUGHNESSTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + material.node_tree.links.new(bsdf.inputs['Roughness'], texture.outputs['Color']) + elif "SPECULARTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + material.node_tree.links.new(bsdf.inputs['Specular'], texture.outputs['Color']) + elif "MRS0" in texture_slot or "MRSTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + material.node_tree.links.new(separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(bsdf.inputs['Metallic'], separate_rgb.outputs['R']) + material.node_tree.links.new(bsdf.inputs['Roughness'], separate_rgb.outputs['G']) + material.node_tree.links.new(bsdf.inputs['Specular'], separate_rgb.outputs['B']) + elif "MRO_MIX0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + material.node_tree.links.new(separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(multiply.inputs['Color2'], separate_rgb.outputs['B']) + material.node_tree.links.new(bsdf.inputs['Metallic'], separate_rgb.outputs['R']) + material.node_tree.links.new(bsdf.inputs['Roughness'], separate_rgb.outputs['G']) + elif "EMISSIVECOLOR0" in texture_slot or "EMISSIVE0" in texture_slot or "EMISSIVETEXTURE0" in texture_slot: + if not texture_metadata.uri.upper().endswith("WHITE.TGA"): + material.node_tree.links.new(bsdf.inputs['Emission'], texture.outputs['Color']) + elif "TRANSPARENCY0" in texture_slot or "OPACITYMASK0" in texture_slot or "OPACITYMASKTEXTURE0" in texture_slot or "TRANSPARENCYTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + material.node_tree.links.new(bsdf.inputs['Alpha'], texture.outputs['Color']) + material.blend_method = 'CLIP' + elif "OCCLUSION0" in texture_slot or "OCCLUSIONTEXTURE0" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + material.node_tree.links.new(multiply.inputs['Color2'], texture.outputs['Color']) + uv_map = material.node_tree.nodes.new('ShaderNodeUVMap') + if has_light_map: + uv_map.uv_map = "mapLM" + else: + uv_map.uv_map = "map1" + material.node_tree.links.new(texture.inputs['Vector'], uv_map.outputs['UV']) + elif "AOTO0" in texture_slot: + material.blend_method = 'CLIP' + texture.image.colorspace_settings.name = 'Non-Color' + aoto_separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + material.node_tree.links.new(aoto_separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(bsdf.inputs['Alpha'], aoto_separate_rgb.outputs['R']) + material.node_tree.links.new(multiply.inputs['Color2'], aoto_separate_rgb.outputs['G']) + elif "NOTO0" in texture_slot: + # Handle occlusion portion + texture.image.colorspace_settings.name = 'Non-Color' + uv_map = material.node_tree.nodes.new('ShaderNodeUVMap') + if has_light_map > 1: + uv_map.uv_map = "mapLM" + else: + uv_map.uv_map = "map1" + material.node_tree.links.new(texture.inputs['Vector'], uv_map.outputs['UV']) + + separate_rgb = material.node_tree.nodes.new('ShaderNodeSeparateRGB') + material.node_tree.links.new(separate_rgb.inputs['Image'], texture.outputs['Color']) + material.node_tree.links.new(multiply.inputs['Color2'], separate_rgb.outputs['B']) + + # Handle other channels + texture = material.node_tree.nodes.new('ShaderNodeTexImage') + texture.image = bpy.data.images.load(texture_path, check_existing=True) + texture.image.colorspace_settings.name = 'Non-Color' + noto_texture = texture + + normalise_group = self._setup_split_normal_group() + normalise = material.node_tree.nodes.new('ShaderNodeGroup') + normalise.node_tree = normalise_group + + if has_normal1: + material.node_tree.links.new(bump.inputs['Normal'], normalise.outputs['Normal']) + else: + material.node_tree.links.new(bsdf.inputs['Normal'], normalise.outputs['Normal']) + + material.node_tree.links.new(normalise.inputs['Color'], texture.outputs['Color']) + elif "NORMAL1" in texture_slot or "NORMALTEXTURE1" in texture_slot: + texture.image.colorspace_settings.name = 'Non-Color' + normalise_group = self._setup_normalise_group() + normalise = material.node_tree.nodes.new('ShaderNodeGroup') + normalise.node_tree = normalise_group + map1 = material.node_tree.nodes.new('ShaderNodeUVMap') + map1.uv_map = "map1" + uv_scale = material.node_tree.nodes.new('ShaderNodeMapping') + + detail_uvscale_buffer = gmtl.get_buffer("DetailUVScale") + if detail_uvscale_buffer is not None: + uv_scale.inputs[3].default_value[0] = detail_uvscale_buffer[0] + uv_scale.inputs[3].default_value[1] = detail_uvscale_buffer[ + len(detail_uvscale_buffer) - 1] + + material.node_tree.links.new(bump.inputs['Height'], normalise.outputs['Normal']) + material.node_tree.links.new(normalise.inputs['Color'], texture.outputs['Color']) + material.node_tree.links.new(texture.inputs['Vector'], uv_scale.outputs['Vector']) + material.node_tree.links.new(uv_scale.inputs['Vector'], map1.outputs['UV']) + elif "COLORCHIP0" in texture_slot and not has_basecolor: + co_texture = texture + material.node_tree.links.new(multiply.inputs['Color1'], co_texture.outputs['Color']) + else: + self.context.texture_slots[texture_slot] = True + + if co_texture is not None and aoto_separate_rgb is not None: + material.node_tree.links.new(co_texture.inputs['Vector'], aoto_separate_rgb.outputs['B']) + + if noto_texture is not None and co_texture is not None: + material.node_tree.links.new(co_texture.inputs['Vector'], noto_texture.outputs['Alpha']) + + return material + + @staticmethod + def _setup_normalise_group() -> NodeTree: + name = "Normalise" + group = bpy.data.node_groups.get(name) + if group: + return group + + group = bpy.data.node_groups.new(name, 'ShaderNodeTree') + group_inputs = group.nodes.new('NodeGroupInput') + group.inputs.new('NodeSocketColor', "Color") + + group_outputs = group.nodes.new('NodeGroupOutput') + group.outputs.new('NodeSocketVector', "Normal") + + separate_rgb = group.nodes.new('ShaderNodeSeparateRGB') + combine_rgb = group.nodes.new('ShaderNodeCombineRGB') + less_than = group.nodes.new('ShaderNodeMath') + less_than.operation = 'LESS_THAN' + less_than.inputs[1].default_value = 0.01 + maximum = group.nodes.new('ShaderNodeMath') + maximum.operation = 'MAXIMUM' + normal_map = group.nodes.new('ShaderNodeNormalMap') + + group.links.new(separate_rgb.inputs['Image'], group_inputs.outputs['Color']) + group.links.new(combine_rgb.inputs['R'], separate_rgb.outputs['R']) + group.links.new(combine_rgb.inputs['G'], separate_rgb.outputs['G']) + group.links.new(normal_map.inputs['Color'], combine_rgb.outputs['Image']) + group.links.new(less_than.inputs[0], separate_rgb.outputs['B']) + group.links.new(maximum.inputs[0], separate_rgb.outputs['B']) + group.links.new(maximum.inputs[1], less_than.outputs['Value']) + group.links.new(combine_rgb.inputs['B'], maximum.outputs['Value']) + group.links.new(group_outputs.inputs['Normal'], normal_map.outputs['Normal']) + + return group + + @staticmethod + def _setup_split_normal_group() -> NodeTree: + name = "RG Normals" + group = bpy.data.node_groups.get(name) + if group: + return group + + group = bpy.data.node_groups.new(name, 'ShaderNodeTree') + group_inputs = group.nodes.new('NodeGroupInput') + group.inputs.new('NodeSocketColor', "Color") + + group_outputs = group.nodes.new('NodeGroupOutput') + group.outputs.new('NodeSocketVector', "Normal") + + separate_rgb = group.nodes.new('ShaderNodeSeparateRGB') + combine_rgb = group.nodes.new('ShaderNodeCombineRGB') + combine_rgb.inputs[2].default_value = 1.0 + normal_map = group.nodes.new('ShaderNodeNormalMap') + + group.links.new(separate_rgb.inputs['Image'], group_inputs.outputs['Color']) + group.links.new(combine_rgb.inputs['R'], separate_rgb.outputs['R']) + group.links.new(combine_rgb.inputs['G'], separate_rgb.outputs['G']) + group.links.new(normal_map.inputs['Color'], combine_rgb.outputs['Image']) + group.links.new(group_outputs.inputs['Normal'], normal_map.outputs['Normal']) + + return group diff --git a/Flagrum.Blender/import_export/import_context.py b/Flagrum.Blender/import_export/import_context.py index 077436c9..b92d7002 100644 --- a/Flagrum.Blender/import_export/import_context.py +++ b/Flagrum.Blender/import_export/import_context.py @@ -4,17 +4,27 @@ import bpy from bpy.types import Material +from .gfxbin.gfxbinheader import GfxbinHeader + @dataclass(init=False) class ImportContext: gfxbin_path: str + import_lods: bool + import_vems: bool + path_without_extension: str amdl_path: str collection: bpy.types.Collection materials: dict[str, Material] texture_slots: dict[str, bool] + base_directory: str + base_uri: str - def __init__(self, gfxbin_file_path): + def __init__(self, gfxbin_file_path, import_lods, import_vems): self.gfxbin_path = gfxbin_file_path + self.import_lods = import_lods + self.import_vems = import_vems + self.path_without_extension = gfxbin_file_path.replace(".gmdl.gfxbin", "") self.materials = {} self.texture_slots = {} @@ -33,3 +43,126 @@ def __init__(self, gfxbin_file_path): group_name += string self.collection = bpy.data.collections.new(group_name) + + def set_base_directory(self, header: GfxbinHeader): + # Get the URI of the first gpubin + gpubin_uri = None + for key in header.dependencies: + if header.dependencies[key].endswith(".gpubin"): + gpubin_uri = header.dependencies[key] + break + + self.base_uri = gpubin_uri[:gpubin_uri.rfind('/')] + tokens = self.gfxbin_path.split('\\')[:-1] + + base_directory = "" + for i in range(len(tokens)): + base_directory += tokens[i] + '\\' + + self.base_directory = base_directory[:-1] + + def get_absolute_path_from_uri(self, uri: str): + if uri.endswith(".tif") or uri.endswith(".exr") or uri.endswith(".png") or uri.endswith(".dds") or uri.endswith( + ".btex"): + return self._resolve_texture_path(uri) + else: + path = self._get_absolute_path_from_uri(uri) + + if not os.path.exists(path): + print(f"[WARNING] File did not exist at {path}") + return None + else: + return path + + def _get_absolute_path_from_uri(self, uri: str): + # Get tokens for the part of the URIs that match + tokens1 = uri.replace("://", "/").split("/") + tokens2 = self.base_uri.replace("://", "/").split("/") + target_tokens = [] + + for i in range(min(len(tokens1), len(tokens2))): + if tokens1[i] == tokens2[i]: + target_tokens.append(tokens1[i]) + else: + break + + # Get the folder name of the deepest matching folder + if len(target_tokens) > 0: + target_token = target_tokens[-1] + else: + target_token = "" + + # Get the index of the highest folder the URIs have in common + index = -1 + counter = 0 + base_tokens = self.base_uri.replace("://", "/").split("/") + + for i in range(len(base_tokens) - 1, -1, -1): + if base_tokens[i] == target_token: + index = i + break + + counter += 1 + + if index == -1: + return None + + # Calculate the absolute path of the highest common folder + base_path = "" + base_path_tokens = self.base_directory.split('\\') + if counter > 0: + base_path_tokens = base_path_tokens[:-counter] + for i in range(len(base_path_tokens)): + base_path += base_path_tokens[i] + "\\" + + base_path = base_path[:-1] + + # Assemble the common URI start + target = "" + for i in range(len(target_tokens)): + target += target_tokens[i] + if i == 0: + target += "://" + else: + target += "/" + + target = target[:-1] + + # Calculate the final absolute path + remaining_path = uri.replace(target, "").replace("://", "/").replace("/", "\\") + return base_path + remaining_path.replace(".gmtl", ".gmtl.gfxbin") + + def _resolve_texture_path(self, uri: str): + extensions = ["dds", "tga", "png"] + + high = uri[:uri.rfind('.')] + "_$h" + uri[uri.rfind('.'):] + highest = high.replace("/sourceimages/", "/highimages/") + medium = uri[:uri.rfind('.')] + "_$m1" + uri[uri.rfind('.'):] + low = uri + + uris = [highest, high, medium, low] + paths_checked = [] + + for i in range(len(uris)): + path = self._get_absolute_path_from_uri(uris[i]) + if path is not None: + for j in range(len(extensions)): + without_extension = path[:path.rfind(".")] + with_extension = without_extension + "." + extensions[j] + paths_checked.append(with_extension) + + if os.path.exists(with_extension): + return with_extension + else: + name = without_extension.split('\\')[-1] + udim = f"{without_extension}\\{name}.1001.{extensions[j]}" + paths_checked.append(udim) + if os.path.exists(udim): + return udim + + print("") + print(f"[WARNING] Could not find texture for {uri} - checked:") + for i in range(len(paths_checked)): + print(f" {paths_checked[i]}") + print("") + return None diff --git a/Flagrum.Blender/import_export/interop.py b/Flagrum.Blender/import_export/interop.py index 637796ef..02fb04a2 100644 --- a/Flagrum.Blender/import_export/interop.py +++ b/Flagrum.Blender/import_export/interop.py @@ -40,9 +40,10 @@ def import_material_inputs(gfxbin_path) -> dict[str, list[float]]: @staticmethod def import_mesh(gfxbin_path): - tempfile_path = tempfile.NamedTemporaryFile().name + ".json" - command = "import -i \"" + gfxbin_path + "\" -o \"" + tempfile_path + "\"" - Interop.run_command(command) + tempfile_path = "C:\\Users\\Kieran\\Desktop\\Models2\\hu000\\model_000\\temp.json" + # tempfile_path = tempfile.NamedTemporaryFile().name + ".json" + # command = "import -i \"" + gfxbin_path + "\" -o \"" + tempfile_path + "\"" + # Interop.run_command(command) import_file = open(tempfile_path, mode='r') import_data = import_file.read() diff --git a/Flagrum.Blender/import_export/menu.py b/Flagrum.Blender/import_export/menu.py index 51eb8d7b..1c703818 100644 --- a/Flagrum.Blender/import_export/menu.py +++ b/Flagrum.Blender/import_export/menu.py @@ -10,8 +10,8 @@ from mathutils import Matrix, Euler, Vector from .generate_armature import generate_armature -from .generate_mesh import generate_mesh from .generate_terrain import generate_terrain +from .gmdlimporter import GmdlImporter from .import_context import ImportContext from .interop import Interop from .pack_mesh import pack_mesh @@ -31,22 +31,33 @@ class ImportOperator(Operator, ImportHelper): options={'HIDDEN'} ) - def execute(self, context): - import_context = ImportContext(self.filepath) + import_lods: BoolProperty( + name="Import LODs", + description="If checked, Flagrum will import all LOD meshes with this model.", + default=False + ) - mesh_data = Interop.import_mesh(import_context.gfxbin_path) + import_vems: BoolProperty( + name="Import VEMs", + description="If checked, Flagrum will import all VEM meshes with this model.", + default=False + ) - if len(mesh_data.BoneTable.__dict__) > 0: - armature_data = import_armature_data(import_context) - generate_armature(import_context, armature_data) + def draw(self, context): + layout = self.layout + layout.label(text="GMDL Import Options") + layout.prop(data=self, property="import_lods") + layout.prop(data=self, property="import_vems") - for mesh in mesh_data.Meshes: - generate_mesh(import_context, - collection=import_context.collection, - mesh_data=mesh, - bone_table=mesh_data.BoneTable.__dict__, - parts=mesh_data.Parts.__dict__) + def execute(self, context): + import_context = ImportContext(self.filepath, self.import_lods, self.import_vems) + armature_data = import_armature_data(import_context) + if armature_data is not None: + generate_armature(import_context, armature_data) + + importer = GmdlImporter(import_context) + importer.run() return {'FINISHED'} @@ -128,9 +139,6 @@ def execute(self, context): if mesh_data is not None: self.import_mesh(context, mesh_data, model, meshes) - for texture_slot in context.texture_slots: - print(texture_slot) - return {'FINISHED'} def import_mesh(self, context: ImportContext, mesh_data: Gpubin, model: EnvironmentModelMetadata, meshes): diff --git a/Flagrum.Blender/utilities/timer.py b/Flagrum.Blender/utilities/timer.py new file mode 100644 index 00000000..798188ff --- /dev/null +++ b/Flagrum.Blender/utilities/timer.py @@ -0,0 +1,14 @@ +import time +from dataclasses import dataclass + + +@dataclass +class Timer: + start: time + + def __init__(self): + self.start = time.time() + + def print(self, process_name: str): + print(f"{process_name} took {time.time() - self.start} seconds") + self.start = time.time() diff --git a/Flagrum.Console/GfxbinTests.cs b/Flagrum.Console/GfxbinTests.cs deleted file mode 100644 index b22f396a..00000000 --- a/Flagrum.Console/GfxbinTests.cs +++ /dev/null @@ -1,1007 +0,0 @@ -// using System; -// using System.Collections.Generic; -// using System.IO; -// using System.IO.Compression; -// using System.Linq; -// using System.Text; -// using Flagrum.Core.Archive; -// using Flagrum.Core.Gfxbin.Btex; -// using Flagrum.Core.Gfxbin.Data; -// using Flagrum.Core.Gfxbin.Gmdl; -// using Flagrum.Core.Gfxbin.Gmdl.Components; -// using Flagrum.Core.Gfxbin.Gmdl.Constructs; -// using Flagrum.Core.Gfxbin.Gmdl.Templates; -// using Flagrum.Core.Gfxbin.Gmtl; -// using Flagrum.Core.Utilities; -// using Microsoft.Extensions.Logging; -// using Newtonsoft.Json; -// - -using System.IO; -using System.Linq; -using System.Text; -using Flagrum.Core.Archive; -using Flagrum.Core.Gfxbin.Gmdl; -using Flagrum.Core.Gfxbin.Gmtl; - -namespace Flagrum.Console; - -// -// public class UselessLogger : ILogger -// { -// public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, -// Func formatter) { } -// -// public bool IsEnabled(LogLevel logLevel) -// { -// return false; -// } -// -// public IDisposable BeginScope(TState state) -// { -// return default!; -// } -// } -// -public static class GfxbinTests -{ - // public static void BuildGameAssetMod() - // { - // var previewImage = File.ReadAllBytes($"{IOHelper.GetExecutingDirectory()}\\Resources\\preview.png"); - // var mod = new Binmod - // { - // Type = (int)BinmodType.StyleEdit, - // Target = (int)OutfitMultiTarget.Gloves, - // GameMenuTitle = "Default Outfit", - // WorkshopTitle = "Default Outfit", - // Description = "So default!", - // Uuid = "2fd5d8d5-0e1a-4ac5-8d80-a7a8ccb2715e", - // Index = 17120955, - // ModDirectoryName = "2fd5d8d5-0e1a-4ac5-8d80-a7a8ccb2715e", - // Model1Name = "slim", - // Model2Name = "chubby" - // }; - // - // var packer = new Packer(); - // - // packer.AddFile(Modmeta.Build(mod), $"data://mod/{mod.ModDirectoryName}/index.modmeta"); - // var exml = EntityPackageBuilder.BuildExml(mod); - // packer.AddFile(exml, "data://$mod/temp.ebex"); - // - // var tempFile = Path.GetTempFileName(); - // var tempFile2 = Path.GetTempFileName(); - // File.WriteAllBytes(tempFile, previewImage); - // BtexConverter.Convert(btexConverterPath, tempFile, tempFile2, BtexConverter.TextureType.Thumbnail); - // var btex = File.ReadAllBytes(tempFile2); - // - // packer.AddFile(previewImage, $"data://mod/{mod.ModDirectoryName}/$preview.png.bin"); - // packer.AddFile(btex, $"data://mod/{mod.ModDirectoryName}/$preview.png"); - // - // foreach (var texturePath in Directory.EnumerateFiles("C:\\Modding\\GameAssetTesting\\nh00_010\\sourceimages")) - // { - // packer.AddFile(File.ReadAllBytes(texturePath), texturePath - // .Replace("C:\\Modding\\GameAssetTesting\\nh00_010", $"data://mod/{mod.ModDirectoryName}") - // .Replace('\\', '/') - // .Replace(".btex", ".tif")); - // } - // - // var gfx = "C:\\Users\\Kieran\\Desktop\\Mods\\Noctis\\character\\nh\\nh00\\model_010\\nh00_010.gmdl.gfxbin"; - // var gpu = gfx.Replace(".gmdl.gfxbin", ".gpubin"); - // var reader = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)); - // var model = reader.Read(); - // var reference = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref"); - // var assetUri = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "asset_uri"); - // reference.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl"; - // assetUri.Path = $"data://mod/{mod.ModDirectoryName}/"; - // var gpubin = model.Header.Dependencies.FirstOrDefault(d => d.Path.EndsWith(".gpubin")); - // var index = model.Header.Hashes.IndexOf(ulong.Parse(gpubin.PathHash)); - // gpubin.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; - // var gpubinHash = Cryptography.HashFileUri64(gpubin.Path); - // gpubin.PathHash = gpubinHash.ToString(); - // model.Header.Hashes[index] = gpubinHash; - // model.AssetHash = gpubinHash; - // model.Header.Dependencies.Clear(); - // - // foreach (var materialPath in Directory.EnumerateFiles("C:\\Modding\\GameAssetTesting\\nh00_010\\materials")) - // { - // var materialUri = materialPath - // .Replace("C:\\Modding\\GameAssetTesting\\nh00_010", $"data://mod/{mod.ModDirectoryName}") - // .Replace('\\', '/') - // .Replace(".gmtl.gfxbin", ".gmtl"); - // - // model.Header.Dependencies.Add(new DependencyPath - // {Path = materialUri, PathHash = Cryptography.HashFileUri64(materialUri).ToString()}); - // - // var materialReader = new MaterialReader(materialPath); - // var material = materialReader.Read(); - // var oldMaterialHash = - // Cryptography.HashFileUri64(material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref").Path); - // var matchingMeshes = model.MeshObjects[0].Meshes.Where(m => m.DefaultMaterialHash == oldMaterialHash); - // if (!matchingMeshes.Any()) - // { - // throw new Exception("REEEEEEE"); - // } - // - // foreach (var matchingMesh in matchingMeshes) - // { - // matchingMesh.DefaultMaterialHash = Cryptography.HashFileUri64(materialUri); - // } - // - // foreach (var texture in material.Textures.Where(t => t.Path.Contains("character/nh/nh00"))) - // { - // texture.Path = texture.Path.Replace("data://character/nh/nh00", "data://mod"); - // texture.PathHash = Cryptography.Hash32(texture.Path); - // texture.ResourceFileHash = Cryptography.HashFileUri64(texture.Path); - // } - // - // var dependencies = new List(); - // dependencies.AddRange(material.ShaderBinaries.Where(s => s.ResourceFileHash > 0).Select(s => - // new DependencyPath {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); - // dependencies.AddRange(material.Textures.Where(s => s.ResourceFileHash > 0).Select(s => new DependencyPath - // {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); - // dependencies.Add(new DependencyPath - // {PathHash = "asset_uri", Path = $"data://mod/{mod.ModDirectoryName}/materials/"}); - // dependencies.Add(new DependencyPath {PathHash = "ref", Path = materialUri}); - // material.Header.Dependencies = dependencies - // .DistinctBy(d => d.PathHash) - // .OrderBy(d => d.PathHash) - // .ToList(); - // material.Header.Hashes = material.Header.Dependencies - // .Where(d => ulong.TryParse(d.PathHash, out _)) - // .Select(d => ulong.Parse(d.PathHash)) - // .OrderBy(h => h) - // .ToList(); - // - // var materialWriter = new MaterialWriter(material); - // packer.AddFile(materialWriter.Write(), materialUri); - // } - // - // var gpubinPath = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; - // model.Header.Dependencies.Add(new DependencyPath - // {Path = gpubinPath, PathHash = Cryptography.HashFileUri64(gpubinPath).ToString()}); - // model.Header.Dependencies.Add(reference); - // model.Header.Dependencies.Add(assetUri); - // model.Header.Dependencies = model.Header.Dependencies - // .OrderBy(d => d.PathHash) - // .ToList(); - // model.Header.Hashes = model.Header.Dependencies - // .Where(d => ulong.TryParse(d.PathHash, out _)) - // .Select(d => ulong.Parse(d.PathHash)) - // .OrderBy(h => h) - // .ToList(); - // - // var writer = new ModelWriter(model); - // var (gfxData, gpuData) = writer.Write(); - // - // packer.AddFile(gfxData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl.gfxbin"); - // packer.AddFile(gpuData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"); - // packer.WriteToFile($"C:\\Modding\\{mod.Uuid}.ffxvbinmod"); - // } - public static void CheckModel() - { - var unpacker = - new Unpacker( - @"C:\Users\Kieran\Documents\My Games\FINAL FANTASY XV\Steam\76561198079211203\mod\2fd5d8d5-0e1a-4ac5-8d80-a7a8ccb2715e.ffxvbinmod"); - var gfx = unpacker.UnpackFileByQuery("slim.gmdl", out _); - var gpu = unpacker.UnpackFileByQuery("slim.gpubin", out _); - //File.WriteAllBytes(@"C:\Modding\slim.gmdl.gfxbin", gfx); - //File.WriteAllBytes(@"C:\Modding\slim.gpubin", gpu); - var reader = new ModelReader(gfx, gpu); - var model = reader.Read(); - var x = true; - } - - public static void CheckMaterial() - { - var unpacker = - new Unpacker( - @"C:\Users\Kieran\Documents\My Games\FINAL FANTASY XV\Steam\76561198079211203\mod\4d0d078e-3042-4814-bd15-845b6b5b7b89.ffxvbinmod"); - var gfx = unpacker.UnpackFileByQuery("full_mat.gmtl", out var uri); - var reader = new MaterialReader(gfx); - var material = reader.Read(); - var x = true; - } - -// public static void Add010ToMod() -// { -// var modDirectoryName = "d0120e15-3b15-4821-845c-c7d6b94b1a72"; -// var modelName = "binmod_prep"; -// var unpacker = new Unpacker($"C:\\Modding\\ModelReplacementTesting\\{modDirectoryName}.earc"); -// var packer = unpacker.ToPacker(); -// -// var gfx = "C:\\Users\\Kieran\\Desktop\\Mods\\Noctis\\character\\nh\\nh00\\model_010\\nh00_010.gmdl.gfxbin"; -// var gpu = gfx.Replace(".gmdl.gfxbin", ".gpubin"); -// var reader = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)); -// var model = reader.Read(); -// -// var allVertices = model.MeshObjects[0].Meshes -// .SelectMany(m => m.VertexPositions) -// .ToList(); -// model.Aabb = new Aabb( -// new Vector3( -// allVertices.Min(v => v.X), -// allVertices.Min(v => v.Y), -// allVertices.Min(v => v.Z)), -// new Vector3( -// allVertices.Max(v => v.X), -// allVertices.Max(v => v.Y), -// allVertices.Max(v => v.Z))); -// -// foreach (var mesh in model.MeshObjects[0].Meshes) -// { -// mesh.ColorMaps.Clear(); -// -// mesh.Aabb = new Aabb( -// new Vector3( -// mesh.VertexPositions.Min(v => v.X), -// mesh.VertexPositions.Min(v => v.Y), -// mesh.VertexPositions.Min(v => v.Z)), -// new Vector3( -// mesh.VertexPositions.Max(v => v.X), -// mesh.VertexPositions.Max(v => v.Y), -// mesh.VertexPositions.Max(v => v.Z))); -// -// var center = new Vector3( -// (mesh.Aabb.Min.X + mesh.Aabb.Max.X) / 2, -// (mesh.Aabb.Min.Y + mesh.Aabb.Min.Y) / 2, -// (mesh.Aabb.Min.Z + mesh.Aabb.Max.Z) / 2 -// ); -// -// mesh.OrientedBB = new OrientedBB( -// center, -// new Vector3(mesh.Aabb.Max.X - center.X, 0, 0), -// new Vector3(0, mesh.Aabb.Max.Y - center.Y, 0), -// new Vector3(0, 0, mesh.Aabb.Max.Z - center.Z) -// ); -// } -// -// -// model.Header.Dependencies.Clear(); -// -// var gpubinPath = $"data://mod/{modDirectoryName}/{modelName}.gpubin"; -// var materialUri = $"data://mod/{modDirectoryName}/materials/bodyshape_mat.gmtl"; -// model.AssetHash = Cryptography.HashFileUri64(gpubinPath); -// foreach (var mesh in model.MeshObjects[0].Meshes) -// { -// mesh.DefaultMaterialHash = Cryptography.HashFileUri64(materialUri); -// mesh.VertexLayoutType = VertexLayoutType.Skinning_4Bones; -// } -// -// model.Header.Dependencies.Add(new DependencyPath -// {Path = materialUri, PathHash = Cryptography.HashFileUri64(materialUri).ToString()}); -// model.Header.Dependencies.Add(new DependencyPath -// {Path = gpubinPath, PathHash = Cryptography.HashFileUri64(gpubinPath).ToString()}); -// model.Header.Dependencies.Add(new DependencyPath -// {PathHash = "asset_uri", Path = $"data://mod/{modDirectoryName}/"}); -// model.Header.Dependencies.Add(new DependencyPath -// {PathHash = "ref", Path = $"data://mod/{modDirectoryName}/{modelName}.gmdl"}); -// model.Header.Dependencies = model.Header.Dependencies -// .OrderBy(d => d.PathHash) -// .ToList(); -// model.Header.Hashes = model.Header.Dependencies -// .Where(d => ulong.TryParse(d.PathHash, out _)) -// .Select(d => ulong.Parse(d.PathHash)) -// .OrderBy(h => h) -// .ToList(); -// -// var writer = new ModelWriter(model); -// var (gfxData, gpuData) = writer.Write(); -// packer.UpdateFile($"{modelName}.gmdl", gfxData); -// packer.UpdateFile($"{modelName}.gpubin", gpuData); -// -// packer.WriteToFile($"C:\\Modding\\ModelReplacementTesting\\{modDirectoryName}.ffxvbinmod"); -// } -// -// public static void BuildGameAssetWeaponMod() -// { -// var btexConverterPath = -// $"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\\squareenix\\ffxvmodtool\\luminousstudio\\luminous\\sdk\\bin\\BTexConverter.exe"; -// -// var previewImage = File.ReadAllBytes($"{IOHelper.GetExecutingDirectory()}\\Resources\\preview.png"); -// var mod = new Binmod -// { -// Type = (int)BinmodType.Weapon, -// Target = (int)WeaponSoloTarget.Sword, -// GameMenuTitle = "Angery Sword", -// WorkshopTitle = "Angery Sword", -// Description = "So default!", -// Uuid = "33684db3-62c7-4a32-be4b-0deb7c293005", -// Index = 17197479, -// ModDirectoryName = "33684db3-62c7-4a32-be4b-0deb7c293005", -// ModelName = "angery_sword", -// Attack = 30, -// Critical = 2, -// MaxHp = 93, -// MaxMp = 19, -// Vitality = 12 -// }; -// -// var packer = new Packer(); -// -// packer.AddFile(Modmeta.Build(mod), $"data://mod/{mod.ModDirectoryName}/index.modmeta"); -// var exml = EntityPackageBuilder.BuildExml(mod); -// packer.AddFile(exml, "data://$mod/temp.ebex"); -// -// var tempFile = Path.GetTempFileName(); -// var tempFile2 = Path.GetTempFileName(); -// File.WriteAllBytes(tempFile, previewImage); -// BtexConverter.Convert(btexConverterPath, tempFile, tempFile2, BtexConverter.TextureType.Thumbnail); -// var btex = File.ReadAllBytes(tempFile2); -// -// packer.AddFile(previewImage, $"data://mod/{mod.ModDirectoryName}/$preview.png.bin"); -// packer.AddFile(btex, $"data://mod/{mod.ModDirectoryName}/$preview.png"); -// -// foreach (var texturePath in Directory.EnumerateFiles( -// "C:\\Modding\\Extractions\\character\\we\\we01\\model_100\\sourceimages")) -// { -// packer.AddFile(File.ReadAllBytes(texturePath), texturePath -// .Replace("C:\\Modding\\Extractions\\character\\we\\we01\\model_100", -// $"data://mod/{mod.ModDirectoryName}") -// .Replace('\\', '/') -// .Replace(".btex", ".tif")); -// } -// -// var gfx = "C:\\Modding\\Extractions\\character\\we\\we01\\model_100\\we01_100.gmdl.gfxbin"; -// var gpu = gfx.Replace(".gmdl.gfxbin", ".gpubin"); -// var reader = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)); -// var model = reader.Read(); -// foreach (var mesh in model.MeshObjects[0].Meshes) -// { -// mesh.MaterialType = MaterialType.FourWeights; -// } -// -// var reference = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref"); -// var assetUri = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "asset_uri"); -// reference.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl"; -// assetUri.Path = $"data://mod/{mod.ModDirectoryName}/"; -// var gpubin = model.Header.Dependencies.FirstOrDefault(d => d.Path.EndsWith(".gpubin")); -// var index = model.Header.Hashes.IndexOf(ulong.Parse(gpubin.PathHash)); -// gpubin.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; -// var gpubinHash = Cryptography.HashFileUri64(gpubin.Path); -// gpubin.PathHash = gpubinHash.ToString(); -// model.Header.Hashes[index] = gpubinHash; -// model.AssetHash = gpubinHash; -// model.Header.Dependencies.Clear(); -// -// foreach (var materialPath in Directory -// .EnumerateFiles("C:\\Modding\\Extractions\\character\\we\\we01\\model_100\\materials") -// .Where(f => !f.Contains("backup"))) -// { -// var materialUri = materialPath -// .Replace("C:\\Modding\\Extractions\\character\\we\\we01\\model_100", -// $"data://mod/{mod.ModDirectoryName}") -// .Replace('\\', '/') -// .Replace(".gmtl.gfxbin", ".gmtl"); -// -// model.Header.Dependencies.Add(new DependencyPath -// {Path = materialUri, PathHash = Cryptography.HashFileUri64(materialUri).ToString()}); -// -// var materialReader = new MaterialReader(materialPath); -// var material = materialReader.Read(); -// var oldMaterialHash = -// Cryptography.HashFileUri64(material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref").Path); -// var matchingMeshes = model.MeshObjects[0].Meshes; //.Where(m => m.DefaultMaterialHash == oldMaterialHash); -// if (!matchingMeshes.Any()) -// { -// throw new Exception("REEEEEEE"); -// } -// -// foreach (var matchingMesh in matchingMeshes) -// { -// matchingMesh.DefaultMaterialHash = Cryptography.HashFileUri64(materialUri); -// } -// -// foreach (var texture in material.Textures.Where(t => t.Path.Contains("character/we/we01"))) -// { -// texture.Path = texture.Path.Replace("data://character/we/we01", "data://mod"); -// texture.PathHash = Cryptography.Hash32(texture.Path); -// texture.ResourceFileHash = Cryptography.HashFileUri64(texture.Path); -// } -// -// var dependencies = new List(); -// dependencies.AddRange(material.ShaderBinaries.Where(s => s.ResourceFileHash > 0).Select(s => -// new DependencyPath {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); -// dependencies.AddRange(material.Textures.Where(s => s.ResourceFileHash > 0).Select(s => new DependencyPath -// {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); -// dependencies.Add(new DependencyPath -// {PathHash = "asset_uri", Path = $"data://mod/{mod.ModDirectoryName}/materials/"}); -// dependencies.Add(new DependencyPath {PathHash = "ref", Path = materialUri}); -// material.Header.Dependencies = dependencies -// .DistinctBy(d => d.PathHash) -// .OrderBy(d => d.PathHash) -// .ToList(); -// material.Header.Hashes = material.Header.Dependencies -// .Where(d => ulong.TryParse(d.PathHash, out _)) -// .Select(d => ulong.Parse(d.PathHash)) -// .OrderBy(h => h) -// .ToList(); -// -// var materialWriter = new MaterialWriter(material); -// packer.AddFile(materialWriter.Write(), materialUri); -// } -// -// var gpubinPath = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; -// model.Header.Dependencies.Add(new DependencyPath -// {Path = gpubinPath, PathHash = Cryptography.HashFileUri64(gpubinPath).ToString()}); -// model.Header.Dependencies.Add(reference); -// model.Header.Dependencies.Add(assetUri); -// model.Header.Dependencies = model.Header.Dependencies -// .OrderBy(d => d.PathHash) -// .ToList(); -// model.Header.Hashes = model.Header.Dependencies -// .Where(d => ulong.TryParse(d.PathHash, out _)) -// .Select(d => ulong.Parse(d.PathHash)) -// .OrderBy(h => h) -// .ToList(); -// -// var writer = new ModelWriter(model); -// var (gfxData, gpuData) = writer.Write(); -// -// packer.AddFile(gfxData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl.gfxbin"); -// packer.AddFile(gpuData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"); -// packer.WriteToFile($"C:\\Modding\\{mod.Uuid}.ffxvbinmod"); -// } -// -// public static void BuildGameAssetMod() -// { -// var btexConverterPath = -// $"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\\squareenix\\ffxvmodtool\\luminousstudio\\luminous\\sdk\\bin\\BTexConverter.exe"; -// -// var previewImage = File.ReadAllBytes($"{IOHelper.GetExecutingDirectory()}\\Resources\\preview.png"); -// var mod = new Binmod -// { -// Type = (int)BinmodType.Character, -// Target = (int)ModelReplacementTarget.Gladiolus, -// GameMenuTitle = "Default Outfit", -// WorkshopTitle = "Default Outfit", -// Description = "So default!", -// Uuid = "b9c5399e-61d1-4552-89ea-08b4089205b1", -// Index = 17217034, -// ModDirectoryName = "model_010", -// ModelName = "nh00_010", -// OriginalGmdls = -// new List(ModelReplacementPresets.GetOriginalGmdls((int)ModelReplacementTarget.Gladiolus)), -// OriginalGmdlCount = ModelReplacementPresets.GetOriginalGmdls((int)ModelReplacementTarget.Gladiolus).Length -// }; -// -// var packer = new Packer(); -// -// packer.AddFile(Modmeta.Build(mod), $"data://mod/{mod.ModDirectoryName}/index.modmeta"); -// var exml = EntityPackageBuilder.BuildExml(mod); -// packer.AddFile(exml, "data://$mod/temp.ebex"); -// -// var tempFile = Path.GetTempFileName(); -// var tempFile2 = Path.GetTempFileName(); -// File.WriteAllBytes(tempFile, previewImage); -// BtexConverter.Convert(btexConverterPath, tempFile, tempFile2, BtexConverter.TextureType.Thumbnail); -// var btex = File.ReadAllBytes(tempFile2); -// -// packer.AddFile(previewImage, $"data://mod/{mod.ModDirectoryName}/$preview.png.bin"); -// packer.AddFile(btex, $"data://mod/{mod.ModDirectoryName}/$preview.png"); -// -// foreach (var texturePath in Directory.EnumerateFiles("C:\\Modding\\GameAssetTesting\\nh00_010\\sourceimages")) -// { -// packer.AddFile(File.ReadAllBytes(texturePath), texturePath -// .Replace("C:\\Modding\\GameAssetTesting\\nh00_010", $"data://mod/{mod.ModDirectoryName}") -// .Replace('\\', '/') -// .Replace(".btex", ".tif")); -// } -// -// var gfx = "C:\\Users\\Kieran\\Desktop\\Mods\\Noctis\\character\\nh\\nh00\\model_010\\nh00_010.gmdl.gfxbin"; -// var gpu = gfx.Replace(".gmdl.gfxbin", ".gpubin"); -// var reader = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)); -// var model = reader.Read(); -// var reference = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref"); -// var assetUri = model.Header.Dependencies.FirstOrDefault(d => d.PathHash == "asset_uri"); -// reference.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl"; -// assetUri.Path = $"data://mod/{mod.ModDirectoryName}/"; -// var gpubin = model.Header.Dependencies.FirstOrDefault(d => d.Path.EndsWith(".gpubin")); -// var index = model.Header.Hashes.IndexOf(ulong.Parse(gpubin.PathHash)); -// gpubin.Path = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; -// var gpubinHash = Cryptography.HashFileUri64(gpubin.Path); -// gpubin.PathHash = gpubinHash.ToString(); -// model.Header.Hashes[index] = gpubinHash; -// model.AssetHash = gpubinHash; -// model.Header.Dependencies.Clear(); -// -// foreach (var materialPath in Directory.EnumerateFiles("C:\\Modding\\GameAssetTesting\\nh00_010\\materials")) -// { -// var materialUri = materialPath -// .Replace("C:\\Modding\\GameAssetTesting\\nh00_010", $"data://mod/{mod.ModDirectoryName}") -// .Replace('\\', '/') -// .Replace(".gmtl.gfxbin", ".gmtl"); -// -// model.Header.Dependencies.Add(new DependencyPath -// {Path = materialUri, PathHash = Cryptography.HashFileUri64(materialUri).ToString()}); -// -// var materialReader = new MaterialReader(materialPath); -// var material = materialReader.Read(); -// var oldMaterialHash = -// Cryptography.HashFileUri64(material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref").Path); -// var matchingMeshes = model.MeshObjects[0].Meshes.Where(m => m.DefaultMaterialHash == oldMaterialHash); -// if (!matchingMeshes.Any()) -// { -// throw new Exception("REEEEEEE"); -// } -// -// foreach (var matchingMesh in matchingMeshes) -// { -// matchingMesh.DefaultMaterialHash = Cryptography.HashFileUri64(materialUri); -// } -// -// foreach (var texture in material.Textures.Where(t => t.Path.Contains("character/nh/nh00"))) -// { -// texture.Path = texture.Path.Replace("data://character/nh/nh00", "data://mod"); -// texture.PathHash = Cryptography.Hash32(texture.Path); -// texture.ResourceFileHash = Cryptography.HashFileUri64(texture.Path); -// } -// -// var dependencies = new List(); -// dependencies.AddRange(material.ShaderBinaries.Where(s => s.ResourceFileHash > 0).Select(s => -// new DependencyPath {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); -// dependencies.AddRange(material.Textures.Where(s => s.ResourceFileHash > 0).Select(s => new DependencyPath -// {Path = s.Path, PathHash = s.ResourceFileHash.ToString()})); -// dependencies.Add(new DependencyPath -// {PathHash = "asset_uri", Path = $"data://mod/{mod.ModDirectoryName}/materials/"}); -// dependencies.Add(new DependencyPath {PathHash = "ref", Path = materialUri}); -// material.Header.Dependencies = dependencies -// .DistinctBy(d => d.PathHash) -// .OrderBy(d => d.PathHash) -// .ToList(); -// material.Header.Hashes = material.Header.Dependencies -// .Where(d => ulong.TryParse(d.PathHash, out _)) -// .Select(d => ulong.Parse(d.PathHash)) -// .OrderBy(h => h) -// .ToList(); -// -// var materialWriter = new MaterialWriter(material); -// packer.AddFile(materialWriter.Write(), materialUri); -// } -// -// var gpubinPath = $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"; -// model.Header.Dependencies.Add(new DependencyPath -// {Path = gpubinPath, PathHash = Cryptography.HashFileUri64(gpubinPath).ToString()}); -// model.Header.Dependencies.Add(reference); -// model.Header.Dependencies.Add(assetUri); -// model.Header.Dependencies = model.Header.Dependencies -// .OrderBy(d => d.PathHash) -// .ToList(); -// model.Header.Hashes = model.Header.Dependencies -// .Where(d => ulong.TryParse(d.PathHash, out _)) -// .Select(d => ulong.Parse(d.PathHash)) -// .OrderBy(h => h) -// .ToList(); -// -// var writer = new ModelWriter(model); -// var (gfxData, gpuData) = writer.Write(); -// -// packer.AddFile(gfxData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gmdl.gfxbin"); -// packer.AddFile(gpuData, $"data://mod/{mod.ModDirectoryName}/{mod.ModelName}.gpubin"); -// packer.WriteToFile($"C:\\Modding\\{mod.Uuid}.ffxvbinmod"); -// } -// -// public static void CheckMod() -// { -// var outPath = "C:\\Testing\\ModelReplacements\\SinglePlayerSword\\"; -// var unpacker = -// new Unpacker( -// "C:\\Testing\\ModelReplacements\\SinglePlayerSword\\33684db3-62c7-4a32-be4b-0deb7c293005.ffxvbinmod"); -// var gfx = unpacker.UnpackFileByQuery("khopesh.fbx", out _); -// var gpu = unpacker.UnpackFileByQuery("khopesh.gpubin", out _); -// File.WriteAllBytes($"{outPath}khopesh.gmdl.gfxbin", gfx); -// var reader = new ModelReader(gfx, gpu); -// var model = reader.Read(); -// var x = true; -// } -// -// public static void FixModelReplacementMod() -// { -// var btexConverterPath = -// $"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\\squareenix\\ffxvmodtool\\luminousstudio\\luminous\\sdk\\bin\\BTexConverter.exe"; -// -// var modDirectory = "f749648e-54dd-4159-940d-65ff379663d4"; -// var modelName = "ardynmankini3"; -// -// var unpacker = new Unpacker("C:\\Testing\\ModelReplacements\\SinglePlayerSword\\sword_1.ffxvbinmod"); -// var moFiles = unpacker.UnpackFilesByQuery("data://shader"); -// -// var fbxDefault = unpacker.UnpackFileByQuery("material.gmtl", out _); -// unpacker = new Unpacker($"C:\\Modding\\ModelReplacementTesting\\{modDirectory}.earc"); -// var packer = unpacker.ToPacker(); -// -// var reader = new MaterialReader(fbxDefault); -// var material = reader.Read(); -// var oldDependency = material.Header.Dependencies.FirstOrDefault(d => d.Path.EndsWith("_d.png")); -// var oldIndex = material.Header.Hashes.IndexOf(ulong.Parse(oldDependency.PathHash)); -// var oldTexture = material.Textures.FirstOrDefault(t => t.Path == oldDependency.Path); -// oldTexture.Path = $"data://mod/{modDirectory}/sourceimages/chest_basecolor0_texture_b.btex"; -// oldTexture.PathHash = Cryptography.Hash32(oldTexture.Path); -// oldTexture.ResourceFileHash = Cryptography.HashFileUri64(oldTexture.Path); -// material.Header.Hashes[oldIndex] = oldTexture.ResourceFileHash; -// oldDependency.Path = oldTexture.Path; -// oldDependency.PathHash = oldTexture.ResourceFileHash.ToString(); -// var assetUri = material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "asset_uri"); -// assetUri.Path = $"data://mod/{modDirectory}/materials/"; -// var reference = material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref"); -// reference.Path = $"data://mod/{modDirectory}/materials/chest_mat.gmtl"; -// material.Name = "chest_mat"; -// material.NameHash = Cryptography.Hash32(material.Name); -// var writer = new MaterialWriter(material); -// -// packer.RemoveFile("chest_mat.gmtl"); -// packer.AddFile(writer.Write(), $"data://mod/{modDirectory}/materials/chest_mat.gmtl"); -// foreach (var (uri, data) in moFiles) -// { -// packer.AddFile(data, uri); -// } -// -// packer.WriteToFile($"C:\\Modding\\ModelReplacementTesting\\{modDirectory}.ffxvbinmod"); -// } -// -// public static void FixWeaponMod() -// { -// var btexConverterPath = -// $"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\\squareenix\\ffxvmodtool\\luminousstudio\\luminous\\sdk\\bin\\BTexConverter.exe"; -// -// var unpacker = new Unpacker("C:\\Testing\\ModelReplacements\\SinglePlayerSword\\sword_1.ffxvbinmod"); -// var moFiles = unpacker.UnpackFilesByQuery("data://shader"); -// -// var fbxDefault = unpacker.UnpackFileByQuery("material.gmtl", out _); -// unpacker = new Unpacker("C:\\Modding\\WeaponTesting\\24d5d6ab-e8f4-443f-a5e1-a8830aff7924.earc"); -// var packer = unpacker.ToPacker(); -// -// var reader = new MaterialReader(fbxDefault); -// var material = reader.Read(); -// var oldDependency = material.Header.Dependencies.FirstOrDefault(d => d.Path.EndsWith("_d.png")); -// var oldIndex = material.Header.Hashes.IndexOf(ulong.Parse(oldDependency.PathHash)); -// var oldTexture = material.Textures.FirstOrDefault(t => t.Path == oldDependency.Path); -// oldTexture.Path = -// "data://mod/24d5d6ab-e8f4-443f-a5e1-a8830aff7924/sourceimages/body_ashape_basecolor0_texture_b.btex"; -// oldTexture.PathHash = Cryptography.Hash32(oldTexture.Path); -// oldTexture.ResourceFileHash = Cryptography.HashFileUri64(oldTexture.Path); -// material.Header.Hashes[oldIndex] = oldTexture.ResourceFileHash; -// oldDependency.Path = oldTexture.Path; -// oldDependency.PathHash = oldTexture.ResourceFileHash.ToString(); -// var assetUri = material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "asset_uri"); -// assetUri.Path = "data://mod/24d5d6ab-e8f4-443f-a5e1-a8830aff7924/materials/"; -// var reference = material.Header.Dependencies.FirstOrDefault(d => d.PathHash == "ref"); -// reference.Path = "data://mod/24d5d6ab-e8f4-443f-a5e1-a8830aff7924/materials/body_ashape_mat.gmtl"; -// material.Name = "body_ashape_mat"; -// material.NameHash = Cryptography.Hash32(material.Name); -// var writer = new MaterialWriter(material); -// -// packer.RemoveFile("body_ashape_mat.gmtl"); -// packer.AddFile(writer.Write(), -// "data://mod/24d5d6ab-e8f4-443f-a5e1-a8830aff7924/materials/body_ashape_mat.gmtl"); -// foreach (var (uri, data) in moFiles) -// { -// packer.AddFile(data, uri); -// } -// -// packer.WriteToFile("C:\\Modding\\WeaponTesting\\24d5d6ab-e8f4-443f-a5e1-a8830aff7924.ffxvbinmod"); -// } -// -// public static void BuildMod() -// { -// //CheckMod(); -// -// var btexConverterPath = -// $"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\\squareenix\\ffxvmodtool\\luminousstudio\\luminous\\sdk\\bin\\BTexConverter.exe"; -// -// var previewImage = File.ReadAllBytes($"{IOHelper.GetExecutingDirectory()}\\Resources\\preview.png"); -// var mod = new Binmod -// { -// Type = (int)BinmodType.Weapon, -// Target = (int)WeaponSoloTarget.Sword, -// GameMenuTitle = "Angery Sword", -// WorkshopTitle = "Angery Sword", -// Description = "So angry!", -// Uuid = "33684db3-62c7-4a32-be4b-0deb7c293005", -// Index = 17197479, -// ModDirectoryName = "sword_1", -// ModelName = "khopesh", -// Attack = 30, -// MaxHp = 93, -// MaxMp = 19, -// Critical = 2, -// Vitality = 12 -// }; -// -// // var builder = new BinmodBuilder(btexConverterPath, mod, previewImage); -// //var fmd = File.ReadAllBytes("C:\\Users\\Kieran\\Desktop\\angery_sword.fmd"); -// // builder.AddFmd(btexConverterPath, fmd, new UselessLogger()); -// // builder.WriteToFile("C:\\Testing\\ModelReplacements\\SinglePlayerSword\\33684db3-62c7-4a32-be4b-0deb7c293005.ffxvbinmod"); -// -// var unpacker = new Unpacker("C:\\Testing\\ModelReplacements\\SinglePlayerSword\\sword_1.ffxvbinmod"); -// var material = unpacker.UnpackFileByQuery("material.gmtl", out _); -// var packer = unpacker.ToPacker(); -// -// packer.UpdateFile("index.modmeta", Modmeta.Build(mod)); -// var exml = EntityPackageBuilder.BuildExml(mod); -// packer.UpdateFile("temp.ebex", exml); -// -// var tempFile = Path.GetTempFileName(); -// var tempFile2 = Path.GetTempFileName(); -// File.WriteAllBytes(tempFile, previewImage); -// BtexConverter.Convert(btexConverterPath, tempFile, tempFile2, BtexConverter.TextureType.Thumbnail); -// var btex = File.ReadAllBytes(tempFile2); -// -// packer.UpdateFile("$preview.png.bin", previewImage); -// packer.UpdateFile("$preview.png", btex); -// -// var fmd = File.ReadAllBytes("C:\\Users\\Kieran\\Desktop\\angery_sword.fmd"); -// using var memoryStream = new MemoryStream(fmd); -// using var archive = new ZipArchive(memoryStream, ZipArchiveMode.Read); -// -// var dataEntry = archive.GetEntry("data.json"); -// using var dataStream = new MemoryStream(); -// var stream = dataEntry.Open(); -// stream.CopyTo(dataStream); -// -// var json = Encoding.UTF8.GetString(dataStream.ToArray()); -// var gpubin = JsonConvert.DeserializeObject(json); -// -// var model = OutfitTemplate.Build(mod.ModDirectoryName, mod.ModelName, gpubin); -// var replacer = new ModelReplacer(model, gpubin, BinmodTypeHelper.GetModmetaTargetName(mod.Type, mod.Target)); -// model = replacer.Replace(); -// -// foreach (var bone in model.BoneHeaders) -// { -// bone.Name = "Bone"; -// bone.LodIndex = 4294967295; -// } -// -// foreach (var mesh in model.MeshObjects[0].Meshes) -// { -// mesh.MaterialType = MaterialType.OneWeight; -// mesh.BoneIds = new[] {0u}; -// for (var index = 0; index < mesh.WeightIndices[0].Count; index++) -// { -// var weightArray = mesh.WeightIndices[0][index]; -// for (var i = 0; i < weightArray.Length; i++) -// { -// weightArray[i] = 0; -// } -// } -// } -// -// var writer = new ModelWriter(model); -// var (gfxData, gpuData) = writer.Write(); -// -// var outPath = "C:\\Testing\\ModelReplacements\\SinglePlayerSword\\khopesh.gmdl.gfxbin"; -// File.WriteAllBytes(outPath, gfxData); -// -// packer.UpdateFile("khopesh.fbx", gfxData); -// packer.UpdateFile("khopesh.gpubin", gpuData); -// packer.RemoveFile("material.gmtl"); -// packer.AddFile(material, "data://mod/sword_1/materials/body_ashape_mat.gmtl"); -// -// var diffusePath = -// "C:\\Modding\\WeaponTesting\\mod\\24d5d6ab-e8f4-443f-a5e1-a8830aff7924\\sourceimages\\body_ashape_basecolor0_texture_b.btex"; -// packer.UpdateFile("khopesh_d.png", File.ReadAllBytes(diffusePath)); -// -// packer.WriteToFile( -// "C:\\Testing\\ModelReplacements\\SinglePlayerSword\\33684db3-62c7-4a32-be4b-0deb7c293005.ffxvbinmod"); -// } -// -// public static void GetBoneTable() -// { -// var gfx = "C:\\Testing\\Exineris\\mod\\b376d00b-e6ae-497d-a004-485914158a9b\\ignis.gmdl.gfxbin"; -// var gpu = "C:\\Testing\\Exineris\\mod\\b376d00b-e6ae-497d-a004-485914158a9b\\ignis.gpubin"; -// -// var reader = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)); -// var model = reader.Read(); -// -// foreach (var bone in model.BoneHeaders) -// { -// System.Console.WriteLine($"{bone.Name}: {bone.LodIndex >> 16}"); -// } -// } -// - public static void CheckMaterialDefaults(string path) - { - var reader = new MaterialReader(path); - var material = reader.Read(); - - var builder = new StringBuilder(); - foreach (var input in material.InterfaceInputs.Where(i => i.InterfaceIndex == 0)) - { - builder.AppendLine($"{input.ShaderGenName}: {string.Join(", ", input.Values)}"); - } - - File.WriteAllText(@"C:\Modding\" + path.Split('\\').Last().Replace(".gmtl.gfxbin", "_input_defaults.txt"), - builder.ToString()); - - builder = new StringBuilder(); - foreach (var texture in material.Textures.Where(t => !t.Path.EndsWith(".sb"))) - { - builder.AppendLine($"{texture.Name}\n{texture.Path}\n\n"); - } - - File.WriteAllText(@"C:\Modding\" + path.Split('\\').Last().Replace(".gmtl.gfxbin", "_texture_defaults.txt"), - builder.ToString()); - } -// -// public static void Compare() -// { -// var gfxbin = "C:\\Testing\\extract\\mod\\7594c633-8944-4542-bc12-627b444fdcc4\\prom_shirtless.gmdl.gfxbin"; -// var gpubin = "C:\\Testing\\extract\\mod\\7594c633-8944-4542-bc12-627b444fdcc4\\prom_shirtless.gpubin"; -// //var gfxbin = "C:\\Users\\Kieran\\Desktop\\Mods\\Noctis\\character\\nh\\nh00\\model_010\\nh00_010.gmdl.gfxbin"; -// //var gpubin = "C:\\Users\\Kieran\\Desktop\\Mods\\Noctis\\character\\nh\\nh00\\model_010\\nh00_010.gpubin"; -// //var moGfxbin = "C:\\Testing\\Gfxbin\\mod\\noctis_custom_2\\main.gmdl.gfxbin"; -// //var moGpubin = "C:\\Testing\\Gfxbin\\mod\\noctis_custom_2\\main.gpubin"; -// //var moGfxbin = "C:\\Testing\\Gfxbin\\noctis_custom_2_backup_main.gmdl.gfxbin"; -// //var moGpubin = "C:\\Testing\\Gfxbin\\noctis_custom_2_backup_main.gpubin"; -// -// //var reader = new ModelReader(File.ReadAllBytes(moGfxbin), File.ReadAllBytes(moGpubin)); -// //var mo = reader.Read(); -// var reader = new ModelReader(File.ReadAllBytes(gfxbin), File.ReadAllBytes(gpubin)); -// var model = reader.Read(); -// -// // foreach (var dependency in model.Header.Dependencies) -// // { -// // System.Console.WriteLine($"{dependency.PathHash} - {dependency.Path}"); -// // } -// // -// // System.Console.WriteLine($"\n{model.AssetHash}"); -// -// System.Console.WriteLine($"[OG Asset] AABB-min: {model.Aabb.Min.X}, {model.Aabb.Min.Y}, {model.Aabb.Min.Z}"); -// // System.Console.WriteLine($"[MO Asset] AABB-min: {mo.Aabb.Min.X}, {mo.Aabb.Min.Y}, {mo.Aabb.Min.Z}"); -// // -// System.Console.WriteLine($"[OG Asset] AABB-max: {model.Aabb.Max.X}, {model.Aabb.Max.Y}, {model.Aabb.Max.Z}"); -// // System.Console.WriteLine($"[MO Asset] AABB-max: {mo.Aabb.Max.X}, {mo.Aabb.Max.Y}, {mo.Aabb.Max.Z}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Name: {model.Name}"); -// // System.Console.WriteLine($"[MO Asset] Name: {mo.Name}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Unknown1: {model.Unknown1}"); -// //System.Console.WriteLine($"[MO Asset] Unknown1: {mo.Unknown1}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Asset Hash: {model.AssetHash}"); -// // System.Console.WriteLine($"[MO Asset] Asset Hash: {mo.AssetHash}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Child Class Format: {model.ChildClassFormat}"); -// // System.Console.WriteLine($"[MO Asset] Child Class Format: {mo.ChildClassFormat}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Instance Name Format: {model.InstanceNameFormat}"); -// // System.Console.WriteLine($"[MO Asset] Instance Name Format: {mo.InstanceNameFormat}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Shader Class Format: {model.ShaderClassFormat}"); -// // System.Console.WriteLine($"[MO Asset] Shader Class Format: {mo.ShaderClassFormat}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Shader Parameter List Format: {model.ShaderParameterListFormat}"); -// // System.Console.WriteLine($"[MO Asset] Shader Parameter List Format: {mo.ShaderParameterListFormat}\n"); -// // -// System.Console.WriteLine( -// $"[OG Asset] Shader Sampler Description Format: {model.ShaderSamplerDescriptionFormat}"); -// // System.Console.WriteLine( -// // $"[MO Asset] Shader Sampler Description Format: {mo.ShaderSamplerDescriptionFormat}\n"); -// // -// System.Console.WriteLine($"[OG Asset] Has PSD Path: {model.HasPsdPath}"); -// // System.Console.WriteLine($"[MO Asset] Has PSD Path: {mo.HasPsdPath}\n"); -// // -// System.Console.WriteLine($"[OG Asset] PSD Path Hash: {model.PsdPathHash}"); -// // System.Console.WriteLine($"[MO Asset] PSD Path Hash: {mo.PsdPathHash}\n"); -// -// foreach (var node in model.NodeTable) -// { -// System.Console.WriteLine($"Node name: {node.Name}"); -// System.Console.WriteLine($"{node.Matrix.Rows[0].X}, {node.Matrix.Rows[0].Y}, {node.Matrix.Rows[0].Z}"); -// System.Console.WriteLine($"{node.Matrix.Rows[1].X}, {node.Matrix.Rows[1].Y}, {node.Matrix.Rows[1].Z}"); -// System.Console.WriteLine($"{node.Matrix.Rows[2].X}, {node.Matrix.Rows[2].Y}, {node.Matrix.Rows[2].Z}"); -// System.Console.WriteLine($"{node.Matrix.Rows[3].X}, {node.Matrix.Rows[3].Y}, {node.Matrix.Rows[3].Z}"); -// } -// -// foreach (var meshObject in model.MeshObjects) -// { -// foreach (var mesh in meshObject.Meshes) -// { -// System.Console.WriteLine($"{mesh.Name}"); -// -// // foreach (var geometry in mesh.SubGeometries) -// // { -// // System.Console.WriteLine($"AABB: {geometry.Aabb.Min}, {geometry.Aabb.Max}"); -// // System.Console.WriteLine($"Draw order: {geometry.DrawOrder}"); -// // System.Console.WriteLine($"Primitive count: {geometry.PrimitiveCount}"); -// // System.Console.WriteLine($"Start index: {geometry.StartIndex}"); -// // System.Console.WriteLine($"Cluster index bit flag: {geometry.ClusterIndexBitFlag}"); -// // } -// -// System.Console.WriteLine($"AABB Min: {mesh.Aabb.Min.X}, {mesh.Aabb.Min.Y}, {mesh.Aabb.Min.Z}"); -// System.Console.WriteLine($"AABB Max: {mesh.Aabb.Max.X}, {mesh.Aabb.Max.Y}, {mesh.Aabb.Max.Z}"); -// -// System.Console.WriteLine($"Flag: {mesh.Flag}"); -// System.Console.WriteLine($"Flags: {mesh.Flags}"); -// System.Console.WriteLine($"Instance Number: {mesh.InstanceNumber}"); -// System.Console.WriteLine($"LodNear: {mesh.LodNear}"); -// System.Console.WriteLine($"LodFar: {mesh.LodFar}"); -// System.Console.WriteLine($"LodFade: {mesh.LodFade}"); -// System.Console.WriteLine($"Breakable bone index: {mesh.BreakableBoneIndex}"); -// System.Console.WriteLine($"Default Material Hash: {mesh.DefaultMaterialHash}"); -// System.Console.WriteLine($"Draw priority offset: {mesh.DrawPriorityOffset}"); -// System.Console.WriteLine($"Vertex layout type: {mesh.VertexLayoutType}"); -// System.Console.WriteLine($"Low Lod Shadow Cascade No: {mesh.LowLodShadowCascadeNo}"); -// System.Console.WriteLine($"Is Oriented BB: {mesh.IsOrientedBB}"); -// System.Console.WriteLine($"Primitive Type: {mesh.PrimitiveType}"); -// System.Console.WriteLine($"Unknown1: {mesh.Unknown1}"); -// System.Console.WriteLine($"Unknown2: {mesh.Unknown2}"); -// System.Console.WriteLine($"Unknown3: {mesh.Unknown3}"); -// System.Console.WriteLine($"Unknown4: {mesh.Unknown4}"); -// System.Console.WriteLine($"Unknown5: {mesh.Unknown5}"); -// System.Console.WriteLine($"Unknown6: {mesh.Unknown6}"); -// -// System.Console.WriteLine( -// $"OBB: {mesh.OrientedBB.XHalfExtent}, {mesh.OrientedBB.YHalfExtent}, {mesh.OrientedBB.ZHalfExtent}, {mesh.OrientedBB.Center}"); -// } -// } -// // -// // foreach (var part in model.Parts) -// // { -// // System.Console.WriteLine($"[OG Asset] Flags: {part.Flags}"); -// // System.Console.WriteLine($"[OG Asset] Id: {part.Id}"); -// // System.Console.WriteLine($"[OG Asset] Name: {part.Name}"); -// // System.Console.WriteLine($"[OG Asset] Unknown: {part.Unknown}\n"); -// // } -// // -// // foreach (var part in mo.Parts) -// // { -// // System.Console.WriteLine($"[MO Asset] Flags: {part.Flags}"); -// // System.Console.WriteLine($"[MO Asset] Id: {part.Id}"); -// // System.Console.WriteLine($"[MO Asset] Name: {part.Name}"); -// // System.Console.WriteLine($"[MO Asset] Unknown: {part.Unknown}\n"); -// // } -// -// System.Console.WriteLine( -// $"Mesh count: {model.MeshObjects[0].Meshes.Count}, {model.MeshObjects[0].Meshes.Count}"); -// for (var i = 0; i < model.MeshObjects[0].Meshes.Count; i++) -// { -// //var moMesh = mo.MeshObjects[0].Meshes[i]; -// // var mesh = model.MeshObjects[0].Meshes[i]; -// // -// // System.Console.WriteLine($"Mesh name: {mesh.Name}"); -// // foreach (var stream in mesh.VertexStreamDescriptions) -// // { -// // System.Console.WriteLine($"Slot: {stream.Slot}"); -// // System.Console.WriteLine($"Stride: {stream.Stride}"); -// // System.Console.WriteLine($"Type: {stream.Type}"); -// // -// // foreach (var desc in stream.VertexElementDescriptions) -// // { -// // System.Console.WriteLine($"\tSemantic: {desc.Semantic}"); -// // System.Console.WriteLine($"\tFormat: {desc.Format}"); -// // System.Console.WriteLine($"\tOffset: {desc.Offset}"); -// // } -// // } -// -// // System.Console.WriteLine($"Vertex count: {moMesh.VertexCount}, {mesh.VertexCount}"); -// // System.Console.WriteLine($"Face index count: {moMesh.FaceIndices.Length}, {mesh.FaceIndices.Length}"); -// -// // for (var j = 0; j < Math.Max(moMesh.FaceIndices.Length, mesh.FaceIndices.Length); j++) -// // { -// // var moFaces = new[] {moMesh.FaceIndices[j, 0], moMesh.FaceIndices[j, 1], moMesh.FaceIndices[j, 2]}; -// // var faces = new[] {mesh.FaceIndices[j, 0], mesh.FaceIndices[j, 1], mesh.FaceIndices[j, 2]}; -// // System.Console.WriteLine($"MO: [{moFaces[0]}, {moFaces[1]}, {moFaces[2]}] O: [{faces[0]}, {faces[1]}, {faces[2]}]"); -// // } -// -// // for (var j = 0; j < moMesh.WeightValues[0].Count; j++) -// // { -// // var map1 = moMesh.WeightValues[0][j]; -// // var map2 = moMesh.WeightValues[1][j]; -// // var sum = map1.Sum(s => s) + map2.Sum(s => s); -// // System.Console.WriteLine( -// // $"[{map1[0]}, {map1[1]}, {map1[2]}, {map1[3]}], [{map2[0]}, {map2[1]}, {map2[2]}, {map2[3]}], Sum: {sum}"); -// // } -// -// // for (var j = 0; j < Math.Max(moMesh.WeightIndices[0].Count, mesh.WeightIndices[0].Count); j++) -// // { -// // var moWeight = moMesh.WeightValues[0][j]; -// // var moIndex = moMesh.WeightIndices[0][j]; -// // var weight = mesh.WeightValues[0][j]; -// // var index = mesh.WeightIndices[0][j]; -// // -// // System.Console.WriteLine($"Indices: [{moIndex[0]}, {moIndex[1]}, {moIndex[2]}, {moIndex[3]}], [{index[0]}, {index[1]}, {index[2]}, {index[3]}]"); -// // System.Console.WriteLine($"Weights: [{moWeight[0]}, {moWeight[1]}, {moWeight[2]}, {moWeight[3]}], [{weight[0]}, {weight[1]}, {weight[2]}, {weight[3]}]"); -// // } -// } -// } -} \ No newline at end of file diff --git a/Flagrum.Console/Program.cs b/Flagrum.Console/Program.cs index c15cac0e..58bf38c3 100644 --- a/Flagrum.Console/Program.cs +++ b/Flagrum.Console/Program.cs @@ -1,19 +1,53 @@ -using System.IO; -using System.Linq; -using System.Text; -using Flagrum.Console.Scripts.Archive; -using Flagrum.Console.Scripts.Terrain; -using Flagrum.Console.Utilities; +using System; +using System.IO; using Flagrum.Core.Archive; -using Flagrum.Core.Ebex.Xmb2; -using Flagrum.Core.Gfxbin.Btex; -using Flagrum.Web.Persistence; +using Flagrum.Core.Utilities; +using Flagrum.Core.Utilities.Types; +using Flagrum.Web.Features.ModManager.Data; using Flagrum.Web.Services; -using Microsoft.Extensions.Logging.Abstractions; +using K4os.Compression.LZ4.Streams; -using var unpacker = new Unpacker(@"C:\Program Files (x86)\Steam\steamapps\common\FORSPOKEN Demo\datas\c000_003.earc"); -var data = unpacker.UnpackFileByQuery( - "data://asset/environment/common/props/co_bottle06/sourceimages/co_bottle06_a_n_$h.tif", out _); +var fragment = new FmodFragment(); +fragment.Read(@"C:\Users\Kieran\AppData\Local\Flagrum\earc\d52176d1-2d22-4311-be79-5ff8831976ed\2\2.ffg"); + +var stream = new MemoryStream(fragment.Data); +using var lz4Stream = LZ4Stream.Decode(stream); +using var destinationStream = new MemoryStream(); +lz4Stream.CopyTo(destinationStream); +stream.Dispose(); +var buffer = destinationStream.ToArray(); + +File.WriteAllBytes(@"C:\Users\Kieran\Downloads\test.btex", buffer); +return; + +Console.WriteLine(((LuminousGame)75759744).HasFlag((LuminousGame)67108864)); +return; + +var uri = "data://asset/character/hu/hu000/model_000/sourceimages/tshirts/hu000_000_inner_shirts_1001_b_$m1.tif"; +Console.WriteLine(Cryptography.HashFileUri64(uri)); +return; + +var converter = new TextureConverter(LuminousGame.Forspoken); +var btex = converter.ToBtex(new BtexBuildRequest +{ + Name = "Terada", + PixelFormat = BtexFormat.BC1_UNORM, + ImageFlags = 121, + MipLevels = 0, + SourceFormat = BtexSourceFormat.Wic, + SourceData = File.ReadAllBytes(@"C:\Users\Kieran\Downloads\hu000_000_inner_shirts_1001_b_h_EvilTerada.png"), + AddSedbHeader = false +}); + +File.WriteAllBytes(@"C:\Users\Kieran\Downloads\hu000_000_inner_shirts_1001_b_h_EvilTerada.btex", btex); +return; + +using var archive = + new EbonyArchive( + @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\menu\first_ff\script\menuswfentry_first_ff.backup"); +archive.WriteToFile( + @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\menu\first_ff\script\menuswfentry_first_ff.earc", + LuminousGame.FFXV); return; //IndexingScripts.DumpUniqueRelativeExtensions(); diff --git a/Flagrum.Console/Program2.cs b/Flagrum.Console/Program2.cs deleted file mode 100644 index 9916f8c0..00000000 --- a/Flagrum.Console/Program2.cs +++ /dev/null @@ -1,223 +0,0 @@ -// using System.IO; -// using Flagrum.Core.Animation; -// using Flagrum.Core.Gfxbin.Gmdl; -// using Flagrum.Web.Persistence; -// using Flagrum.Web.Persistence.Entities; -// using Flagrum.Web.Services; -// -// namespace Flagrum.Console; -// -// public class HeightFieldEntity -// { -// public string Name { get; set; } -// public float[] Position { get; set; } = new float[3]; -// public int Width { get; set; } -// public int Height { get; set; } -// public float[] Pixels { get; set; } -// } -// -// public class Program2 -// { -// public static void Main(string[] args) -// { -// -// -// // var results = context.AssetUris -// // .Where(a => a.Uri.StartsWith("data://level")) -// // .ToList() -// // .Select(a => a.Uri[(a.Uri.LastIndexOf('.') + 1)..]) -// // .Distinct(); -// // foreach (var extension in results) -// // { -// // System.Console.WriteLine(extension); -// // } -// -// // var data = context.GetFileByUri("data://environment/world/models/terrainmaterial_highspec.gmdl"); -// // var data2 = context.GetFileByUri("data://environment/world/models/terrainmaterial_highspec.gpubin"); -// // var model = new ModelReader(data, data2).Read(); -// // bool x = true; -// -// // var xmb2 = File.ReadAllBytes(@"C:\Modding\HebTest\area_leide_h.ebex"); -// // using var stream = new MemoryStream(xmb2); -// // var package = Xmb2Document.GetRootElement(stream); -// // var objects = package.GetElementByName("objects"); -// // -// // var tiles = new List(); -// // foreach (var element in objects.GetElements()) -// // { -// // var type = element.GetAttributeByName("type").GetTextValue(); -// // if (type == "Black.Entity.Actor.HeightFieldEntity") -// // { -// // var name = element.GetAttributeByName("name").GetTextValue(); -// // var path = -// // $@"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\environment\world\heightmaps\lod00\{name}.heb"; -// // -// // if (!File.Exists(path)) -// // { -// // continue; -// // } -// // -// // var heb = new HebReader(path).Read(); -// // foreach (var (extension, data) in BtexConverter.HebToDds(heb)) -// // { -// // if (extension == "json") -// // { -// // var position = element.GetElementByName("position_").GetFloat4Value(); -// // var (width, height, buffer) = ((uint, uint, float[]))data; -// // tiles.Add(new HeightFieldEntity -// // { -// // Name = name, -// // Position = new[] {position[0], position[1], position[2]}, -// // Width = (int)width, -// // Height = (int)height, -// // Pixels = buffer -// // }); -// // } -// // } -// // } -// // } -// // File.WriteAllText(@"C:\Modding\HebTest\Leide.json", JsonConvert.SerializeObject(tiles)); -// -// // var reader = -// // new HebReader( -// // @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\environment\world\heightmaps\lod00\height_x40_y30.backup"); -// // var heb = reader.Read(); -// // //heb.ImageHeaders.RemoveAt(2); -// // var image = TexHelper.Instance.LoadFromTGAFile(@"C:\Modding\HebTest\x40_y30\merged_mask_map.tga"); -// // using var stream = new MemoryStream(); -// // using var ddsStream = image.SaveToDDSMemory(DDS_FLAGS.NONE); -// // ddsStream.CopyTo(stream); -// // var dds = stream.ToArray(); -// // var newData = new byte[dds.Length - 128]; -// // Array.Copy(dds, 128, newData, 0, newData.Length); -// // heb.ImageHeaders[2].DdsData = newData; -// // var writer = new HebWriter(); -// // var result = writer.Write(heb); -// // File.WriteAllBytes(@"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\environment\world\heightmaps\lod00\height_x40_y30.heb", result); -// -// -// // var counter = 0; -// // foreach (var (extension, data) in BtexConverter.HebToDds(heb)) -// // { -// // System.Console.WriteLine(counter); -// // -// // if (extension == "dds") -// // { -// // var pinnedData = GCHandle.Alloc(data, GCHandleType.Pinned); -// // var pointer = pinnedData.AddrOfPinnedObject(); -// // -// // var image = TexHelper.Instance.LoadFromDDSMemory(pointer, ((byte[])data).Length, DDS_FLAGS.NONE); -// // //image = image.Convert(DXGI_FORMAT.R32G32B32A32_FLOAT, TEX_FILTER_FLAGS.CUBIC, 0.5f); -// // -// // pinnedData.Free(); -// // -// // var metadata = image.GetMetadata(); -// // if (metadata.Format != DXGI_FORMAT.R8G8B8A8_UNORM) -// // { -// // image = image.Decompress(DXGI_FORMAT.R8G8B8A8_UNORM); -// // } -// // -// // using var stream = new MemoryStream(); -// // //using var ddsStream = -// // // image.SaveToWICMemory(0, WIC_FLAGS.FORCE_SRGB, TexHelper.Instance.GetWICCodec(WICCodecs.PNG)); -// // //using var ddsStream = image.SaveToDDSMemory(DDS_FLAGS.NONE); -// // //using var ddsStream = image.SaveToHDRMemory(0); -// // using var ddsStream = image.SaveToTGAMemory(0); -// // ddsStream.CopyTo(stream); -// // File.WriteAllBytes(@$"C:\Modding\HebTest\x40_y30\{counter}.tga", stream.ToArray()); -// // } -// // -// // counter++; -// // } -// - -// -// // var gfx = @"C:\Users\Kieran\Desktop\Environments\Altissia\al_ar_castle01_typ07c.gmdl.gfxbin"; -// // var gpu = gfx.Replace(".gmdl.gfxbin", ".gpubin"); -// // var model = new ModelReader(File.ReadAllBytes(gfx), File.ReadAllBytes(gpu)).Read(); -// // -// // foreach (var meshObject in model.MeshObjects) -// // { -// // foreach (var mesh in meshObject.Meshes) -// // { -// // foreach (var uvMap in mesh.UVMaps) -// // { -// // foreach (var coord in uvMap.UVs) -// // { -// // if (coord.U == Half.NaN || coord.U == Half.NegativeInfinity || coord.U == Half.PositiveInfinity -// // || coord.V == Half.NaN || coord.V == Half.NegativeInfinity || -// // coord.V == Half.PositiveInfinity) -// // { -// // System.Console.WriteLine($"{meshObject.Name}, {mesh.Name} - U: {coord.U}, V: {coord.V}"); -// // } -// // } -// // } -// // } -// // } -// // -// // var x = true; -// -// // var finder = new FileFinder(); -// // finder.FindByQuery( -// // file => file.Uri.EndsWith(".amdl"), -// // file => System.Console.WriteLine($"{file.Uri.Split('/').Last()}\t\t{file.Uri}") -// // ); -// -// // var input = @"C:\Modding\teal.png"; -// // var output = @"C:\Modding\teal.btex"; -// // var converter = new TextureConverter(); -// // var btex = converter.ToBtex("teal", "png", TextureType.Mrs, File.ReadAllBytes(input)); -// // File.WriteAllBytes(output, btex); -// // var gfx = File.ReadAllBytes(@"C:\Users\Kieran\Desktop\Models2\le_ar_gqshop1\le_ar_gqshop1.gmdl.gfxbin"); -// // var gpu = File.ReadAllBytes(@"C:\Users\Kieran\Desktop\Models2\le_ar_gqshop1\le_ar_gqshop1.gpubin"); -// // var model = new ModelReader(gfx, gpu).Read(); -// // var x = true; -// //Visit(@"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas"); -// // var json = File.ReadAllText(@"C:\Modding\map3.json"); -// // var map = JsonConvert.DeserializeObject>>(json); -// // -// // var start = DateTime.Now; -// // -// // var root = new NamedTreeNode("data://", null); -// // foreach (var (archive, uris) in map) -// // { -// // foreach (var uri in uris) -// // { -// // var tokens = uri.Split('/'); -// // var currentDirectory = root; -// // foreach (var token in tokens) -// // { -// // var subdirectory = currentDirectory.Children -// // .FirstOrDefault(c => c.Name == token) ?? currentDirectory.AddChild(token); -// // -// // currentDirectory = subdirectory; -// // } -// // } -// // } -// // -// // var serializer = new DataContractSerializer(typeof(NamedTreeNode)); -// // using var fileStream = new FileStream(@"C:\Modding\Tree.xml", FileMode.Create, FileAccess.Write); -// // serializer.WriteObject(fileStream, root); -// // -// // //File.WriteAllText(@"C:\Modding\UriDirectoryMap.json", JsonConvert.SerializeObject(root)); -// // System.Console.WriteLine((DateTime.Now - start).TotalMilliseconds); -// } -// -// private static void Visit(string directory) -// { -// foreach (var file in Directory.EnumerateFiles(directory)) -// { -// if (!file.EndsWith(".earc") && !file.EndsWith(".heb") && !file.EndsWith(".hephysx") && -// !file.EndsWith(".bk2") && !file.EndsWith(".sab")) -// { -// System.Console.WriteLine(file); -// } -// } -// -// foreach (var subdirectory in Directory.EnumerateDirectories(directory)) -// { -// Visit(subdirectory); -// } -// } -// } - diff --git a/Flagrum.Console/Scripts/Archive/IndexingScripts.cs b/Flagrum.Console/Scripts/Archive/IndexingScripts.cs index 66cf7ad0..a2354fe4 100644 --- a/Flagrum.Console/Scripts/Archive/IndexingScripts.cs +++ b/Flagrum.Console/Scripts/Archive/IndexingScripts.cs @@ -11,14 +11,40 @@ namespace Flagrum.Console.Scripts.Archive; public class IndexingScripts { - private readonly SettingsService _settings = new(); - private readonly ConcurrentDictionary> _assets = new(); + private readonly ConcurrentDictionary> _assets = + new(); + private readonly ProfileService _profile = new(); + + public static void DumpDifferingExtensions() + { + var script = new IndexingScripts(); + var relativePaths = script.GetRelativePaths().ToList(); + + var extensions = relativePaths.Select(u => + { + var fileName = u.relativePath.Split('\\').Last(); + var relativeExtension = fileName[fileName.IndexOf('.')..]; + fileName = u.uri.Split('/').Last(); + var extension = fileName[fileName.IndexOf('.')..]; + return (relativeExtension, extension); + }) + .DistinctBy(e => e.extension) + .ToList(); + + foreach (var (relativeExtension, extension) in extensions + .OrderBy(e => e.relativeExtension) + .Where(e => e.extension != e.relativeExtension)) + { + System.Console.WriteLine("{\"" + extension[1..] + "\", \"" + relativeExtension[1..] + "\"},"); + } + } + public static void DumpUniqueRelativeExtensions() { var script = new IndexingScripts(); var relativePaths = script.GetRelativePaths().ToList(); - + var extensions = relativePaths.Select(u => { var fileName = u.relativePath.Split('\\').Last(); @@ -33,14 +59,35 @@ public static void DumpUniqueRelativeExtensions() foreach (var (relativeExtension, extension) in extensions .OrderBy(e => e.relativeExtension)) { - System.Console.WriteLine("\"" + relativeExtension + "\", "); + System.Console.WriteLine("\"" + relativeExtension[1..] + "\","); + } + } + + public static void DumpUniqueUriExtensions() + { + var script = new IndexingScripts(); + var relativePaths = script.GetRelativePaths().ToList(); + + var extensions = relativePaths.Select(u => + { + var fileName = u.uri.Split('/').Last(); + var extension = fileName[fileName.IndexOf('.')..]; + return extension; + }) + .DistinctBy(e => e) + .ToList(); + + foreach (var extension in extensions + .OrderBy(e => e)) + { + System.Console.WriteLine($"\"{extension[1..]}\","); } } private IEnumerable<(string relativePath, string uri)> GetRelativePaths() { - MapDirectory(_settings.GameDataDirectory); - Parallel.ForEach(Directory.EnumerateDirectories(_settings.GameDataDirectory), GenerateMapRecursively); + MapDirectory(_profile.GameDataDirectory); + Parallel.ForEach(Directory.EnumerateDirectories(_profile.GameDataDirectory), GenerateMapRecursively); return _assets.SelectMany(kvp => kvp.Value); } @@ -49,16 +96,16 @@ private void GenerateMapRecursively(string directory) MapDirectory(directory); Parallel.ForEach(Directory.EnumerateDirectories(directory), GenerateMapRecursively); } - + private void MapDirectory(string directory) { foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) { - using var unpacker = new Unpacker(file); - var archive = new ArchiveLocation {Path = file.Replace(_settings.GameDataDirectory + "\\", "")}; + using var unpacker = new EbonyArchive(file); + var archive = new ArchiveLocation {Path = file.Replace(_profile.GameDataDirectory + "\\", "")}; _assets.TryAdd(archive, unpacker.Files - .Where(f => !f.Flags.HasFlag(ArchiveFileFlag.Reference)) - .Select(f => (f.RelativePath, f.Uri))); + .Where(f => !f.Value.Flags.HasFlag(ArchiveFileFlag.Reference)) + .Select(f => (f.Value.RelativePath, f.Value.Uri))); } } } \ No newline at end of file diff --git a/Flagrum.Console/Scripts/Forspoken/ModScripts.cs b/Flagrum.Console/Scripts/Forspoken/ModScripts.cs new file mode 100644 index 00000000..9f611b17 --- /dev/null +++ b/Flagrum.Console/Scripts/Forspoken/ModScripts.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.IO; +using Flagrum.Core.Archive; +using Flagrum.Core.Utilities; +using Flagrum.Core.Utilities.Types; + +namespace Flagrum.Console.Scripts.Forspoken; + +public static class ModScripts +{ + private const string ForspokenDirectory = @"C:\Program Files (x86)\Steam\steamapps\common\FORSPOKEN Demo\datas"; + private const string AssetsDirectory = @"C:\Modding\Forspoken\Assets"; + + private static EbonyReplace _erep; + private static EbonyArchive _modArchive; + + private static readonly List<(string originalUri, string replacementUri, string replacementFilePath)> + _replacements = new() + { + ("data://asset/character/hu/hu000/model_000/sourceimages/tshirts/hu000_000_inner_shirts_1001_b.tif", + "data://mods/hu000_000_inner_shirts_1001_b.tif", + AssetPath("hu000_000_inner_shirts_1001_b_$h.btex")), + + ("data://asset/character/hu/hu000/model_000/sourceimages/tshirts/hu000_000_inner_shirts_1001_b_$h.tif", + "data://mods/hu000_000_inner_shirts_1001_b_$h.tif", + AssetPath("hu000_000_inner_shirts_1001_b_$h.btex")), + + ("data://asset/character/hu/hu000/model_000/sourceimages/tshirts/hu000_000_inner_shirts_1001_b_$m1.tif", + "data://mods/hu000_000_inner_shirts_1001_b_$m1.tif", + AssetPath("hu000_000_inner_shirts_1001_b_$h.btex")) + }; + + public static void CreateMod() + { + // Load up c000.earc and the erep file + using var c000 = new EbonyArchive(AbsolutePath("c000.backup")); + c000.WriteToFile(AbsolutePath("c000.earc"), LuminousGame.Forspoken); + return; + + var erepFile = c000["data://c000.erep"]; + _erep = new EbonyReplace(erepFile.GetReadableData()); + + // Create a new mod earc + _modArchive = new EbonyArchive(false); + _modArchive.SetFlags(EbonyArchiveFlags.HasLooseData); + + foreach (var (originalUri, replacementUri, replacementFilePath) in _replacements) + { + AddReplacement(originalUri, replacementUri, replacementFilePath); + } + + _modArchive.WriteToFile(AbsolutePath("mods.earc"), LuminousGame.Forspoken); + _modArchive.Dispose(); + + // Write the updated erep to the archive + using var stream = new MemoryStream(); + _erep.Write(stream); + //erepFile.SetRawData(stream.ToArray()); + + // Add reference to the new mod archive and write updated c000.earc to disk + //c000.AddFile("data://mods.ebex@", ArchiveFileFlag.Autoload | ArchiveFileFlag.Reference, Array.Empty()); + //c000["data://mods.ebex@"].RelativePath = "$archives/mods.earc"; + c000.WriteToFile(AbsolutePath("c000.earc"), LuminousGame.Forspoken); + } + + private static void AddReplacement(string originalUri, string replacementUri, string replacementFilePath) + { + // Add new file to the mod archive + var data = File.ReadAllBytes(replacementFilePath); + _modArchive.AddFile(replacementUri, ArchiveFileFlag.Compressed | ArchiveFileFlag.LZ4Compression, data); + + // Add replacement record to the erep + _erep.Replacements.Add(Cryptography.HashFileUri64(originalUri), Cryptography.HashFileUri64(replacementUri)); + } + + private static string AbsolutePath(string relativePath) + { + return $@"{ForspokenDirectory}\{relativePath}"; + } + + private static string AssetPath(string relativePath) + { + return $@"{AssetsDirectory}\{relativePath}"; + } +} \ No newline at end of file diff --git a/Flagrum.Console/Scripts/Terrain/HebScripts.cs b/Flagrum.Console/Scripts/Terrain/HebScripts.cs index 7fd4d3f9..84edf0b8 100644 --- a/Flagrum.Console/Scripts/Terrain/HebScripts.cs +++ b/Flagrum.Console/Scripts/Terrain/HebScripts.cs @@ -1,73 +1,73 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Linq; -using DirectXTexNet; -using Flagrum.Console.Utilities; -using Flagrum.Core.Gfxbin.Btex; -using Flagrum.Web.Services; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Flagrum.Console.Scripts.Terrain; - -public static class HebScripts -{ - /// - /// Dumps all images in a HEB file to the given directory in TGA format - /// - /// File path of the HEB file to dump from - /// Path to the directory to dump the images in - public static void DumpImages(string hebPath, string outputDirectory) - { - var heb = HebHeader.FromData(File.ReadAllBytes(hebPath)); - var images = new TerrainPacker(NullLogger.Instance, new SettingsService()).HebToImages(heb); - - foreach (var imageBase in images) - { - if (imageBase is HebImageData image) - { - File.WriteAllBytes($@"{outputDirectory}\{image.Index}.{image.Extension}", image.Data); - } - } - } - - /// - /// Replaces the merged mask map of a HEB file with a given replacement TGA file and repacks the HEB file in place - /// - /// File path of the HEB file to edit - /// File path of the replacement merged mask map TGA - public static void ReplaceMergedMaskMap(string hebPath, string replacementMergedMaskMapTgaPath) - { - var timer = Stopwatch.StartNew(); - var backupPath = hebPath.Replace(".heb", ".backup"); - - Log.WriteLine($"Creating backup of HEB file at {backupPath}"); - if (!File.Exists(backupPath)) - { - File.Copy(hebPath, backupPath); - } - - Log.WriteLine("Loading HEB data..."); - var heb = HebHeader.FromData(File.ReadAllBytes(hebPath)); - var mask = heb.ImageHeaders.FirstOrDefault(i => i.Type == HebImageType.TYPE_MERGED_MASK_MAP); - - if (mask == null) - { - Log.WriteLine("ERROR: HEB file provided does not contain a merged mask map", ConsoleColor.Red); - return; - } - - Log.WriteLine("Converting input file to DDS..."); - var dds = new TextureConverter().TargaToDds(File.ReadAllBytes(replacementMergedMaskMapTgaPath), - mask.MipCount, - (DXGI_FORMAT)(int)BtexConverter.FormatMap[mask.TextureFormat]); - - mask.DdsData = dds; - - Log.WriteLine($"Repacking HEB and writing to {hebPath}"); - File.WriteAllBytes(hebPath, HebHeader.ToData(heb)); - - timer.Stop(); - Log.WriteLine($"ReplaceMergedMaskMap completed after {timer.ElapsedMilliseconds} millseconds"); - } -} \ No newline at end of file +// using System; +// using System.Diagnostics; +// using System.IO; +// using System.Linq; +// using DirectXTexNet; +// using Flagrum.Console.Utilities; +// using Flagrum.Core.Gfxbin.Btex; +// using Flagrum.Web.Services; +// using Microsoft.Extensions.Logging.Abstractions; +// +// namespace Flagrum.Console.Scripts.Terrain; +// +// public static class HebScripts +// { +// /// +// /// Dumps all images in a HEB file to the given directory in TGA format +// /// +// /// File path of the HEB file to dump from +// /// Path to the directory to dump the images in +// public static void DumpImages(string hebPath, string outputDirectory) +// { +// var heb = HebHeader.FromData(File.ReadAllBytes(hebPath)); +// var images = new TerrainPacker(NullLogger.Instance, new ProfileService()).HebToImages(heb); +// +// foreach (var imageBase in images) +// { +// if (imageBase is HebImageData image) +// { +// File.WriteAllBytes($@"{outputDirectory}\{image.Index}.{image.Extension}", image.Data); +// } +// } +// } +// +// /// +// /// Replaces the merged mask map of a HEB file with a given replacement TGA file and repacks the HEB file in place +// /// +// /// File path of the HEB file to edit +// /// File path of the replacement merged mask map TGA +// public static void ReplaceMergedMaskMap(string hebPath, string replacementMergedMaskMapTgaPath) +// { +// var timer = Stopwatch.StartNew(); +// var backupPath = hebPath.Replace(".heb", ".backup"); +// +// Log.WriteLine($"Creating backup of HEB file at {backupPath}"); +// if (!File.Exists(backupPath)) +// { +// File.Copy(hebPath, backupPath); +// } +// +// Log.WriteLine("Loading HEB data..."); +// var heb = HebHeader.FromData(File.ReadAllBytes(hebPath)); +// var mask = heb.ImageHeaders.FirstOrDefault(i => i.Type == HebImageType.TYPE_MERGED_MASK_MAP); +// +// if (mask == null) +// { +// Log.WriteLine("ERROR: HEB file provided does not contain a merged mask map", ConsoleColor.Red); +// return; +// } +// +// Log.WriteLine("Converting input file to DDS..."); +// var dds = new TextureConverter().TargaToDds(File.ReadAllBytes(replacementMergedMaskMapTgaPath), +// mask.MipCount, +// (DXGI_FORMAT)(int)BtexConverter.FormatMap[mask.TextureFormat]); +// +// mask.DdsData = dds; +// +// Log.WriteLine($"Repacking HEB and writing to {hebPath}"); +// File.WriteAllBytes(hebPath, HebHeader.ToData(heb)); +// +// timer.Stop(); +// Log.WriteLine($"ReplaceMergedMaskMap completed after {timer.ElapsedMilliseconds} millseconds"); +// } +// } \ No newline at end of file diff --git a/Flagrum.Console/Scripts/Terrain/TerrainPaletteScripts.cs b/Flagrum.Console/Scripts/Terrain/TerrainPaletteScripts.cs index e6859cd7..b2c377a0 100644 --- a/Flagrum.Console/Scripts/Terrain/TerrainPaletteScripts.cs +++ b/Flagrum.Console/Scripts/Terrain/TerrainPaletteScripts.cs @@ -14,7 +14,7 @@ public static class TerrainPaletteScripts /// The URI of the TPD file to read out public static void DumpTerrainTextureTable(string tpdUri) { - using var context = new FlagrumDbContext(new SettingsService()); + using var context = new FlagrumDbContext(new ProfileService()); var data = context.GetFileByUri(tpdUri); var tpd = TerrainPaletteData.Read(data); diff --git a/Flagrum.Console/Scripts/Texture/TextureConversionScripts.cs b/Flagrum.Console/Scripts/Texture/TextureConversionScripts.cs index 17655ed0..a88b1f29 100644 --- a/Flagrum.Console/Scripts/Texture/TextureConversionScripts.cs +++ b/Flagrum.Console/Scripts/Texture/TextureConversionScripts.cs @@ -1,42 +1,43 @@ -using System.IO; -using System.Runtime.InteropServices; -using DirectXTexNet; -using Flagrum.Core.Gfxbin.Btex; -using Flagrum.Web.Persistence; -using Flagrum.Web.Persistence.Entities; -using Flagrum.Web.Services; - -namespace Flagrum.Console.Scripts.Texture; - -public static class TextureConversionScripts -{ - /// - /// Outputs all images in a btex texture array to PNG files - /// - /// The URI for the texture array btex file - /// The absolute path to the directory to dump the PNG files in - public static void DumpPngTextureArrayFromBtex(string btexArrayUri, string outputDirectoryPath) - { - using var context = new FlagrumDbContext(new SettingsService()); - var data = context.GetFileByUri(btexArrayUri); - data = BtexConverter.BtexToDds(data); - - var pinnedData = GCHandle.Alloc(data, GCHandleType.Pinned); - var pointer = pinnedData.AddrOfPinnedObject(); - - var image = TexHelper.Instance.LoadFromDDSMemory(pointer, data.Length, DDS_FLAGS.NONE); - - pinnedData.Free(); - - for (var i = 0; i < image.GetMetadata().ArraySize; i++) - { - var result = image.Decompress(i * 11, DXGI_FORMAT.R8G8B8A8_UNORM); - using var stream = new MemoryStream(); - using var ddsStream = - result.SaveToWICMemory(0, WIC_FLAGS.FORCE_SRGB, TexHelper.Instance.GetWICCodec(WICCodecs.PNG)); - ddsStream.CopyTo(stream); - - File.WriteAllBytes($@"{outputDirectoryPath}\{i}.png", stream.ToArray()); - } - } -} \ No newline at end of file +// using System.IO; +// using System.Runtime.InteropServices; +// using DirectXTexNet; +// using Flagrum.Core.Gfxbin.Btex; +// using Flagrum.Core.Utilities.Types; +// using Flagrum.Web.Persistence; +// using Flagrum.Web.Persistence.Entities; +// using Flagrum.Web.Services; +// +// namespace Flagrum.Console.Scripts.Texture; +// +// public static class TextureConversionScripts +// { +// /// +// /// Outputs all images in a btex texture array to PNG files +// /// +// /// The URI for the texture array btex file +// /// The absolute path to the directory to dump the PNG files in +// public static void DumpPngTextureArrayFromBtex(string btexArrayUri, string outputDirectoryPath) +// { +// using var context = new FlagrumDbContext(new ProfileService()); +// var data = context.GetFileByUri(btexArrayUri); +// data = BtexConverter.BtexToDds(data, FlagrumProfileType.FFXV); +// +// var pinnedData = GCHandle.Alloc(data, GCHandleType.Pinned); +// var pointer = pinnedData.AddrOfPinnedObject(); +// +// var image = TexHelper.Instance.LoadFromDDSMemory(pointer, data.Length, DDS_FLAGS.NONE); +// +// pinnedData.Free(); +// +// for (var i = 0; i < image.GetMetadata().ArraySize; i++) +// { +// var result = image.Decompress(i * 11, DXGI_FORMAT.R8G8B8A8_UNORM); +// using var stream = new MemoryStream(); +// using var ddsStream = +// result.SaveToWICMemory(0, WIC_FLAGS.FORCE_SRGB, TexHelper.Instance.GetWICCodec(WICCodecs.PNG)); +// ddsStream.CopyTo(stream); +// +// File.WriteAllBytes($@"{outputDirectoryPath}\{i}.png", stream.ToArray()); +// } +// } +// } \ No newline at end of file diff --git a/Flagrum.Console/Utilities/FileFinder.cs b/Flagrum.Console/Utilities/FileFinder.cs index 9660dea4..c57464f5 100644 --- a/Flagrum.Console/Utilities/FileFinder.cs +++ b/Flagrum.Console/Utilities/FileFinder.cs @@ -23,7 +23,9 @@ public class FileData public class FileFinder { - private const string DataDirectory = @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas"; + //private const string DataDirectory = @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas"; + private const string DataDirectory = + @"F:\Forspoken\Forspoken.Digital.Deluxe.Edition.Steam.Rip-InsaneRamZes\FORSPOKEN"; private ConcurrentBag _map; @@ -90,8 +92,9 @@ private void MapDirectory(string directory) if (file.EndsWith(".earc")) { - using var unpacker = new Unpacker(file); - foreach (var archiveFile in unpacker.Files.Where(f => !f.Flags.HasFlag(ArchiveFileFlag.Reference))) + using var unpacker = new EbonyArchive(file); + foreach (var (_, archiveFile) in unpacker.Files.Where(f => + !f.Value.Flags.HasFlag(ArchiveFileFlag.Reference))) { _map.Add(new FileData { @@ -105,38 +108,33 @@ private void MapDirectory(string directory) } } - public void FindByQuery(Func condition, Action onMatch, bool unpackMatchedFiles) + public void FindByQuery(Func, bool> condition, Action onMatch) { System.Console.WriteLine("Starting search..."); var watch = Stopwatch.StartNew(); var startDirectory = DataDirectory; Parallel.ForEach(Directory.EnumerateDirectories(startDirectory), - directory => { FindRecursively(directory, condition, onMatch, unpackMatchedFiles); }); + directory => { FindRecursively(directory, condition, onMatch); }); watch.Stop(); System.Console.WriteLine($"Search finished after {watch.ElapsedMilliseconds} milliseconds."); } - private void FindRecursively(string directory, Func condition, Action onMatch, - bool unpackMatchedFiles) + private void FindRecursively(string directory, Func, bool> condition, + Action onMatch) { foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) { - using var unpacker = new Unpacker(file); - foreach (var archiveFile in unpacker.Files.Where(condition)) + using var unpacker = new EbonyArchive(file); + foreach (var (_, archiveFile) in unpacker.Files.Where(condition)) { - if (unpackMatchedFiles) - { - unpacker.ReadFileData(archiveFile); - } - onMatch(archiveFile); } } Parallel.ForEach(Directory.EnumerateDirectories(directory), - subdirectory => { FindRecursively(subdirectory, condition, onMatch, unpackMatchedFiles); }); + subdirectory => { FindRecursively(subdirectory, condition, onMatch); }); } public static List GetUrisByString(string query, string extension) @@ -144,9 +142,9 @@ public static List GetUrisByString(string query, string extension) var list = new List(); var finder = new FileFinder(); finder.FindByQuery( - file => file.Uri.Contains(query, StringComparison.OrdinalIgnoreCase) && file.Uri.EndsWith("." + extension), - file => { list.Add(file.Uri); }, - false); + file => file.Value.Uri.Contains(query, StringComparison.OrdinalIgnoreCase) && + file.Value.Uri.EndsWith("." + extension), + file => { list.Add(file.Uri); }); return list; } @@ -155,16 +153,16 @@ public static void FindUriByString(string query) { var finder = new FileFinder(); finder.FindByQuery( - file => file.Uri.Contains(query, StringComparison.OrdinalIgnoreCase) && file.Uri.EndsWith(".btex"), - file => { System.Console.WriteLine(file.Uri); }, - false); + file => file.Value.Uri.Contains(query, StringComparison.OrdinalIgnoreCase) && + file.Value.Uri.EndsWith(".btex"), + file => { System.Console.WriteLine(file.Uri); }); } public static void FindStringInExml(string query) { var finder = new FileFinder(); finder.FindByQuery( - file => file.Uri.EndsWith(".ebex") || file.Uri.EndsWith(".prefab"), + file => file.Value.Uri.EndsWith(".ebex") || file.Value.Uri.EndsWith(".prefab"), file => { var builder = new StringBuilder(); @@ -174,15 +172,14 @@ public static void FindStringInExml(string query) { System.Console.WriteLine(file.Uri); } - }, - true); + }); } public static void FindStringInMaterialTextures(string query) { var finder = new FileFinder(); finder.FindByQuery( - file => file.Uri.EndsWith(".gmtl"), + file => file.Value.Uri.EndsWith(".gmtl"), file => { var reader = new MaterialReader(file.GetReadableData()).Read(); @@ -190,7 +187,6 @@ public static void FindStringInMaterialTextures(string query) { System.Console.WriteLine(file.Uri); } - }, - true); + }); } } \ No newline at end of file diff --git a/Flagrum.Console/Utilities/MaterialFinder.cs b/Flagrum.Console/Utilities/MaterialFinder.cs index e1ef95fa..912d028e 100644 --- a/Flagrum.Console/Utilities/MaterialFinder.cs +++ b/Flagrum.Console/Utilities/MaterialFinder.cs @@ -1,282 +1,283 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Flagrum.Core.Archive; -using Flagrum.Core.Gfxbin.Gmdl; -using Flagrum.Core.Gfxbin.Gmdl.Components; -using Flagrum.Core.Gfxbin.Gmtl; -using Flagrum.Core.Utilities; -using Newtonsoft.Json; +// using System; +// using System.Collections.Concurrent; +// using System.Collections.Generic; +// using System.Diagnostics; +// using System.IO; +// using System.Linq; +// using System.Threading.Tasks; +// using Flagrum.Core.Archive; +// using Flagrum.Core.Gfxbin.Gmdl; +// using Flagrum.Core.Gfxbin.Gmdl.Components; +// using Flagrum.Core.Gfxbin.Gmtl; +// using Flagrum.Core.Utilities; +// using Newtonsoft.Json; +// +// namespace Flagrum.Console.Utilities; +// +// public class MaterialFinder +// { +// public const string DataDirectory = +// @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\character"; +// +// private readonly ConcurrentDictionary _hashes = new(); +// private readonly ConcurrentDictionary _limits = new(); +// +// private ConcurrentBag<(string, string)> _matches; +// +// public void MakeTemplate() +// { +// const string templateName = "NAMED_HUMAN_OUTFIT"; +// const string sourceEarcPath = @$"{DataDirectory}\nh\nh02\model_030\materials\autoexternal.earc"; +// const string sourceMaterialPath = "data://character/nh/nh02/model_030/materials/nh02_030_cloth_00_mat.gmtl"; +// +// using var unpacker = new EbonyArchive(sourceEarcPath); +// var sourceBytes = unpacker.UnpackFile(sourceMaterialPath, out _); +// var sourceMaterial = new MaterialReader(sourceBytes).Read(); +// +// var outJson = JsonConvert.SerializeObject(sourceMaterial); +// File.WriteAllText(@$"C:\Modding\MaterialTesting\{templateName}.json", outJson); +// +// MaterialToPython.ConvertFromJsonFile(@$"C:\Modding\MaterialTesting\{templateName}.json", +// @$"C:\Modding\MaterialTesting\{templateName}.py"); +// } +// +// public void MakeGlassTemplate() +// { +// // Get Iggy glasses material from the game files +// const string iggyPath = $"{DataDirectory}\\nh\\nh03\\model_000\\materials\\autoexternal.earc"; +// using var iggyUnpacker = new EbonyArchive(iggyPath); +// var iggyBytes = iggyUnpacker.UnpackFile("nh03_000_basic_01_mat.gmtl", out _); +// var glass = new MaterialReader(iggyBytes).Read(); +// +// const string path = +// @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\character\nh\nh03\model_304\materials\autoexternal.earc"; +// +// // Shiva ribbon +// //const string path = $"{DataDirectory}\\sm\\sm03\\model_000\\materials\\autoexternal.earc"; +// +// using var unpacker = new EbonyArchive(path); +// var materialBytes = unpacker.UnpackFile("nh03_304_basic_01_mat.gmtl", out _); +// var material = new MaterialReader(materialBytes).Read(); +// +// // Overwrite material input values with glass input values +// foreach (var input in glass.InterfaceInputs) +// { +// var match = material.InterfaceInputs.FirstOrDefault(i => i.Name == input.Name); +// if (match == null) +// { +// System.Console.WriteLine($"No match found for input {input.ShaderGenName}"); +// } +// else +// { +// match.Values = input.Values; +// } +// } +// +// // Save new template to file +// var outJson = JsonConvert.SerializeObject(material); +// File.WriteAllText(@"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.json", outJson); +// +// MaterialToPython.ConvertFromJsonFile(@"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.json", +// @"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.py"); +// } +// +// public void Find() +// { +// System.Console.WriteLine("Starting search..."); +// var watch = Stopwatch.StartNew(); +// +// _matches = new ConcurrentBag<(string, string)>(); +// Parallel.ForEach(Directory.EnumerateDirectories(DataDirectory), FindRecursively); +// //FindRecursively(dataDirectory); +// File.WriteAllText(@"C:\Modding\MaterialTesting\Cloth.txt", _matches +// .Distinct() +// .Aggregate("", (current, next) => current + next.Item1 + " - " + next.Item2 + "\r\n")); +// watch.Stop(); +// System.Console.WriteLine($"Search finished after {watch.ElapsedMilliseconds} milliseconds."); +// } +// +// public void FindWeightLimits() +// { +// // var materials = new[] +// // { +// // "CHR_NhBasic_Material", +// // "CHR_nhSkin_Material", +// // "CHR_NhHairBlendTips_Material", +// // "CHR_NhEye_Material", +// // "CHR_Transparency_Material", +// // "CHR_AhBasic_Material", +// // "CHR_AhSkin_Material", +// // "CHR_AhCloth_Material", +// // "CHR_AhHair_Material", +// // "CHR_AhEye_Material", +// // "CHR_AhTransparency_Material", +// // "CHR_Basic_Material" +// // }; +// // +// // Parallel.ForEach(materials, material => { FindMaterialHashRecursively(DataDirectory, material); }); +// // +// // var builder = new StringBuilder(); +// // builder.AppendLine("var hashes = new Dictionary"); +// // builder.AppendLine("{"); +// // foreach (var (material, hash) in _hashes) +// // { +// // builder.AppendLine("{\"" + material + "\", " + hash + "}"); +// // } +// // +// // builder.AppendLine("}"); +// // System.Console.Write(builder.ToString()); +// // return; +// +// var hashes = new Dictionary +// { +// {"CHR_Basic_Material", 1364795739100408041}, +// {"CHR_NhBasic_Material", 1364796605585357796}, +// {"CHR_Transparency_Material", 1364801387760584455}, +// {"CHR_AhBasic_Material", 1364792899576541549}, +// {"CHR_AhEye_Material", 1364799758182926714}, +// {"CHR_AhCloth_Material", 1364798494917927011}, +// {"CHR_NhEye_Material", 1364797471743020018}, +// {"CHR_AhHair_Material", 1364796879206193149}, +// {"CHR_AhTransparency_Material", 1364787128362217975}, +// {"CHR_AhSkin_Material", 1364792619619205877}, +// {"CHR_NhHairBlendTips_Material", 1364788630667498024}, +// {"CHR_nhSkin_Material", 1364789999862915357} +// }; +// +// Parallel.ForEach(hashes, kvp => +// { +// var (name, hash) = kvp; +// FindWeightLimitRecursively(DataDirectory, name, hash); +// }); +// // foreach (var (name, hash) in hashes) +// // { +// // FindWeightLimitRecursively(DataDirectory, name, hash); +// // } +// +// foreach (var (material, limit) in _limits) +// { +// System.Console.WriteLine($"{material}: {limit}"); +// } +// } +// +// private void FindRecursively(string directory) +// { +// foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) +// { +// using var unpacker = new EbonyArchive(file); +// var materials = unpacker.UnpackFilesByQuery(".gmtl"); +// +// foreach (var (materialUri, materialData) in materials) +// { +// try +// { +// var reader = new MaterialReader(materialData); +// var material = reader.Read(); +// if (material.Interfaces.Any(i => i.Name == "CHR_NhBasic_Material") +// && !material.Textures.Any(t => t.ShaderGenName.Contains("MultiMask")) +// && material.InterfaceInputs.Any(i => +// i.ShaderGenName.Contains("BaseColor") && +// i.Values.All(v => v == 1.0))) +// { +// _matches.Add((materialUri, file)); +// } +// } +// catch +// { +// System.Console.WriteLine($"Failed to read material {materialUri}"); +// } +// } +// } +// +// foreach (var subdirectory in Directory.EnumerateDirectories(directory)) +// { +// FindRecursively(subdirectory); +// } +// } +// +// public void FindMaterialHashRecursively(string directory, string materialName) +// { +// foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) +// { +// using var unpacker = new EbonyArchive(file); +// var materials = unpacker.UnpackFilesByQuery(".gmtl"); +// +// foreach (var (materialUri, materialData) in materials) +// { +// try +// { +// var reader = new MaterialReader(materialData); +// var material = reader.Read(); +// if (material.Interfaces.Any(i => i.Name.Equals(materialName, StringComparison.OrdinalIgnoreCase))) +// { +// var hash = Cryptography +// .HashFileUri64(material.Header.Dependencies +// ?.FirstOrDefault(d => d.PathHash == "ref")?.Path); +// _hashes.TryAdd(materialName, hash); +// return; +// } +// } +// catch (Exception ex) +// { +// System.Console.WriteLine($"Failed to read material {materialUri}"); +// System.Console.WriteLine(ex.Message); +// } +// } +// } +// +// foreach (var subdirectory in Directory.EnumerateDirectories(directory)) +// { +// FindMaterialHashRecursively(subdirectory, materialName); +// } +// } +// +// private void FindWeightLimitRecursively(string directory, string materialName, ulong materialHash) +// { +// foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) +// { +// using var unpacker = new EbonyArchive(file); +// var models = unpacker.UnpackFilesByQuery(".gmdl"); +// var gpubins = unpacker.UnpackFilesByQuery(".gpubin"); +// +// foreach (var (modelUri, modelData) in models) +// { +// try +// { +// var (gpuUri, gpuData) = gpubins.FirstOrDefault(g => g.Key == modelUri.Replace(".gmdl", ".gpubin")); +// if (gpuData == null) +// { +// System.Console.WriteLine($"Failed to find gpubin for {modelUri}"); +// } +// +// var reader = new ModelReader(modelData, gpuData); +// var model = reader.Read(); +// foreach (var meshObject in model.MeshObjects) +// { +// foreach (var mesh in meshObject.Meshes) +// { +// if (mesh.DefaultMaterialHash == materialHash) +// { +// System.Console.WriteLine( +// $"Found VertexLayoutType {mesh.VertexLayoutType} for {materialName}"); +// _limits.TryAdd(materialName, mesh.VertexLayoutType); +// return; +// } +// } +// } +// } +// catch (Exception ex) +// { +// System.Console.WriteLine($"Failed to read model {modelUri}"); +// System.Console.WriteLine(ex.StackTrace); +// return; +// } +// } +// } +// +// foreach (var subdirectory in Directory.EnumerateDirectories(directory)) +// { +// FindWeightLimitRecursively(subdirectory, materialName, materialHash); +// } +// } +// } -namespace Flagrum.Console.Utilities; - -public class MaterialFinder -{ - public const string DataDirectory = - @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\character"; - - private readonly ConcurrentDictionary _hashes = new(); - private readonly ConcurrentDictionary _limits = new(); - - private ConcurrentBag<(string, string)> _matches; - - public void MakeTemplate() - { - const string templateName = "NAMED_HUMAN_OUTFIT"; - const string sourceEarcPath = @$"{DataDirectory}\nh\nh02\model_030\materials\autoexternal.earc"; - const string sourceMaterialPath = "data://character/nh/nh02/model_030/materials/nh02_030_cloth_00_mat.gmtl"; - - using var unpacker = new Unpacker(sourceEarcPath); - var sourceBytes = unpacker.UnpackFileByQuery(sourceMaterialPath, out _); - var sourceMaterial = new MaterialReader(sourceBytes).Read(); - - var outJson = JsonConvert.SerializeObject(sourceMaterial); - File.WriteAllText(@$"C:\Modding\MaterialTesting\{templateName}.json", outJson); - - MaterialToPython.ConvertFromJsonFile(@$"C:\Modding\MaterialTesting\{templateName}.json", - @$"C:\Modding\MaterialTesting\{templateName}.py"); - } - - public void MakeGlassTemplate() - { - // Get Iggy glasses material from the game files - const string iggyPath = $"{DataDirectory}\\nh\\nh03\\model_000\\materials\\autoexternal.earc"; - using var iggyUnpacker = new Unpacker(iggyPath); - var iggyBytes = iggyUnpacker.UnpackFileByQuery("nh03_000_basic_01_mat.gmtl", out _); - var glass = new MaterialReader(iggyBytes).Read(); - - const string path = - @"C:\Program Files (x86)\Steam\steamapps\common\FINAL FANTASY XV\datas\character\nh\nh03\model_304\materials\autoexternal.earc"; - - // Shiva ribbon - //const string path = $"{DataDirectory}\\sm\\sm03\\model_000\\materials\\autoexternal.earc"; - - using var unpacker = new Unpacker(path); - var materialBytes = unpacker.UnpackFileByQuery("nh03_304_basic_01_mat.gmtl", out _); - var material = new MaterialReader(materialBytes).Read(); - - // Overwrite material input values with glass input values - foreach (var input in glass.InterfaceInputs) - { - var match = material.InterfaceInputs.FirstOrDefault(i => i.Name == input.Name); - if (match == null) - { - System.Console.WriteLine($"No match found for input {input.ShaderGenName}"); - } - else - { - match.Values = input.Values; - } - } - - // Save new template to file - var outJson = JsonConvert.SerializeObject(material); - File.WriteAllText(@"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.json", outJson); - - MaterialToPython.ConvertFromJsonFile(@"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.json", - @"C:\Modding\MaterialTesting\NAMED_HUMAN_GLASS.py"); - } - - public void Find() - { - System.Console.WriteLine("Starting search..."); - var watch = Stopwatch.StartNew(); - - _matches = new ConcurrentBag<(string, string)>(); - Parallel.ForEach(Directory.EnumerateDirectories(DataDirectory), FindRecursively); - //FindRecursively(dataDirectory); - File.WriteAllText(@"C:\Modding\MaterialTesting\Cloth.txt", _matches - .Distinct() - .Aggregate("", (current, next) => current + next.Item1 + " - " + next.Item2 + "\r\n")); - watch.Stop(); - System.Console.WriteLine($"Search finished after {watch.ElapsedMilliseconds} milliseconds."); - } - - public void FindWeightLimits() - { - // var materials = new[] - // { - // "CHR_NhBasic_Material", - // "CHR_nhSkin_Material", - // "CHR_NhHairBlendTips_Material", - // "CHR_NhEye_Material", - // "CHR_Transparency_Material", - // "CHR_AhBasic_Material", - // "CHR_AhSkin_Material", - // "CHR_AhCloth_Material", - // "CHR_AhHair_Material", - // "CHR_AhEye_Material", - // "CHR_AhTransparency_Material", - // "CHR_Basic_Material" - // }; - // - // Parallel.ForEach(materials, material => { FindMaterialHashRecursively(DataDirectory, material); }); - // - // var builder = new StringBuilder(); - // builder.AppendLine("var hashes = new Dictionary"); - // builder.AppendLine("{"); - // foreach (var (material, hash) in _hashes) - // { - // builder.AppendLine("{\"" + material + "\", " + hash + "}"); - // } - // - // builder.AppendLine("}"); - // System.Console.Write(builder.ToString()); - // return; - - var hashes = new Dictionary - { - {"CHR_Basic_Material", 1364795739100408041}, - {"CHR_NhBasic_Material", 1364796605585357796}, - {"CHR_Transparency_Material", 1364801387760584455}, - {"CHR_AhBasic_Material", 1364792899576541549}, - {"CHR_AhEye_Material", 1364799758182926714}, - {"CHR_AhCloth_Material", 1364798494917927011}, - {"CHR_NhEye_Material", 1364797471743020018}, - {"CHR_AhHair_Material", 1364796879206193149}, - {"CHR_AhTransparency_Material", 1364787128362217975}, - {"CHR_AhSkin_Material", 1364792619619205877}, - {"CHR_NhHairBlendTips_Material", 1364788630667498024}, - {"CHR_nhSkin_Material", 1364789999862915357} - }; - - Parallel.ForEach(hashes, kvp => - { - var (name, hash) = kvp; - FindWeightLimitRecursively(DataDirectory, name, hash); - }); - // foreach (var (name, hash) in hashes) - // { - // FindWeightLimitRecursively(DataDirectory, name, hash); - // } - - foreach (var (material, limit) in _limits) - { - System.Console.WriteLine($"{material}: {limit}"); - } - } - - private void FindRecursively(string directory) - { - foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) - { - using var unpacker = new Unpacker(file); - var materials = unpacker.UnpackFilesByQuery(".gmtl"); - - foreach (var (materialUri, materialData) in materials) - { - try - { - var reader = new MaterialReader(materialData); - var material = reader.Read(); - if (material.Interfaces.Any(i => i.Name == "CHR_NhBasic_Material") - && !material.Textures.Any(t => t.ShaderGenName.Contains("MultiMask")) - && material.InterfaceInputs.Any(i => - i.ShaderGenName.Contains("BaseColor") && - i.Values.All(v => v == 1.0))) - { - _matches.Add((materialUri, file)); - } - } - catch - { - System.Console.WriteLine($"Failed to read material {materialUri}"); - } - } - } - - foreach (var subdirectory in Directory.EnumerateDirectories(directory)) - { - FindRecursively(subdirectory); - } - } - - public void FindMaterialHashRecursively(string directory, string materialName) - { - foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) - { - using var unpacker = new Unpacker(file); - var materials = unpacker.UnpackFilesByQuery(".gmtl"); - - foreach (var (materialUri, materialData) in materials) - { - try - { - var reader = new MaterialReader(materialData); - var material = reader.Read(); - if (material.Interfaces.Any(i => i.Name.Equals(materialName, StringComparison.OrdinalIgnoreCase))) - { - var hash = Cryptography - .HashFileUri64(material.Header.Dependencies - ?.FirstOrDefault(d => d.PathHash == "ref")?.Path); - _hashes.TryAdd(materialName, hash); - return; - } - } - catch (Exception ex) - { - System.Console.WriteLine($"Failed to read material {materialUri}"); - System.Console.WriteLine(ex.Message); - } - } - } - - foreach (var subdirectory in Directory.EnumerateDirectories(directory)) - { - FindMaterialHashRecursively(subdirectory, materialName); - } - } - - private void FindWeightLimitRecursively(string directory, string materialName, ulong materialHash) - { - foreach (var file in Directory.EnumerateFiles(directory, "*.earc")) - { - using var unpacker = new Unpacker(file); - var models = unpacker.UnpackFilesByQuery(".gmdl"); - var gpubins = unpacker.UnpackFilesByQuery(".gpubin"); - - foreach (var (modelUri, modelData) in models) - { - try - { - var (gpuUri, gpuData) = gpubins.FirstOrDefault(g => g.Key == modelUri.Replace(".gmdl", ".gpubin")); - if (gpuData == null) - { - System.Console.WriteLine($"Failed to find gpubin for {modelUri}"); - } - - var reader = new ModelReader(modelData, gpuData); - var model = reader.Read(); - foreach (var meshObject in model.MeshObjects) - { - foreach (var mesh in meshObject.Meshes) - { - if (mesh.DefaultMaterialHash == materialHash) - { - System.Console.WriteLine( - $"Found VertexLayoutType {mesh.VertexLayoutType} for {materialName}"); - _limits.TryAdd(materialName, mesh.VertexLayoutType); - return; - } - } - } - } - catch (Exception ex) - { - System.Console.WriteLine($"Failed to read model {modelUri}"); - System.Console.WriteLine(ex.StackTrace); - return; - } - } - } - - foreach (var subdirectory in Directory.EnumerateDirectories(directory)) - { - FindWeightLimitRecursively(subdirectory, materialName, materialHash); - } - } -} \ No newline at end of file diff --git a/Flagrum.Core/Archive/ArchiveFile.cs b/Flagrum.Core/Archive/ArchiveFile.cs index ccd9bbcb..865fe205 100644 --- a/Flagrum.Core/Archive/ArchiveFile.cs +++ b/Flagrum.Core/Archive/ArchiveFile.cs @@ -1,7 +1,9 @@ using System; using System.IO; using System.Linq; +using Flagrum.Core.Archive.DataSource; using Flagrum.Core.Utilities; +using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; using ZLibNet; @@ -30,7 +32,7 @@ public class ArchiveFile public const uint HeaderSize = 40; public const ulong HeaderHash = 14695981039346656037; - private byte[] _buffer; + private IArchiveFileDataSource _dataSource; public ArchiveFile() { } @@ -102,10 +104,8 @@ public ArchiveFile(string uri, string relativePath, uint size, uint processedSiz public ArchiveFileFlag Flags { get; set; } public byte LocalizationType { get; set; } public byte Locale { get; set; } - public ushort Key { get; set; } - public bool HasData => _buffer?.Length > 0; public bool IsDataEncrypted { get; set; } public bool IsDataCompressed { get; set; } @@ -151,7 +151,7 @@ public static byte[] GetUnprocessedData(ArchiveFileFlag flags, uint originalSize public byte[] GetReadableData() { - var buffer = _buffer; + var buffer = _dataSource.GetData(); if (Key > 0) { @@ -191,12 +191,12 @@ public byte[] GetReadableData() } else { - buffer = DecompressData(); + buffer = DecompressData(buffer); } } else if (IsDataEncrypted) { - buffer = Cryptography.Decrypt(_buffer); + buffer = Cryptography.Decrypt(buffer); } if (ProcessedSize > Size) @@ -211,14 +211,16 @@ public byte[] GetReadableData() public byte[] GetDataForExport() { - if (_buffer == null) + if (_dataSource == null) { return Array.Empty(); } + var buffer = _dataSource.GetData(); + if (!IsDataEncrypted && !IsDataCompressed) { - Size = (uint)_buffer.Length; + Size = _dataSource.Size; } if (!(Flags.HasFlag(ArchiveFileFlag.Encrypted) || Flags.HasFlag(ArchiveFileFlag.Compressed))) @@ -228,16 +230,30 @@ public byte[] GetDataForExport() if (!IsDataEncrypted && Flags.HasFlag(ArchiveFileFlag.Encrypted)) { - var encryptedData = Cryptography.Encrypt(_buffer); + var encryptedData = Cryptography.Encrypt(buffer); ProcessedSize = (uint)encryptedData.Length; return encryptedData; } if (!IsDataCompressed && Flags.HasFlag(ArchiveFileFlag.Compressed)) { - var compressedData = CompressData(_buffer); + byte[] compressedData; + + if (Flags.HasFlag(ArchiveFileFlag.LZ4Compression)) + { + using var stream = new MemoryStream(buffer); + using var destinationStream = new MemoryStream(); + var lz4Stream = LZ4Stream.Encode(destinationStream, LZ4Level.L12_MAX); + stream.CopyTo(lz4Stream); + lz4Stream.Dispose(); + compressedData = destinationStream.ToArray(); + } + else + { + compressedData = CompressData(buffer); + } - if (compressedData.Length >= _buffer.Length) + if (compressedData.Length >= buffer.Length) { Flags &= ~ArchiveFileFlag.Compressed; Key = 0; @@ -250,47 +266,38 @@ public byte[] GetDataForExport() } } - return _buffer; + return buffer; } public byte[] GetRawData() { - return _buffer; + return _dataSource.GetData(); } public void SetDataByFlags(byte[] data) { - _buffer = data; + _dataSource = new MemoryDataSource(data); IsDataCompressed = Flags.HasFlag(ArchiveFileFlag.Compressed); IsDataEncrypted = Flags.HasFlag(ArchiveFileFlag.Encrypted); } public void SetProcessedData(uint originalSize, byte[] data) { - _buffer = data; + _dataSource = new MemoryDataSource(data); Size = originalSize; ProcessedSize = (uint)data.Length; } public void SetRawData(byte[] data) { - _buffer = data; + _dataSource = new MemoryDataSource(data); IsDataCompressed = false; IsDataEncrypted = false; } - public void SetCompressedData(byte[] data) - { - _buffer = data; - IsDataCompressed = true; - IsDataEncrypted = false; - } - - public void SetEncryptedData(byte[] data) + public void SetRawData(Stream stream) { - _buffer = data; - IsDataCompressed = false; - IsDataEncrypted = true; + _dataSource = new OpenFileDataSource(stream, DataOffset, ProcessedSize); } private ArchiveFileFlag GetDefaultBinmodFlags() @@ -380,7 +387,7 @@ private byte[] CompressData(byte[] data) return memoryStream.ToArray(); } - private byte[] DecompressData() + private byte[] DecompressData(byte[] dataSourceBuffer) { const int chunkSize = 128 * 1024; var chunks = Size / chunkSize; @@ -391,7 +398,7 @@ private byte[] DecompressData() chunks++; } - using var memoryStream = new MemoryStream(_buffer); + using var memoryStream = new MemoryStream(dataSourceBuffer); using var outStream = new MemoryStream(); using var writer = new BinaryWriter(outStream); @@ -455,7 +462,7 @@ private void FixRelativePath() } else if (RelativePath.EndsWith(".ebex@")) { - RelativePath = RelativePath.Replace(".ebex@", ".earc"); + RelativePath = "$archives/" + RelativePath.Replace(".ebex@", ".earc"); } else if (RelativePath.EndsWith(".prefab@")) { diff --git a/Flagrum.Core/Archive/ArchiveHelper.cs b/Flagrum.Core/Archive/ArchiveHelper.cs index bfb6a00d..2c20e899 100644 --- a/Flagrum.Core/Archive/ArchiveHelper.cs +++ b/Flagrum.Core/Archive/ArchiveHelper.cs @@ -1,5 +1,5 @@ using System.Collections.Generic; -using Flagrum.Core.Gfxbin.Gmdl; +using Flagrum.Core.Utilities.Types; namespace Flagrum.Core.Archive; @@ -61,6 +61,7 @@ public class RelativeExtensionMap private readonly Dictionary _map = new() { {"aiia.xml", "aiia.ref"}, + {"alist@", "earc"}, {"tif", "btex"}, {"dds", "btex"}, {"exr", "btex"}, @@ -68,6 +69,8 @@ public class RelativeExtensionMap {"png", "btex"}, {"ccf", "ccb"}, {"ebex", "exml"}, + {"ebex@", "earc"}, + {"htpk", "earc"}, {"prefab", "exml"}, {"folg", "folgbin"}, {"gmdl", "gmdl.gfxbin"}, @@ -80,7 +83,13 @@ public class RelativeExtensionMap {"tpd", "tpdbin"}, {"max", "win.mab"}, {"sax", "win.sab"}, - {"wth2", "wth2b"} + {"wth2", "wth2b"}, + {"wpcp", "wpcpbin"}, + {"navalist@", "earc"}, + {"vfxlist@", "earc"}, + {"pka@", "earc"}, + {"anmgph@", "earc"}, + {"aig", "exml"} }; public string this[string uriExtension] @@ -96,7 +105,13 @@ public string this[string uriExtension] .Replace("aiia_erudition.", "") .Replace("tif.", "") .Replace("tif_$h.", "") - .Replace("dds.", ""); + .Replace("dds.", "") + .Replace("1024.", "") + .Replace("2048.", "") + .Replace("4096.", "") + .Replace("512.", "") + .Replace("256.", "") + .Replace("r.", ""); return _map.TryGetValue(uriExtension, out var extension) ? extension : uriExtension; } @@ -105,187 +120,180 @@ public string this[string uriExtension] public static class ArchiveHelper { - public static string[] RelativeExtensions => new[] + public static string[] RelativeExtensions(LuminousGame type) { - "2dcol", - "acd", - "aiia", - "aiia.dbg", - "aiia.ref", - "amdl", - "ani", - "anmgph", - "apx", - "autoext", - "bcfg", - "blos", - "bmdl", - "bnm", - "bnmwnd", - "bod", - "btex", - "ccb", - "clsn", - "clsx", - "cmdl", - "dat", - "db3", - "dml", - "dx11.fntbin", - "dyn", - "elx", - "exml", - "folgbin", - "gmdl.gfxbin", - "gmtl.gfxbin", - "gpubin", - "ies", - "ikts", - "irr", - "kab", - "kdi", - "lik", - "listb", - "lnkani", - "lsd", - "mid", - "nav", - "nav_cell_ref", - "nav_ref", - "nav_waypoint", - "nav_world_map", - "nav_world_splitter", - "pka", - "pngbin", - "prt", - "prtd", - "prtx", - "ps.sb", - "rag", - "sbmdl", - "ssd", - "style", - "swfb", - "tcd", - "tcm", - "tcophysx", - "tmsbin", - "tnav", - "tpd", - "tpdbin", - "tspack", - "txt", - "uam", - "umbra", - "ups", - "vegy", - "vfol", - "vfuncs", - "vfx", - "vhlist", - "vlink", - "vs.sb", - "win.config", - "win.mab", - "win.sab", - "win32.bin", - "win32.bins", - "win32.msgbin", - "wth2b", - "xml" - }; - - public static string[] UriExtensions => new[] - { - "2dcol", - "acd", - "aiia", - "amdl", - "ani", - "anmgph", - "apx", - "autoext", - "bcfg", - "blos", - "bmdl", - "bnm", - "bnmwnd", - "bod", - "btex", - "ccf", - "clsn", - "clsx", - "cmdl", - "dat", - "db3", - "dbg", - "dds", - "dml", - "dyn", - "ebex", - "elx", - "exr", - "fntbin", - "folg", - "gmdl", - "gmtl", - "gpubin", - "ies", - "ikts", - "irr", - "kab", - "kdi", - "lik", - "listb", - "lnkani", - "lsd", - "max", - "mid", - "nav", - "nav_cell_ref", - "nav_context", - "nav_waypoint", - "nav_world", - "nav_world_splitter", - "pka", - "png", - "pngbin", - "prefab", - "prt", - "prtd", - "prtx", - "ps.sb", - "rag", - "sax", - "sbmdl", - "ssd", - "style", - "swf", - "tcd", - "tcm", - "tco", - "tga", - "tif", - "tms", - "tnav", - "tpd", - "tspack", - "txt", - "uam", - "umbra", - "ups", - "vegy", - "vfol", - "vfuncs", - "vfx", - "vhlist", - "vlink", - "vs.sb", - "win.config", - "win32.bin", - "win32.bins", - "win32.msgbin", - "wth2", - "xml" - }; + if (type == LuminousGame.FFXV) + { + return new[] + { + "2dcol", + "acd", + "aiia", + "aiia.dbg", + "aiia.ref", + "amdl", + "ani", + "anmgph", + "apx", + "autoext", + "bcfg", + "blos", + "bmdl", + "bnm", + "bnmwnd", + "bod", + "btex", + "ccb", + "clsn", + "clsx", + "cmdl", + "dat", + "db3", + "dml", + "dx11.fntbin", + "dyn", + "elx", + "exml", + "folgbin", + "gmdl.gfxbin", + "gmtl.gfxbin", + "gpubin", + "ies", + "ikts", + "irr", + "kab", + "kdi", + "lik", + "listb", + "lnkani", + "lsd", + "mid", + "nav", + "nav_cell_ref", + "nav_ref", + "nav_waypoint", + "nav_world_map", + "nav_world_splitter", + "pka", + "pngbin", + "prt", + "prtd", + "prtx", + "ps.sb", + "rag", + "sbmdl", + "ssd", + "style", + "swfb", + "tcd", + "tcm", + "tcophysx", + "tmsbin", + "tnav", + "tpd", + "tpdbin", + "tspack", + "txt", + "uam", + "umbra", + "ups", + "vegy", + "vfol", + "vfuncs", + "vfx", + "vhlist", + "vlink", + "vs.sb", + "win.config", + "win.mab", + "win.sab", + "win32.bin", + "win32.bins", + "win32.msgbin", + "wth2b", + "xml" + }; + } + + return new[] + { + "acd", + "amdl", + "ani", + "anmgph", + "apb", + "atlas", + "bin", + "blast", + "blastc", + "bnm", + "btex", + "ccb", + "ch.sb", + "clsn", + "cmdl", + "customdata", + "data", + "earc", + "elx", + "empath", + "erep", + "exml", + "file_list", + "folgbin", + "gmdl_hair", + "gmdl.gfxbin", + "gmtl.gfxbin", + "gpubin", + "heb", + "hephysx", + "id2s", + "json", + "kdi", + "layer_info", + "layer_pcd", + "lipmap", + "list", + "lsd", + "n3d2p_raw", + "nav", + "nav_cell_ref", + "nav_connectivity", + "nav_debug_single", + "nav_edgelinks", + "nav_smp", + "nav_world_deps", + "nav_world_map", + "navalist", + "navcelllist", + "parambin", + "pka", + "pkr", + "pmdl", + "ps.sb", + "psocache", + "ragdoll", + "res_info", + "sapb", + "sbd", + "tpdbin", + "tspack", + "txt", + "uifn", + "uip", + "vegy", + "vfx", + "vfxlist", + "vs.sb", + "win.config", + "win.sab", + "wld_prc", + "wlod", + "wlodn", + "wpcm", + "wpcpbin", + "wped", + "wpvd" + }; + } } \ No newline at end of file diff --git a/Flagrum.Core/Archive/DataSource/IArchiveFileDataSource.cs b/Flagrum.Core/Archive/DataSource/IArchiveFileDataSource.cs new file mode 100644 index 00000000..e7875c3f --- /dev/null +++ b/Flagrum.Core/Archive/DataSource/IArchiveFileDataSource.cs @@ -0,0 +1,7 @@ +namespace Flagrum.Core.Archive.DataSource; + +public interface IArchiveFileDataSource +{ + public uint Size { get; } + public byte[] GetData(); +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/DataSource/MemoryDataSource.cs b/Flagrum.Core/Archive/DataSource/MemoryDataSource.cs new file mode 100644 index 00000000..9d3bb0e5 --- /dev/null +++ b/Flagrum.Core/Archive/DataSource/MemoryDataSource.cs @@ -0,0 +1,18 @@ +namespace Flagrum.Core.Archive.DataSource; + +public class MemoryDataSource : IArchiveFileDataSource +{ + private readonly byte[] _buffer; + + public MemoryDataSource(byte[] buffer) + { + _buffer = buffer; + } + + public uint Size => (uint)_buffer.Length; + + public byte[] GetData() + { + return _buffer; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/DataSource/OpenFileDataSource.cs b/Flagrum.Core/Archive/DataSource/OpenFileDataSource.cs new file mode 100644 index 00000000..8e4a6a1b --- /dev/null +++ b/Flagrum.Core/Archive/DataSource/OpenFileDataSource.cs @@ -0,0 +1,32 @@ +using System.IO; + +namespace Flagrum.Core.Archive.DataSource; + +public class OpenFileDataSource : IArchiveFileDataSource +{ + private readonly Stream _stream; + private readonly ulong _offset; + private readonly uint _size; + + public OpenFileDataSource(Stream stream, ulong offset, uint size) + { + _stream = stream; + _offset = offset; + _size = size; + } + + public uint Size => _size; + + public byte[] GetData() + { + var buffer = new byte[_size]; + + lock (_stream) + { + _stream.Seek((long)_offset, SeekOrigin.Begin); + _ = _stream.Read(buffer); + } + + return buffer; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/EbonyArchive.cs b/Flagrum.Core/Archive/EbonyArchive.cs new file mode 100644 index 00000000..29c2c2e0 --- /dev/null +++ b/Flagrum.Core/Archive/EbonyArchive.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using Flagrum.Core.Utilities; + +namespace Flagrum.Core.Archive; + +public partial class EbonyArchive : IDisposable +{ + private const uint PointerSize = 8; + + private readonly EbonyArchiveHeader _header; + private readonly string _sourcePath; + private readonly Stream _stream; + + public EbonyArchive(bool isProtectedArchive) + { + _header = new EbonyArchiveHeader + { + Version = EbonyArchiveHeader.ArchiveVersion + }; + + if (isProtectedArchive) + { + _header.Version |= EbonyArchiveHeader.ProtectedArchiveVersion; + } + + Files = new ConcurrentDictionary(); + } + + public EbonyArchive(byte[] archive) + { + _stream = new MemoryStream(archive); + _header = ReadHeader(); + Files = new ConcurrentDictionary(ReadFileHeaders()); + } + + public EbonyArchive(string archivePath) + { + _sourcePath = archivePath; + _stream = new FileStream(archivePath, FileMode.Open, FileAccess.Read); + _header = ReadHeader(); + Files = new ConcurrentDictionary(ReadFileHeaders()); + } + + public ConcurrentDictionary Files { get; private set; } + public bool IsProtectedArchive => _header.IsProtectedArchive; + + public ArchiveFile this[string uri] + { + get + { + var hash = Cryptography.HashFileUri64(uri); + return Files[hash]; + } + + private set + { + var hash = Cryptography.HashFileUri64(uri); + Files[hash] = value; + } + } + + [SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize")] + public void Dispose() + { + _stream?.Dispose(); + } + + public void SetFlags(EbonyArchiveFlags flags) + { + _header.Flags = flags; + } + + public bool HasFile(string uri) + { + var hash = Cryptography.HashFileUri64(uri); + return Files.ContainsKey(hash); + } + + public static byte[] GetFileByLocation(string earcLocation, string uri) + { + using var unpacker = new EbonyArchive(earcLocation); + return unpacker[uri].GetReadableData(); + } + + public void AddProcessedFile(string uri, ArchiveFileFlag flags, byte[] data, uint size, ushort key, + string relativePathOverride) + { + if (string.IsNullOrWhiteSpace(relativePathOverride)) + { + relativePathOverride = null; + } + + var file = new ArchiveFile(uri, relativePathOverride) + { + Flags = flags, + Size = size, + Key = key + }; + + if (data != null) + { + file.ProcessedSize = (uint)data.Length; + file.SetDataByFlags(data); + } + + var hash = Cryptography.HashFileUri64(uri); + Files.TryAdd(hash, file); + } + + public void AddFile(string uri, ArchiveFileFlag flags, byte[] data) + { + var file = new ArchiveFile(uri) + { + Flags = flags + }; + + if (data != null) + { + file.SetRawData(data); + } + + var hash = Cryptography.HashFileUri64(uri); + Files.TryAdd(hash, file); + } + + public void AddFile(byte[] data, string uri) + { + var file = new ArchiveFile(uri); + file.SetRawData(data); + + var hash = Cryptography.HashFileUri64(uri); + Files.TryAdd(hash, file); + } + + public void UpdateFile(string uri, byte[] data) + { + this[uri].SetRawData(data); + } + + public void RemoveFile(string uri) + { + var hash = Cryptography.HashFileUri64(uri); + Files.Remove(hash, out _); + } + + public void UpdateFileWithProcessedData(string uri, uint originalSize, byte[] data) + { + this[uri].SetProcessedData(originalSize, data); + } + + public void AddFileFromBackup(string uri, string relativePath, uint size, ArchiveFileFlag flags, ushort key, + byte[] data) + { + var file = new ArchiveFile(uri, relativePath, size, (uint)data.Length, flags, 0, 0, key); + file.SetDataByFlags(data); + this[uri] = file; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/ArchiveHeader.cs b/Flagrum.Core/Archive/EbonyArchiveHeader.cs similarity index 64% rename from Flagrum.Core/Archive/ArchiveHeader.cs rename to Flagrum.Core/Archive/EbonyArchiveHeader.cs index 612e0b91..9ed4dfa0 100644 --- a/Flagrum.Core/Archive/ArchiveHeader.cs +++ b/Flagrum.Core/Archive/EbonyArchiveHeader.cs @@ -1,45 +1,41 @@ using System; -using System.Text; namespace Flagrum.Core.Archive; [Flags] -public enum ArchiveHeaderFlags +public enum EbonyArchiveFlags : uint { + None = 0, HasLooseData = 1, HasLocaleData = 2, DebugArchive = 4, Copyguard = 8 } -public class ArchiveHeader +public class EbonyArchiveHeader { public const uint ArchiveVersion = 196628; public const uint ProtectedArchiveVersion = 2147483648; + private const uint DefaultChunkSize = 128; + private const uint DefaultBlockSize = 512; + public const uint Size = 64; - public const uint DefaultChunkSize = 128; public const ulong CopyguardHash = 10026789885951819402; public const uint ProtectVersionHash = 2147483648; + public const uint MagicValue = 0x46415243; - public ArchiveHeader() - { - Version = ProtectedArchiveVersion | ArchiveVersion; - } - - public static byte[] DefaultTag { get; } = Encoding.UTF8.GetBytes("CRAF"); - - public byte[] Tag { get; set; } + public uint Magic { get; set; } public uint Version { get; set; } public ushort VersionMajor { get; set; } public ushort VersionMinor { get; set; } public uint FileCount { get; set; } - public uint BlockSize { get; set; } + public uint BlockSize { get; set; } = DefaultBlockSize; public uint FileHeadersOffset { get; set; } public uint UriListOffset { get; set; } public uint PathListOffset { get; set; } public uint DataOffset { get; set; } - public ArchiveHeaderFlags Flags { get; set; } - public uint ChunkSize { get; set; } + public EbonyArchiveFlags Flags { get; set; } + public uint ChunkSize { get; set; } = DefaultChunkSize; public ulong Hash { get; set; } public bool IsProtectedArchive => (Version & ProtectVersionHash) > 0; diff --git a/Flagrum.Core/Archive/EbonyArchiveManager.cs b/Flagrum.Core/Archive/EbonyArchiveManager.cs new file mode 100644 index 00000000..4f322b4e --- /dev/null +++ b/Flagrum.Core/Archive/EbonyArchiveManager.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Flagrum.Core.Utilities; + +namespace Flagrum.Core.Archive; + +public class EbonyArchiveManager : IDisposable +{ + private readonly Dictionary _archives = new(); + + public EbonyArchive Open(string filePath) + { + EbonyArchive archive; + var hash = Cryptography.Hash64(filePath); + + lock (_archives) + { + if (_archives.TryGetValue(hash, out archive)) + { + return archive; + } + + archive = new EbonyArchive(filePath); + _archives[hash] = archive; + } + + return archive; + } + + [SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize")] + public void Dispose() + { + foreach (var (_, archive) in _archives) + { + archive?.Dispose(); + } + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/EbonyArchiveReader.cs b/Flagrum.Core/Archive/EbonyArchiveReader.cs new file mode 100644 index 00000000..c0f4c6ed --- /dev/null +++ b/Flagrum.Core/Archive/EbonyArchiveReader.cs @@ -0,0 +1,89 @@ +using System.Collections.Generic; +using System.IO; +using System.Text; +using Flagrum.Core.Utilities; + +namespace Flagrum.Core.Archive; + +public partial class EbonyArchive +{ + private IDictionary ReadFileHeaders() + { + using var reader = new BinaryReader(_stream, Encoding.UTF8, true); + var result = new Dictionary(); + var hash = ArchiveFile.HeaderHash ^ _header.Hash; + + if (_header.Flags.HasFlag(EbonyArchiveFlags.Copyguard)) + { + hash ^= EbonyArchiveHeader.CopyguardHash; + } + + for (var i = 0; i < _header.FileCount; i++) + { + _stream.Seek(_header.FileHeadersOffset + i * ArchiveFile.HeaderSize, SeekOrigin.Begin); + + var file = new ArchiveFile + { + UriAndTypeHash = reader.ReadUInt64(), + Size = reader.ReadUInt32(), + ProcessedSize = reader.ReadUInt32(), + Flags = (ArchiveFileFlag)reader.ReadInt32(), + UriOffset = reader.ReadUInt32(), + DataOffset = reader.ReadUInt64(), + RelativePathOffset = reader.ReadUInt32(), + LocalizationType = reader.ReadByte(), + Locale = reader.ReadByte(), + Key = reader.ReadUInt16() + }; + + if (!file.Flags.HasFlag(ArchiveFileFlag.MaskProtected) && _header.IsProtectedArchive) + { + var subhash = Cryptography.MergeHashes(hash, file.UriAndTypeHash); + file.Size ^= (uint)(subhash >> 32); + file.ProcessedSize ^= (uint)subhash; + hash = Cryptography.MergeHashes(subhash, ~file.UriAndTypeHash); + file.DataOffset ^= hash; + } + + _stream.Seek(file.UriOffset, SeekOrigin.Begin); + file.Uri = reader.ReadNullTerminatedString(); + + _stream.Seek(file.RelativePathOffset, SeekOrigin.Begin); + file.RelativePath = reader.ReadNullTerminatedString(); + + file.DeconstructUriAndTypeHash(); + + file.IsDataCompressed = file.Flags.HasFlag(ArchiveFileFlag.Compressed); + file.IsDataEncrypted = file.Flags.HasFlag(ArchiveFileFlag.Encrypted); + file.SetRawData(_stream); + + result[Cryptography.HashFileUri64(file.Uri)] = file; + } + + return result; + } + + private EbonyArchiveHeader ReadHeader() + { + var header = new EbonyArchiveHeader(); + using var reader = new BinaryReader(_stream, Encoding.UTF8, true); + + header.Magic = reader.ReadUInt32(); + header.Version = reader.ReadUInt32(); + header.FileCount = reader.ReadUInt32(); + header.BlockSize = reader.ReadUInt32(); + header.FileHeadersOffset = reader.ReadUInt32(); + header.UriListOffset = reader.ReadUInt32(); + header.PathListOffset = reader.ReadUInt32(); + header.DataOffset = reader.ReadUInt32(); + header.Flags = (EbonyArchiveFlags)reader.ReadInt32(); + header.ChunkSize = reader.ReadUInt32(); + header.Hash = reader.ReadUInt64(); + + var version = header.Version & ~EbonyArchiveHeader.ProtectVersionHash; + header.VersionMajor = (ushort)(version >> 16); + header.VersionMinor = (ushort)(version & ushort.MaxValue); + + return header; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/EbonyArchiveWriter.cs b/Flagrum.Core/Archive/EbonyArchiveWriter.cs new file mode 100644 index 00000000..c8429598 --- /dev/null +++ b/Flagrum.Core/Archive/EbonyArchiveWriter.cs @@ -0,0 +1,389 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using CommunityToolkit.HighPerformance.Buffers; +using Flagrum.Core.Utilities; +using Flagrum.Core.Utilities.Types; + +namespace Flagrum.Core.Archive; + +public partial class EbonyArchive +{ + private List _sorted; + + /// + /// Writes the archive back to the path it originates from + /// + /// Throws if called on an archive that was not instantiated with a file path + public void WriteToSource(LuminousGame targetGame) + { + if (_sourcePath == null) + { + throw new InvalidOperationException( + "This method can only be called on archives that were opened from a file"); + } + + WriteToFile(_sourcePath, targetGame); + } + + public void WriteToFile(string path, LuminousGame targetGame) + { + using var archiveStream = new MemoryStream(); + + if (_header.Flags.HasFlag(EbonyArchiveFlags.Copyguard)) + { + _header.Version |= EbonyArchiveHeader.ProtectVersionHash; + } + + _sorted = Files + .Select(kvp => kvp.Value) + .OrderBy(f => f.Flags.HasFlag(ArchiveFileFlag.Autoload)) + .ThenBy(f => f.Uri.EndsWith(".autoext")) + .ThenBy(f => f.Flags.HasFlag(ArchiveFileFlag.Reference)) + .ThenBy(f => f.TypeHash) + .ThenBy(f => f.UriHash) + .ToList(); + + _header.UriListOffset = EbonyArchiveHeader.Size + + Serialization.GetAlignment((uint)_sorted.Count * ArchiveFile.HeaderSize, + PointerSize); + + var endOfUriList = SerializeUriList(out var uriListStream); + archiveStream.Seek(_header.UriListOffset, SeekOrigin.Begin); + uriListStream.CopyTo(archiveStream); + uriListStream.Dispose(); + + _header.PathListOffset = + Serialization.GetAlignment((uint)(_header.UriListOffset + endOfUriList), PointerSize); + + var endOfPathList = SerializePathList(out var pathListStream); + archiveStream.Seek(_header.PathListOffset, SeekOrigin.Begin); + pathListStream.CopyTo(archiveStream); + pathListStream.Dispose(); + + _header.DataOffset = + Serialization.GetAlignment(_header.PathListOffset + (uint)endOfPathList, _header.BlockSize); + + archiveStream.Seek(_header.DataOffset, SeekOrigin.Begin); + SerializeFileData(archiveStream, out var endOfFile); + _stream?.Dispose(); + + var fileHeaderStream = SerializeFileHeaders(); + archiveStream.Seek(EbonyArchiveHeader.Size, SeekOrigin.Begin); + fileHeaderStream.CopyTo(archiveStream); + fileHeaderStream.Dispose(); + + var headerStream = targetGame switch + { + LuminousGame.FFXV => SerializeHeader(), + LuminousGame.Forspoken => SerializeHeaderWitch(archiveStream, endOfFile), + _ => throw new ArgumentOutOfRangeException(nameof(targetGame), targetGame, + $"Unsupported {nameof(LuminousGame)}") + }; + + archiveStream.Seek(0, SeekOrigin.Begin); + headerStream.CopyTo(archiveStream); + headerStream.Dispose(); + + archiveStream.Seek(0, SeekOrigin.Begin); + using var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write); + archiveStream.CopyTo(fileStream); + } + + private Stream SerializeHeader() + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + + writer.Write(EbonyArchiveHeader.MagicValue); + writer.Write(_header.Version); + writer.Write((uint)_sorted.Count); + writer.Write(_header.BlockSize); + writer.Write(EbonyArchiveHeader.Size); + writer.Write(_header.UriListOffset); + writer.Write(_header.PathListOffset); + writer.Write(_header.DataOffset); + writer.Write((uint)_header.Flags); + writer.Write(_header.ChunkSize); + + // Archive hash must be zero before the whole header is hashed + writer.Write(0UL); + + // Constant padding + writer.Write(new byte[16]); + + _header.Hash = Cryptography.Base64Hash(stream.ToArray()); + writer.Dispose(); + + stream = new MemoryStream(); + writer = new BinaryWriter(stream, Encoding.UTF8, true); + + writer.Write(EbonyArchiveHeader.MagicValue); + writer.Write(_header.Version); + writer.Write((uint)_sorted.Count); + writer.Write(_header.BlockSize); + writer.Write(EbonyArchiveHeader.Size); + writer.Write(_header.UriListOffset); + writer.Write(_header.PathListOffset); + writer.Write(_header.DataOffset); + writer.Write((uint)_header.Flags); + writer.Write(_header.ChunkSize); + writer.Write(_header.Hash); + + // Constant padding + stream.Write(new byte[16]); + + stream.Seek(0, SeekOrigin.Begin); + writer.Dispose(); + return stream; + } + + private Stream SerializeFileHeaders() + { + var stream = new MemoryStream(); + var hash = ArchiveFile.HeaderHash ^ _header.Hash; + + if (_header.Flags.HasFlag(EbonyArchiveFlags.Copyguard)) + { + hash ^= EbonyArchiveHeader.CopyguardHash; + } + + foreach (var file in _sorted) + { + var size = file.Size; + var processedSize = file.ProcessedSize; + var dataOffset = file.DataOffset; + + if (!file.Flags.HasFlag(ArchiveFileFlag.MaskProtected) && _header.IsProtectedArchive) + { + hash = Cryptography.MergeHashes(hash, file.UriAndTypeHash); + size ^= (uint)(hash >> 32); + processedSize ^= (uint)hash; + hash = Cryptography.MergeHashes(hash, ~file.UriAndTypeHash); + dataOffset ^= hash; + } + + stream.Write(BitConverter.GetBytes(file.UriAndTypeHash)); + stream.Write(BitConverter.GetBytes(size)); + stream.Write(BitConverter.GetBytes(processedSize)); + stream.Write(BitConverter.GetBytes((uint)file.Flags)); + stream.Write(BitConverter.GetBytes(file.UriOffset)); + stream.Write(BitConverter.GetBytes(dataOffset)); + stream.Write(BitConverter.GetBytes(file.RelativePathOffset)); + stream.WriteByte(file.LocalizationType); + stream.WriteByte(file.Locale); + stream.Write(BitConverter.GetBytes(file.Key)); + } + + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + + private int SerializeUriList(out Stream uriListStream) + { + uriListStream = new MemoryStream(); + var currentUriOffset = 0; + + foreach (var file in _sorted) + { + var size = EncodeString(file.Uri, out var bytes); + file.UriOffset = _header.UriListOffset + (uint)currentUriOffset; + uriListStream.Seek(currentUriOffset, SeekOrigin.Begin); + uriListStream.Write(bytes, 0, size); + currentUriOffset += (int)Serialization.GetAlignment((uint)size, PointerSize); + } + + uriListStream.Seek(0, SeekOrigin.Begin); + return currentUriOffset; + } + + private int SerializePathList(out Stream pathListStream) + { + pathListStream = new MemoryStream(); + var currentPathOffset = 0; + + foreach (var file in _sorted) + { + var size = EncodeString(file.RelativePath, out var bytes); + file.RelativePathOffset = _header.PathListOffset + (uint)currentPathOffset; + pathListStream.Seek(currentPathOffset, SeekOrigin.Begin); + pathListStream.Write(bytes, 0, size); + currentPathOffset += (int)Serialization.GetAlignment((uint)size, PointerSize); + } + + pathListStream.Seek(0, SeekOrigin.Begin); + return currentPathOffset; + } + + private void SerializeFileData(Stream stream, out long endOfFile) + { + endOfFile = 0L; + var rng = new Random((int)_header.Hash); + var currentDataOffset = 0UL; + + for (var i = 0; i < _sorted.Count; i++) + { + var file = _sorted.ElementAt(i); + + if (file.Key == 0 && _header.IsProtectedArchive) + { + var hashCode = file.Uri.GetHashCode(); + file.Key = (ushort)((hashCode >> 16) ^ hashCode); + if (file.Key == 0) + { + file.Key = 57005; + } + } + + if (!file.Flags.HasFlag(ArchiveFileFlag.Compressed) && !file.Flags.HasFlag(ArchiveFileFlag.Encrypted)) + { + file.Key = 0; + } + + file.DataOffset = _header.DataOffset + currentDataOffset; + + var fileData = file.GetDataForExport(); + stream.Write(fileData, 0, fileData.Length); + + if (i == _sorted.Count - 1) + { + endOfFile = stream.Position; + } + + var finalSize = Serialization.GetAlignment(file.ProcessedSize, _header.BlockSize); + var paddingSize = finalSize - file.ProcessedSize; + var padding = new byte[paddingSize]; + rng.NextBytes(padding); + stream.Write(padding, 0, (int)paddingSize); + + currentDataOffset += finalSize; + } + } + + private Stream SerializeHeaderWitch(Stream archiveStream, long endOfFile) + { + _header.Hash = CalculateArchiveHash(archiveStream, endOfFile); + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream, Encoding.UTF8, true); + + writer.Write(EbonyArchiveHeader.MagicValue); + writer.Write(_header.Version); + writer.Write((uint)_sorted.Count); + writer.Write(_header.BlockSize); + writer.Write(EbonyArchiveHeader.Size); + writer.Write(_header.UriListOffset); + writer.Write(_header.PathListOffset); + writer.Write(_header.DataOffset); + writer.Write((uint)_header.Flags); + writer.Write(_header.ChunkSize); + writer.Write(_header.Hash); + + // Constant padding + stream.Write(new byte[16]); + + stream.Seek(0, SeekOrigin.Begin); + writer.Dispose(); + return stream; + } + + private int EncodeString(string value, out byte[] bytes) + { + var stringBufferSize = value.Length + 1 > 256 ? value.Length + 1 : 256; + + var stringBuffer = new char[stringBufferSize]; + bytes = new byte[stringBufferSize * 4]; + + uint i; + for (i = 0; i < value.Length; ++i) + { + stringBuffer[i] = value[(int)i]; + } + + stringBuffer[i] = char.MinValue; + + return Encoding.UTF8.GetEncoder().GetBytes(stringBuffer, 0, value.Length, bytes, 0, true); + } + + /// + /// Forspoken uses a different method of hashing than FFXV + /// See https://github.com/yretenai/Witch/blob/develop/Scarlet/Archive/EbonyArchive.cs + /// + private ulong CalculateArchiveHash(Stream stream, long dataSize, bool force = false) + { + if (dataSize > int.MaxValue && !force) + { + return 0; + } + + const uint CHUNK_SIZE = 0x8000000; + const ulong XOR_CONST = 0x7575757575757575UL; + const uint STATIC_SEED0 = 0xb16949df; + const uint STATIC_SEED1 = 0x104098f5; + const uint STATIC_SEED2 = 0x9eb9b68b; + const uint STATIC_SEED3 = 0x3120f7cb; + + var position = stream.Position; + + stream.Position = EbonyArchiveHeader.Size; + + try + { + var size = dataSize - EbonyArchiveHeader.Size; + + // hash up to MAX_SIZE (128 MiB) of data in CHUNK_SIZE (8MiB) chunks and hash them into a 128-bit hash array. + var blocks = (int)(Align(size, CHUNK_SIZE) >> 27); // 0x8000000 -> 0x1 + + using var chunk = MemoryOwner.Allocate((int)CHUNK_SIZE); + using var hashList = MemoryOwner.Allocate((blocks + 1) << 4); // 1 -> 16 + for (var i = 0; i < blocks; ++i) + { + var offset = (long)i << 27; // 1 -> 0x8000000 + var length = (int)Math.Min(size - offset, CHUNK_SIZE); + _ = stream.Read(chunk.Span[..length]); + MD5.HashData(chunk.Span[..length]).CopyTo(hashList.Span[(i << 4)..]); + } + + var hashSeed = MemoryMarshal.Cast(hashList.Span[^16..]); + if (blocks > 8) + { + var seed = (ulong)size ^ XOR_CONST; + // xorshift64. + for (var i = 0; i < 4; ++i) + { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + hashSeed[i] = (uint)seed; + } + } + else + { + // some magic numbers fished up from leviathan's egg pond or someth + hashSeed[0] = STATIC_SEED0; + hashSeed[1] = STATIC_SEED1; + hashSeed[2] = STATIC_SEED2; + hashSeed[3] = STATIC_SEED3; + } + + // SHA256 the hash list and overlay the 256-bit hash into 64-bits. + // one might wonder, why would you not just use Murmur64 or xxHash64 as it's faster + // since we only care about data integrity, we don't need the security benefits of SHA + var sha = MemoryMarshal.Cast(SHA256.HashData(hashList.Span).AsSpan()); + return sha[0] ^ sha[1] ^ sha[2] ^ sha[3]; + } + finally + { + stream.Position = position; + } + } + + private long Align(long value, long n) + { + return unchecked(value + (n - 1)) & ~(n - 1); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/EbonyReplace.cs b/Flagrum.Core/Archive/EbonyReplace.cs new file mode 100644 index 00000000..89c34444 --- /dev/null +++ b/Flagrum.Core/Archive/EbonyReplace.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +namespace Flagrum.Core.Archive; + +public class EbonyReplace +{ + public EbonyReplace() { } + + public EbonyReplace(byte[] erep) + { + Replacements = new Dictionary(); + + var data = MemoryMarshal.Cast(erep.AsSpan()); + for (var i = 0; i < data.Length; i += 2) + { + Replacements.Add(data[i], data[i + 1]); + } + } + + public Dictionary Replacements { get; set; } + + public byte[] ToArray() + { + using var stream = new MemoryStream(); + Write(stream, true); + return stream.ToArray(); + } + + public void Write(Stream destination, bool leaveOpen = false) + { + using var writer = new BinaryWriter(destination, Encoding.UTF8, leaveOpen); + + foreach (var (original, replacement) in Replacements) + { + writer.Write(original); + writer.Write(replacement); + } + } +} \ No newline at end of file diff --git a/Flagrum.Core/Archive/Packer.cs b/Flagrum.Core/Archive/Packer.cs deleted file mode 100644 index 4269ab75..00000000 --- a/Flagrum.Core/Archive/Packer.cs +++ /dev/null @@ -1,431 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using Flagrum.Core.Services.Logging; -using Flagrum.Core.Utilities; - -namespace Flagrum.Core.Archive; - -public class Packer -{ - private const uint PointerSize = 8; - private const uint BlockSize = 512; - private readonly ArchiveHeader _header; - - private readonly Logger _logger; - - public Packer() - { - _logger = new ConsoleLogger(); - _header = new ArchiveHeader(); - Files = new ConcurrentCollection(); - } - - private Packer(ArchiveHeader header, List files) : this() - { - _header = header; - Files = new ConcurrentCollection(files); - } - - public ConcurrentCollection Files { get; private set; } - - public bool IsProtectedArchive => _header.IsProtectedArchive; - - public static Packer FromUnpacker(ArchiveHeader header, List files) - { - return new Packer(header, files); - } - - public void SetFlags(ArchiveHeaderFlags flags) - { - _header.Flags = flags; - } - - public bool HasFile(string uri) - { - return Files.Any(f => f.Uri == uri); - } - - public void AddCompressedFile(string uri, byte[] data, bool autoload = false) - { - var file = new ArchiveFile(uri) - { - Flags = ArchiveFileFlag.Compressed - }; - - if (autoload) - { - file.Flags |= ArchiveFileFlag.Autoload; - } - - file.SetRawData(data); - Files.Add(file); - } - - public void AddProcessedFile(string uri, ArchiveFileFlag flags, byte[] data, uint size, ushort key, string relativePathOverride) - { - if (string.IsNullOrWhiteSpace(relativePathOverride)) - { - relativePathOverride = null; - } - - var file = new ArchiveFile(uri, relativePathOverride) - { - Flags = flags, - Size = size, - Key = key - }; - - if (data != null) - { - file.ProcessedSize = (uint)data.Length; - file.SetDataByFlags(data); - } - - Files.Add(file); - } - - public void AddFile(string uri, ArchiveFileFlag flags, byte[] data) - { - var file = new ArchiveFile(uri); - file.Flags = flags; - - if (data != null) - { - file.SetRawData(data); - } - - Files.Add(file); - } - - public void AddFile(byte[] data, string uri) - { - var file = new ArchiveFile(uri); - file.SetRawData(data); - - Files.Add(file); - } - - public void AddReference(string uri, bool autoload) - { - var file = new ArchiveFile(uri); - file.Flags = ArchiveFileFlag.Reference; - - if (autoload) - { - file.Flags |= ArchiveFileFlag.Autoload; - } - - Files.Add(file); - } - - public void UpdateFile(string query, byte[] data) - { - var match = Files.FirstOrDefault(f => f.Uri.EndsWith(query)); - if (match != null) - { - match.SetRawData(data); - } - else - { - throw new ArgumentException($"Could not find file ending with \"{query}\" in the archive.", - nameof(query)); - } - } - - public void UpdateFileWithProcessedData(string query, byte[] data) - { - var match = Files.FirstOrDefault(f => f.Uri.EndsWith(query)); - if (match != null) - { - match.SetDataByFlags(data); - } - else - { - throw new ArgumentException($"Could not find file ending with \"{query}\" in the archive.", - nameof(query)); - } - } - - public void RemoveFile(string uri) - { - var match = Files.FirstOrDefault(f => f.Uri.Equals(uri, StringComparison.OrdinalIgnoreCase)); - if (match != null) - { - Files.Remove(match); - } - else - { - throw new ArgumentException("Could not find file in the archive", nameof(uri)); - } - } - - public void UpdateFileWithProcessedData(string uri, uint originalSize, byte[] data) - { - var match = Files.FirstOrDefault(f => f.Uri.Equals(uri, StringComparison.OrdinalIgnoreCase)); - if (match != null) - { - match.SetProcessedData(originalSize, data); - } - else - { - throw new FileNotFoundException("File was not found in the archive", uri); - } - } - - public void AddFileFromBackup(string uri, string relativePath, uint size, ArchiveFileFlag flags, ushort key, - byte[] data) - { - var file = new ArchiveFile(uri, relativePath, size, (uint)data.Length, flags, 0, 0, key); - file.SetDataByFlags(data); - Files.Add(file); - } - - public void WriteToFile(string path) - { - using var archiveStream = new MemoryStream(); - - _logger.LogInformation("Packing archive..."); - - if (_header.Flags.HasFlag(ArchiveHeaderFlags.Copyguard)) - { - _header.Version |= ArchiveHeader.ProtectVersionHash; - } - - Files = new ConcurrentCollection(Files - .OrderBy(f => f.Flags.HasFlag(ArchiveFileFlag.Autoload)) - .ThenBy(f => f.Uri.EndsWith(".autoext")) - .ThenBy(f => f.Flags.HasFlag(ArchiveFileFlag.Reference)) - .ThenBy(f => f.TypeHash) - .ThenBy(f => f.UriHash)); - - _header.UriListOffset = ArchiveHeader.Size + - Serialization.GetAlignment((uint)Files.Count * ArchiveFile.HeaderSize, - PointerSize); - - var endOfUriList = SerializeUriList(out var uriListStream); - archiveStream.Seek(_header.UriListOffset, SeekOrigin.Begin); - uriListStream.CopyTo(archiveStream); - uriListStream.Dispose(); - - _header.PathListOffset = - Serialization.GetAlignment((uint)(_header.UriListOffset + endOfUriList), PointerSize); - - var endOfPathList = SerializePathList(out var pathListStream); - archiveStream.Seek(_header.PathListOffset, SeekOrigin.Begin); - pathListStream.CopyTo(archiveStream); - pathListStream.Dispose(); - - _header.DataOffset = Serialization.GetAlignment(_header.PathListOffset + (uint)endOfPathList, BlockSize); - - var dataStream = SerializeFileData(); - archiveStream.Seek(_header.DataOffset, SeekOrigin.Begin); - dataStream.CopyTo(archiveStream); - dataStream.Dispose(); - - var headerStream = SerializeHeader(); - archiveStream.Seek(0, SeekOrigin.Begin); - headerStream.CopyTo(archiveStream); - headerStream.Dispose(); - - var fileHeaderStream = SerializeFileHeaders(); - archiveStream.Seek(ArchiveHeader.Size, SeekOrigin.Begin); - fileHeaderStream.CopyTo(archiveStream); - fileHeaderStream.Dispose(); - - File.WriteAllBytes(path, archiveStream.ToArray()); - } - - private Stream SerializeHeader() - { - _logger.LogInformation("Serializing Archive Header"); - - var stream = new MemoryStream(); - - stream.Write(ArchiveHeader.DefaultTag); - stream.Write(BitConverter.GetBytes(_header.Version)); - stream.Write(BitConverter.GetBytes((uint)Files.Count)); - stream.Write(BitConverter.GetBytes(BlockSize)); - stream.Write(BitConverter.GetBytes(ArchiveHeader.Size)); - stream.Write(BitConverter.GetBytes(_header.UriListOffset)); - stream.Write(BitConverter.GetBytes(_header.PathListOffset)); - stream.Write(BitConverter.GetBytes(_header.DataOffset)); - stream.Write(BitConverter.GetBytes((uint)_header.Flags)); - stream.Write(BitConverter.GetBytes(ArchiveHeader.DefaultChunkSize)); - - // Archive hash must be zero before the whole header is hashed - stream.Write(BitConverter.GetBytes((ulong)0)); - - // Constant padding - stream.Write(new byte[16]); - - _header.Hash = Cryptography.Base64Hash(stream.ToArray()); - stream.Dispose(); - - stream = new MemoryStream(); - - stream.Write(ArchiveHeader.DefaultTag); - stream.Write(BitConverter.GetBytes(_header.Version)); - stream.Write(BitConverter.GetBytes((uint)Files.Count)); - stream.Write(BitConverter.GetBytes(BlockSize)); - stream.Write(BitConverter.GetBytes(ArchiveHeader.Size)); - stream.Write(BitConverter.GetBytes(_header.UriListOffset)); - stream.Write(BitConverter.GetBytes(_header.PathListOffset)); - stream.Write(BitConverter.GetBytes(_header.DataOffset)); - stream.Write(BitConverter.GetBytes((uint)_header.Flags)); - stream.Write(BitConverter.GetBytes(ArchiveHeader.DefaultChunkSize)); - stream.Write(BitConverter.GetBytes(_header.Hash)); - - // Constant padding - stream.Write(new byte[16]); - - stream.Seek(0, SeekOrigin.Begin); - return stream; - } - - private Stream SerializeFileHeaders() - { - _logger.LogInformation("Serializing File Headers"); - - var stream = new MemoryStream(); - var hash = ArchiveFile.HeaderHash ^ _header.Hash; - - if (_header.Flags.HasFlag(ArchiveHeaderFlags.Copyguard)) - { - hash ^= ArchiveHeader.CopyguardHash; - } - - foreach (var file in Files) - { - var size = file.Size; - var processedSize = file.ProcessedSize; - var dataOffset = file.DataOffset; - - if (!file.Flags.HasFlag(ArchiveFileFlag.MaskProtected)) - { - hash = Cryptography.MergeHashes(hash, file.UriAndTypeHash); - size ^= (uint)(hash >> 32); - processedSize ^= (uint)hash; - hash = Cryptography.MergeHashes(hash, ~file.UriAndTypeHash); - dataOffset ^= hash; - } - - stream.Write(BitConverter.GetBytes(file.UriAndTypeHash)); - stream.Write(BitConverter.GetBytes(size)); - stream.Write(BitConverter.GetBytes(processedSize)); - stream.Write(BitConverter.GetBytes((uint)file.Flags)); - stream.Write(BitConverter.GetBytes(file.UriOffset)); - stream.Write(BitConverter.GetBytes(dataOffset)); - stream.Write(BitConverter.GetBytes(file.RelativePathOffset)); - stream.WriteByte(file.LocalizationType); - stream.WriteByte(file.Locale); - stream.Write(BitConverter.GetBytes(file.Key)); - } - - stream.Seek(0, SeekOrigin.Begin); - return stream; - } - - private int SerializeUriList(out Stream uriListStream) - { - _logger.LogInformation("Serializing File URIs"); - - uriListStream = new MemoryStream(); - var currentUriOffset = 0; - - foreach (var file in Files) - { - var size = EncodeString(file.Uri, out var bytes); - file.UriOffset = _header.UriListOffset + (uint)currentUriOffset; - uriListStream.Seek(currentUriOffset, SeekOrigin.Begin); - uriListStream.Write(bytes, 0, size); - currentUriOffset += (int)Serialization.GetAlignment((uint)size, PointerSize); - } - - uriListStream.Seek(0, SeekOrigin.Begin); - return currentUriOffset; - } - - private int SerializePathList(out Stream pathListStream) - { - _logger.LogInformation("Serializing File Paths"); - - pathListStream = new MemoryStream(); - var currentPathOffset = 0; - - foreach (var file in Files) - { - var size = EncodeString(file.RelativePath, out var bytes); - file.RelativePathOffset = _header.PathListOffset + (uint)currentPathOffset; - pathListStream.Seek(currentPathOffset, SeekOrigin.Begin); - pathListStream.Write(bytes, 0, size); - currentPathOffset += (int)Serialization.GetAlignment((uint)size, PointerSize); - } - - pathListStream.Seek(0, SeekOrigin.Begin); - return currentPathOffset; - } - - private Stream SerializeFileData() - { - _logger.LogInformation("Serializing File Data"); - - var stream = new MemoryStream(); - var rng = new Random((int)_header.Hash); - var currentDataOffset = 0L; - - foreach (var file in Files) - { - if (file.Key == 0 && _header.IsProtectedArchive) - { - var hashCode = file.Uri.GetHashCode(); - file.Key = (ushort)((hashCode >> 16) ^ hashCode); - if (file.Key == 0) - { - file.Key = 57005; - } - } - - if (!file.Flags.HasFlag(ArchiveFileFlag.Compressed) && !file.Flags.HasFlag(ArchiveFileFlag.Encrypted)) - { - file.Key = 0; - } - - file.DataOffset = _header.DataOffset + (uint)currentDataOffset; - - var fileData = file.GetDataForExport(); - stream.Write(fileData, 0, fileData.Length); - - var finalSize = Serialization.GetAlignment(file.ProcessedSize, BlockSize); - var paddingSize = finalSize - file.ProcessedSize; - var padding = new byte[paddingSize]; - rng.NextBytes(padding); - stream.Write(padding, 0, (int)paddingSize); - - currentDataOffset += finalSize; - } - - stream.Seek(0, SeekOrigin.Begin); - return stream; - } - - private int EncodeString(string value, out byte[] bytes) - { - var stringBufferSize = value.Length + 1 > 256 ? value.Length + 1 : 256; - - var stringBuffer = new char[stringBufferSize]; - bytes = new byte[stringBufferSize * 4]; - - uint i; - for (i = 0; i < value.Length; ++i) - { - stringBuffer[i] = value[(int)i]; - } - - stringBuffer[i] = char.MinValue; - - return Encoding.UTF8.GetEncoder().GetBytes(stringBuffer, 0, value.Length, bytes, 0, true); - } -} \ No newline at end of file diff --git a/Flagrum.Core/Archive/Unpacker.cs b/Flagrum.Core/Archive/Unpacker.cs deleted file mode 100644 index ddadbb3c..00000000 --- a/Flagrum.Core/Archive/Unpacker.cs +++ /dev/null @@ -1,287 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using Flagrum.Core.Utilities; - -namespace Flagrum.Core.Archive; - -public class Unpacker : IDisposable -{ - private readonly ArchiveHeader _header; - - private readonly Stream _stream; - private List _files; - - public Unpacker(byte[] archive) - { - _stream = new MemoryStream(archive); - _header = ReadHeader(); - } - - public Unpacker(string archivePath) - { - _stream = new FileStream(archivePath, FileMode.Open, FileAccess.Read); - _header = ReadHeader(); - } - - public List Files => _files ??= ReadFileHeaders().ToList(); - public bool IsProtectedArchive => _header.IsProtectedArchive; - - public void Dispose() - { - _stream?.Dispose(); - } - - public bool HasFile(string uri) - { - _files ??= ReadFileHeaders().ToList(); - return _files.Any(f => f.Uri == uri); - } - - public string GetUriByQuery(string query) - { - _files ??= ReadFileHeaders().ToList(); - return _files.FirstOrDefault(f => f.Uri.EndsWith(query))?.Uri; - } - - /// - /// Retrieves the data for one file in the archive - /// - /// A string that must be contained in the URI - /// Buffer containing the file data - public byte[] UnpackFileByQuery(string query, out string uri) - { - _files ??= ReadFileHeaders().ToList(); - - var match = _files.FirstOrDefault(f => f.Uri.Contains(query, StringComparison.OrdinalIgnoreCase)); - if (match != null) - { - uri = match.Uri; - if (!match.HasData) - { - ReadFileData(match); - } - - return match.GetReadableData(); - } - - uri = null; - return Array.Empty(); - } - - public byte[] UnpackRawByUri(string uri) - { - _files ??= ReadFileHeaders().ToList(); - - var match = _files.FirstOrDefault(f => f.Uri.Equals(uri, StringComparison.OrdinalIgnoreCase)); - if (match != null) - { - if (!match.HasData) - { - ReadFileData(match); - } - - return match.GetRawData(); - } - - throw new FileNotFoundException("The file at the given URI was not found", uri); - } - - public Dictionary UnpackFilesByQuery(string query) - { - _files ??= ReadFileHeaders().ToList(); - - var result = new Dictionary(); - var matches = _files.Where(f => f.Uri.Contains(query)); - foreach (var match in matches) - { - if (!match.HasData) - { - ReadFileData(match); - } - - result.Add(match.Uri, match.GetReadableData()); - } - - return result; - } - - public Packer ToPacker() - { - _files ??= ReadFileHeaders().ToList(); - ReadDataForAllFiles(); - var packer = Packer.FromUnpacker(_header, _files); - Dispose(); - return packer; - } - - public void ReadDataForAllFiles() - { - foreach (var file in _files.Where(file => !file.HasData)) - { - ReadFileData(file); - } - } - - public void ReadFileData(ArchiveFile file) - { - byte[] buffer; - - lock (_stream) - { - _stream.Seek((long)file.DataOffset, SeekOrigin.Begin); - buffer = new byte[file.ProcessedSize]; - _ = _stream.Read(buffer, 0, (int)file.ProcessedSize); - } - - if (file.Flags.HasFlag(ArchiveFileFlag.Compressed)) - { - file.SetCompressedData(buffer); - } - else if (file.Flags.HasFlag(ArchiveFileFlag.Encrypted)) - { - file.SetEncryptedData(buffer); - } - else - { - file.SetRawData(buffer); - } - } - - private IEnumerable ReadFileHeaders() - { - var hash = ArchiveFile.HeaderHash ^ _header.Hash; - - if (_header.Flags.HasFlag(ArchiveHeaderFlags.Copyguard)) - { - hash ^= ArchiveHeader.CopyguardHash; - } - - for (var i = 0; i < _header.FileCount; i++) - { - _stream.Seek(_header.FileHeadersOffset + i * ArchiveFile.HeaderSize, SeekOrigin.Begin); - - var file = new ArchiveFile - { - UriAndTypeHash = ReadUint64(), - Size = ReadUint32(), - ProcessedSize = ReadUint32(), - Flags = (ArchiveFileFlag)ReadInt32(), - UriOffset = ReadUint32(), - DataOffset = ReadUint64(), - RelativePathOffset = ReadUint32(), - LocalizationType = ReadByte(), - Locale = ReadByte(), - Key = ReadUint16() - }; - - if (!file.Flags.HasFlag(ArchiveFileFlag.MaskProtected) && _header.IsProtectedArchive) - { - var subhash = Cryptography.MergeHashes(hash, file.UriAndTypeHash); - file.Size ^= (uint)(subhash >> 32); - file.ProcessedSize ^= (uint)subhash; - hash = Cryptography.MergeHashes(subhash, ~file.UriAndTypeHash); - file.DataOffset ^= hash; - } - - _stream.Seek(file.UriOffset, SeekOrigin.Begin); - file.Uri = ReadString(); - - _stream.Seek(file.RelativePathOffset, SeekOrigin.Begin); - file.RelativePath = ReadString(); - - file.DeconstructUriAndTypeHash(); - - yield return file; - } - } - - private ArchiveHeader ReadHeader() - { - var header = new ArchiveHeader(); - - var buffer = new byte[4]; - _stream.Read(buffer, 0, 4); - header.Tag = buffer; - - header.Version = ReadUint32(); - header.FileCount = ReadUint32(); - header.BlockSize = ReadUint32(); - header.FileHeadersOffset = ReadUint32(); - header.UriListOffset = ReadUint32(); - header.PathListOffset = ReadUint32(); - header.DataOffset = ReadUint32(); - header.Flags = (ArchiveHeaderFlags)ReadInt32(); - header.ChunkSize = ReadUint32(); - header.Hash = ReadUint64(); - - var version = header.Version & ~ArchiveHeader.ProtectVersionHash; - header.VersionMajor = (ushort)(version >> 16); - header.VersionMinor = (ushort)(version & ushort.MaxValue); - - return header; - } - - public static byte[] GetFileByLocation(string earcLocation, string fileQuery) - { - using var unpacker = new Unpacker(earcLocation); - return unpacker.UnpackFileByQuery(fileQuery, out _); - } - - private byte ReadByte() - { - var buffer = new byte[1]; - _stream.Read(buffer, 0, 1); - return buffer[0]; - } - - private ushort ReadUint16() - { - var buffer = new byte[2]; - _stream.Read(buffer, 0, 2); - return BitConverter.ToUInt16(buffer); - } - - private uint ReadUint32() - { - var buffer = new byte[4]; - _stream.Read(buffer, 0, 4); - return BitConverter.ToUInt32(buffer); - } - - private int ReadInt32() - { - var buffer = new byte[4]; - _stream.Read(buffer, 0, 4); - return BitConverter.ToInt32(buffer); - } - - private ulong ReadUint64() - { - var buffer = new byte[8]; - _stream.Read(buffer, 0, 8); - return BitConverter.ToUInt64(buffer); - } - - private string ReadString() - { - var builder = new StringBuilder(); - - var buffer = new byte[1]; - while (true) - { - _stream.Read(buffer, 0, 1); - - if (buffer[0] == byte.MinValue) - { - break; - } - - builder.Append(Convert.ToChar(buffer[0])); - } - - return builder.ToString(); - } -} \ No newline at end of file diff --git a/Flagrum.Core/Flagrum.Core.csproj b/Flagrum.Core/Flagrum.Core.csproj index f8bb183c..c2c28731 100644 --- a/Flagrum.Core/Flagrum.Core.csproj +++ b/Flagrum.Core/Flagrum.Core.csproj @@ -6,8 +6,10 @@ + + diff --git a/Flagrum.Core/Gfxbin/Btex/BtexConverter.cs b/Flagrum.Core/Gfxbin/Btex/BtexConverter.cs index abb45788..725fc03f 100644 --- a/Flagrum.Core/Gfxbin/Btex/BtexConverter.cs +++ b/Flagrum.Core/Gfxbin/Btex/BtexConverter.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using Flagrum.Core.Utilities; +using Flagrum.Core.Utilities.Types; namespace Flagrum.Core.Gfxbin.Btex; @@ -39,10 +40,11 @@ static BtexConverter() public static FallbackMap FormatMap { get; } = new(DxgiFormat.BC1_UNORM, BtexFormat.BC1_UNORM); - public static byte[] BtexToDds(byte[] btex) + public static byte[] BtexToDds(byte[] btex, LuminousGame profile, + Func calculatePitch) { byte[] withoutSedb; - + if (btex[0] == 'B' && btex[1] == 'T' && btex[2] == 'E' && btex[3] == 'X') { withoutSedb = btex; @@ -54,16 +56,74 @@ public static byte[] BtexToDds(byte[] btex) Array.Copy(btex, 128, withoutSedb, 0, withoutSedb.Length); } - var btexHeader = ReadBtexHeader(withoutSedb); + var btexHeader = ReadBtexHeader(withoutSedb, profile == LuminousGame.FFXV); - if (btexHeader.ArraySize > 1) + if (profile == LuminousGame.FFXV) { + if (btexHeader.ArraySize > 1) + { + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream); + + foreach (var mip in btexHeader.MipMaps.SelectMany(m => m)) + { + writer.Write(btexHeader.Data, (int)mip.Offset, (int)mip.Size); + } + + btexHeader.Data = stream.ToArray(); + } + } + else + { + var offset = 0; using var stream = new MemoryStream(); using var writer = new BinaryWriter(stream); - foreach (var mip in btexHeader.MipMaps.SelectMany(m => m)) + try + { + for (var a = 0; a < btexHeader.ArraySize; a++) + { + var format = FormatMap[btexHeader.Format]; + var formatBlockSize = GetBlockSize(format); + uint width = btexHeader.Width; + uint height = btexHeader.Height; + + for (var i = 0; i < btexHeader.MipMapCount; i++) + { + var mipSize = (int)((width + 3) / 4) * (int)((height + 3) / 4) * formatBlockSize; + var size = (uint)Math.Max(formatBlockSize, mipSize); + + var blockSize = size == formatBlockSize + ? formatBlockSize + : Math.Min(256, Math.Max(formatBlockSize, (int)size / (int)(Math.Min(width, height) / 4))); + + var blockCount = size / blockSize; + + for (var j = 0; j < blockCount; j++) + { + writer.Write(btexHeader.Data, offset, blockSize); + offset += blockSize; + + if (size == formatBlockSize && j > blockCount - 4) + { + offset += 512 - blockSize; + } + else + { + offset += 256 - blockSize; + } + } + + width /= 2; + height /= 2; + } + } + } + catch { - writer.Write(btexHeader.Data, (int)mip.Offset, (int)mip.Size); + // Cringe to avoid dealing with stupid BTEX edge-cases with Forspoken direct storage nonsense + btexHeader.ArraySize = 1; + btexHeader.MipMapCount = 1; } btexHeader.Data = stream.ToArray(); @@ -89,74 +149,6 @@ public static byte[] BtexToDds(byte[] btex) return WriteDds(ddsHeader, btexHeader.Data); } - public static IEnumerable BtexArrayToDds(byte[] btex) - { - // Remove SEDB header - var withoutSedb = new byte[btex.Length - 128]; - Array.Copy(btex, 128, withoutSedb, 0, withoutSedb.Length); - - var btexHeader = ReadBtexHeader(withoutSedb); - - var ddsHeader = new DdsHeader - { - Height = btexHeader.Height, - Width = btexHeader.Width, - PitchOrLinearSize = btexHeader.Pitch, - Depth = btexHeader.Depth, - MipMapCount = 1, //btexHeader.MipMapCount, - Flags = DDSFlags.Texture | DDSFlags.Pitch | DDSFlags.Depth | DDSFlags.MipMapCount, - PixelFormat = new PixelFormat(), - DX10Header = new DX10 - { - ArraySize = 1, //btexHeader.ArraySize, - Format = FormatMap[btexHeader.Format], - ResourceDimension = btexHeader.Dimension + 1u - } - }; - - foreach (var mips in btexHeader.MipMaps) - { - var start = (int)mips[0].Offset; - var end = (int)(start + mips[0].Size); - yield return WriteDds(ddsHeader, btexHeader.Data[start..end]); - } - } - - public static byte[] ReplaceInTextureArray(byte[] btex, List btexMips, int index) - { - // Remove SEDB header - var withoutSedb = new byte[btex.Length - 128]; - Array.Copy(btex, 128, withoutSedb, 0, withoutSedb.Length); - var btexHeader = ReadBtexHeader(withoutSedb); - - for (var i = 0; i < btexMips.Count; i++) - { - var mipBtex = btexMips[i]; - File.WriteAllBytes(@$"C:\Modding\Wiz\Terrain\TestDump\{i}.btex", mipBtex); - var mipWithoutSedb = new byte[mipBtex.Length - 128]; - Array.Copy(mipBtex, 128, mipWithoutSedb, 0, mipWithoutSedb.Length); - var mipBtexHeader = ReadBtexHeader(mipWithoutSedb); - - var mipmap = btexHeader.MipMaps[index][i]; - if (mipmap.Size != mipBtexHeader.MipMaps[0][0].Size) - { - Console.WriteLine( - $"Mip {i} of size {mipBtexHeader.MipMaps[0][0].Size} is not equal to {mipmap.Size}—will not continue"); - break; - } - - var start = btexHeader.HeaderSize + 128 + mipmap.Offset; - var end = start + mipmap.Size; - - for (var j = start; j < end; j++) - { - btex[j] = mipBtexHeader.Data[j - start]; - } - } - - return btex; - } - public static byte[] DdsToBtex(TextureType type, string fileName, byte[] dds) { var ddsHeader = ReadDdsHeader(dds, out var ddsContent); @@ -210,7 +202,8 @@ public static byte[] DdsToBtex(TextureType type, string fileName, byte[] dds) return stream.ToArray(); } - public static byte[] DdsToBtex(string fileName, byte[] dds, byte imageFlags, long rowPitch, DxgiFormat format) + public static byte[] DdsToBtex(string fileName, byte[] dds, byte imageFlags, long rowPitch, DxgiFormat format, + bool addSedbHeader) { var ddsHeader = ReadDdsHeader(dds, out var ddsContent); var btexHeader = new BtexHeader @@ -227,38 +220,112 @@ public static byte[] DdsToBtex(string fileName, byte[] dds, byte imageFlags, lon p_ImageFileSize = (uint)ddsContent.Length, p_ImageFlags = imageFlags, p_Name = fileName, - p_SurfaceCount = (ushort)ddsHeader.MipMapCount + p_SurfaceCount = addSedbHeader ? (ushort)ddsHeader.MipMapCount : (ushort)0 }; - uint width = btexHeader.Width; - uint height = btexHeader.Height; + if (addSedbHeader) + { + btexHeader.HeaderSize = 128; - var mips = new List(); - for (var i = 0; i < btexHeader.MipMapCount; i++) + uint width = btexHeader.Width; + uint height = btexHeader.Height; + + var mips = new List(); + for (var i = 0; i < btexHeader.MipMapCount; i++) + { + mips.Add(new BtexMipMap + { + Offset = i == 0 ? 0 : mips[i - 1].Offset + mips[i - 1].Size, + Size = (uint)Math.Max(GetBlockSize(format), CalculatePitch(width, format) * height) + }); + + width /= 2; + height /= 2; + } + + btexHeader.MipMaps.Add(mips); + } + else { - mips.Add(new BtexMipMap + btexHeader.HeaderSize = 128; + btexHeader.p_Name = ""; + btexHeader.Pitch = 0; // Terada why? + + var offset = 0u; + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream); + + for (var a = 0; a < btexHeader.ArraySize; a++) { - Offset = i == 0 ? 0 : mips[i - 1].Offset + mips[i - 1].Size, - Size = (uint)Math.Max(GetBlockSize(format), CalculatePitch(width, format) * height) - }); + var formatBlockSize = GetBlockSize(format); + uint width = btexHeader.Width; + uint height = btexHeader.Height; - width /= 2; - height /= 2; + for (var i = 0; i < btexHeader.MipMapCount; i++) + { + var mipSize = (int)((width + 3) / 4) * (int)((height + 3) / 4) * formatBlockSize; + var size = (uint)Math.Max(formatBlockSize, mipSize); + + var blockSize = size == formatBlockSize + ? formatBlockSize + : Math.Min(256, Math.Max(formatBlockSize, (int)size / (int)(Math.Max(width, height) / 4))); + + var blockCount = size / blockSize; + + for (var j = 0; j < blockCount; j++) + { + writer.Write(btexHeader.Data, (int)offset, blockSize); + offset += (uint)blockSize; + + if (a == btexHeader.ArraySize - 1 && i == btexHeader.MipMapCount - 1) + { + btexHeader.p_PlatformDataSize = (uint)stream.Position; + writer.Align(256, 0x00); + btexHeader.p_ImageFileSize = (uint)stream.Position; + } + else if (size == formatBlockSize && j > blockCount - 4) + { + var paddingSize = 512 - blockSize; + if (paddingSize > 0) + { + writer.Write(new byte[paddingSize]); + } + } + else + { + var paddingSize = 256 - blockSize; + if (paddingSize > 0) + { + writer.Write(new byte[paddingSize]); + } + } + } + + width /= 2; + height /= 2; + } + } + + btexHeader.Data = stream.ToArray(); } - btexHeader.MipMaps.Add(mips); var btex = WriteBtex(btexHeader); - var sedbHeader = new SedbBtexHeader + if (addSedbHeader) { - FileSize = (ulong)btex.Length + 128 // Size of this header itself - }; + var sedbHeader = new SedbBtexHeader + { + FileSize = (ulong)btex.Length + 128 // Size of this header itself + }; - using var stream = new MemoryStream(); - using var writer = new BinaryWriter(stream, Encoding.UTF8); - writer.Write(sedbHeader.ToBytes()); - writer.Write(btex); - return stream.ToArray(); + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream, Encoding.UTF8); + writer.Write(sedbHeader.ToBytes()); + writer.Write(btex); + return stream.ToArray(); + } + + return btex; } public static byte[] WriteDds(DdsHeader header, byte[] data) @@ -435,8 +502,15 @@ private static int GetBlockSize(DxgiFormat format) }; } - public static BtexHeader ReadBtexHeader(byte[] btex) + public static BtexHeader ReadBtexHeader(byte[] btex, bool readMipData, bool readImageData = true) { + // Skip the SEDBbtex header if it is present + var tag = Encoding.UTF8.GetString(btex[..4]); + if (tag == "SEDB") + { + btex = btex[128..]; + } + var header = new BtexHeader(); using var memoryStream = new MemoryStream(btex); using var reader = new BinaryReader(memoryStream, Encoding.UTF8); @@ -480,41 +554,48 @@ public static BtexHeader ReadBtexHeader(byte[] btex) header.p_TileMode = reader.ReadUInt32(); header.ArraySize = reader.ReadUInt32(); - for (var i = 0; i < header.ArraySize; i++) + if (readMipData) { - var mips = new List(); - for (var j = 0; j < header.MipMapCount; j++) + for (var i = 0; i < header.ArraySize; i++) { - mips.Add(new BtexMipMap + var mips = new List(); + for (var j = 0; j < header.MipMapCount; j++) { - Offset = reader.ReadUInt32(), - Size = reader.ReadUInt32() - }); + mips.Add(new BtexMipMap + { + Offset = reader.ReadUInt32(), + Size = reader.ReadUInt32() + }); + } + + header.MipMaps.Add(mips); } - header.MipMaps.Add(mips); - } + memoryStream.Seek(header.p_ImageHeaderOffset + header.p_NameOffset, SeekOrigin.Begin); - memoryStream.Seek(header.p_ImageHeaderOffset + header.p_NameOffset, SeekOrigin.Begin); - - var nameBytes = new List(); - byte current = 0x00; - - do - { - current = reader.ReadByte(); + var nameBytes = new List(); + byte current = 0x00; - if (current != 0x00) + do { - nameBytes.Add(current); - } - } while (current != 0x00); + current = reader.ReadByte(); + + if (current != 0x00) + { + nameBytes.Add(current); + } + } while (current != 0x00); - header.p_Name = Encoding.UTF8.GetString(nameBytes.ToArray()); + header.p_Name = Encoding.UTF8.GetString(nameBytes.ToArray()); + } memoryStream.Seek(header.HeaderSize, SeekOrigin.Begin); - header.Data = reader.ReadBytes((int)(memoryStream.Length - memoryStream.Position)); + if (readImageData) + { + header.Data = reader.ReadBytes((int)(memoryStream.Length - memoryStream.Position)); + } + return header; } @@ -587,13 +668,13 @@ public static byte[] AddTextureToArray(byte[] btex, byte[] btexArray) var withoutSedb = new byte[btex.Length - 128]; Array.Copy(btex, 128, withoutSedb, 0, withoutSedb.Length); - var btexHeader = ReadBtexHeader(withoutSedb); + var btexHeader = ReadBtexHeader(withoutSedb, true); // Remove SEDB header var withoutSedbArray = new byte[btexArray.Length - 128]; Array.Copy(btexArray, 128, withoutSedbArray, 0, withoutSedbArray.Length); - var btexArrayHeader = ReadBtexHeader(withoutSedbArray); + var btexArrayHeader = ReadBtexHeader(withoutSedbArray, true); // Update metadata btexArrayHeader.ArraySize++; diff --git a/Flagrum.Core/Gfxbin/Btex/ImageFormat.cs b/Flagrum.Core/Gfxbin/Btex/ImageFormat.cs new file mode 100644 index 00000000..2f54ea62 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Btex/ImageFormat.cs @@ -0,0 +1,22 @@ +using System; +using System.Linq; +using Flagrum.Core.Utilities.Types; + +namespace Flagrum.Core.Gfxbin.Btex; + +public class ImageFormat : Enum +{ + private ImageFormat(string value) : base(value) { } + + public static ImageFormat Png { get; } = new("png"); + public static ImageFormat Targa { get; } = new("tga"); + public static ImageFormat Dds { get; } = new("dds"); + public static ImageFormat Btex { get; } = new("btex"); + + public static implicit operator string(ImageFormat format) => format.Value; + + public static explicit operator ImageFormat(string value) + { + return GetAll().First(f => f.Value.Equals(value, StringComparison.OrdinalIgnoreCase)); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/Components/Mesh.cs b/Flagrum.Core/Gfxbin/Gmdl/Components/Mesh.cs index 6bb4cb84..43afb237 100644 --- a/Flagrum.Core/Gfxbin/Gmdl/Components/Mesh.cs +++ b/Flagrum.Core/Gfxbin/Gmdl/Components/Mesh.cs @@ -70,8 +70,7 @@ public enum PrimitiveType public enum IndexType { IndexType16 = 0x0, - IndexType32 = 0x1, - IndexTypeNum = 0x2 + IndexType32 = 0x1 } public class Mesh diff --git a/Flagrum.Core/Gfxbin/Gmdl/Components/VertexElementDescription.cs b/Flagrum.Core/Gfxbin/Gmdl/Components/VertexElementDescription.cs index 682dfdb9..741dcca9 100644 --- a/Flagrum.Core/Gfxbin/Gmdl/Components/VertexElementDescription.cs +++ b/Flagrum.Core/Gfxbin/Gmdl/Components/VertexElementDescription.cs @@ -1,4 +1,7 @@ -namespace Flagrum.Core.Gfxbin.Gmdl.Components; +using System.Linq; +using Flagrum.Core.Utilities.Types; + +namespace Flagrum.Core.Gfxbin.Gmdl.Components; public enum VertexElementFormat { @@ -51,6 +54,43 @@ public enum VertexElementFormat Num = 0x2E } +public class VertexElementSemantic : Enum +{ + private VertexElementSemantic(string value) : base(value) { } + + public static VertexElementSemantic Position0 => new("POSITION0"); + public static VertexElementSemantic Normal0 => new("NORMAL0"); + public static VertexElementSemantic Color0 => new("COLOR0"); + public static VertexElementSemantic Color1 => new("COLOR1"); + public static VertexElementSemantic Color2 => new("COLOR2"); + public static VertexElementSemantic Color3 => new("COLOR3"); + public static VertexElementSemantic TexCoord0 => new("TEXCOORD0"); + public static VertexElementSemantic TexCoord1 => new("TEXCOORD1"); + public static VertexElementSemantic TexCoord2 => new("TEXCOORD2"); + public static VertexElementSemantic TexCoord3 => new("TEXCOORD3"); + public static VertexElementSemantic TexCoord4 => new("TEXCOORD4"); + public static VertexElementSemantic TexCoord5 => new("TEXCOORD5"); + public static VertexElementSemantic TexCoord6 => new("TEXCOORD6"); + public static VertexElementSemantic TexCoord7 => new("TEXCOORD7"); + public static VertexElementSemantic BlendWeight0 => new("BLENDWEIGHT0"); + public static VertexElementSemantic BlendWeight1 => new("BLENDWEIGHT1"); + public static VertexElementSemantic BlendIndices0 => new("BLENDINDICES0"); + public static VertexElementSemantic BlendIndices1 => new("BLENDINDICES1"); + public static VertexElementSemantic Tangent0 => new("TANGENT0"); + public static VertexElementSemantic Tangent1 => new("TANGENT1"); + public static VertexElementSemantic Binormal0 => new("BINORMAL0"); + public static VertexElementSemantic Binormal1 => new("BINORMAL1"); + public static VertexElementSemantic Normal2Factors => new("NORMAL2FACTORS0"); + public static VertexElementSemantic Normal4Factors => new("NORMAL4FACTORS0"); + public static VertexElementSemantic FogCoord0 => new("FOGCOORD0"); + public static VertexElementSemantic PSize0 => new("PSIZE0"); + + public static explicit operator VertexElementSemantic(string name) + { + return GetAll().First(e => e.Value == name); + } +} + public class VertexElementDescription { // Positions of each of the vertices in the mesh diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gfxbin.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gfxbin.cs new file mode 100644 index 00000000..d1a1a24e --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gfxbin.cs @@ -0,0 +1,321 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Flagrum.Core.Gfxbin.Gmdl.Components; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class Gfxbin +{ + public Gfxbin(byte[] gfxbinData) + { + using var stream = new MemoryStream(gfxbinData); + using var reader = new MessagePackReader(stream); + + Header = new GfxbinHeader(); + Header.Read(reader); + + reader.DataVersion = Header.Version; + + Aabb = reader.ReadAABB(); + + if (Header.Version < 20220707) + { + InstanceNameFormat = reader.Read(); + ShaderClassFormat = reader.Read(); + ShaderSamplerDescriptionFormat = reader.Read>(); + ShaderParameterListFormat = reader.Read>(); + ChildClassFormat = reader.Read>(); + } + + Bones = reader.Read>(); + Nodes = reader.Read>(); + + if (Header.Version >= 20220707) + { + Unknown = reader.Read(); + GpubinCount = reader.Read(); + } + else + { + GpubinCount = 1; + } + + GpubinHashes = new List(); + for (var i = 0UL; i < GpubinCount; i++) + { + GpubinHashes.Add(reader.Read()); + } + + MeshObjects = reader.Read>(); + + Unknown2 = reader.Read(); + Name = reader.Read(); + + Parts = reader.Read>(); + + if (Header.Version >= 20220707) + { + Unknown3 = reader.Read(); + Unknown4 = reader.Read(); + Unknown5 = reader.Read(); + Unknown6 = reader.Read(); + Unknown7 = reader.Read(); + + GpubinSizeCount = reader.Read(); + GpubinSizes = new List(); + for (var i = 0; i < GpubinSizeCount; i++) + { + GpubinSizes.Add(reader.Read()); + } + } + } + + public GfxbinHeader Header { get; set; } + public GfxbinAABB Aabb { get; set; } + + public string InstanceNameFormat { get; set; } + public string ShaderClassFormat { get; set; } + public IList ShaderSamplerDescriptionFormat { get; set; } + public IList ShaderParameterListFormat { get; set; } + public IList ChildClassFormat { get; set; } + + public IList Bones { get; set; } + public IList Nodes { get; set; } + + public float Unknown { get; set; } + + public ulong GpubinCount { get; set; } + public IList GpubinHashes { get; set; } + public IList MeshObjects { get; set; } + + public bool Unknown2 { get; set; } + public string Name { get; set; } + + public IList Parts { get; set; } + + public float Unknown3 { get; set; } + public float Unknown4 { get; set; } + public float Unknown5 { get; set; } + public float Unknown6 { get; set; } + public float Unknown7 { get; set; } + + public uint GpubinSizeCount { get; set; } + public IList GpubinSizes { get; set; } + + // public void ReadVertexData(List gpubins) + // { + // //var gpubin = new Gpubin(gpubins); + // // foreach (var meshObject in MeshObjects) + // // { + // // foreach (var mesh in meshObject.Meshes) + // // { + // // mesh.FaceIndices = gpubin.UnpackFaceIndices(mesh.FaceIndexOffset, mesh.FaceIndexCount, mesh.FaceIndexType); + // // gpubin.UnpackVertexStreams(mesh); + // // } + // // } + // + // Parallel.ForEach(MeshObjects, + // meshObject => + // { + // Parallel.ForEach(meshObject.Meshes, + // mesh => + // { + // Parallel.Invoke( + // () => + // { + // mesh.FaceIndices = gpubin.UnpackFaceIndices(mesh.FaceIndexOffset, mesh.FaceIndexCount, + // mesh.FaceIndexType); + // }, () => gpubin.UnpackVertexStreams(mesh)); + // }); + // }); + // } + + public void ReadVertexData2(List gpubins) + { + Parallel.ForEach(MeshObjects, meshObject => + { + Parallel.ForEach(meshObject.Meshes, mesh => + { + Parallel.Invoke(() => + { + using var stream = new MemoryStream(gpubins[(int)mesh.GpubinIndex]); + using var reader = new BinaryReader(stream); + + var faceCount = mesh.FaceIndexCount / 3; + mesh.FaceIndices = new uint[faceCount, 3]; + + stream.Seek(mesh.FaceIndexOffset, SeekOrigin.Begin); + + switch (mesh.FaceIndexType) + { + case IndexType.IndexType16: + for (var i = 0; i < faceCount; i++) + { + mesh.FaceIndices[i, 0] = reader.ReadUInt16(); + mesh.FaceIndices[i, 1] = reader.ReadUInt16(); + mesh.FaceIndices[i, 2] = reader.ReadUInt16(); + } + + break; + case IndexType.IndexType32: + for (var i = 0; i < faceCount; i++) + { + mesh.FaceIndices[i, 0] = reader.ReadUInt32(); + mesh.FaceIndices[i, 1] = reader.ReadUInt32(); + mesh.FaceIndices[i, 2] = reader.ReadUInt32(); + } + + break; + default: + throw new ArgumentOutOfRangeException(nameof(mesh.FaceIndexType), mesh.FaceIndexType, + "Unsupported face index type"); + } + }, () => + { + Parallel.ForEach(mesh.VertexStreams, vertexStream => + { + var semantics = new Dictionary(); + + using var stream = new MemoryStream(gpubins[(int)mesh.GpubinIndex]); + using var reader = new BinaryReader(stream); + + stream.Seek(mesh.VertexBufferOffset + vertexStream.Offset, SeekOrigin.Begin); + + vertexStream.Elements = vertexStream.Elements.OrderBy(e => e.Offset).ToList(); + foreach (var element in vertexStream.Elements) + { + var type = GetElementType(element.Format); + var listType = typeof(List<>).MakeGenericType(type); + semantics[element.Semantic] = (IList)Activator.CreateInstance(listType); + } + + for (var i = 0; i < mesh.VertexCount; i++) + { + foreach (var element in vertexStream.Elements) + { + switch (element.Format) + { + case VertexElementFormat.XYZ32_Float: + semantics[element.Semantic].Add(new[] + { + reader.ReadSingle(), + reader.ReadSingle(), + reader.ReadSingle() + }); + break; + case VertexElementFormat.XY16_SintN: + semantics[element.Semantic].Add(new[] + { + reader.ReadInt16() * (1f / 0x7FFF), + reader.ReadInt16() * (1f / 0x7FFF) + }); + break; + case VertexElementFormat.XY16_UintN: + semantics[element.Semantic].Add(new[] + { + (float)reader.ReadUInt16() / 0xFFFF, + (float)reader.ReadUInt16() / 0xFFFF + }); + break; + case VertexElementFormat.XY16_Float: + semantics[element.Semantic].Add(new[] + { + (float)reader.ReadHalf(), + (float)reader.ReadHalf() + }); + break; + case VertexElementFormat.XY32_Float: + semantics[element.Semantic].Add(new[] + { + reader.ReadSingle(), + reader.ReadSingle() + }); + break; + case VertexElementFormat.XYZW8_UintN: + case VertexElementFormat.XYZW8_Uint: + semantics[element.Semantic].Add(new[] + { + (float)reader.ReadByte() / 0xFF, + (float)reader.ReadByte() / 0xFF, + (float)reader.ReadByte() / 0xFF, + (float)reader.ReadByte() / 0xFF + }); + break; + case VertexElementFormat.XYZW8_SintN: + case VertexElementFormat.XYZW8_Sint: + semantics[element.Semantic].Add(new[] + { + reader.ReadSByte() * (1f / 0x7F), + reader.ReadSByte() * (1f / 0x7F), + reader.ReadSByte() * (1f / 0x7F), + reader.ReadSByte() * (1f / 0x7F) + }); + break; + case VertexElementFormat.XYZW16_Uint: + semantics[element.Semantic].Add(new[] + { + reader.ReadUInt16(), + reader.ReadUInt16(), + reader.ReadUInt16(), + reader.ReadUInt16() + }); + break; + case VertexElementFormat.XYZW32_Uint: + semantics[element.Semantic].Add(new[] + { + reader.ReadUInt32(), + reader.ReadUInt32(), + reader.ReadUInt32(), + reader.ReadUInt32() + }); + break; + case VertexElementFormat.XYZW16_Float: + semantics[element.Semantic].Add(new[] + { + reader.ReadHalf(), + reader.ReadHalf(), + reader.ReadHalf(), + reader.ReadHalf() + }); + break; + default: + throw new ArgumentOutOfRangeException(nameof(element.Format), element.Format, + $"Unsupported vertex element format for semantic {element.Semantic}"); + } + } + } + + foreach (var (semantic, list) in semantics) + { + mesh.Semantics[semantic] = list; + } + }); + }); + }); + }); + } + + private Type GetElementType(VertexElementFormat format) + { + return format switch + { + VertexElementFormat.XYZ32_Float => typeof(float[]), + VertexElementFormat.XY16_SintN => typeof(float[]), + VertexElementFormat.XY16_UintN => typeof(float[]), + VertexElementFormat.XY16_Float => typeof(float[]), + VertexElementFormat.XY32_Float => typeof(float[]), + VertexElementFormat.XYZW8_UintN => typeof(float[]), + VertexElementFormat.XYZW8_Uint => typeof(float[]), + VertexElementFormat.XYZW8_SintN => typeof(float[]), + VertexElementFormat.XYZW8_Sint => typeof(float[]), + VertexElementFormat.XYZW16_Uint => typeof(ushort[]), + VertexElementFormat.XYZW32_Uint => typeof(uint[]), + VertexElementFormat.XYZW16_Float => typeof(Half[]), + _ => typeof(float[]) + }; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinAABB.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinAABB.cs new file mode 100644 index 00000000..acd204fc --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinAABB.cs @@ -0,0 +1,9 @@ +using Flagrum.Core.Gfxbin.Gmdl.Constructs; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinAABB +{ + public Vector3 Start { get; set; } + public Vector3 End { get; set; } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinBone.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinBone.cs new file mode 100644 index 00000000..94e8f3e4 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinBone.cs @@ -0,0 +1,23 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinBone : IMessagePackItem +{ + public string Name { get; set; } + public uint LodIndex { get; set; } + public uint UniqueIndex { get; set; } + + public void Read(MessagePackReader reader) + { + Name = reader.Read(); + LodIndex = reader.Read(); + + if (reader.DataVersion >= 20220707) + { + UniqueIndex = reader.Read(); + } + else + { + UniqueIndex = LodIndex >> 16; + } + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinHeader.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinHeader.cs new file mode 100644 index 00000000..d593ca1b --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinHeader.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinHeader +{ + public uint Version { get; set; } + public Dictionary Dependencies { get; set; } + public IList Hashes { get; set; } + + public void Read(MessagePackReader reader) + { + Version = reader.Read(); + Dependencies = reader.Read>(); + Hashes = reader.Read>(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMesh.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMesh.cs new file mode 100644 index 00000000..12bc6393 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMesh.cs @@ -0,0 +1,148 @@ +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Flagrum.Core.Gfxbin.Gmdl.Components; +using Flagrum.Core.Gfxbin.Gmdl.Constructs; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinMesh : IMessagePackItem +{ + public string Name { get; set; } + public uint Unknown { get; set; } + public IList BoneIds { get; set; } + public VertexLayoutType VertexLayoutType { get; set; } + public bool Unknown2 { get; set; } + public GfxbinAABB AABB { get; set; } + public bool IsOrientedBB { get; set; } + public OrientedBB OrientedBB { get; set; } + public PrimitiveType PrimitiveType { get; set; } + public uint FaceIndexCount { get; set; } + public IndexType FaceIndexType { get; set; } + public uint GpubinIndex { get; set; } + public uint FaceIndexOffset { get; set; } + public uint FaceIndexSize { get; set; } + public uint VertexCount { get; set; } + public IList VertexStreams { get; set; } + public uint VertexBufferOffset { get; set; } + public uint VertexBufferSize { get; set; } + public uint InstanceNumber { get; set; } + public IList Subgeometries { get; set; } + public uint Unknown6 { get; set; } + public uint UnknownOffset { get; set; } + public uint UnknownSize { get; set; } + public ulong MaterialHash { get; set; } + public uint DrawPriorityOffset { get; set; } + public bool Unknown7 { get; set; } + public bool Unknown8 { get; set; } + public float LodNear { get; set; } + public float LodFar { get; set; } + public float LodFade { get; set; } + public bool Unknown11 { get; set; } + public bool Unknown12 { get; set; } + public uint PartsId { get; set; } + public IList Parts { get; set; } + public bool Unknown9 { get; set; } + public uint Flags { get; set; } + public bool Unknown10 { get; set; } + public uint BreakableBoneIndex { get; set; } + public uint LowLodShadowCascadeNo { get; set; } + + public uint[,] FaceIndices { get; set; } + + public ConcurrentDictionary Semantics { get; set; } = new(); + // public List VertexPositions { get; set; } = new(); + // public List Normals { get; set; } = new(); + // public List Tangents { get; set; } = new(); + // public List ColorMaps { get; set; } = new(); + // public List UVMaps { get; set; } = new(); + // public List> WeightIndices { get; set; } = new(); + // public List> WeightValues { get; set; } = new(); + + public void Read(MessagePackReader reader) + { + Name = reader.Read(); + + if (reader.DataVersion < 20220707) + { + Unknown = reader.Read(); + BoneIds = reader.Read>(); + } + + VertexLayoutType = (VertexLayoutType)reader.Read(); + Unknown2 = reader.Read(); + AABB = reader.ReadAABB(); + IsOrientedBB = reader.Read(); + OrientedBB = new OrientedBB( + reader.ReadVector3(), + reader.ReadVector3(), + reader.ReadVector3(), + reader.ReadVector3() + ); + + PrimitiveType = (PrimitiveType)reader.Read(); + FaceIndexCount = reader.Read(); + FaceIndexType = (IndexType)reader.Read(); + + if (reader.DataVersion >= 20220707) + { + GpubinIndex = reader.Read(); + } + + FaceIndexOffset = reader.Read(); + FaceIndexSize = reader.Read(); + VertexCount = reader.Read(); + VertexStreams = reader.Read>(); + VertexBufferOffset = reader.Read(); + VertexBufferSize = reader.Read(); + + if (reader.DataVersion < 20220707) + { + InstanceNumber = reader.Read(); + } + + if (reader.DataVersion >= 20220707) + { + var subgeometryCount = reader.Read(); + for (var i = 0; i < subgeometryCount; i++) + { + var subgeometry = new GfxbinSubgeometry(); + subgeometry.Read(reader); + Subgeometries.Add(subgeometry); + } + } + else + { + Subgeometries = reader.Read>(); + } + + if (reader.DataVersion >= 20220707) + { + Unknown6 = reader.Read(); + UnknownOffset = reader.Read(); + UnknownSize = reader.Read(); + } + + MaterialHash = reader.Read(); + DrawPriorityOffset = reader.Read(); + Unknown7 = reader.Read(); + Unknown8 = reader.Read(); + LodNear = reader.Read(); + LodFar = reader.Read(); + LodFade = reader.Read(); + + if (reader.DataVersion < 20220707) + { + Unknown11 = reader.Read(); + Unknown12 = reader.Read(); + } + + PartsId = reader.Read(); + Parts = reader.Read>(); + Unknown9 = reader.Read(); + Flags = reader.Read(); + Unknown10 = reader.Read(); + BreakableBoneIndex = reader.Read(); + LowLodShadowCascadeNo = reader.Read(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshObject.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshObject.cs new file mode 100644 index 00000000..814b8523 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshObject.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinMeshObject : IMessagePackItem, IMessagePackDifferentFirstItem +{ + public bool Unknown { get; set; } + public string Name { get; set; } + public IList Clusters { get; set; } + public IList Meshes { get; set; } + public bool IsFirst { get; set; } + + public void Read(MessagePackReader reader) + { + if (!IsFirst) + { + Unknown = reader.Read(); + } + + Name = reader.Read(); + Clusters = reader.Read>(); + Meshes = reader.Read>(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshPart.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshPart.cs new file mode 100644 index 00000000..b6f3126f --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinMeshPart.cs @@ -0,0 +1,15 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinMeshPart : IMessagePackItem +{ + public uint PartsId { get; set; } + public uint StartIndex { get; set; } + public uint IndexCount { get; set; } + + public void Read(MessagePackReader reader) + { + PartsId = reader.Read(); + StartIndex = reader.Read(); + IndexCount = reader.Read(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinModelPart.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinModelPart.cs new file mode 100644 index 00000000..3e8f7562 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinModelPart.cs @@ -0,0 +1,17 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinModelPart : IMessagePackItem +{ + public string Name { get; set; } + public uint Id { get; set; } + public string Unknown { get; set; } + public bool Flags { get; set; } + + public void Read(MessagePackReader reader) + { + Name = reader.Read(); + Id = reader.Read(); + Unknown = reader.Read(); + Flags = reader.Read(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinNode.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinNode.cs new file mode 100644 index 00000000..6f74f26d --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinNode.cs @@ -0,0 +1,42 @@ +using Flagrum.Core.Gfxbin.Gmdl.Constructs; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinNode : IMessagePackItem, IMessagePackDifferentFirstItem +{ + public Matrix Matrix { get; set; } + public float Unknown { get; set; } + public string Name { get; set; } + public int Unknown2 { get; set; } + public int Unknown3 { get; set; } + public int Unknown4 { get; set; } + public bool IsFirst { get; set; } + + public void Read(MessagePackReader reader) + { + Matrix = new Matrix( + reader.ReadVector3(), + reader.ReadVector3(), + reader.ReadVector3(), + reader.ReadVector3() + ); + + if (IsFirst) + { + Unknown = 0.0f; + } + else if (reader.DataVersion >= 20220707) + { + Unknown = reader.Read(); + } + + Name = reader.Read(); + + if (reader.DataVersion >= 20220707) + { + Unknown2 = reader.Read(); + Unknown3 = reader.Read(); + Unknown4 = reader.Read(); + } + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinSubgeometry.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinSubgeometry.cs new file mode 100644 index 00000000..a0283c1b --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinSubgeometry.cs @@ -0,0 +1,19 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinSubgeometry : IMessagePackItem +{ + public GfxbinAABB AABB { get; set; } + public uint StartIndex { get; set; } + public uint PrimitiveCount { get; set; } + public uint ClusterIndexBitFlag { get; set; } + public uint DrawOrder { get; set; } + + public void Read(MessagePackReader reader) + { + AABB = reader.ReadAABB(); + StartIndex = reader.Read(); + PrimitiveCount = reader.Read(); + ClusterIndexBitFlag = reader.Read(); + DrawOrder = reader.Read(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexElement.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexElement.cs new file mode 100644 index 00000000..f087a18b --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexElement.cs @@ -0,0 +1,17 @@ +using Flagrum.Core.Gfxbin.Gmdl.Components; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinVertexElement : IMessagePackItem +{ + public uint Offset { get; set; } + public VertexElementSemantic Semantic { get; set; } + public VertexElementFormat Format { get; set; } + + public void Read(MessagePackReader reader) + { + Offset = reader.Read(); + Semantic = (VertexElementSemantic)reader.Read(); + Format = (VertexElementFormat)reader.Read(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexStream.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexStream.cs new file mode 100644 index 00000000..eb70b491 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/GfxbinVertexStream.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using Flagrum.Core.Gfxbin.Gmdl.Components; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class GfxbinVertexStream : IMessagePackItem +{ + public VertexStreamSlot Slot { get; set; } + public VertexStreamType Type { get; set; } + public uint Stride { get; set; } + public uint Offset { get; set; } + public IList Elements { get; set; } + + public void Read(MessagePackReader reader) + { + Slot = (VertexStreamSlot)reader.Read(); + Type = (VertexStreamType)reader.Read(); + Stride = reader.Read(); + Offset = reader.Read(); + Elements = reader.Read>(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gpubin.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gpubin.cs new file mode 100644 index 00000000..d87bc836 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/Gpubin.cs @@ -0,0 +1,321 @@ +// using System; +// using System.Collections.Generic; +// using System.IO; +// using System.Threading.Tasks; +// using Flagrum.Core.Gfxbin.Gmdl.Components; +// using Flagrum.Core.Gfxbin.Gmdl.Constructs; +// +// namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; +// +// public class Gpubin +// { +// private readonly List _buffers; +// +// private readonly Dictionary _formatParameters; +// +// public Gpubin(List gpubins) +// { +// _buffers = gpubins; +// _formatParameters = new Dictionary +// { +// {VertexElementFormat.X8_Uint, (typeof(byte), 1)}, +// {VertexElementFormat.XYZW8_Uint, (typeof(byte), 4)}, +// {VertexElementFormat.X8_Sint, (typeof(sbyte), 1)}, +// {VertexElementFormat.XYZW8_Sint, (typeof(sbyte), 4)}, +// +// {VertexElementFormat.X8_UintN, (typeof(byte), 1)}, +// {VertexElementFormat.XYZW8_UintN, (typeof(byte), 4)}, +// {VertexElementFormat.X8_SintN, (typeof(sbyte), 1)}, +// {VertexElementFormat.XYZW8_SintN, (typeof(sbyte), 4)}, +// +// {VertexElementFormat.X16_Sint, (typeof(short), 1)}, +// {VertexElementFormat.XY16_Sint, (typeof(short), 2)}, +// {VertexElementFormat.XYZW16_Sint, (typeof(short), 4)}, +// +// {VertexElementFormat.X16_Uint, (typeof(ushort), 1)}, +// {VertexElementFormat.XY16_Uint, (typeof(ushort), 2)}, +// {VertexElementFormat.XYZW16_Uint, (typeof(ushort), 4)}, +// +// {VertexElementFormat.X16_SintN, (typeof(short), 1)}, +// {VertexElementFormat.XY16_SintN, (typeof(short), 2)}, +// {VertexElementFormat.XYZW16_SintN, (typeof(short), 4)}, +// +// {VertexElementFormat.X16_UintN, (typeof(ushort), 1)}, +// {VertexElementFormat.XY16_UintN, (typeof(ushort), 2)}, +// {VertexElementFormat.XYZW16_UintN, (typeof(ushort), 4)}, +// +// {VertexElementFormat.X32_Sint, (typeof(int), 1)}, +// {VertexElementFormat.XY32_Sint, (typeof(int), 2)}, +// {VertexElementFormat.XYZ32_Sint, (typeof(int), 3)}, +// {VertexElementFormat.XYZW32_Sint, (typeof(int), 4)}, +// +// {VertexElementFormat.X32_Uint, (typeof(uint), 1)}, +// {VertexElementFormat.XY32_Uint, (typeof(uint), 2)}, +// {VertexElementFormat.XYZ32_Uint, (typeof(uint), 3)}, +// {VertexElementFormat.XYZW32_Uint, (typeof(uint), 4)}, +// +// {VertexElementFormat.X32_SintN, (typeof(int), 1)}, +// {VertexElementFormat.XY32_SintN, (typeof(int), 2)}, +// {VertexElementFormat.XYZ32_SintN, (typeof(int), 3)}, +// {VertexElementFormat.XYZW32_SintN, (typeof(int), 4)}, +// +// {VertexElementFormat.X32_UintN, (typeof(uint), 1)}, +// {VertexElementFormat.XY32_UintN, (typeof(uint), 2)}, +// {VertexElementFormat.XYZ32_UintN, (typeof(uint), 3)}, +// {VertexElementFormat.XYZW32_UintN, (typeof(uint), 4)}, +// +// {VertexElementFormat.X16_Float, (typeof(Half), 1)}, +// {VertexElementFormat.XY16_Float, (typeof(Half), 2)}, +// {VertexElementFormat.XYZW16_Float, (typeof(Half), 4)}, +// +// {VertexElementFormat.X32_Float, (typeof(float), 1)}, +// {VertexElementFormat.XY32_Float, (typeof(float), 2)}, +// {VertexElementFormat.XYZ32_Float, (typeof(float), 3)}, +// {VertexElementFormat.XYZW32_Float, (typeof(float), 4)} +// }; +// } +// +// public uint[,] UnpackFaceIndices(uint gpubinOffset, uint faceIndicesCount, IndexType faceIndexType) +// { +// using var stream = new MemoryStream(_buffers[0]); +// using var reader = new BinaryReader(stream); +// +// var faceCount = faceIndicesCount / 3; // All faces are tris on gmdl meshes +// var faceIndices = new uint[faceCount, 3]; +// +// stream.Seek(gpubinOffset, SeekOrigin.Begin); +// +// // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault +// switch (faceIndexType) +// { +// case IndexType.IndexType16: +// for (var i = 0; i < faceCount; i++) +// { +// faceIndices[i, 0] = reader.ReadUInt16(); +// faceIndices[i, 1] = reader.ReadUInt16(); +// faceIndices[i, 2] = reader.ReadUInt16(); +// } +// +// break; +// case IndexType.IndexType32: +// for (var i = 0; i < faceCount; i++) +// { +// faceIndices[i, 0] = reader.ReadUInt32(); +// faceIndices[i, 1] = reader.ReadUInt32(); +// faceIndices[i, 2] = reader.ReadUInt32(); +// } +// +// break; +// default: +// throw new ArgumentOutOfRangeException(nameof(faceIndexType), faceIndexType, null); +// } +// +// return faceIndices; +// } +// +// public void UnpackVertexStreams(GfxbinMesh mesh) +// { +// Parallel.ForEach(mesh.VertexStreams, vertexStream => +// { +// using var stream = new MemoryStream(_buffers[(int)mesh.GpubinIndex]); +// using var reader = new BinaryReader(stream); +// +// stream.Seek(mesh.VertexBufferOffset + vertexStream.Offset, SeekOrigin.Begin); +// +// for (var i = 0; i < mesh.VertexCount; i++) +// { +// foreach (var element in vertexStream.Elements) +// { +// switch (element.Semantic) +// { +// case VertexElementDescription.Position0: +// var position = new Vector3 +// { +// X = ReadSingle(reader), +// Y = ReadSingle(reader), +// Z = ReadSingle(reader) +// }; +// mesh.VertexPositions.Add(position); +// break; +// case VertexElementDescription.Normal0: +// var normal = new Normal +// { +// X = reader.ReadSByte(), +// Y = reader.ReadSByte(), +// Z = reader.ReadSByte(), +// W = reader.ReadSByte() +// }; +// mesh.Normals.Add(normal); +// break; +// case VertexElementDescription.Tangent0: +// var tangent = new Normal +// { +// X = reader.ReadSByte(), +// Y = reader.ReadSByte(), +// Z = reader.ReadSByte(), +// W = reader.ReadSByte() +// }; +// mesh.Tangents.Add(tangent); +// break; +// case VertexElementDescription.Color0: +// case VertexElementDescription.Color1: +// case VertexElementDescription.Color2: +// case VertexElementDescription.Color3: +// var colorIndex = int.Parse(element.Semantic[^1].ToString()); +// while (mesh.ColorMaps.Count - 1 < colorIndex) +// { +// mesh.ColorMaps.Add(new ColorMap {Colors = new List()}); +// } +// +// var colorMap = mesh.ColorMaps[colorIndex]; +// var color = new Color4 +// { +// R = reader.ReadByte(), +// G = reader.ReadByte(), +// B = reader.ReadByte(), +// A = reader.ReadByte() +// }; +// colorMap.Colors.Add(color); +// break; +// case VertexElementDescription.TexCoord0: +// case VertexElementDescription.TexCoord1: +// case VertexElementDescription.TexCoord2: +// case VertexElementDescription.TexCoord3: +// case VertexElementDescription.TexCoord4: +// case VertexElementDescription.TexCoord5: +// case VertexElementDescription.TexCoord6: +// case VertexElementDescription.TexCoord7: +// var texCoordIndex = int.Parse(element.Semantic[^1].ToString()); +// while (mesh.UVMaps.Count - 1 < texCoordIndex) +// { +// mesh.UVMaps.Add(new UVMap {UVs = new List()}); +// } +// +// var uvMap = mesh.UVMaps[texCoordIndex]; +// var uv = new UV +// { +// U = ReadHalf(reader), +// V = ReadHalf(reader) +// }; +// +// uvMap.UVs.Add(uv); +// break; +// case VertexElementDescription.BlendWeight0: +// case VertexElementDescription.BlendWeight1: +// var weightIndex = int.Parse(element.Semantic[^1].ToString()); +// while (mesh.WeightValues.Count - 1 < weightIndex) +// { +// mesh.WeightValues.Add(new List()); +// } +// +// var weightMap = mesh.WeightValues[weightIndex]; +// var weight = new[] +// {reader.ReadByte(), reader.ReadByte(), reader.ReadByte(), reader.ReadByte()}; +// weightMap.Add(weight); +// break; +// case VertexElementDescription.BlendIndices0: +// case VertexElementDescription.BlendIndices1: +// var weightIndicesIndex = int.Parse(element.Semantic[^1].ToString()); +// while (mesh.WeightIndices.Count - 1 < weightIndicesIndex) +// { +// mesh.WeightIndices.Add(new List()); +// } +// +// var weightIndexMap = mesh.WeightIndices[weightIndicesIndex]; +// var weightIndices = new[] +// { +// ReadWeightIndex(reader, element.Format), +// ReadWeightIndex(reader, element.Format), +// ReadWeightIndex(reader, element.Format), +// ReadWeightIndex(reader, element.Format) +// }; +// weightIndexMap.Add(weightIndices); +// break; +// default: +// // TODO: Read the remaining semantics into proper data structures +// ReadPastVertexElement(reader, element.Format); +// break; +// } +// } +// } +// }); +// } +// +// private ushort ReadWeightIndex(BinaryReader reader, VertexElementFormat format) +// { +// var (type, _) = _formatParameters[format]; +// return type == typeof(byte) ? reader.ReadByte() : reader.ReadUInt16(); +// } +// +// private void ReadPastVertexElement(BinaryReader reader, VertexElementFormat format) +// { +// var (type, elementCount) = _formatParameters[format]; +// +// for (var i = 0; i < elementCount; i++) +// { +// if (type == typeof(byte)) +// { +// reader.ReadByte(); +// } +// else if (type == typeof(sbyte)) +// { +// reader.ReadSByte(); +// } +// else if (type == typeof(short)) +// { +// reader.ReadInt16(); +// } +// else if (type == typeof(ushort)) +// { +// reader.ReadUInt16(); +// } +// else if (type == typeof(int)) +// { +// reader.ReadInt32(); +// } +// else if (type == typeof(uint)) +// { +// reader.ReadUInt32(); +// } +// else if (type == typeof(Half)) +// { +// ReadHalf(reader); +// } +// else if (type == typeof(float)) +// { +// ReadSingle(reader); +// } +// } +// } +// +// private float ReadSingle(BinaryReader reader) +// { +// var value = reader.ReadSingle(); +// +// if (float.IsNaN(value) +// || float.IsInfinity(value) +// || float.IsNegativeInfinity(value) +// || float.IsPositiveInfinity(value)) +// { +// return 0.0f; +// } +// +// return value; +// } +// +// private Half ReadHalf(BinaryReader reader) +// { +// var value = reader.ReadHalf(); +// +// if (Half.IsNaN(value) +// || Half.IsInfinity(value) +// || Half.IsNegativeInfinity(value) +// || Half.IsPositiveInfinity(value)) +// { +// return (Half)0; +// } +// +// return value; +// } +// } + diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackDifferentFirstItem.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackDifferentFirstItem.cs new file mode 100644 index 00000000..49c35259 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackDifferentFirstItem.cs @@ -0,0 +1,6 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public interface IMessagePackDifferentFirstItem +{ + bool IsFirst { get; set; } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackItem.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackItem.cs new file mode 100644 index 00000000..3c91b714 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/IMessagePackItem.cs @@ -0,0 +1,6 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public interface IMessagePackItem +{ + void Read(MessagePackReader reader); +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackReader.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackReader.cs new file mode 100644 index 00000000..6407a6de --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackReader.cs @@ -0,0 +1,210 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text; +using Flagrum.Core.Gfxbin.Gmdl.Constructs; + +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public class MessagePackReader : IDisposable +{ + private readonly BinaryReader _reader; + private readonly Stream _stream; + + public MessagePackReader(Stream stream, bool leaveOpen = false) + { + _stream = stream; + _reader = new BinaryReader(_stream, Encoding.UTF8, leaveOpen); + } + + public uint DataVersion { get; set; } + + [SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize")] + public void Dispose() + { + _reader?.Dispose(); + } + + public Vector3 ReadVector3() + { + return new Vector3( + Read(), + Read(), + Read() + ); + } + + public GfxbinAABB ReadAABB() + { + return new GfxbinAABB + { + Start = ReadVector3(), + End = ReadVector3() + }; + } + + public TValue Read() + { + var type = (MessagePackType)_reader.ReadByte(); + + switch (type) + { + case >= MessagePackType.PositiveFixIntStart and <= MessagePackType.PositiveFixIntEnd: + return (TValue)Convert.ChangeType((byte)type & 0x7F, typeof(TValue)); + case >= MessagePackType.FixMapStart and <= MessagePackType.FixMapEnd: + var fixMapTypes = typeof(TValue).GetGenericArguments(); + return (TValue)ReadMap(fixMapTypes[0], fixMapTypes[1], (uint)((byte)type & 0xF)); + case >= MessagePackType.FixArrayStart and <= MessagePackType.FixArrayEnd: + var array8Type = typeof(TValue).GetGenericArguments()[0]; + return (TValue)ReadArray(array8Type, (uint)((byte)type & 0xF)); + case >= MessagePackType.FixStrStart and <= MessagePackType.FixStrEnd: + var lengthFix = (byte)type & 0x1F; + return lengthFix == 0 + ? (TValue)(object)"" + : (TValue)(object)new string(_reader.ReadChars(lengthFix)[..^1]); + case MessagePackType.Nil: + return (TValue)(object)null; + case MessagePackType.Unused: + throw new NotImplementedException("Why does your file even have this!?"); + case MessagePackType.False: + return (TValue)(object)false; + case MessagePackType.True: + return (TValue)(object)true; + case MessagePackType.Bin8: + var size8 = _reader.ReadByte(); + return (TValue)(object)_reader.ReadBytes(size8); + case MessagePackType.Bin16: + var size16 = _reader.ReadUInt16(); + return (TValue)(object)_reader.ReadBytes(size16); + case MessagePackType.Bin32: + var size32 = _reader.ReadUInt32(); + return (TValue)(object)_reader.ReadBytes((int)size32); + case MessagePackType.Ext8: + throw new NotImplementedException(); + case MessagePackType.Ext16: + throw new NotImplementedException(); + case MessagePackType.Ext32: + throw new NotImplementedException(); + case MessagePackType.Float32: + return (TValue)(object)_reader.ReadSingle(); + case MessagePackType.Float64: + return (TValue)(object)_reader.ReadDouble(); + case MessagePackType.Uint8: + return (TValue)Convert.ChangeType(_reader.ReadByte(), typeof(TValue)); + case MessagePackType.Uint16: + return (TValue)Convert.ChangeType(_reader.ReadUInt16(), typeof(TValue)); + case MessagePackType.Uint32: + return (TValue)Convert.ChangeType(_reader.ReadUInt32(), typeof(TValue)); + case MessagePackType.Uint64: + return (TValue)(object)_reader.ReadUInt64(); + case MessagePackType.Int8: + return (TValue)(object)_reader.ReadSByte(); + case MessagePackType.Int16: + return (TValue)Convert.ChangeType(_reader.ReadInt16(), typeof(TValue)); + case MessagePackType.Int32: + return (TValue)(object)_reader.ReadInt32(); + case MessagePackType.Int64: + return (TValue)(object)_reader.ReadInt64(); + case MessagePackType.FixExt1: + throw new NotImplementedException(); + case MessagePackType.FixExt2: + throw new NotImplementedException(); + case MessagePackType.FixExt4: + throw new NotImplementedException(); + case MessagePackType.FixExt8: + throw new NotImplementedException(); + case MessagePackType.FixExt16: + throw new NotImplementedException(); + case MessagePackType.Str8: + var length8 = _reader.ReadByte(); + return (TValue)(object)new string(_reader.ReadChars(length8)[..^1]); + case MessagePackType.Str16: + var length16 = _reader.ReadUInt16(); + return (TValue)(object)new string(_reader.ReadChars(length16)[..^1]); + case MessagePackType.Str32: + var length32 = _reader.ReadUInt32(); + return (TValue)(object)new string(_reader.ReadChars((int)length32)[..^1]); + case MessagePackType.Array16: + var array16Size = _reader.ReadUInt16(); + var array16Type = typeof(TValue).GetGenericArguments()[0]; + return (TValue)ReadArray(array16Type, array16Size); + case MessagePackType.Array32: + var array32Size = _reader.ReadUInt32(); + var array32Type = typeof(TValue).GetGenericArguments()[0]; + return (TValue)ReadArray(array32Type, array32Size); + case MessagePackType.Map16: + var map16Size = _reader.ReadUInt16(); + var map16Types = typeof(TValue).GetGenericArguments(); + return (TValue)ReadMap(map16Types[0], map16Types[1], map16Size); + case MessagePackType.Map32: + var map32Size = _reader.ReadUInt32(); + var map32Types = typeof(TValue).GetGenericArguments(); + return (TValue)ReadMap(map32Types[0], map32Types[1], map32Size); + case >= MessagePackType.NegativeFixIntStart or <= MessagePackType.NegativeFixIntEnd: + return (TValue)(object)-((byte)type & 0x1F); + } + } + + private object ReadMap(Type key, Type value, uint size) + { + var type = typeof(Dictionary<,>).MakeGenericType(key, value); + var dictionary = (IDictionary)Activator.CreateInstance(type); + + for (var i = 0; i < size; i++) + { + var method = typeof(MessagePackReader).GetMethod(nameof(Read))!; + var keyMethod = method.MakeGenericMethod(key); + var valueMethod = method.MakeGenericMethod(value); + + dictionary!.Add( + keyMethod.Invoke(this, Array.Empty())!, + valueMethod.Invoke(this, Array.Empty()) + ); + } + + return dictionary; + } + + private object ReadArray(Type type, uint size) + { + var listType = typeof(List<>).MakeGenericType(type); + var list = (IList)Activator.CreateInstance(listType)!; + + for (var i = 0; i < size; i++) + { + if (type.IsAssignableTo(typeof(IMessagePackItem))) + { + var item = (IMessagePackItem)Activator.CreateInstance(type)!; + + if (type.IsAssignableTo(typeof(IMessagePackDifferentFirstItem))) + { + ((IMessagePackDifferentFirstItem)item).IsFirst = i == 0; + } + + item.Read(this); + list.Add(item); + } + else + { + var method = typeof(MessagePackReader).GetMethod(nameof(Read))!.MakeGenericMethod(type); + list.Add(method.Invoke(this, Array.Empty())); + } + } + + return list; + } + + private string GetTypeString(MessagePackType type) + { + return type switch + { + >= MessagePackType.PositiveFixIntStart and <= MessagePackType.PositiveFixIntEnd => "PositiveFixInt", + >= MessagePackType.FixMapStart and <= MessagePackType.FixMapEnd => "FixMap", + >= MessagePackType.FixArrayStart and <= MessagePackType.FixArrayEnd => "FixArray", + >= MessagePackType.FixStrStart and <= MessagePackType.FixStrEnd => "FixString", + >= MessagePackType.NegativeFixIntStart or <= MessagePackType.NegativeFixIntEnd => "NegativeFixInt" + }; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackType.cs b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackType.cs new file mode 100644 index 00000000..a8d87da5 --- /dev/null +++ b/Flagrum.Core/Gfxbin/Gmdl/MessagePack/MessagePackType.cs @@ -0,0 +1,47 @@ +namespace Flagrum.Core.Gfxbin.Gmdl.MessagePack; + +public enum MessagePackType : byte +{ + PositiveFixIntStart = 0x00, + PositiveFixIntEnd = 0x7F, + FixMapStart = 0x80, + FixMapEnd = 0x8F, + FixArrayStart = 0x90, + FixArrayEnd = 0x9F, + FixStrStart = 0xA0, + FixStrEnd = 0xBF, + Nil = 0xC0, + Unused = 0xC1, + False = 0xC2, + True = 0xC3, + Bin8 = 0xC4, + Bin16 = 0xC5, + Bin32 = 0xC6, + Ext8 = 0xC7, + Ext16 = 0xC8, + Ext32 = 0xC9, + Float32 = 0xCA, + Float64 = 0xCB, + Uint8 = 0xCC, + Uint16 = 0xCD, + Uint32 = 0xCE, + Uint64 = 0xCF, + Int8 = 0xD0, + Int16 = 0xD1, + Int32 = 0xD2, + Int64 = 0xD3, + FixExt1 = 0xD4, + FixExt2 = 0xD5, + FixExt4 = 0xD6, + FixExt8 = 0xD7, + FixExt16 = 0xD8, + Str8 = 0xD9, + Str16 = 0xDA, + Str32 = 0xDB, + Array16 = 0xDC, + Array32 = 0xDD, + Map16 = 0xDE, + Map32 = 0xDF, + NegativeFixIntStart = 0xE0, + NegativeFixIntEnd = 0xFF +} \ No newline at end of file diff --git a/Flagrum.Core/Gfxbin/Gmtl/MaterialReader.cs b/Flagrum.Core/Gfxbin/Gmtl/MaterialReader.cs index 3960ecef..61fc8697 100644 --- a/Flagrum.Core/Gfxbin/Gmtl/MaterialReader.cs +++ b/Flagrum.Core/Gfxbin/Gmtl/MaterialReader.cs @@ -46,6 +46,11 @@ private void ReadMaterialData() { _reader.UnpackUInt64(out var nameOffset); + if (_material.Header.Version >= 20220707) + { + _reader.UnpackUInt16(out _); + } + _reader.UnpackUInt16(out var matUniformCount); _reader.UnpackUInt16(out var matBufferCount); _reader.UnpackUInt16(out var matTextureCount); @@ -61,6 +66,12 @@ private void ReadMaterialData() _reader.UnpackUInt32(out var gpuDataSize); _reader.UnpackUInt32(out _); // String buffer size _reader.UnpackUInt32(out var nameHash); + + if (_material.Header.Version >= 20220707) + { + _reader.UnpackUInt32(out _); + } + _reader.UnpackUInt32(out var blendType); _reader.UnpackFloat32(out var blendFactor); @@ -330,7 +341,7 @@ private void ReadInterfaceInputParameters() { return; } - + _reader.UnpackBlob(out var buffer, out _); foreach (var input in _material.InterfaceInputs) diff --git a/Flagrum.Core/Utilities/BoyerMoore.cs b/Flagrum.Core/Utilities/BoyerMoore.cs new file mode 100644 index 00000000..f1261c6c --- /dev/null +++ b/Flagrum.Core/Utilities/BoyerMoore.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; + +namespace Flagrum.Core.Utilities; + +public class BoyerMoore +{ + private readonly int[] charTable; + private readonly byte[] needle; + private readonly int[] offsetTable; + + public BoyerMoore(byte[] needle) + { + this.needle = needle; + charTable = makeByteTable(needle); + offsetTable = makeOffsetTable(needle); + } + + public IEnumerable Search(byte[] haystack) + { + if (needle.Length == 0) + { + yield break; + } + + for (var i = needle.Length - 1; i < haystack.Length;) + { + int j; + + for (j = needle.Length - 1; needle[j] == haystack[i]; --i, --j) + { + if (j != 0) + { + continue; + } + + yield return i; + i += needle.Length - 1; + break; + } + + i += Math.Max(offsetTable[needle.Length - 1 - j], charTable[haystack[i]]); + } + } + + private static int[] makeByteTable(byte[] needle) + { + const int ALPHABET_SIZE = 256; + var table = new int[ALPHABET_SIZE]; + + for (var i = 0; i < table.Length; ++i) + { + table[i] = needle.Length; + } + + for (var i = 0; i < needle.Length - 1; ++i) + { + table[needle[i]] = needle.Length - 1 - i; + } + + return table; + } + + private static int[] makeOffsetTable(byte[] needle) + { + var table = new int[needle.Length]; + var lastPrefixPosition = needle.Length; + + for (var i = needle.Length - 1; i >= 0; --i) + { + if (isPrefix(needle, i + 1)) + { + lastPrefixPosition = i + 1; + } + + table[needle.Length - 1 - i] = lastPrefixPosition - i + needle.Length - 1; + } + + for (var i = 0; i < needle.Length - 1; ++i) + { + var slen = suffixLength(needle, i); + table[slen] = needle.Length - 1 - i + slen; + } + + return table; + } + + private static bool isPrefix(byte[] needle, int p) + { + for (int i = p, j = 0; i < needle.Length; ++i, ++j) + { + if (needle[i] != needle[j]) + { + return false; + } + } + + return true; + } + + private static int suffixLength(byte[] needle, int p) + { + var len = 0; + + for (int i = p, j = needle.Length - 1; i >= 0 && needle[i] == needle[j]; --i, --j) + { + ++len; + } + + return len; + } +} \ No newline at end of file diff --git a/Flagrum.Core/Utilities/Cryptography.cs b/Flagrum.Core/Utilities/Cryptography.cs index f68e9491..67695eeb 100644 --- a/Flagrum.Core/Utilities/Cryptography.cs +++ b/Flagrum.Core/Utilities/Cryptography.cs @@ -1,102 +1,101 @@ using System; using System.Linq; +using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; -namespace Flagrum.Core.Utilities +namespace Flagrum.Core.Utilities; + +public static class Cryptography { - public static class Cryptography + private const uint HashSeed32 = 2166136261; + private const uint HashPrime32 = 16777619; + private const ulong HashSeed64 = 1469598103934665603; + private const ulong HashPrime64 = 1099511628211; + + private static readonly byte[] _aesKey = {156, 108, 93, 65, 21, 82, 63, 23, 90, 211, 248, 183, 117, 88, 30, 207}; + + private static readonly byte[] _ememKey = + {0x50, 0x16, 0xec, 0xa2, 0x58, 0x3d, 0x8e, 0xdd, 0x44, 0xfc, 0x15, 0x78, 0x4c, 0x9e, 0x2c, 0xcb}; + + public static byte[] Encrypt(byte[] data) { - private const uint HashSeed32 = 2166136261; - private const uint HashPrime32 = 16777619; - private const ulong HashSeed64 = 1469598103934665603; - private const ulong HashPrime64 = 1099511628211; + var unencryptedSize = (data.Length + 15) & -16; + var encryptedSize = unencryptedSize + 33; // 16 for IV, 16 empty, 1 for flag - private static readonly byte[] _aesKey = new byte[16] - {156, 108, 93, 65, 21, 82, 63, 23, 90, 211, 248, 183, 117, 88, 30, 207}; + var unencryptedData = new byte[unencryptedSize]; + var encryptedData = new byte[encryptedSize]; + Buffer.BlockCopy(data, 0, unencryptedData, 0, data.Length); - public static byte[] Encrypt(byte[] data) - { - var unencryptedSize = (data.Length + 15) & -16; - var encryptedSize = unencryptedSize + 33; // 16 for IV, 16 empty, 1 for flag + var aes = new AesManaged {Key = _aesKey}; + aes.GenerateIV(); + var encryptor = aes.CreateEncryptor(); + encryptor.TransformBlock(unencryptedData, 0, unencryptedSize, encryptedData, 0); - var unencryptedData = new byte[unencryptedSize]; - var encryptedData = new byte[encryptedSize]; - Buffer.BlockCopy(data, 0, unencryptedData, 0, data.Length); + Buffer.BlockCopy(aes.IV, 0, encryptedData, unencryptedSize, aes.IV.Length); - var aes = new AesManaged {Key = _aesKey}; - aes.GenerateIV(); - var encryptor = aes.CreateEncryptor(); - encryptor.TransformBlock(unencryptedData, 0, unencryptedSize, encryptedData, 0); + // 1 signifies that the data is encrypted + encryptedData[unencryptedSize + 32] = 1; - Buffer.BlockCopy(aes.IV, 0, encryptedData, unencryptedSize, aes.IV.Length); + return encryptedData; + } - // 1 signifies that the data is encrypted - encryptedData[unencryptedSize + 32] = 1; + public static byte[] Decrypt(byte[] data) + { + var size = data.Length - 33; - return encryptedData; - } + var iv = new byte[16]; + Array.Copy(data, size, iv, 0, 16); - public static byte[] Decrypt(byte[] data) + var aes = new AesManaged { - var size = data.Length - 33; + Key = _aesKey, + IV = iv, + Padding = PaddingMode.None + }; - var iv = new byte[16]; - Array.Copy(data, size, iv, 0, 16); + var decryptedBuffer = new byte[size]; + var encryptedBuffer = new byte[size]; - var aes = new AesManaged - { - Key = _aesKey, - IV = iv, - Padding = PaddingMode.None - }; + Array.Copy(data, 0, encryptedBuffer, 0, size); - var decryptedBuffer = new byte[size]; - var encryptedBuffer = new byte[size]; + var decryptor = aes.CreateDecryptor(); + decryptor.TransformBlock(encryptedBuffer, 0, size, decryptedBuffer, 0); - Array.Copy(data, 0, encryptedBuffer, 0, size); + return decryptedBuffer; + } - var decryptor = aes.CreateDecryptor(); - decryptor.TransformBlock(encryptedBuffer, 0, size, decryptedBuffer, 0); + public static ulong Hash64(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + return bytes.Aggregate(HashSeed64, (current, b) => (current ^ b) * HashPrime64); + } - return decryptedBuffer; - } + public static uint Hash32(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + return bytes.Aggregate(HashSeed32, (current, b) => (uint)(((int)current ^ b) * HashPrime32)); + } - public static ulong Hash64(string value) - { - var bytes = Encoding.UTF8.GetBytes(value); - return bytes.Aggregate(HashSeed64, (current, b) => (current ^ b) * HashPrime64); - } + public static ulong Base64Hash(byte[] bytes) + { + var hash = SHA256.HashData(bytes); + var buffer = MemoryMarshal.Cast(hash.AsSpan()); + return buffer[0] ^ buffer[1] ^ buffer[3]; + } - public static uint Hash32(string value) - { - var bytes = Encoding.UTF8.GetBytes(value); - return bytes.Aggregate(HashSeed32, (current, b) => (uint)(((int)current ^ b) * HashPrime32)); - } + public static ulong MergeHashes(ulong hash1, ulong hash2) + { + return (hash1 * 1099511628211) ^ hash2; + } - public static ulong Base64Hash(byte[] bytes) - { - var provider = new SHA256CryptoServiceProvider(); - var hash = provider.ComputeHash(bytes); - var long1 = BitConverter.ToUInt64(hash, 0); - var long2 = BitConverter.ToUInt64(hash, 8); - var long3 = BitConverter.ToUInt64(hash, 24); - return long1 ^ long2 ^ long3; - } - - public static ulong MergeHashes(ulong hash1, ulong hash2) - { - return (hash1 * 1099511628211) ^ hash2; - } + public static ulong HashFileUri64(string uri) + { + var tokens = uri.Replace("data://", "").Split('/').Last().Split('.'); + var extension = tokens.Length > 2 ? string.Join('.', tokens, 1, tokens.Length - 1) : tokens.Last(); - public static ulong HashFileUri64(string uri) - { - var tokens = uri.Replace("data://", "").Split('/').Last().Split('.'); - var extension = tokens.Length > 2 ? string.Join('.', tokens, 1, tokens.Length - 1) : tokens.Last(); - - var uriHash = Hash64(uri); - var typeHash = Hash64(extension); - return (ulong)(((long)uriHash & 17592186044415L) | (((long)typeHash << 44) & -17592186044416L)); - } + var uriHash = Hash64(uri); + var typeHash = Hash64(extension); + return (ulong)(((long)uriHash & 17592186044415L) | (((long)typeHash << 44) & -17592186044416L)); } } \ No newline at end of file diff --git a/Flagrum.Core/Utilities/Extensions.cs b/Flagrum.Core/Utilities/Extensions.cs index 4ccacafc..036aa7b2 100644 --- a/Flagrum.Core/Utilities/Extensions.cs +++ b/Flagrum.Core/Utilities/Extensions.cs @@ -1,8 +1,11 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; using System.Numerics; +using System.Reflection; using System.Text.RegularExpressions; namespace Flagrum.Core.Utilities; @@ -97,4 +100,80 @@ public static string SpacePascalCase(this string pascalCaseString) { return Regex.Replace(pascalCaseString, @"(?<=[A-Za-z])(?=[A-Z][a-z])|(?<=[a-z0-9])(?=[0-9]?[A-Z])", " "); } + + public static string WithTrailingZeros(this int value, int digits) + { + var result = value.ToString(); + while (result.Length < digits) + { + result = "0" + result; + } + + return result; + } + + /// + /// Creates a new object of the same type as the source, and copies the property values over. + /// WARNING: May not cater to all property types + /// + public static T DeepClone(this T @object) where T : new() + { + var typeSource = @object.GetType(); + var clone = Activator.CreateInstance(typeSource); + + var propertyInfo = + typeSource.GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); + + foreach (var property in propertyInfo) + { + if (property.CanWrite) + { + if (property.PropertyType.IsValueType || property.PropertyType.IsEnum || + property.PropertyType == typeof(string)) + { + property.SetValue(clone, property.GetValue(@object)); + } + else if (typeof(IEnumerable).IsAssignableFrom(property.PropertyType)) + { + var itemType = property.PropertyType + .GetInterfaces() + .Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + .Select(i => i.GetGenericArguments()[0]) + .FirstOrDefault()!; + + var countMethodInfo = typeof(Enumerable).GetMethods().Single(method => + method.Name == "Count" && method.IsStatic && method.IsGenericMethod && + method.GetParameters().Length == 1); + var elementAtMethodInfo = typeof(Enumerable).GetMethods().Single(method => + method.Name == "ElementAt" && method.IsStatic && method.IsGenericMethod && + method.GetParameters().Length == 2 && + method.GetParameters()[1].ParameterType == typeof(int)); + var enumerable = property.GetValue(@object) as IEnumerable; + + var countMethod = countMethodInfo.MakeGenericMethod(itemType); + var elementAtMethod = elementAtMethodInfo.MakeGenericMethod(itemType); + var count = (int)countMethod.Invoke(enumerable, new object[] {enumerable})!; + var cloneArray = Array.CreateInstance(itemType, count); + + for (var i = 0; i < count; i++) + { + cloneArray.SetValue( + elementAtMethod.Invoke(enumerable, new object[] {enumerable, i}).DeepClone(), i); + } + + // var type = typeof(IEnumerable<>).MakeGenericType(itemType); + // var constructor = property.PropertyType.GetConstructor(new[] {type})!; + // var enumerableClone = constructor.Invoke((object[])cloneArray); + property.SetValue(clone, cloneArray); + } + else + { + var objPropertyValue = property.GetValue(@object); + property.SetValue(clone, objPropertyValue?.DeepClone()); + } + } + } + + return (T)clone; + } } \ No newline at end of file diff --git a/Flagrum.Core/Utilities/Serialization.cs b/Flagrum.Core/Utilities/Serialization.cs index 8e76b0bc..ffec5a18 100644 --- a/Flagrum.Core/Utilities/Serialization.cs +++ b/Flagrum.Core/Utilities/Serialization.cs @@ -11,7 +11,7 @@ public static uint Align(uint offset, uint blockSize) { return blockSize + blockSize * (offset / blockSize) - offset; } - + /// /// Aligns current offset to the given block size /// @@ -66,7 +66,7 @@ public static string ToBase64(this string input) var bytes = Encoding.UTF8.GetBytes(input); return Convert.ToBase64String(bytes); } - + public static void Align(this BinaryWriter writer, int blockSize, byte paddingByte) { var size = Align((uint)writer.BaseStream.Position, (uint)blockSize); @@ -74,13 +74,13 @@ public static void Align(this BinaryWriter writer, int blockSize, byte paddingBy { return; } - + for (var i = 0; i < size; i++) { writer.Write(paddingByte); } } - + public static void Align(this BinaryReader reader, int blockSize) { var size = Align((uint)reader.BaseStream.Position, (uint)blockSize); @@ -91,13 +91,14 @@ public static void Align(this BinaryReader reader, int blockSize) reader.BaseStream.Seek(size, SeekOrigin.Current); } - + public static void CopyTo(this FileStream source, FileStream destination, uint count) { var bytesRemaining = count; while (bytesRemaining > 0) { - var readSize = Math.Min(4096, bytesRemaining); + // 81920 is the default buffer size FileStream uses, so good enough for me! :) + var readSize = Math.Min(81920, bytesRemaining); var buffer = new byte[readSize]; _ = source.Read(buffer); diff --git a/Flagrum.Core/Utilities/Types/Enum.cs b/Flagrum.Core/Utilities/Types/Enum.cs new file mode 100644 index 00000000..21b62b4c --- /dev/null +++ b/Flagrum.Core/Utilities/Types/Enum.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Flagrum.Core.Utilities.Types; + +public abstract class Enum : IComparable where TValue : IComparable +{ + protected Enum(TValue value) + { + Value = value; + } + + public TValue Value { get; } + + public int CompareTo(object other) + { + return Value.CompareTo(other); + } + + public override string ToString() + { + return Value.ToString(); + } + + public static IEnumerable GetAll() where TEnum : Enum + { + var fields = typeof(TEnum).GetProperties(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly); + return fields.Select(f => f.GetValue(null)).Cast(); + } + + public override bool Equals(object obj) + { + if (obj is not Enum other) + { + return false; + } + + var typeMatches = GetType() == obj.GetType(); + var valueMatches = Value.Equals(other.Value); + + return typeMatches && valueMatches; + } + + public override int GetHashCode() + { + return Value.GetHashCode(); + } +} \ No newline at end of file diff --git a/Flagrum.Core/Utilities/Types/LuminousGame.cs b/Flagrum.Core/Utilities/Types/LuminousGame.cs new file mode 100644 index 00000000..13eda8dc --- /dev/null +++ b/Flagrum.Core/Utilities/Types/LuminousGame.cs @@ -0,0 +1,7 @@ +namespace Flagrum.Core.Utilities.Types; + +public enum LuminousGame +{ + FFXV, + Forspoken +} \ No newline at end of file diff --git a/Flagrum.Desktop/App.xaml.cs b/Flagrum.Desktop/App.xaml.cs index ab8beb58..671dcf50 100644 --- a/Flagrum.Desktop/App.xaml.cs +++ b/Flagrum.Desktop/App.xaml.cs @@ -50,11 +50,63 @@ public App() SetFileTypeAssociation(); // Migrate the database if required - using var context = new FlagrumDbContext(); + Profile = new ProfileService(); + AssetExplorerNode.Profile = Profile; + using var context = new FlagrumDbContext(Profile); context.Database.MigrateAsync().Wait(); + // If this is the first time using the profiles system + if (Profile.DidMigrateThisSession) + { + // Move the paths out into the profiles file + var gamePath = context.GetString(StateKey.GamePath); + var binmodListPath = context.GetString(StateKey.BinmodListPath); + + Profile.GamePath = gamePath; + Profile.BinmodListPath = binmodListPath; + + context.DeleteStateKey(StateKey.GamePath); + context.DeleteStateKey(StateKey.BinmodListPath); + + var oldRoot = context.AssetExplorerNodes + .Where(n => n.ParentId == null) + .Select(n => n.Id) + .First(); + + var newRoot = new AssetExplorerNode {Name = ""}; + context.AssetExplorerNodes.Add(newRoot); + context.SaveChanges(); + + var oldRootEntity = context.AssetExplorerNodes.Find(oldRoot)!; + oldRootEntity.ParentId = newRoot.Id; + oldRootEntity.Name = "data:"; + context.SaveChanges(); + + // Update the mod paths to point to the new locations + foreach (var file in context.EarcModReplacements) + { + if (file.ReplacementFilePath?.StartsWith($@"{Profile.FlagrumDirectory}\earc") == true) + { + file.ReplacementFilePath = file.ReplacementFilePath.Replace($@"{Profile.FlagrumDirectory}\earc", Profile.EarcModsDirectory); + } + } + + context.SaveChanges(); + + foreach (var file in context.EarcModLooseFile) + { + if (file.FilePath?.StartsWith($@"{Profile.FlagrumDirectory}\earc") == true) + { + file.FilePath = file.FilePath.Replace($@"{Profile.FlagrumDirectory}\earc", Profile.EarcModsDirectory); + } + } + + context.SaveChanges(); + } + // Start loading asset explorer nodes so the user won't be waiting too long - AppState = new AppStateService(new SettingsService()); + var appStateContext = new FlagrumDbContext(Profile); + AppState = new AppStateService(Profile, appStateContext, new UriMapper(appStateContext, Profile)); AppState.LoadNodes(); // Set default key bindings for 3D viewer @@ -85,7 +137,8 @@ public App() } } - public AppStateService AppState { get; set; } + public AppStateService AppState { get; } + public ProfileService Profile { get; } [DllImport("shell32.dll", SetLastError = true)] private static extern void SetCurrentProcessExplicitAppUserModelID([MarshalAs(UnmanagedType.LPWStr)] string appId); @@ -152,7 +205,7 @@ private void App_OnStartup(object sender, StartupEventArgs e) fmodPath = e.Args[0]; } - var window = new MainWindow(AppState, fmodPath); + var window = new MainWindow(Profile, AppState, fmodPath); window.Show(); } } \ No newline at end of file diff --git a/Flagrum.Desktop/Architecture/RelayCommand.cs b/Flagrum.Desktop/Architecture/RelayCommand.cs new file mode 100644 index 00000000..7b3f9a13 --- /dev/null +++ b/Flagrum.Desktop/Architecture/RelayCommand.cs @@ -0,0 +1,29 @@ +using System; +using System.Windows.Input; + +namespace Flagrum.Desktop.Architecture; + +public class RelayCommand : ICommand +{ + private readonly Action _action; + private readonly Func _canExecute; + + public RelayCommand(Action action) : this(action, () => true) {} + public RelayCommand(Action action, Func canExecute) + { + _action = action; + _canExecute = canExecute; + } + + public bool CanExecute(object? parameter) + { + return _canExecute(); + } + + public void Execute(object? parameter) + { + _action(); + } + + public event EventHandler? CanExecuteChanged; +} \ No newline at end of file diff --git a/Flagrum.Desktop/Architecture/ViewportHelper.cs b/Flagrum.Desktop/Architecture/ViewportHelper.cs index 6501510d..51c16d3b 100644 --- a/Flagrum.Desktop/Architecture/ViewportHelper.cs +++ b/Flagrum.Desktop/Architecture/ViewportHelper.cs @@ -5,6 +5,8 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Flagrum.Core.Gfxbin.Gmdl.Components; +using Flagrum.Core.Gfxbin.Gmdl.MessagePack; using Flagrum.Core.Gfxbin.Gmtl; using Flagrum.Web.Features.AssetExplorer.Data; using Flagrum.Web.Persistence; @@ -15,7 +17,6 @@ using HelixToolkit.SharpDX.Core.Model.Scene; using HelixToolkit.Wpf.SharpDX; using SharpDX; -using ModelReader = Flagrum.Core.Gfxbin.Gmdl.ModelReader; namespace Flagrum.Desktop.Architecture; @@ -29,10 +30,10 @@ public ViewportHelper( MainViewModel mainViewModel) { _mainViewModel = mainViewModel; - _context = new FlagrumDbContext(new SettingsService()); + _context = new FlagrumDbContext(new ProfileService()); } - public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view, string? inputPath) + public async Task ChangeModel(IAssetExplorerNode gmdlNode, AssetExplorerView view) { _mainViewModel.Viewer.Reset(); @@ -77,27 +78,44 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view // throw; // } - var model = new ModelReader(gmdl, gpubin).Read(); - using var context = new FlagrumDbContext(_context.Settings); + //var model = new ModelReader(gmdl, gpubin).Read(); + //var model = new Gfxbin(gmdl, gpubin); + var gmdl = gmdlNode.Data; + var model = new Gfxbin(gmdl); + + var gpubinUris = model.Header.Dependencies + .Where(d => d.Value.EndsWith(".gpubin")) + .Select(d => d.Value) + .ToList(); + + var gpubins = gpubinUris.Select(uri => gmdlNode.Parent.Children + .Single(c => c.Path.EndsWith(uri.Split('\\', '/').Last(), StringComparison.OrdinalIgnoreCase))) + .OrderBy(f => f.Name) + .Select(gpubin => gpubin.Data) + .ToList(); + + model.ReadVertexData2(gpubins); + + await using var context = new FlagrumDbContext(_context.Profile); var fidelityInt = context.GetInt(StateKey.ViewportTextureFidelity); - _fidelity = fidelityInt == -1 ? ModelViewerTextureFidelity.Low : (ModelViewerTextureFidelity)fidelityInt; + _fidelity = fidelityInt == -1 ? ModelViewerTextureFidelity.None : (ModelViewerTextureFidelity)fidelityInt; // Calculate paths - var gpubinUri = model.Header.Dependencies.First(d => d.Path.EndsWith(".gpubin")).Path; + var gpubinUri = gpubinUris.First(); var modelContext = new FileSystemModelContext { RootUri = gpubinUri[..gpubinUri.LastIndexOf('/')], - RootDirectory = inputPath == null ? null : string.Join('\\', inputPath.Split('\\')[..^1]) + RootDirectory = string.Join('\\', gmdlNode.Path.Split('\\')[..^1]) }; Parallel.ForEach(model.MeshObjects, meshObject => { - Parallel.ForEach(meshObject.Meshes.Where(m => m.LodNear == 0), mesh => + Parallel.ForEach(meshObject.Meshes.Where(m => m.LodNear == 0 && (m.Flags & 67108864) == 0), mesh => { FlagrumDbContext? context = null; if (view == AssetExplorerView.GameView) { - context = new FlagrumDbContext(_context.Settings); + context = new FlagrumDbContext(_context.Profile); } object? contextObject = view == AssetExplorerView.FileSystem @@ -113,40 +131,53 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view for (var j = 0; j < 3; j++) { var index = (int)mesh.FaceIndices[i, j]; - + if (index < mesh.VertexCount) { triangle.Add(index); } } - + if (triangle.Count == 3) { faceIndices.AddRange(triangle); } } - - var textureCoordinates = mesh.UVMaps.Count > 0 - ? mesh.UVMaps[0].UVs.Select(uv => new Vector2((float)uv.U, (float)uv.V)).ToList() - : mesh.VertexPositions.Select(uv => new Vector2(0f, 0f)).ToList(); + + // var textureCoordinates = mesh.UVMaps.Count > 0 + // ? mesh.UVMaps[0].UVs.Select(uv => new Vector2((float)uv.U, (float)uv.V)).ToList() + // : mesh.VertexPositions.Select(uv => new Vector2(0f, 0f)).ToList(); + // + // builder.Append( + // mesh.VertexPositions.Select(v => new Vector3(v.X, v.Y, v.Z)).ToList(), + // faceIndices, + // mesh.Normals.Select(n => new Vector3( + // n.X > 0 ? n.X / 127f : n.X / 128f, + // n.Y > 0 ? n.Y / 127f : n.Y / 128f, + // n.Z > 0 ? n.Z / 127f : n.Z / 128f)) + // .ToList(), + // textureCoordinates); builder.Append( - mesh.VertexPositions.Select(v => new Vector3(v.X, v.Y, v.Z)).ToList(), + ((IList)mesh.Semantics[VertexElementSemantic.Position0]).Select(v => new Vector3(v)) + .ToList(), faceIndices, - mesh.Normals.Select(n => new Vector3(n.X, n.Y, n.Z)).ToList(), - textureCoordinates); + ((IList)mesh.Semantics[VertexElementSemantic.Normal0]).Select(n => new Vector3(n[..3])) + .ToList(), + ((IList)mesh.Semantics[VertexElementSemantic.TexCoord0]).Select(c => new Vector2(c)) + .ToList()); var meshNode = new BoneSkinMeshNode(); var material = new PBRMaterial { - AlbedoColor = Color4.White + AlbedoColor = Color4.White, }; if (_fidelity != ModelViewerTextureFidelity.None) { var materialUri = model.Header.Dependencies - .FirstOrDefault(d => d.PathHash == mesh.DefaultMaterialHash.ToString()) - !.Path; + .FirstOrDefault(d => d.Key == mesh.MaterialHash.ToString()) + !.Value; if (materialUri != null) { @@ -158,11 +189,13 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view var gfxbin = new MaterialReader(materialData).Read(); var diffuseUri = gfxbin.Textures - .FirstOrDefault(t => t.ShaderGenName.Contains("BaseColor0")) + .FirstOrDefault(t => + t.ShaderGenName.Contains("BaseColor0") || t.ShaderGenName.Contains("BaseColorTexture0")) ?.Path; var normalUri = gfxbin.Textures - .FirstOrDefault(t => t.ShaderGenName.Contains("Normal0")) + .FirstOrDefault(t => + t.ShaderGenName.Contains("Normal0") || t.ShaderGenName.Contains("NormalTexture0")) ?.Path; if (diffuseUri != null && normalUri != null) @@ -190,7 +223,7 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view var diffuse = GetDataByUri(diffuseUri, contextObject, view); var normal = GetDataByUri(normalUri, contextObject, view); - var textureConverter = new TextureConverter(); + var textureConverter = new TextureConverter(new ProfileService().Current.Type); TextureModel? albedoMap = null; TextureModel? normalMap = null; @@ -201,7 +234,7 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view { var tag = Encoding.UTF8.GetString(diffuse[..8]); - if (tag == "SEDBbtex") + if (tag == "SEDBbtex" || tag.StartsWith("BTEX")) { diffuse = textureConverter.BtexToPng(diffuse); } @@ -217,7 +250,7 @@ public async Task ChangeModel(byte[] gmdl, byte[] gpubin, AssetExplorerView view { var tag = Encoding.UTF8.GetString(normal[..8]); - if (tag == "SEDBbtex") + if (tag == "SEDBbtex" || tag.StartsWith("BTEX")) { normal = textureConverter.BtexToPng(normal); } diff --git a/Flagrum.Desktop/Flagrum.Desktop.csproj b/Flagrum.Desktop/Flagrum.Desktop.csproj index a72a9017..7180ead6 100644 --- a/Flagrum.Desktop/Flagrum.Desktop.csproj +++ b/Flagrum.Desktop/Flagrum.Desktop.csproj @@ -8,8 +8,8 @@ Resources\flagrum.ico embedded Flagrum - 1.3.5 - 1.3.5 + 1.4.1 + 1.4.1 @@ -73,7 +73,7 @@ - PreserveNewest + PreserveNewest @@ -81,48 +81,51 @@ PreserveNewest + + PreserveNewest + - - PublicResXFileCodeGenerator - Localisation.Designer.cs - - - PublicResXFileCodeGenerator - Localisation.ja-JP.Designer.cs - - - ResXFileCodeGenerator - Localisation.zh-cn.Designer.cs - - - ResXFileCodeGenerator - Localisation.zh-hk.Designer.cs - + + PublicResXFileCodeGenerator + Localisation.Designer.cs + + + PublicResXFileCodeGenerator + Localisation.ja-JP.Designer.cs + + + ResXFileCodeGenerator + Localisation.zh-cn.Designer.cs + + + ResXFileCodeGenerator + Localisation.zh-hk.Designer.cs + - - True - True - Localisation.resx - - - True - True - Localisation.ja-JP.resx - - - True - True - Localisation.zh-cn.resx - - - True - True - Localisation.zh-hk.resx - + + True + True + Localisation.resx + + + True + True + Localisation.ja-JP.resx + + + True + True + Localisation.zh-cn.resx + + + True + True + Localisation.zh-hk.resx + diff --git a/Flagrum.Desktop/MainViewModel.cs b/Flagrum.Desktop/MainViewModel.cs index 3e4e5b3e..1496b9a4 100644 --- a/Flagrum.Desktop/MainViewModel.cs +++ b/Flagrum.Desktop/MainViewModel.cs @@ -55,7 +55,7 @@ public MainViewModel() FarPlaneDistance = 10000 }; - using var context = new FlagrumDbContext(new SettingsService()); + using var context = new FlagrumDbContext(new ProfileService()); var viewportRotateModifierKey = context.GetEnum(StateKey.ViewportRotateModifierKey); var viewportRotateMouseAction = context.GetEnum(StateKey.ViewportRotateMouseAction); @@ -78,6 +78,15 @@ public bool HasWebView2Runtime set => SetValue(ref _hasWebView2Runtime, value); } + private ICommand? _patreonLink; + public ICommand PatreonLink => _patreonLink ??= new RelayCommand(() => + { + Process.Start(new ProcessStartInfo("https://www.patreon.com/Flagrum") + { + UseShellExecute = true + }); + }); + public int ViewportLeft { get => _viewportLeft; diff --git a/Flagrum.Desktop/MainWindow.xaml b/Flagrum.Desktop/MainWindow.xaml index 9c8f8631..26e8d28e 100644 --- a/Flagrum.Desktop/MainWindow.xaml +++ b/Flagrum.Desktop/MainWindow.xaml @@ -48,6 +48,7 @@ + @@ -58,7 +59,18 @@ Foreground="#E7E5E4" FontSize="14" VerticalAlignment="Center" Padding="0, 5, 15, 8" /> - - -