diff --git a/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs b/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs index a5f0486..3a2e719 100644 --- a/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs +++ b/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs @@ -10,10 +10,11 @@ namespace StarBreaker.Cli; [Command("dcb-extract", Description = "Extracts a DataCore binary file into separate xml files")] public class DataCoreExtractCommand : ICommand { - private static readonly string[] _dataCoreFiles = [@"Data\Game2.dcb", @"Data\Game.dcb"]; - [CommandOption("p4k", 'p', Description = "Path to the Game.p4k")] - public required string P4kFile { get; init; } + public string? P4kFile { get; init; } + + [CommandOption("dcb", 'd', Description = "Path to the Game.dcb")] + public string? DcbFile { get; init; } [CommandOption("output", 'o', Description = "Path to the output directory")] public required string OutputDirectory { get; init; } @@ -23,16 +24,35 @@ public class DataCoreExtractCommand : ICommand public ValueTask ExecuteAsync(IConsole console) { - var p4k = P4k.P4kFile.FromFile(P4kFile); - console.Output.WriteLine("P4k loaded."); + if (P4kFile == null && DcbFile == null) + { + console.Output.WriteLine("P4k and DCB files are required."); + return default; + } + if (!string.IsNullOrEmpty(P4kFile) && !string.IsNullOrEmpty(DcbFile)) + { + console.Output.WriteLine("Only one of P4k and DCB files can be specified."); + return default; + } + Stream? dcbStream = null; - foreach (var file in _dataCoreFiles) + if (!string.IsNullOrEmpty(DcbFile)) + { + dcbStream = File.OpenRead(DcbFile); + console.Output.WriteLine("DCB loaded."); + } + else if (!string.IsNullOrEmpty(P4kFile)) { - if (!p4k.FileExists(file)) continue; + var p4k = P4k.P4kFile.FromFile(P4kFile); + console.Output.WriteLine("P4k loaded."); + foreach (var file in DataCoreUtils.KnownPaths) + { + if (!p4k.FileExists(file)) continue; - dcbStream = p4k.OpenRead(file); - console.Output.WriteLine($"{file} found"); - break; + dcbStream = p4k.OpenRead(file); + console.Output.WriteLine($"{file} found"); + break; + } } if (dcbStream == null) diff --git a/src/StarBreaker.Cli/DiffCommand.cs b/src/StarBreaker.Cli/DiffCommand.cs new file mode 100644 index 0000000..26d48da --- /dev/null +++ b/src/StarBreaker.Cli/DiffCommand.cs @@ -0,0 +1,81 @@ +using System.IO.Compression; +using CliFx; +using CliFx.Attributes; +using CliFx.Infrastructure; +using StarBreaker.DataCore; + +namespace StarBreaker.Cli; + +[Command("diff", Description = "Dumps game information into plain text files for comparison")] +public class DiffCommand : ICommand +{ + [CommandOption("game", 'g', Description = "Path to the game folder")] + public required string GameFolder { get; init; } + + [CommandOption("output", 'o', Description = "Path to the output directory")] + public required string OutputDirectory { get; init; } + + public async ValueTask ExecuteAsync(IConsole console) + { + // Hide output from subcommands + var fakeConsole = new FakeConsole(); + + var p4kFile = Path.Combine(GameFolder, "Data.p4k"); + var exeFile = Path.Combine(GameFolder, "Bin64", "StarCitizen.exe"); + + var dumpP4k = new DumpP4kCommand + { + P4kFile = p4kFile, + OutputDirectory = Path.Combine(OutputDirectory, "P4k") + }; + await dumpP4k.ExecuteAsync(fakeConsole); + + var dcbExtract = new DataCoreExtractCommand + { + P4kFile = p4kFile, + OutputDirectory = Path.Combine(OutputDirectory, "DataCore") + }; + await dcbExtract.ExecuteAsync(fakeConsole); + + var extractProtobufs = new ExtractProtobufsCommand + { + Input = exeFile, + Output = Path.Combine(OutputDirectory, "Protobuf") + }; + await extractProtobufs.ExecuteAsync(fakeConsole); + + var extractDescriptor = new ExtractDescriptorSetCommand + { + Input = exeFile, + Output = Path.Combine(OutputDirectory, "Protobuf", "descriptor_set.bin") + }; + await extractDescriptor.ExecuteAsync(fakeConsole); + + await ExtractDataCoreIntoZip(p4kFile, Path.Combine(OutputDirectory, "DataCore", "DataCore.zip")); + + await console.Output.WriteLineAsync("Done."); + } + + private static async Task ExtractDataCoreIntoZip(string p4kFile, string zipPath) + { + var p4k = P4k.P4kFile.FromFile(p4kFile); + Stream? dcbStream = null; + string? dcbFile = null; + foreach (var file in DataCoreUtils.KnownPaths) + { + if (!p4k.FileExists(file)) continue; + + dcbFile = file; + dcbStream = p4k.OpenRead(file); + break; + } + + if (dcbStream == null || dcbFile == null) + throw new InvalidOperationException("DataCore not found."); + + using var zip = new ZipArchive(File.Create(zipPath), ZipArchiveMode.Create); + var entry = zip.CreateEntry(Path.GetFileName(dcbFile), CompressionLevel.SmallestSize); + await using var entryStream = entry.Open(); + await dcbStream.CopyToAsync(entryStream); + } +} \ No newline at end of file diff --git a/src/StarBreaker.Cli/P4kCommands/DumpP4kCommand.cs b/src/StarBreaker.Cli/P4kCommands/DumpP4kCommand.cs new file mode 100644 index 0000000..f05f312 --- /dev/null +++ b/src/StarBreaker.Cli/P4kCommands/DumpP4kCommand.cs @@ -0,0 +1,27 @@ +using System.Text; +using CliFx; +using CliFx.Attributes; +using CliFx.Infrastructure; +using StarBreaker.Cli.Utils; +using StarBreaker.P4k; + +namespace StarBreaker.Cli; + +[Command("p4k-dump", Description = "Dumps the contents a Game.p4k file")] +public class DumpP4kCommand : ICommand +{ + [CommandOption("p4k", 'p', Description = "Path to the Game.p4k")] + public required string P4kFile { get; init; } + + [CommandOption("output", 'o', Description = "Path to the output directory")] + public required string OutputDirectory { get; init; } + + public ValueTask ExecuteAsync(IConsole console) + { + var p4k = P4k.P4kFile.FromFile(P4kFile); + var p4kExtractor = new P4kExtractor(p4k); + p4kExtractor.ExtractDummies(OutputDirectory, new ProgressBar(console)); + + return default; + } +} \ No newline at end of file diff --git a/src/StarBreaker.Cli/Program.cs b/src/StarBreaker.Cli/Program.cs index 278c1b9..3fda1ba 100644 --- a/src/StarBreaker.Cli/Program.cs +++ b/src/StarBreaker.Cli/Program.cs @@ -4,6 +4,7 @@ return await new CliApplicationBuilder() .SetExecutableName("StarBreaker.Cli") .AddCommand() + .AddCommand() .AddCommand() .AddCommand() .AddCommand() @@ -12,6 +13,7 @@ .AddCommand() .AddCommand() .AddCommand() + .AddCommand() .AddCommand() .AddCommand() .AddCommand() diff --git a/src/StarBreaker.Cli/Properties/launchSettings.json b/src/StarBreaker.Cli/Properties/launchSettings.json index e2e673b..4fc18cf 100644 --- a/src/StarBreaker.Cli/Properties/launchSettings.json +++ b/src/StarBreaker.Cli/Properties/launchSettings.json @@ -42,7 +42,15 @@ }, "dcb-extract": { "commandName": "Project", - "commandLineArgs": "dcb-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\"" + "commandLineArgs": "dcb-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\"" + }, + "dcb-extract2": { + "commandName": "Project", + "commandLineArgs": "dcb-extract -d \"D:\\StarCitizen\\P4k\\Data\\Game2.dcb\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\"" + }, + "diff": { + "commandName": "Project", + "commandLineArgs": "diff -g \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\" -o \"C:\\Development\\StarCitizen\\StarCitizenDiff\"" }, "dcb-extract-old": { "commandName": "Project", @@ -58,15 +66,19 @@ }, "p4k-extract": { "commandName": "Project", - "commandLineArgs": "p4k-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" -o \"D:\\StarCitizen\\P4k\"" + "commandLineArgs": "p4k-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"D:\\StarCitizen\\P4k\"" + }, + "p4k-dump": { + "commandName": "Project", + "commandLineArgs": "p4k-dump -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\"" }, "proto-extract": { "commandName": "Project", - "commandLineArgs": "proto-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\protos\"" + "commandLineArgs": "proto-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\protos\"" }, "proto-set-extract": { "commandName": "Project", - "commandLineArgs": "proto-set-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\descriptorSet.bin\"" + "commandLineArgs": "proto-set-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\descriptorSet.bin\"" } } } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataCoreUtils.cs b/src/StarBreaker.DataCore/DataCoreUtils.cs new file mode 100644 index 0000000..e5239b9 --- /dev/null +++ b/src/StarBreaker.DataCore/DataCoreUtils.cs @@ -0,0 +1,12 @@ +using System.IO.Enumeration; + +namespace StarBreaker.DataCore; + +public static class DataCoreUtils +{ + public static readonly string[] KnownPaths = [@"Data\Game2.dcb", @"Data\Game.dcb"]; + public static bool IsDataCoreFile(string path) + { + return FileSystemName.MatchesSimpleExpression("Data\\*.dcb", path); + } +} \ No newline at end of file diff --git a/src/StarBreaker.P4k/P4kExtractor.cs b/src/StarBreaker.P4k/P4kExtractor.cs index e5da805..b7830d4 100644 --- a/src/StarBreaker.P4k/P4kExtractor.cs +++ b/src/StarBreaker.P4k/P4kExtractor.cs @@ -1,4 +1,5 @@ using System.IO.Enumeration; +using System.Text; namespace StarBreaker.P4k; @@ -52,8 +53,8 @@ public void Extract(string outputDir, string? filter = null, IProgress? return; var entryPath = Path.Combine(outputDir, entry.Name); - if (File.Exists(entryPath)) - return; + //if (File.Exists(entryPath)) + // return; Directory.CreateDirectory(Path.GetDirectoryName(entryPath) ?? throw new InvalidOperationException()); using (var writeStream = new FileStream(entryPath, FileMode.Create, FileAccess.Write, FileShare.None, @@ -78,4 +79,71 @@ public void Extract(string outputDir, string? filter = null, IProgress? progress?.Report(1); } + + public void ExtractDummies(string outputDir, IProgress? progress = null) + { + //TODO: if the filter is for *.dds, make sure to include *.dds.N too. Maybe do the pre processing before we filter? + + var numberOfEntries = _p4KFile.Entries.Length; + var fivePercent = numberOfEntries / 20; + var processedEntries = 0; + + progress?.Report(0); + + var lockObject = new Lock(); + + //TODO: Preprocessing step: + // 1. start with the list of total files + // 2. run the following according to the filter: + // 3. find one-shot single file procesors + // 4. find file -> multiple file processors + // 5. find multiple file -> single file unsplit processors - remove from the list so we don't double process + // run it! + Parallel.ForEach(_p4KFile.Entries, + entry => + { + var entryPath = Path.Combine(outputDir, entry.Name) + ".ini"; + Directory.CreateDirectory(Path.GetDirectoryName(entryPath) ?? throw new InvalidOperationException()); + //write metadata to the file instead of the actual data + var sb = new StringBuilder(); + + sb.Append("CRC32: 0x"); + sb.Append(entry.Crc32.ToString("X8")); + sb.AppendLine(); + + sb.Append("LastModified: "); + sb.Append(entry.LastModified.ToString("s")); + sb.AppendLine(); + + sb.Append("UncompressedSize: "); + sb.Append(entry.UncompressedSize); + sb.AppendLine(); + + sb.Append("CompressedSize: "); + sb.Append(entry.CompressedSize); + sb.AppendLine(); + + sb.Append("CompressionType: "); + sb.Append(entry.CompressionMethod); + sb.AppendLine(); + + sb.Append("IsCrypted: "); + sb.Append(entry.IsCrypted); + sb.AppendLine(); + + File.WriteAllText(entryPath, sb.ToString()); + + Interlocked.Increment(ref processedEntries); + if (processedEntries == numberOfEntries || processedEntries % fivePercent == 0) + { + using (lockObject.EnterScope()) + { + progress?.Report(processedEntries / (double)numberOfEntries); + } + } + } + ); + + progress?.Report(1); + } } \ No newline at end of file diff --git a/src/StarBreaker.P4k/P4kFile.cs b/src/StarBreaker.P4k/P4kFile.cs index 1fdfa49..0b13a3e 100644 --- a/src/StarBreaker.P4k/P4kFile.cs +++ b/src/StarBreaker.P4k/P4kFile.cs @@ -1,5 +1,5 @@ using System.Buffers; -using System.IO.Enumeration; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using System.Threading.Channels; @@ -39,7 +39,7 @@ private P4kFile(string path, ZipEntry[] entries, ZipNode root) public static P4kFile FromFile(string filePath, IProgress? progress = null) { progress?.Report(0); - using var reader = new BinaryReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, 4096), Encoding.UTF8, false); + using var reader = new BinaryReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096), Encoding.UTF8, false); var eocdLocation = reader.BaseStream.Locate(EOCDRecord.Magic); reader.BaseStream.Seek(eocdLocation, SeekOrigin.Begin); @@ -168,7 +168,8 @@ private static ZipEntry ReadEntry(BinaryReader reader) header.CompressionMethod, isCrypted, localHeaderOffset, - header.LastModifiedTimeAndDate + header.LastModifiedTimeAndDate, + header.Crc32 ); } finally @@ -188,9 +189,14 @@ private StreamSegment OpenInternal(ZipEntry entry) var localHeader = p4kStream.Read(); - p4kStream.Seek(localHeader.FileNameLength + localHeader.ExtraFieldLength, SeekOrigin.Current); + var offset = entry.Offset + + sizeof(uint) + + (ulong)Unsafe.SizeOf() + + localHeader.FileNameLength + + localHeader.ExtraFieldLength; + var length = entry.CompressedSize; - return new StreamSegment(p4kStream, p4kStream.Position, (long)entry.CompressedSize, false); + return new StreamSegment(p4kStream, (long)offset, (long)length, false); } // Remarks: Streams returned by this method might not support seeking or length. diff --git a/src/StarBreaker.P4k/Zip/ZipEntry.cs b/src/StarBreaker.P4k/Zip/ZipEntry.cs index 10227c8..f03224a 100644 --- a/src/StarBreaker.P4k/Zip/ZipEntry.cs +++ b/src/StarBreaker.P4k/Zip/ZipEntry.cs @@ -16,6 +16,7 @@ public sealed class ZipEntry public bool IsCrypted { get; } public ulong Offset { get; } public DateTime LastModified => FromDosDateTime(_dosDateTime); + public uint Crc32 { get; } public ZipEntry( string name, @@ -24,7 +25,8 @@ public ZipEntry( ushort compressionMethod, bool isCrypted, ulong offset, - uint lastModifiedDateTime + uint lastModifiedDateTime, + uint crc32 ) { Name = name; @@ -34,6 +36,7 @@ uint lastModifiedDateTime IsCrypted = isCrypted; Offset = offset; _dosDateTime = lastModifiedDateTime; + Crc32 = crc32; } //https://source.dot.net/#System.IO.Compression/System/IO/Compression/ZipHelper.cs,76523e345de18cc8