add dump command

src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs

@@ -10,10 +10,11 @@ namespace StarBreaker.Cli;
 [Command("dcb-extract", Description = "Extracts a DataCore binary file into separate xml files")]
 public class DataCoreExtractCommand : ICommand
 {
-    private static readonly string[] _dataCoreFiles = [@"Data\Game2.dcb", @"Data\Game.dcb"];
-
     [CommandOption("p4k", 'p', Description = "Path to the Game.p4k")]
-    public required string P4kFile { get; init; }
+    public string? P4kFile { get; init; }
+
+    [CommandOption("dcb", 'd', Description = "Path to the Game.dcb")]
+    public string? DcbFile { get; init; }
 
     [CommandOption("output", 'o', Description = "Path to the output directory")]
     public required string OutputDirectory { get; init; }
@@ -23,16 +24,35 @@ public class DataCoreExtractCommand : ICommand
 
     public ValueTask ExecuteAsync(IConsole console)
     {
-        var p4k = P4k.P4kFile.FromFile(P4kFile);
-        console.Output.WriteLine("P4k loaded.");
+        if (P4kFile == null && DcbFile == null)
+        {
+            console.Output.WriteLine("P4k and DCB files are required.");
+            return default;
+        }
+        if (!string.IsNullOrEmpty(P4kFile) && !string.IsNullOrEmpty(DcbFile))
+        {
+            console.Output.WriteLine("Only one of P4k and DCB files can be specified.");
+            return default;
+        }
+
         Stream? dcbStream = null;
-        foreach (var file in _dataCoreFiles)
+        if (!string.IsNullOrEmpty(DcbFile))
+        {
+            dcbStream = File.OpenRead(DcbFile);
+            console.Output.WriteLine("DCB loaded.");
+        }
+        else if (!string.IsNullOrEmpty(P4kFile))
         {
-            if (!p4k.FileExists(file)) continue;
+            var p4k = P4k.P4kFile.FromFile(P4kFile);
+            console.Output.WriteLine("P4k loaded.");
+            foreach (var file in DataCoreUtils.KnownPaths)
+            {
+                if (!p4k.FileExists(file)) continue;
 
-            dcbStream = p4k.OpenRead(file);
-            console.Output.WriteLine($"{file} found");
-            break;
+                dcbStream = p4k.OpenRead(file);
+                console.Output.WriteLine($"{file} found");
+                break;
+            }
         }
 
         if (dcbStream == null)

src/StarBreaker.Cli/DiffCommand.cs

@@ -0,0 +1,81 @@
+using System.IO.Compression;
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Infrastructure;
+using StarBreaker.DataCore;
+
+namespace StarBreaker.Cli;
+
+[Command("diff", Description = "Dumps game information into plain text files for comparison")]
+public class DiffCommand : ICommand
+{
+    [CommandOption("game", 'g', Description = "Path to the game folder")]
+    public required string GameFolder { get; init; }
+
+    [CommandOption("output", 'o', Description = "Path to the output directory")]
+    public required string OutputDirectory { get; init; }
+
+    public async ValueTask ExecuteAsync(IConsole console)
+    {
+        // Hide output from subcommands
+        var fakeConsole = new FakeConsole();
+
+        var p4kFile = Path.Combine(GameFolder, "Data.p4k");
+        var exeFile = Path.Combine(GameFolder, "Bin64", "StarCitizen.exe");
+
+        var dumpP4k = new DumpP4kCommand
+        {
+            P4kFile = p4kFile,
+            OutputDirectory = Path.Combine(OutputDirectory, "P4k")
+        };
+        await dumpP4k.ExecuteAsync(fakeConsole);
+
+        var dcbExtract = new DataCoreExtractCommand
+        {
+            P4kFile = p4kFile,
+            OutputDirectory = Path.Combine(OutputDirectory, "DataCore")
+        };
+        await dcbExtract.ExecuteAsync(fakeConsole);
+
+        var extractProtobufs = new ExtractProtobufsCommand
+        {
+            Input = exeFile,
+            Output = Path.Combine(OutputDirectory, "Protobuf")
+        };
+        await extractProtobufs.ExecuteAsync(fakeConsole);
+
+        var extractDescriptor = new ExtractDescriptorSetCommand
+        {
+            Input = exeFile,
+            Output = Path.Combine(OutputDirectory, "Protobuf", "descriptor_set.bin")
+        };
+        await extractDescriptor.ExecuteAsync(fakeConsole);
+
+        await ExtractDataCoreIntoZip(p4kFile, Path.Combine(OutputDirectory, "DataCore", "DataCore.zip"));
+
+        await console.Output.WriteLineAsync("Done.");
+    }
+
+    private static async Task ExtractDataCoreIntoZip(string p4kFile, string zipPath)
+    {
+        var p4k = P4k.P4kFile.FromFile(p4kFile);
+        Stream? dcbStream = null;
+        string? dcbFile = null;
+        foreach (var file in DataCoreUtils.KnownPaths)
+        {
+            if (!p4k.FileExists(file)) continue;
+
+            dcbFile = file;
+            dcbStream = p4k.OpenRead(file);
+            break;
+        }
+
+        if (dcbStream == null || dcbFile == null)
+            throw new InvalidOperationException("DataCore not found.");
+
+        using var zip = new ZipArchive(File.Create(zipPath), ZipArchiveMode.Create);
+        var entry = zip.CreateEntry(Path.GetFileName(dcbFile), CompressionLevel.SmallestSize);
+        await using var entryStream = entry.Open();
+        await dcbStream.CopyToAsync(entryStream);
+    }
+}

src/StarBreaker.Cli/P4kCommands/DumpP4kCommand.cs

@@ -0,0 +1,27 @@
+using System.Text;
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Infrastructure;
+using StarBreaker.Cli.Utils;
+using StarBreaker.P4k;
+
+namespace StarBreaker.Cli;
+
+[Command("p4k-dump", Description = "Dumps the contents a Game.p4k file")]
+public class DumpP4kCommand : ICommand
+{
+    [CommandOption("p4k", 'p', Description = "Path to the Game.p4k")]
+    public required string P4kFile { get; init; }
+
+    [CommandOption("output", 'o', Description = "Path to the output directory")]
+    public required string OutputDirectory { get; init; }
+
+    public ValueTask ExecuteAsync(IConsole console)
+    {
+        var p4k = P4k.P4kFile.FromFile(P4kFile);
+        var p4kExtractor = new P4kExtractor(p4k);
+        p4kExtractor.ExtractDummies(OutputDirectory, new ProgressBar(console));
+
+        return default;
+    }
+}

src/StarBreaker.Cli/Program.cs

@@ -4,6 +4,7 @@
 return await new CliApplicationBuilder()
     .SetExecutableName("StarBreaker.Cli")
     .AddCommand<DownloadCommand>()
+    .AddCommand<DiffCommand>()
     .AddCommand<ExportAllCommand>()
     .AddCommand<ImportAllCommand>()
     .AddCommand<ProcessAllCommand>()
@@ -12,6 +13,7 @@
     .AddCommand<WatchImportCommand>()
     .AddCommand<DataCoreExtractCommand>()
     .AddCommand<ExtractP4kCommand>()
+    .AddCommand<DumpP4kCommand>()
     .AddCommand<ExtractProtobufsCommand>()
     .AddCommand<ExtractDescriptorSetCommand>()
     .AddCommand<ConvertCryXmlBCommand>()

src/StarBreaker.Cli/Properties/launchSettings.json

@@ -42,7 +42,15 @@
     },
     "dcb-extract": {
       "commandName": "Project",
-      "commandLineArgs": "dcb-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\""
+      "commandLineArgs": "dcb-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\""
+    },
+    "dcb-extract2": {
+      "commandName": "Project",
+      "commandLineArgs": "dcb-extract -d \"D:\\StarCitizen\\P4k\\Data\\Game2.dcb\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\""
+    },
+    "diff": {
+      "commandName": "Project",
+      "commandLineArgs": "diff -g \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\" -o \"C:\\Development\\StarCitizen\\StarCitizenDiff\""
     },
     "dcb-extract-old": {
       "commandName": "Project",
@@ -58,15 +66,19 @@
     },
     "p4k-extract": {
       "commandName": "Project",
-      "commandLineArgs": "p4k-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" -o \"D:\\StarCitizen\\P4k\""
+      "commandLineArgs": "p4k-extract -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"D:\\StarCitizen\\P4k\""
+    },
+    "p4k-dump": {
+      "commandName": "Project",
+      "commandLineArgs": "p4k-dump -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\""
     },
     "proto-extract": {
       "commandName": "Project",
-      "commandLineArgs": "proto-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\protos\""
+      "commandLineArgs": "proto-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\protos\""
     },
     "proto-set-extract": {
       "commandName": "Project",
-      "commandLineArgs": "proto-set-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\descriptorSet.bin\""
+      "commandLineArgs": "proto-set-extract -i \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Bin64\\StarCitizen.exe\" -o \"C:\\Development\\StarCitizen\\StarCitizenProtobuf\\descriptorSet.bin\""
     }
   }
 }

src/StarBreaker.DataCore/DataCoreUtils.cs

@@ -0,0 +1,12 @@
+using System.IO.Enumeration;
+
+namespace StarBreaker.DataCore;
+
+public static class DataCoreUtils
+{
+    public static readonly string[] KnownPaths = [@"Data\Game2.dcb", @"Data\Game.dcb"];
+    public static bool IsDataCoreFile(string path)
+    {
+        return FileSystemName.MatchesSimpleExpression("Data\\*.dcb", path);
+    }
+}

src/StarBreaker.P4k/P4kExtractor.cs

@@ -1,4 +1,5 @@
 using System.IO.Enumeration;
+using System.Text;
 
 namespace StarBreaker.P4k;
 
@@ -52,8 +53,8 @@ public void Extract(string outputDir, string? filter = null, IProgress<double>?
                     return;
 
                 var entryPath = Path.Combine(outputDir, entry.Name);
-                if (File.Exists(entryPath))
-                    return;
+                //if (File.Exists(entryPath))
+                //    return;
 
                 Directory.CreateDirectory(Path.GetDirectoryName(entryPath) ?? throw new InvalidOperationException());
                 using (var writeStream = new FileStream(entryPath, FileMode.Create, FileAccess.Write, FileShare.None,
@@ -78,4 +79,71 @@ public void Extract(string outputDir, string? filter = null, IProgress<double>?
 
         progress?.Report(1);
     }
+
+    public void ExtractDummies(string outputDir, IProgress<double>? progress = null)
+    {
+        //TODO: if the filter is for *.dds, make sure to include *.dds.N too. Maybe do the pre processing before we filter?
+
+        var numberOfEntries = _p4KFile.Entries.Length;
+        var fivePercent = numberOfEntries / 20;
+        var processedEntries = 0;
+
+        progress?.Report(0);
+
+        var lockObject = new Lock();
+
+        //TODO: Preprocessing step:
+        // 1. start with the list of total files
+        // 2. run the following according to the filter:
+        // 3. find one-shot single file procesors
+        // 4. find file -> multiple file processors
+        // 5. find multiple file -> single file unsplit processors - remove from the list so we don't double process
+        // run it!
+        Parallel.ForEach(_p4KFile.Entries,
+            entry =>
+            {
+                var entryPath = Path.Combine(outputDir, entry.Name) + ".ini";
+                Directory.CreateDirectory(Path.GetDirectoryName(entryPath) ?? throw new InvalidOperationException());
+                //write metadata to the file instead of the actual data
+                var sb = new StringBuilder();
+
+                sb.Append("CRC32: 0x");
+                sb.Append(entry.Crc32.ToString("X8"));
+                sb.AppendLine();
+
+                sb.Append("LastModified: ");
+                sb.Append(entry.LastModified.ToString("s"));
+                sb.AppendLine();
+
+                sb.Append("UncompressedSize: ");
+                sb.Append(entry.UncompressedSize);
+                sb.AppendLine();
+
+                sb.Append("CompressedSize: ");
+                sb.Append(entry.CompressedSize);
+                sb.AppendLine();
+
+                sb.Append("CompressionType: ");
+                sb.Append(entry.CompressionMethod);
+                sb.AppendLine();
+
+                sb.Append("IsCrypted: ");
+                sb.Append(entry.IsCrypted);
+                sb.AppendLine();
+
+                File.WriteAllText(entryPath, sb.ToString());
+
+                Interlocked.Increment(ref processedEntries);
+                if (processedEntries == numberOfEntries || processedEntries % fivePercent == 0)
+                {
+                    using (lockObject.EnterScope())
+                    {
+                        progress?.Report(processedEntries / (double)numberOfEntries);
+                    }
+                }
+            }
+        );
+
+        progress?.Report(1);
+    }
 }

src/StarBreaker.P4k/P4kFile.cs

@@ -1,5 +1,5 @@
 using System.Buffers;
-using System.IO.Enumeration;
+using System.Runtime.CompilerServices;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Channels;
@@ -39,7 +39,7 @@ private P4kFile(string path, ZipEntry[] entries, ZipNode root)
     public static P4kFile FromFile(string filePath, IProgress<double>? progress = null)
     {
         progress?.Report(0);
-        using var reader = new BinaryReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, 4096), Encoding.UTF8, false);
+        using var reader = new BinaryReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096), Encoding.UTF8, false);
 
         var eocdLocation = reader.BaseStream.Locate(EOCDRecord.Magic);
         reader.BaseStream.Seek(eocdLocation, SeekOrigin.Begin);
@@ -168,7 +168,8 @@ private static ZipEntry ReadEntry(BinaryReader reader)
                 header.CompressionMethod,
                 isCrypted,
                 localHeaderOffset,
-                header.LastModifiedTimeAndDate
+                header.LastModifiedTimeAndDate,
+                header.Crc32
             );
         }
         finally
@@ -188,9 +189,14 @@ private StreamSegment OpenInternal(ZipEntry entry)
 
         var localHeader = p4kStream.Read<LocalFileHeader>();
 
-        p4kStream.Seek(localHeader.FileNameLength + localHeader.ExtraFieldLength, SeekOrigin.Current);
+        var offset = entry.Offset
+                     + sizeof(uint)
+                     + (ulong)Unsafe.SizeOf<LocalFileHeader>()
+                     + localHeader.FileNameLength
+                     + localHeader.ExtraFieldLength;
+        var length = entry.CompressedSize;
 
-        return new StreamSegment(p4kStream, p4kStream.Position, (long)entry.CompressedSize, false);
+        return new StreamSegment(p4kStream, (long)offset, (long)length, false);
     }
 
     // Remarks: Streams returned by this method might not support seeking or length.

src/StarBreaker.P4k/Zip/ZipEntry.cs

@@ -16,6 +16,7 @@ public sealed class ZipEntry
     public bool IsCrypted { get; }
     public ulong Offset { get; }
     public DateTime LastModified => FromDosDateTime(_dosDateTime);
+    public uint Crc32 { get; }
 
     public ZipEntry(
         string name,
@@ -24,7 +25,8 @@ public ZipEntry(
         ushort compressionMethod,
         bool isCrypted,
         ulong offset,
-        uint lastModifiedDateTime
+        uint lastModifiedDateTime,
+        uint crc32
     )
     {
         Name = name;
@@ -34,6 +36,7 @@ uint lastModifiedDateTime
         IsCrypted = isCrypted;
         Offset = offset;
         _dosDateTime = lastModifiedDateTime;
+        Crc32 = crc32;
     }
 
     //https://source.dot.net/#System.IO.Compression/System/IO/Compression/ZipHelper.cs,76523e345de18cc8