DDS Overhaul

README.md

@@ -24,18 +24,21 @@ OPTIONS
   --version         Shows version information.
 
 COMMANDS
-  chf-download        Downloads all characters from the website and saves them to the website characters folder.
-  chf-export-all      Exports all modded characters into the Star Citizen folder.
-  chf-export-watch    Watch for new modded characters and export them to the star citizen folder.
-  chf-import-all      Imports all non-modded characters exported from the game into our local characters folder.
-  chf-import-watch    Watch for new characters in the Star Citizen folder and import them.
-  chf-process         Process a character file
-  chf-process-all     Processes all characters in the given folder.
-  dcb-extract         Extracts a DataCore binary file into separate xml files
-  dcb-extract-single  Extracts a DataCore binary file into a single xml
-  p4k-extract         Extracts a Game.p4k file
-  proto-extract       Extracts protobuf definitions from the Star Citizen executable.
-  proto-set-extract   Extracts the protobuf descriptor set from the Star Citizen executable.
+  chf-download      Downloads all characters from the website and saves them to the website characters folder.
+  chf-export-all    Exports all modded characters into the Star Citizen folder.
+  chf-export-watch  Watch for new modded characters and export them to the star citizen folder.
+  chf-import-all    Imports all non-modded characters exported from the game into our local characters folder.
+  chf-import-watch  Watch for new characters in the Star Citizen folder and import them.
+  chf-process       Process a character file
+  chf-process-all   Processes all characters in the given folder.
+  cryxml-convert    Converts a CryXmlB file to a XML file
+  cryxml-convert-all  Converts all CryXmlB files in a folder to XML files
+  dcb-extract       Extracts a DataCore binary file into separate xml files
+  dds-merge         Merges multiple DDS files into a single DDS file
+  dds-merge-all     Merges all DDS files in a folder into a single DDS file
+  p4k-extract       Extracts a Game.p4k file
+  proto-extract     Extracts protobuf definitions from the Star Citizen executable.
+  proto-set-extract  Extracts the protobuf descriptor set from the Star Citizen executable.
 
 You can run `StarBreaker.Cli [command] --help` to show help on a specific command.
 ```

src/StarBreaker.Common/StreamExtensions.cs

@@ -1,6 +1,7 @@
 using System.Buffers;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Text;
 
 namespace StarBreaker.Common;
 
@@ -21,6 +22,32 @@ public static T Read<T>(this Stream stream) where T : unmanaged
         return MemoryMarshal.Read<T>(span);
     }
 
+    public static T[] ReadArray<T>(this Stream stream, int count) where T : unmanaged
+    {
+        var items = new T[count];
+
+        var bytes = MemoryMarshal.Cast<T, byte>(items);
+
+        stream.ReadExactly(bytes);
+
+        return items;
+    }
+
+
+    public static void Write<T>(this Stream stream, T value) where T : unmanaged
+    {
+        var size = Unsafe.SizeOf<T>();
+
+        if (size > 256)
+            throw new Exception("Size is too large");
+
+        Span<byte> span = stackalloc byte[size];
+
+        MemoryMarshal.Write(span, in value);
+
+        stream.Write(span);
+    }
+
     public static void CopyAmountTo(this Stream source, Stream destination, int byteCount)
     {
         var rent = ArrayPool<byte>.Shared.Rent(byteCount);
@@ -35,4 +62,52 @@ public static void CopyAmountTo(this Stream source, Stream destination, int byte
             ArrayPool<byte>.Shared.Return(rent);
         }
     }
+
+    public static long Locate(this Stream stream, ReadOnlySpan<byte> magic, long bytesFromEnd = 0)
+    {
+        const int chunkSize = 4096;
+
+        var rent = ArrayPool<byte>.Shared.Rent(chunkSize);
+        var search = rent.AsSpan();
+        stream.Seek(-bytesFromEnd, SeekOrigin.End);
+
+        try
+        {
+            var location = -1;
+
+            while (location == -1)
+            {
+                // seek to the left by chunkSize + magic.Length.
+                // this is to ensure we don't miss the magic bytes that are split between chunks
+                stream.Seek((rent.Length + magic.Length) * -1, SeekOrigin.Current);
+
+                if (stream.Read(rent, 0, rent.Length) != rent.Length)
+                    throw new Exception("Failed to read end of central directory record");
+
+                location = search.LastIndexOf(magic);
+            }
+
+            return stream.Position + location - rent.Length;
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(rent);
+        }
+    }
+
+    public static string ReadStringOfLength(this Stream stream, int length)
+    {
+        if (length >= 0xffff)
+            throw new Exception("Size is too large");
+
+        if (length == 0)
+            return string.Empty;
+
+        Span<byte> span = stackalloc byte[length];
+
+        if (stream.Read(span) != length)
+            throw new Exception("Failed to read from stream");
+
+        return Encoding.ASCII.GetString(span);
+    }
 }

src/StarBreaker.CryXmlB/CryXml.cs

@@ -39,11 +39,11 @@ public CryXml(Stream stream)
         if (!magic.SequenceEqual(thisMagic))
             throw new Exception("Invalid CryXmlB file");
 
-        var header = br.Read<CryXmlHeader>();
-        _nodes = br.ReadArray<CryXmlNode>((int)header.NodeCount);
-        _childIndices = br.ReadArray<int>((int)header.ChildCount);
-        _attributes = br.ReadArray<CryXmlAttribute>((int)header.AttributeCount);
-        _stringData = br.ReadBytes((int)header.StringDataSize);
+        var header = br.BaseStream.Read<CryXmlHeader>();
+        _nodes = br.BaseStream.ReadArray<CryXmlNode>((int)header.NodeCount);
+        _childIndices = br.BaseStream.ReadArray<int>((int)header.ChildCount);
+        _attributes = br.BaseStream.ReadArray<CryXmlAttribute>((int)header.AttributeCount);
+        _stringData = br.BaseStream.ReadArray<byte>((int)header.StringDataSize);
     }
 
     public static bool IsCryXmlB(ReadOnlySpan<byte> data)
@@ -114,4 +114,21 @@ public void Save(string entry)
         using var writer = XmlWriter.Create(entry, new XmlWriterSettings { Indent = true });
         WriteTo(writer);
     }
+
+    public HashSet<string> EnumerateAllStrings()
+    {
+        var strings = new HashSet<string>();
+        foreach (var node in _nodes)
+        {
+            strings.Add(GetString(_stringData, (int)node.TagStringOffset));
+        }
+
+        foreach (var attribute in _attributes)
+        {
+            strings.Add(GetString(_stringData, (int)attribute.KeyStringOffset));
+            strings.Add(GetString(_stringData, (int)attribute.ValueStringOffset));
+        }
+
+        return strings;
+    }
 }

src/StarBreaker.DataCore/DataCoreDatabase.cs

@@ -90,35 +90,35 @@ public DataCoreDatabase(Stream fs)
         var textLength = reader.ReadUInt32();
         var textLength2 = reader.ReadUInt32();
 
-        StructDefinitions = reader.ReadArray<DataCoreStructDefinition>(structDefinitionCount);
-        PropertyDefinitions = reader.ReadArray<DataCorePropertyDefinition>(propertyDefinitionCount);
-        EnumDefinitions = reader.ReadArray<DataCoreEnumDefinition>(enumDefinitionCount);
-        DataMappings = reader.ReadArray<DataCoreDataMapping>(dataMappingCount);
-        RecordDefinitions = reader.ReadArray<DataCoreRecord>(recordDefinitionCount);
-
-        Int8Values = reader.ReadArray<sbyte>(int8ValueCount);
-        Int16Values = reader.ReadArray<short>(int16ValueCount);
-        Int32Values = reader.ReadArray<int>(int32ValueCount);
-        Int64Values = reader.ReadArray<long>(int64ValueCount);
-
-        UInt8Values = reader.ReadArray<byte>(uint8ValueCount);
-        UInt16Values = reader.ReadArray<ushort>(uint16ValueCount);
-        UInt32Values = reader.ReadArray<uint>(uint32ValueCount);
-        UInt64Values = reader.ReadArray<ulong>(uint64ValueCount);
-
-        BooleanValues = reader.ReadArray<bool>(booleanValueCount);
-        SingleValues = reader.ReadArray<float>(singleValueCount);
-        DoubleValues = reader.ReadArray<double>(doubleValueCount);
-        GuidValues = reader.ReadArray<CigGuid>(guidValueCount);
-
-        StringIdValues = reader.ReadArray<DataCoreStringId>(stringIdValueCount);
-        LocaleValues = reader.ReadArray<DataCoreStringId>(localeValueCount);
-        EnumValues = reader.ReadArray<DataCoreStringId>(enumValueCount);
-
-        StrongValues = reader.ReadArray<DataCorePointer>(strongValueCount);
-        WeakValues = reader.ReadArray<DataCorePointer>(weakValueCount);
-        ReferenceValues = reader.ReadArray<DataCoreReference>(referenceValueCount);
-        EnumOptions = reader.ReadArray<DataCoreStringId2>(enumOptionCount);
+        StructDefinitions = reader.BaseStream.ReadArray<DataCoreStructDefinition>(structDefinitionCount);
+        PropertyDefinitions = reader.BaseStream.ReadArray<DataCorePropertyDefinition>(propertyDefinitionCount);
+        EnumDefinitions = reader.BaseStream.ReadArray<DataCoreEnumDefinition>(enumDefinitionCount);
+        DataMappings = reader.BaseStream.ReadArray<DataCoreDataMapping>(dataMappingCount);
+        RecordDefinitions = reader.BaseStream.ReadArray<DataCoreRecord>(recordDefinitionCount);
+
+        Int8Values = reader.BaseStream.ReadArray<sbyte>(int8ValueCount);
+        Int16Values = reader.BaseStream.ReadArray<short>(int16ValueCount);
+        Int32Values = reader.BaseStream.ReadArray<int>(int32ValueCount);
+        Int64Values = reader.BaseStream.ReadArray<long>(int64ValueCount);
+
+        UInt8Values = reader.BaseStream.ReadArray<byte>(uint8ValueCount);
+        UInt16Values = reader.BaseStream.ReadArray<ushort>(uint16ValueCount);
+        UInt32Values = reader.BaseStream.ReadArray<uint>(uint32ValueCount);
+        UInt64Values = reader.BaseStream.ReadArray<ulong>(uint64ValueCount);
+
+        BooleanValues = reader.BaseStream.ReadArray<bool>(booleanValueCount);
+        SingleValues = reader.BaseStream.ReadArray<float>(singleValueCount);
+        DoubleValues = reader.BaseStream.ReadArray<double>(doubleValueCount);
+        GuidValues = reader.BaseStream.ReadArray<CigGuid>(guidValueCount);
+
+        StringIdValues = reader.BaseStream.ReadArray<DataCoreStringId>(stringIdValueCount);
+        LocaleValues = reader.BaseStream.ReadArray<DataCoreStringId>(localeValueCount);
+        EnumValues = reader.BaseStream.ReadArray<DataCoreStringId>(enumValueCount);
+
+        StrongValues = reader.BaseStream.ReadArray<DataCorePointer>(strongValueCount);
+        WeakValues = reader.BaseStream.ReadArray<DataCorePointer>(weakValueCount);
+        ReferenceValues = reader.BaseStream.ReadArray<DataCoreReference>(referenceValueCount);
+        EnumOptions = reader.BaseStream.ReadArray<DataCoreStringId2>(enumOptionCount);
 
         CachedStrings = ReadStringTable(reader.ReadBytes((int)textLength).AsSpan());
         if (Version >= 6)