diff --git a/src/StarBreaker.DataCore/DataCoreBinary.cs b/src/StarBreaker.DataCore/DataCoreBinary.cs index 74c4050..8c88677 100644 --- a/src/StarBreaker.DataCore/DataCoreBinary.cs +++ b/src/StarBreaker.DataCore/DataCoreBinary.cs @@ -24,9 +24,9 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE { //TODO: do we need to handle different types of arrays? ConversionType.Attribute => GetAttribute(prop, ref reader, context), - ConversionType.ComplexArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ComplexArray", context.ShouldWriteMetadata), - ConversionType.SimpleArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "SimpleArray", context.ShouldWriteMetadata), - ConversionType.ClassArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ClassArray", context.ShouldWriteMetadata), + ConversionType.ComplexArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ComplexArray", context.Options.ShouldWriteMetadata), + ConversionType.SimpleArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "SimpleArray", context.Options.ShouldWriteMetadata), + ConversionType.ClassArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ClassArray", context.Options.ShouldWriteMetadata), _ => throw new InvalidOperationException(nameof(ConversionType)) }); } @@ -48,10 +48,10 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE { arrayNode.Add(prop.DataType switch { - DataType.Reference => GetFromReference(Database.ReferenceValues[i], context)?.WithAttribute("__type", "ArrReference", context.ShouldWriteMetadata), - DataType.WeakPointer => GetWeakPointer(Database.WeakValues[i], context)?.WithAttribute("__type", "ArrWeak", context.ShouldWriteMetadata), - DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context)?.WithAttribute("__type", "ArrStrong", context.ShouldWriteMetadata), - DataType.Class => GetFromInstance(prop.StructIndex, i, context)?.WithAttribute("__type", "ArrClass", context.ShouldWriteMetadata), + DataType.Reference => GetFromReference(Database.ReferenceValues[i], context)?.WithAttribute("__type", "ArrReference", context.Options.ShouldWriteMetadata), + DataType.WeakPointer => GetWeakPointer(Database.WeakValues[i], context)?.WithAttribute("__type", "ArrWeak", context.Options.ShouldWriteMetadata), + DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context)?.WithAttribute("__type", "ArrStrong", context.Options.ShouldWriteMetadata), + DataType.Class => GetFromInstance(prop.StructIndex, i, context)?.WithAttribute("__type", "ArrClass", context.Options.ShouldWriteMetadata), DataType.EnumChoice => new XElement(prop.DataType.ToStringFast(), Database.EnumValues[i].ToString(Database)), DataType.Guid => new XElement(prop.DataType.ToStringFast(), Database.GuidValues[i].ToString()), @@ -79,10 +79,10 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE { return prop.DataType switch { - DataType.Reference => GetFromReference(reader.Read(), context)?.WithAttribute("__type", "AttReference", context.ShouldWriteMetadata), - DataType.WeakPointer => GetWeakPointer(reader.Read(), context)?.WithAttribute("__type", "AttWeak", context.ShouldWriteMetadata), - DataType.StrongPointer => GetFromPointer(reader.Read(), context)?.WithAttribute("__type", "AttStrong", context.ShouldWriteMetadata), - DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context)?.WithAttribute("__type", "AttClass", context.ShouldWriteMetadata), + DataType.Reference => GetFromReference(reader.Read(), context)?.WithAttribute("__type", "AttReference", context.Options.ShouldWriteMetadata), + DataType.WeakPointer => GetWeakPointer(reader.Read(), context)?.WithAttribute("__type", "AttWeak", context.Options.ShouldWriteMetadata), + DataType.StrongPointer => GetFromPointer(reader.Read(), context)?.WithAttribute("__type", "AttStrong", context.Options.ShouldWriteMetadata), + DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context)?.WithAttribute("__type", "AttClass", context.Options.ShouldWriteMetadata), DataType.EnumChoice => new XAttribute(prop.GetName(Database), reader.Read().ToString(Database)), DataType.Guid => new XAttribute(prop.GetName(Database), reader.Read().ToString()), @@ -105,14 +105,14 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE private XElement? GetFromReference(DataCoreReference reference, DataCoreExtractionContext context) { - if (reference.IsInvalid) + if (reference.InstanceIndex == -1 || reference.RecordId == CigGuid.Empty) { - if (!context.ShouldWriteNulls) + if (!context.Options.ShouldWriteNulls) return null; var invalidNode = new XElement("NullReference"); invalidNode.Add(new XAttribute("guid", reference.RecordId.ToString())); - if (context.ShouldWriteMetadata) + if (context.Options.ShouldWriteMetadata) { invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture))); } @@ -127,7 +127,7 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE //if we're referencing a full on file, just add a small mention to it var fileReferenceNode = new XElement("FileReference"); fileReferenceNode.Add(new XAttribute("guid", reference.RecordId.ToString())); - fileReferenceNode.Add(new XAttribute("filePath", ComputeRelativePath(record.GetFileName(Database), context.FileName))); + fileReferenceNode.Add(new XAttribute("filePath", DataCoreUtils.ComputeRelativePath(record.GetFileName(Database), context.FileName))); return fileReferenceNode; } @@ -163,7 +163,7 @@ public XElement GetFromMainRecord(DataCoreRecord record, DataCoreExtractionConte private XElement? GetFromPointer(DataCorePointer pointer, DataCoreExtractionContext context) { - if (pointer.IsInvalid) + if (pointer.InstanceIndex == -1 || pointer.StructIndex == -1) return GetNullPointer(pointer, context); return GetFromInstance(pointer.StructIndex, pointer.InstanceIndex, context); @@ -176,7 +176,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt context.Elements[(structIndex, instanceIndex)] = element; - if (context.ShouldWriteMetadata) + if (context.Options.ShouldWriteMetadata) { element.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture))); element.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture))); @@ -187,7 +187,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt private XElement? GetWeakPointer(DataCorePointer pointer, DataCoreExtractionContext context) { - if (pointer.IsInvalid) + if (pointer.InstanceIndex == -1 || pointer.StructIndex == -1) return GetNullPointer(pointer, context); var pointerId = context.AddWeakPointer(pointer.StructIndex, pointer.InstanceIndex); @@ -198,7 +198,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt var structName = Database.StructDefinitions[pointer.StructIndex].GetName(Database); invalidNode.Add(new XAttribute("structName", structName)); - if (context.ShouldWriteMetadata) + if (context.Options.ShouldWriteMetadata) { invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture))); invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture))); @@ -209,11 +209,11 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt private static XElement? GetNullPointer(DataCorePointer pointer, DataCoreExtractionContext context) { - if (!context.ShouldWriteNulls) + if (!context.Options.ShouldWriteNulls) return null; var invalidNode = new XElement("NullPointer"); - if (context.ShouldWriteMetadata) + if (context.Options.ShouldWriteMetadata) { invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture))); invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture))); @@ -221,16 +221,4 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt return invalidNode; } - - public static string ComputeRelativePath(ReadOnlySpan filePath, ReadOnlySpan contextFileName) - { - var slashes = contextFileName.Count('/'); - var sb = new StringBuilder("file://./"); - - for (var i = 0; i < slashes; i++) - sb.Append("../"); - - sb.Append(filePath); - return sb.ToString(); - } } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataCoreExtractionContext.cs b/src/StarBreaker.DataCore/DataCoreExtractionContext.cs index debf923..aaff093 100644 --- a/src/StarBreaker.DataCore/DataCoreExtractionContext.cs +++ b/src/StarBreaker.DataCore/DataCoreExtractionContext.cs @@ -11,14 +11,14 @@ public sealed class DataCoreExtractionContext public Dictionary<(int structIndex, int instanceIndex), XElement> Elements { get; } public string FileName { get; } - public bool ShouldWriteMetadata { get; } - public bool ShouldWriteNulls { get; } + public DataCoreExtractionOptions Options { get; } - public DataCoreExtractionContext(string fileName, bool shouldWriteMetadata = false, bool shouldWriteNulls = false) + + public DataCoreExtractionContext(string fileName, DataCoreExtractionOptions options) { FileName = fileName; - ShouldWriteMetadata = shouldWriteMetadata; - ShouldWriteNulls = shouldWriteNulls; + Options = options; + Elements = []; _weakPointerIds = []; } diff --git a/src/StarBreaker.DataCore/DataCoreExtractionOptions.cs b/src/StarBreaker.DataCore/DataCoreExtractionOptions.cs new file mode 100644 index 0000000..3f8c1dc --- /dev/null +++ b/src/StarBreaker.DataCore/DataCoreExtractionOptions.cs @@ -0,0 +1,11 @@ +namespace StarBreaker.DataCore; + +public class DataCoreExtractionOptions +{ + public required bool ShouldWriteMetadata { get; init; } + public required bool ShouldWriteEmptyArrays { get; init; } + public required bool ShouldWriteTypeNames { get; init; } + public required bool ShouldWriteBaseTypeNames { get; init; } + public required bool ShouldWriteDataTypes { get; init; } + public required bool ShouldWriteNulls { get; init; } +} \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataCoreUtils.cs b/src/StarBreaker.DataCore/DataCoreUtils.cs index e5239b9..34474ce 100644 --- a/src/StarBreaker.DataCore/DataCoreUtils.cs +++ b/src/StarBreaker.DataCore/DataCoreUtils.cs @@ -1,4 +1,5 @@ using System.IO.Enumeration; +using System.Text; namespace StarBreaker.DataCore; @@ -9,4 +10,16 @@ public static bool IsDataCoreFile(string path) { return FileSystemName.MatchesSimpleExpression("Data\\*.dcb", path); } + + internal static string ComputeRelativePath(ReadOnlySpan filePath, ReadOnlySpan contextFileName) + { + var slashes = contextFileName.Count('/'); + var sb = new StringBuilder("file://./"); + + for (var i = 0; i < slashes; i++) + sb.Append("../"); + + sb.Append(filePath); + return sb.ToString(); + } } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataForge.cs b/src/StarBreaker.DataCore/DataForge.cs index 2f02a15..a4f4576 100644 --- a/src/StarBreaker.DataCore/DataForge.cs +++ b/src/StarBreaker.DataCore/DataForge.cs @@ -29,9 +29,9 @@ public Dictionary GetRecordsByFileName(string? fileNameF return structsPerFileName; } - public XElement GetFromRecord(DataCoreRecord record, bool includeMetadata = false, bool keepNulls = false) + public XElement GetFromRecord(DataCoreRecord record, DataCoreExtractionOptions? options = null) { - var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), includeMetadata, keepNulls); + var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), options ?? GetDefaultExtractionOptions()); return DataCore.GetFromMainRecord(record, context); } @@ -53,7 +53,7 @@ public Dictionary ExportEnums() return result; } - public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProgress? progress = null) + public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProgress? progress = null, DataCoreExtractionOptions? options = null) { var progressValue = 0; var recordsByFileName = GetRecordsByFileName(fileNameFilter); @@ -65,7 +65,7 @@ public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProg Directory.CreateDirectory(Path.GetDirectoryName(filePath)!); - var node = GetFromRecord(record); + var node = GetFromRecord(record, options); node.Save(filePath); @@ -78,7 +78,7 @@ public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProg progress?.Report(1); } - public void ExtractAllParallel(string outputFolder, string? fileNameFilter = null, IProgress? progress = null) + public void ExtractAllParallel(string outputFolder, string? fileNameFilter = null, IProgress? progress = null, DataCoreExtractionOptions? options = null) { var progressValue = 0; var recordsByFileName = GetRecordsByFileName(fileNameFilter); @@ -91,7 +91,7 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul Directory.CreateDirectory(Path.GetDirectoryName(filePath)!); - var node = GetFromRecord(record); + var node = GetFromRecord(record, options); node.Save(filePath); @@ -103,4 +103,38 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul progress?.Report(1); } -} \ No newline at end of file + + public void ExtractUnp4k(string outputFileName, IProgress? progress = null, DataCoreExtractionOptions? options = null) + { + var progressValue = 0; + var total = DataCore.Database.MainRecords.Count; + + var doc = new XDocument(new XElement("DataCore")); + + foreach (var recordId in DataCore.Database.MainRecords) + { + var record = DataCore.Database.GetRecord(recordId); + var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), options ?? GetDefaultExtractionOptions()); + var node = DataCore.GetFromMainRecord(record, context); + + doc.Root?.Add(node); + + var currentProgress = Interlocked.Increment(ref progressValue); + //only report progress every 250 records and when we are done + if (currentProgress == total || currentProgress % 250 == 0) + progress?.Report(currentProgress / (double)total); + } + + doc.Save(outputFileName); + } + + private static DataCoreExtractionOptions GetDefaultExtractionOptions() => new() + { + ShouldWriteMetadata = false, + ShouldWriteEmptyArrays = false, + ShouldWriteTypeNames = false, + ShouldWriteBaseTypeNames = false, + ShouldWriteDataTypes = false, + ShouldWriteNulls = false + }; +} diff --git a/src/StarBreaker.DataCore/Structs/DataCorePointer.cs b/src/StarBreaker.DataCore/Structs/DataCorePointer.cs index 32c6705..0a96988 100644 --- a/src/StarBreaker.DataCore/Structs/DataCorePointer.cs +++ b/src/StarBreaker.DataCore/Structs/DataCorePointer.cs @@ -7,6 +7,4 @@ public readonly record struct DataCorePointer { public readonly int StructIndex; public readonly int InstanceIndex; - - public bool IsInvalid => StructIndex == -1 || InstanceIndex == -1; } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/Structs/DataCoreReference.cs b/src/StarBreaker.DataCore/Structs/DataCoreReference.cs index 057f45b..c15f58e 100644 --- a/src/StarBreaker.DataCore/Structs/DataCoreReference.cs +++ b/src/StarBreaker.DataCore/Structs/DataCoreReference.cs @@ -6,8 +6,6 @@ namespace StarBreaker.DataCore; [StructLayout(LayoutKind.Sequential, Pack = 1)] public readonly record struct DataCoreReference { - public readonly uint InstanceIndex; + public readonly int InstanceIndex; public readonly CigGuid RecordId; - - public bool IsInvalid => RecordId == CigGuid.Empty || InstanceIndex == 0xFFFFFFFF; } \ No newline at end of file