Skip to content

Commit

Permalink
separate extraction options
Browse files Browse the repository at this point in the history
  • Loading branch information
@diogotr7
diogotr7 committed Jan 24, 2025
1 parent 0c81943 commit af35039
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 50 deletions.
54 changes: 21 additions & 33 deletions src/StarBreaker.DataCore/DataCoreBinary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
{
//TODO: do we need to handle different types of arrays?
ConversionType.Attribute => GetAttribute(prop, ref reader, context),
ConversionType.ComplexArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ComplexArray", context.ShouldWriteMetadata),
ConversionType.SimpleArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "SimpleArray", context.ShouldWriteMetadata),
ConversionType.ClassArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ClassArray", context.ShouldWriteMetadata),
ConversionType.ComplexArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ComplexArray", context.Options.ShouldWriteMetadata),
ConversionType.SimpleArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "SimpleArray", context.Options.ShouldWriteMetadata),
ConversionType.ClassArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ClassArray", context.Options.ShouldWriteMetadata),
_ => throw new InvalidOperationException(nameof(ConversionType))
});
}
Expand All @@ -48,10 +48,10 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
{
arrayNode.Add(prop.DataType switch
{
DataType.Reference => GetFromReference(Database.ReferenceValues[i], context)?.WithAttribute("__type", "ArrReference", context.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(Database.WeakValues[i], context)?.WithAttribute("__type", "ArrWeak", context.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context)?.WithAttribute("__type", "ArrStrong", context.ShouldWriteMetadata),
DataType.Class => GetFromInstance(prop.StructIndex, i, context)?.WithAttribute("__type", "ArrClass", context.ShouldWriteMetadata),
DataType.Reference => GetFromReference(Database.ReferenceValues[i], context)?.WithAttribute("__type", "ArrReference", context.Options.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(Database.WeakValues[i], context)?.WithAttribute("__type", "ArrWeak", context.Options.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context)?.WithAttribute("__type", "ArrStrong", context.Options.ShouldWriteMetadata),
DataType.Class => GetFromInstance(prop.StructIndex, i, context)?.WithAttribute("__type", "ArrClass", context.Options.ShouldWriteMetadata),

DataType.EnumChoice => new XElement(prop.DataType.ToStringFast(), Database.EnumValues[i].ToString(Database)),
DataType.Guid => new XElement(prop.DataType.ToStringFast(), Database.GuidValues[i].ToString()),
Expand Down Expand Up @@ -79,10 +79,10 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
{
return prop.DataType switch
{
DataType.Reference => GetFromReference(reader.Read<DataCoreReference>(), context)?.WithAttribute("__type", "AttReference", context.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttWeak", context.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttStrong", context.ShouldWriteMetadata),
DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context)?.WithAttribute("__type", "AttClass", context.ShouldWriteMetadata),
DataType.Reference => GetFromReference(reader.Read<DataCoreReference>(), context)?.WithAttribute("__type", "AttReference", context.Options.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttWeak", context.Options.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttStrong", context.Options.ShouldWriteMetadata),
DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context)?.WithAttribute("__type", "AttClass", context.Options.ShouldWriteMetadata),

DataType.EnumChoice => new XAttribute(prop.GetName(Database), reader.Read<DataCoreStringId>().ToString(Database)),
DataType.Guid => new XAttribute(prop.GetName(Database), reader.Read<CigGuid>().ToString()),
Expand All @@ -105,14 +105,14 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE

private XElement? GetFromReference(DataCoreReference reference, DataCoreExtractionContext context)
{
if (reference.IsInvalid)
if (reference.InstanceIndex == -1 || reference.RecordId == CigGuid.Empty)
{
if (!context.ShouldWriteNulls)
if (!context.Options.ShouldWriteNulls)
return null;

var invalidNode = new XElement("NullReference");
invalidNode.Add(new XAttribute("guid", reference.RecordId.ToString()));
if (context.ShouldWriteMetadata)
if (context.Options.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
}
Expand All @@ -127,7 +127,7 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
//if we're referencing a full on file, just add a small mention to it
var fileReferenceNode = new XElement("FileReference");
fileReferenceNode.Add(new XAttribute("guid", reference.RecordId.ToString()));
fileReferenceNode.Add(new XAttribute("filePath", ComputeRelativePath(record.GetFileName(Database), context.FileName)));
fileReferenceNode.Add(new XAttribute("filePath", DataCoreUtils.ComputeRelativePath(record.GetFileName(Database), context.FileName)));
return fileReferenceNode;
}

Expand Down Expand Up @@ -163,7 +163,7 @@ public XElement GetFromMainRecord(DataCoreRecord record, DataCoreExtractionConte

private XElement? GetFromPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (pointer.IsInvalid)
if (pointer.InstanceIndex == -1 || pointer.StructIndex == -1)
return GetNullPointer(pointer, context);

return GetFromInstance(pointer.StructIndex, pointer.InstanceIndex, context);
Expand All @@ -176,7 +176,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt

context.Elements[(structIndex, instanceIndex)] = element;

if (context.ShouldWriteMetadata)
if (context.Options.ShouldWriteMetadata)
{
element.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture)));
element.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture)));
Expand All @@ -187,7 +187,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt

private XElement? GetWeakPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (pointer.IsInvalid)
if (pointer.InstanceIndex == -1 || pointer.StructIndex == -1)
return GetNullPointer(pointer, context);

var pointerId = context.AddWeakPointer(pointer.StructIndex, pointer.InstanceIndex);
Expand All @@ -198,7 +198,7 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt
var structName = Database.StructDefinitions[pointer.StructIndex].GetName(Database);
invalidNode.Add(new XAttribute("structName", structName));

if (context.ShouldWriteMetadata)
if (context.Options.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
Expand All @@ -209,28 +209,16 @@ private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExt

private static XElement? GetNullPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (!context.ShouldWriteNulls)
if (!context.Options.ShouldWriteNulls)
return null;

var invalidNode = new XElement("NullPointer");
if (context.ShouldWriteMetadata)
if (context.Options.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
}

return invalidNode;
}

public static string ComputeRelativePath(ReadOnlySpan<char> filePath, ReadOnlySpan<char> contextFileName)
{
var slashes = contextFileName.Count('/');
var sb = new StringBuilder("file://./");

for (var i = 0; i < slashes; i++)
sb.Append("../");

sb.Append(filePath);
return sb.ToString();
}
}
10 changes: 5 additions & 5 deletions src/StarBreaker.DataCore/DataCoreExtractionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ public sealed class DataCoreExtractionContext
public Dictionary<(int structIndex, int instanceIndex), XElement> Elements { get; }

public string FileName { get; }
public bool ShouldWriteMetadata { get; }
public bool ShouldWriteNulls { get; }
public DataCoreExtractionOptions Options { get; }

public DataCoreExtractionContext(string fileName, bool shouldWriteMetadata = false, bool shouldWriteNulls = false)

public DataCoreExtractionContext(string fileName, DataCoreExtractionOptions options)
{
FileName = fileName;
ShouldWriteMetadata = shouldWriteMetadata;
ShouldWriteNulls = shouldWriteNulls;
Options = options;

Elements = [];
_weakPointerIds = [];
}
Expand Down
11 changes: 11 additions & 0 deletions src/StarBreaker.DataCore/DataCoreExtractionOptions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
namespace StarBreaker.DataCore;

public class DataCoreExtractionOptions
{
public required bool ShouldWriteMetadata { get; init; }
public required bool ShouldWriteEmptyArrays { get; init; }
public required bool ShouldWriteTypeNames { get; init; }
public required bool ShouldWriteBaseTypeNames { get; init; }
public required bool ShouldWriteDataTypes { get; init; }
public required bool ShouldWriteNulls { get; init; }
}
13 changes: 13 additions & 0 deletions src/StarBreaker.DataCore/DataCoreUtils.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.IO.Enumeration;
using System.Text;

namespace StarBreaker.DataCore;

Expand All @@ -9,4 +10,16 @@ public static bool IsDataCoreFile(string path)
{
return FileSystemName.MatchesSimpleExpression("Data\\*.dcb", path);
}

internal static string ComputeRelativePath(ReadOnlySpan<char> filePath, ReadOnlySpan<char> contextFileName)
{
var slashes = contextFileName.Count('/');
var sb = new StringBuilder("file://./");

for (var i = 0; i < slashes; i++)
sb.Append("../");

sb.Append(filePath);
return sb.ToString();
}
}
48 changes: 41 additions & 7 deletions src/StarBreaker.DataCore/DataForge.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public Dictionary<string, DataCoreRecord> GetRecordsByFileName(string? fileNameF
return structsPerFileName;
}

public XElement GetFromRecord(DataCoreRecord record, bool includeMetadata = false, bool keepNulls = false)
public XElement GetFromRecord(DataCoreRecord record, DataCoreExtractionOptions? options = null)
{
var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), includeMetadata, keepNulls);
var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), options ?? GetDefaultExtractionOptions());
return DataCore.GetFromMainRecord(record, context);
}

Expand All @@ -53,7 +53,7 @@ public Dictionary<string, string[]> ExportEnums()
return result;
}

public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProgress<double>? progress = null)
public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProgress<double>? progress = null, DataCoreExtractionOptions? options = null)
{
var progressValue = 0;
var recordsByFileName = GetRecordsByFileName(fileNameFilter);
Expand All @@ -65,7 +65,7 @@ public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProg

Directory.CreateDirectory(Path.GetDirectoryName(filePath)!);

var node = GetFromRecord(record);
var node = GetFromRecord(record, options);

node.Save(filePath);

Expand All @@ -78,7 +78,7 @@ public void ExtractAll(string outputFolder, string? fileNameFilter = null, IProg
progress?.Report(1);
}

public void ExtractAllParallel(string outputFolder, string? fileNameFilter = null, IProgress<double>? progress = null)
public void ExtractAllParallel(string outputFolder, string? fileNameFilter = null, IProgress<double>? progress = null, DataCoreExtractionOptions? options = null)
{
var progressValue = 0;
var recordsByFileName = GetRecordsByFileName(fileNameFilter);
Expand All @@ -91,7 +91,7 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul

Directory.CreateDirectory(Path.GetDirectoryName(filePath)!);

var node = GetFromRecord(record);
var node = GetFromRecord(record, options);

node.Save(filePath);

Expand All @@ -103,4 +103,38 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul

progress?.Report(1);
}
}

public void ExtractUnp4k(string outputFileName, IProgress<double>? progress = null, DataCoreExtractionOptions? options = null)
{
var progressValue = 0;
var total = DataCore.Database.MainRecords.Count;

var doc = new XDocument(new XElement("DataCore"));

foreach (var recordId in DataCore.Database.MainRecords)
{
var record = DataCore.Database.GetRecord(recordId);
var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), options ?? GetDefaultExtractionOptions());
var node = DataCore.GetFromMainRecord(record, context);

doc.Root?.Add(node);

var currentProgress = Interlocked.Increment(ref progressValue);
//only report progress every 250 records and when we are done
if (currentProgress == total || currentProgress % 250 == 0)
progress?.Report(currentProgress / (double)total);
}

doc.Save(outputFileName);
}

private static DataCoreExtractionOptions GetDefaultExtractionOptions() => new()
{
ShouldWriteMetadata = false,
ShouldWriteEmptyArrays = false,
ShouldWriteTypeNames = false,
ShouldWriteBaseTypeNames = false,
ShouldWriteDataTypes = false,
ShouldWriteNulls = false
};
}
2 changes: 0 additions & 2 deletions src/StarBreaker.DataCore/Structs/DataCorePointer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,4 @@ public readonly record struct DataCorePointer
{
public readonly int StructIndex;
public readonly int InstanceIndex;

public bool IsInvalid => StructIndex == -1 || InstanceIndex == -1;
}
4 changes: 1 addition & 3 deletions src/StarBreaker.DataCore/Structs/DataCoreReference.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ namespace StarBreaker.DataCore;
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public readonly record struct DataCoreReference
{
public readonly uint InstanceIndex;
public readonly int InstanceIndex;
public readonly CigGuid RecordId;

public bool IsInvalid => RecordId == CigGuid.Empty || InstanceIndex == 0xFFFFFFFF;
}

0 comments on commit af35039

Please sign in to comment.