Skip to content

Commit

Permalink
Actually fix DataCore extracting
Browse files Browse the repository at this point in the history
frfr no cap
  • Loading branch information
diogotr7 committed Jan 11, 2025
1 parent e74239c commit cd9fe61
Show file tree
Hide file tree
Showing 5 changed files with 142 additions and 117 deletions.
129 changes: 87 additions & 42 deletions src/StarBreaker.DataCore/DataCoreBinary.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
{
//TODO: do we need to handle different types of arrays?
ConversionType.Attribute => GetAttribute(prop, ref reader, context),
ConversionType.ComplexArray => GetArray(prop, ref reader, context),
ConversionType.SimpleArray => GetArray(prop, ref reader, context),
ConversionType.ClassArray => GetArray(prop, ref reader, context),
ConversionType.ComplexArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ComplexArray", context.ShouldWriteMetadata),
ConversionType.SimpleArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "SimpleArray", context.ShouldWriteMetadata),
ConversionType.ClassArray => GetArray(prop, ref reader, context)?.WithAttribute("__type", "ClassArray", context.ShouldWriteMetadata),
_ => throw new InvalidOperationException(nameof(ConversionType))
});
}
Expand All @@ -48,10 +48,10 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
{
arrayNode.Add(prop.DataType switch
{
DataType.Reference => GetFromReference(Database.ReferenceValues[i], context),
DataType.WeakPointer => GetFromPointer(Database.WeakValues[i], context),
DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context),
DataType.Class => GetFromInstance(prop.StructIndex, i, context),
DataType.Reference => GetFromReference(Database.ReferenceValues[i], context)?.WithAttribute("__type", "ArrReference", context.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(Database.WeakValues[i], context)?.WithAttribute("__type", "ArrWeak", context.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(Database.StrongValues[i], context)?.WithAttribute("__type", "ArrStrong", context.ShouldWriteMetadata),
DataType.Class => GetFromInstance(prop.StructIndex, i, context)?.WithAttribute("__type", "ArrClass", context.ShouldWriteMetadata),

DataType.EnumChoice => new XElement(prop.DataType.ToStringFast(), Database.EnumValues[i].ToString(Database)),
DataType.Guid => new XElement(prop.DataType.ToStringFast(), Database.GuidValues[i].ToString()),
Expand All @@ -75,14 +75,14 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, DataCoreE
return arrayNode;
}

private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, DataCoreExtractionContext context)
private XObject? GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, DataCoreExtractionContext context)
{
return prop.DataType switch
{
DataType.Reference => GetFromReference(reader.Read<DataCoreReference>(), context),
DataType.WeakPointer => GetFromPointer(reader.Read<DataCorePointer>(), context),
DataType.StrongPointer => GetFromPointer(reader.Read<DataCorePointer>(), context),
DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context),
DataType.Reference => GetFromReference(reader.Read<DataCoreReference>(), context)?.WithAttribute("__type", "AttReference", context.ShouldWriteMetadata),
DataType.WeakPointer => GetWeakPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttWeak", context.ShouldWriteMetadata),
DataType.StrongPointer => GetFromPointer(reader.Read<DataCorePointer>(), context)?.WithAttribute("__type", "AttStrong", context.ShouldWriteMetadata),
DataType.Class => GetFromStruct(prop.StructIndex, ref reader, context)?.WithAttribute("__type", "AttClass", context.ShouldWriteMetadata),

DataType.EnumChoice => new XAttribute(prop.GetName(Database), reader.Read<DataCoreStringId>().ToString(Database)),
DataType.Guid => new XAttribute(prop.GetName(Database), reader.Read<CigGuid>().ToString()),
Expand All @@ -103,13 +103,20 @@ private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader rea
};
}

private XElement GetFromReference(DataCoreReference reference, DataCoreExtractionContext context)
private XElement? GetFromReference(DataCoreReference reference, DataCoreExtractionContext context)
{
if (reference.IsInvalid)
{
if (!context.ShouldWriteNulls)
return null;

var invalidNode = new XElement("NullReference");
invalidNode.Add(new XAttribute("__guid", reference.RecordId.ToString()));
invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("guid", reference.RecordId.ToString()));
if (context.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
}

return invalidNode;
}

Expand All @@ -119,59 +126,97 @@ private XElement GetFromReference(DataCoreReference reference, DataCoreExtractio
{
//if we're referencing a full on file, just add a small mention to it
var fileReferenceNode = new XElement("FileReference");
fileReferenceNode.Add(new XAttribute("__guid", reference.RecordId.ToString()));
fileReferenceNode.Add(new XAttribute("__filePath", ComputeRelativePath(record.GetFileName(Database), context.FileName)));
fileReferenceNode.Add(new XAttribute("guid", reference.RecordId.ToString()));
fileReferenceNode.Add(new XAttribute("filePath", ComputeRelativePath(record.GetFileName(Database), context.FileName)));
return fileReferenceNode;
}

return GetFromRecord(record, context);
}

public XElement GetFromRecord(DataCoreRecord record, DataCoreExtractionContext context)
private XElement GetFromRecord(DataCoreRecord record, DataCoreExtractionContext context)
{
var element = GetFromInstance(record.StructIndex, record.InstanceIndex, context);
element.Add(new XAttribute("__recordGuid", record.Id.ToString()));
return element;
return GetFromInstance(record.StructIndex, record.InstanceIndex, context)
.WithAttribute("recordGuid", record.Id.ToString());
}

private XElement GetFromPointer(DataCorePointer pointer, DataCoreExtractionContext context)
public XElement GetFromMainRecord(DataCoreRecord record, DataCoreExtractionContext context)
{
if (pointer.IsInvalid)
if (!Database.MainRecords.Contains(record.Id))
throw new InvalidOperationException("Can only extract main records");

var element = GetFromRecord(record, context);

//add weak pointers ids, so we can actually see what a weak pointer is pointing at
foreach (var weakPtr in context.GetWeakPointers())
{
var invalidNode = new XElement("NullPointer");
invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
return invalidNode;
var pointedAtElement = context.Elements[(weakPtr.structIndex, weakPtr.instanceIndex)];

pointedAtElement.Add(new XAttribute("weakPointerId", context.GetWeakPointerId(weakPtr.structIndex, weakPtr.instanceIndex).ToString(CultureInfo.InvariantCulture)));
}

return element;
}

private XElement? GetFromPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (pointer.IsInvalid)
return GetNullPointer(pointer, context);

return GetFromInstance(pointer.StructIndex, pointer.InstanceIndex, context);
}

private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExtractionContext context)
{
if (context.AlreadyWroteInstance(structIndex, instanceIndex))
{
var instanceReference = new XElement("InstanceReference");
var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]);
var element = GetFromStruct(structIndex, ref reader, context);

instanceReference.Add(new XAttribute("__structName", Database.StructDefinitions[structIndex].GetName(Database)));
instanceReference.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture)));
instanceReference.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture)));
context.Elements[(structIndex, instanceIndex)] = element;

return instanceReference;
if (context.ShouldWriteMetadata)
{
element.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture)));
element.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture)));
}

context.Push(structIndex, instanceIndex);
return element;
}

private XElement? GetWeakPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (pointer.IsInvalid)
return GetNullPointer(pointer, context);

var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]);
var element = GetFromStruct(structIndex, ref reader, context);
var pointerId = context.AddWeakPointer(pointer.StructIndex, pointer.InstanceIndex);

context.Pop();
var invalidNode = new XElement("WeakPointer");

// add some metadata to the element, mostly so we can figure out what a CircularReference is pointing to
element.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture)));
element.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("weakPointerId", pointerId.ToString(CultureInfo.InvariantCulture)));
var structName = Database.StructDefinitions[pointer.StructIndex].GetName(Database);
invalidNode.Add(new XAttribute("structName", structName));

return element;
if (context.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
}

return invalidNode;
}

private static XElement? GetNullPointer(DataCorePointer pointer, DataCoreExtractionContext context)
{
if (!context.ShouldWriteNulls)
return null;

var invalidNode = new XElement("NullPointer");
if (context.ShouldWriteMetadata)
{
invalidNode.Add(new XAttribute("__structIndex", pointer.StructIndex.ToString(CultureInfo.InvariantCulture)));
invalidNode.Add(new XAttribute("__instanceIndex", pointer.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
}

return invalidNode;
}

public static string ComputeRelativePath(string filePath, string contextFileName)
Expand Down
76 changes: 23 additions & 53 deletions src/StarBreaker.DataCore/DataCoreExtractionContext.cs
Original file line number Diff line number Diff line change
@@ -1,69 +1,39 @@
namespace StarBreaker.DataCore;
using System.Runtime.InteropServices;
using System.Xml.Linq;

/// <summary>
/// The strategy to use when resolving repeated references in the data core.
/// </summary>
public enum DataCoreRepeatedReferenceResolutionStrategy
{
/// <summary>
/// Each instance will be written once per file. Subsequent references will be written as a reference to the original instance.
/// </summary>
PerFile,

/// <summary>
/// Each instance will be written once per node. References to the original instance will be used when they are children of the original.
/// Otherwise, the instance will be written again.
/// </summary>
PerNode,
}
namespace StarBreaker.DataCore;

public sealed class DataCoreExtractionContext
{
private readonly HashSet<(int structIndex, int instanceIndex)> _hashSet;
private readonly Stack<(int structIndex, int instanceIndex)> _stack;
private readonly Dictionary<(int structIndex, int instanceIndex), int> _weakPointerIds;
private int _nextWeakPointerId = 0;

public Dictionary<(int structIndex, int instanceIndex), XElement> Elements { get; }

public string FileName { get; }
public DataCoreRepeatedReferenceResolutionStrategy Strategy { get; }
public bool ShouldWriteMetadata { get; }
public bool ShouldWriteNulls { get; }

public DataCoreExtractionContext(string fileName, DataCoreRepeatedReferenceResolutionStrategy strategy)
public DataCoreExtractionContext(string fileName, bool shouldWriteMetadata = false, bool shouldWriteNulls = false)
{
_hashSet = [];
_stack = [];
FileName = fileName;
Strategy = strategy;
ShouldWriteMetadata = shouldWriteMetadata;
ShouldWriteNulls = shouldWriteNulls;
Elements = [];
_weakPointerIds = [];
}

public bool AlreadyWroteInstance(int structIndex, int instanceIndex)
public int AddWeakPointer(int structIndex, int instanceIndex)
{
return Strategy switch
{
DataCoreRepeatedReferenceResolutionStrategy.PerFile => _hashSet.Contains((structIndex, instanceIndex)),
DataCoreRepeatedReferenceResolutionStrategy.PerNode => _stack.Contains((structIndex, instanceIndex)),
_ => throw new ArgumentOutOfRangeException()
};
}
ref var id = ref CollectionsMarshal.GetValueRefOrAddDefault(_weakPointerIds, (structIndex, instanceIndex), out var existed);

public void Push(int structIndex, int instanceIndex)
{
switch (Strategy)
{
case DataCoreRepeatedReferenceResolutionStrategy.PerFile:
_hashSet.Add((structIndex, instanceIndex));
break;
case DataCoreRepeatedReferenceResolutionStrategy.PerNode:
_stack.Push((structIndex, instanceIndex));
break;
default:
throw new ArgumentOutOfRangeException();
}
}

public void Pop()
{
if (Strategy == DataCoreRepeatedReferenceResolutionStrategy.PerNode)
_stack.Pop();
if (!existed)
id = _nextWeakPointerId++;

// Otherwise, do nothing.
return id;
}
}

public int GetWeakPointerId(int structIndex, int instanceIndex) => _weakPointerIds[(structIndex, instanceIndex)];

public IEnumerable<(int structIndex, int instanceIndex)> GetWeakPointers() => _weakPointerIds.Keys;
}
8 changes: 3 additions & 5 deletions src/StarBreaker.DataCore/DataForge.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
using System.IO.Enumeration;
using System.Text;
using System.Xml;
using System.Xml.Linq;

namespace StarBreaker.DataCore;
Expand Down Expand Up @@ -36,9 +34,9 @@ public Dictionary<string, DataCoreRecord> GetRecordsByFileName(string? fileNameF

public XElement GetFromRecord(DataCoreRecord record)
{
//TODO: strategy should be configurable
var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database), DataCoreRepeatedReferenceResolutionStrategy.PerFile);
return DataCore.GetFromRecord(record, context);
//TODO: metadata should be a parameter
var context = new DataCoreExtractionContext(record.GetFileName(DataCore.Database));
return DataCore.GetFromMainRecord(record, context);
}

public Dictionary<string, string[]> ExportEnums()
Expand Down
14 changes: 14 additions & 0 deletions src/StarBreaker.DataCore/XElementExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using System.Xml.Linq;

namespace StarBreaker.DataCore;

public static class XObjectExtensions
{
public static XElement WithAttribute(this XElement xObject, string name, string value, bool write = true)
{
if (write)
xObject.Add(new XAttribute(name, value));

return xObject;
}
}
32 changes: 15 additions & 17 deletions src/StarBreaker.Sandbox/DataCoreSandbox.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,29 @@ public static class DataCoreSandbox
{
public static void Run()
{
var p4k = P4kFile.FromFile(@"C:\Program Files\Roberts Space Industries\StarCitizen\4.0_PREVIEW\Data.p4k");
var p4k = P4kFile.FromFile(@"C:\Program Files\Roberts Space Industries\StarCitizen\PTU\Data.p4k");
var dcbStream = p4k.OpenRead(@"Data\Game2.dcb");
var dcb = new DataForge(dcbStream);
var asdasd = dcb.DataCore.Database.RecordDefinitions.GroupBy(x => x.GetFileName(dcb.DataCore.Database))
.Where(x => x.Count() > 1)
.OrderBy(x => x.Count())
.ToList();

var keys = asdasd.Select(x => $"{x.Key} - {x.Count()}").ToList();
var dump = string.Join("\n", keys);
Directory.CreateDirectory(@"D:\StarCitizen\DataCore\Sandbox");

dcb.ExtractAll(@"D:\StarCitizen\DataCore\Sandbox");

return;

var xxx = dcb.DataCore.Database.RecordDefinitions.Where(x => x.GetFileName(dcb.DataCore.Database).Contains("TagDatabase.Tagdatabase", StringComparison.InvariantCultureIgnoreCase)).ToList();
var yyy = xxx.GroupBy(x => x.InstanceIndex);
var megaMap = dcb.GetRecordsByFileName("*megamap.pu*").Values.Single();
var tagDatabase = dcb.GetRecordsByFileName("*TagDatabase*").Values.Single();
var broker = dcb.GetRecordsByFileName("*missionbroker.pu*").Values.Single();
var unittest = dcb.GetRecordsByFileName("*unittesta*").Values.Single();
var someActorThing = dcb.DataCore.Database.GetRecord(new CigGuid("41d8fb15-72bb-446e-81df-eaecbc01e195"));

dcb.GetFromRecord(broker).Save(@"D:\broker.xml");
dcb.GetFromRecord(unittest).Save(@"D:\unittesta.xml");
dcb.GetFromRecord(someActorThing).Save(@"D:\someActorThing.xml");
dcb.GetFromRecord(tagDatabase).Save(@"D:\tagdatabase.xml");
dcb.GetFromRecord(megaMap).Save(@"D:\megamap.xml");
//var someActorThing = dcb.DataCore.Database.GetRecord(new CigGuid("41d8fb15-72bb-446e-81df-eaecbc01e195"));
var zeroggraph = dcb.GetRecordsByFileName("*playerzerogtraversalgraph*").Values.Single();

dcb.ExtractAllParallel(@"D:\StarCitizen\DataCoreExport");
dcb.GetFromRecord(zeroggraph).Save(@"D:\StarCitizen\DataCore\Sandbox\zeroggraph.xml");
dcb.GetFromRecord(broker).Save(@"D:\StarCitizen\DataCore\Sandbox\broker.xml");
dcb.GetFromRecord(unittest).Save(@"D:\StarCitizen\DataCore\Sandbox\unittesta.xml");
//dcb.GetFromRecord(someActorThing).Save(@"D:\StarCitizen\DataCore\Sandbox\someActorThing.xml");
dcb.GetFromRecord(tagDatabase).Save(@"D:\StarCitizen\DataCore\Sandbox\tagdatabase.xml");
dcb.GetFromRecord(megaMap).Save(@"D:\StarCitizen\DataCore\Sandbox\megamap.xml");
//
}
}

0 comments on commit cd9fe61

Please sign in to comment.