diff --git a/docs/dson.md b/docs/dson.md index 95f47b8..7bdd855 100644 --- a/docs/dson.md +++ b/docs/dson.md @@ -91,7 +91,7 @@ FieldInfo contains several types of information: -----|-----|-----|-----|----- numMeta2Entries|??|Field|fields|fields in canonical order -Note: The size of a field is not sepcified. A heuristic that worked for this application was to find the next biggest offset in Meta2. +Note: The size of a field is not specified. A heuristic that worked for this application was to find the next biggest offset in Meta2. The next biggest offset is not necessarily the offset of the next Meta2 Entry. I assume that the data size isn't needed when the runtime already knows the data size of each field. ### Field @@ -99,7 +99,7 @@ The next biggest offset is not necessarily the offset of the next Meta2 Entry. I **Size (bytes)**|**Type**|**Name**|**Description** -----|-----|-----|----- ??|String|name|field name including `\0` character. -0-3|Raw|alignment|Optional empty bits for aligment depending on type +0-3|Raw|alignment|Optional empty bits for alignment depending on type ??|Raw|data|field data Field data may be 4-byte aligned, depending on the type. @@ -118,11 +118,14 @@ Int|Yes|4|4-byte Integer Float|Yes|4|4-byte Float IntVector|Yes|4+(4*n)|4-byte integer count, then [count] 4-byte integers StringVector|Yes|4+((?\_n)*n)|4-byte count, then [count] string length + null-terminated string -FloatArray|Yes|4*n|arbitary number of 4-byte floats +FloatArray|Yes|4*n|Arbitrary number of 4-byte floats +TwoInt|Yes|8|Two 4-byte integers Notes: * Files are Strings that can be deserialized as another Dson File. They can be identified with their Magic Number (see Header). -* Types can generally not be inferred. DsonField.java and DsonTypes.java contain an approach to efficiently identify the field type nonetheless. This approach hardcodes `FloatArray`, `StringVector`, `IntVector` and `Float` field names, and identifies the other types using a heuristic involving the data size. +* Types can generally not be inferred. DsonField.java and DsonTypes.java contain an approach to efficiently identify the field type nonetheless. This approach hard-codes `FloatArray`, `StringVector`, `IntVector` and `Float` field names, and identifies the other types using a heuristic involving the data size. +* Some files contain duplicate fields within the same object. This implementation ignores them, resulting in a different file size + when re-encoded. The object structure is defined by the order of the fields in data. Beginning with a root object, fields are read in. When an object is encountered, this object is pushed onto the stack. Parsed fields are added to the object on top of the object stack until the object on top of the stack has all its child fields, then, the elements which have all of their child fields are popped from the stack. This is similar to most other structured data formats with the exception that there is no "end object" token. \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2d80b69..f4d7b2b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/de/robojumper/ddsavereader/file/DsonField.java b/src/main/java/de/robojumper/ddsavereader/file/DsonField.java index 00f017c..8ee2f97 100644 --- a/src/main/java/de/robojumper/ddsavereader/file/DsonField.java +++ b/src/main/java/de/robojumper/ddsavereader/file/DsonField.java @@ -92,7 +92,20 @@ public boolean guessType(UnhashBehavior behavior) throws ParseException { } private boolean parseHardcodedType(UnhashBehavior behavior) { - return parseFloatArray() || parseIntVector(behavior) || parseStringVector() || parseFloat(); + return parseFloatArray() || parseIntVector(behavior) || parseStringVector() || parseFloat() || parseTwoInt(); + } + + private boolean parseTwoInt() { + if (DsonTypes.isA(FieldType.TYPE_TWOINT, this::nameIterator)) { + if (alignedSize() == 8) { + type = FieldType.TYPE_TWOINT; + byte[] tempArr = Arrays.copyOfRange(rawData, alignmentSkip(), alignmentSkip() + 8); + ByteBuffer buf = ByteBuffer.wrap(tempArr).order(ByteOrder.LITTLE_ENDIAN); + dataString = "[" + Integer.toString(buf.getInt()) + ", " + Integer.toString(buf.getInt()) + "]"; + return true; + } + } + return false; } private boolean parseFloat() { @@ -126,7 +139,7 @@ private boolean parseStringVector() { bf.position(bf.position() + strlen); if (i < arrLen - 1) { // Skip for alignment, but only if we have things following - bf.position(bf.position() + ((4 - (bf.position() % 4)) % 4)); + bf.position(bf.position() + ((4 - (bf.position() % 4)) % 4)); sb.append(", "); } } @@ -282,10 +295,11 @@ public String getExtraComments() { } return sb.toString(); } - + private Iterator nameIterator() { return new Iterator() { private DsonField field = DsonField.this; + @Override public boolean hasNext() { return field != null; @@ -296,6 +310,7 @@ public String next() { String f = field.name; field = field.parent; return f; - }}; + } + }; } } diff --git a/src/main/java/de/robojumper/ddsavereader/file/DsonFile.java b/src/main/java/de/robojumper/ddsavereader/file/DsonFile.java index 89d515f..98b2053 100644 --- a/src/main/java/de/robojumper/ddsavereader/file/DsonFile.java +++ b/src/main/java/de/robojumper/ddsavereader/file/DsonFile.java @@ -14,11 +14,12 @@ import de.robojumper.ddsavereader.file.DsonFile.Meta2Block.Meta2BlockEntry; public class DsonFile { - + public enum UnhashBehavior { NONE, // Don't unhash, works in all cases UNHASH, // Simple unhash, useful for simply looking at the files - POUNDUNHASH, // Unhash as ###string, useful combination: Reasonable safety against accidental collisions, still somewhat readable + POUNDUNHASH, // Unhash as ###string, useful combination: Reasonable safety against accidental + // collisions, still somewhat readable }; private final static char[] hexArray = "0123456789ABCDEF".toCharArray(); @@ -355,6 +356,39 @@ public String toString() { return getJSonString(0, false); } + // Whether this File has duplicate fields that will get lost when converting to + // string + // This doesn't seem to be causing any issues, but is important for test + // coverage because + // files with duplicate fields will re-encode to a different size. + public boolean hasDuplicateFields() { + Set fields = new HashSet<>(); + for (int i = 0; i < rootFields.size(); i++) { + if (!fields.add(rootFields.get(i).name)) { + return true; + } + if (hasDuplicateFields(rootFields.get(i))) { + return true; + } + } + return false; + } + + private boolean hasDuplicateFields(DsonField field) { + Set fields = new HashSet<>(); + if (field.type == FieldType.TYPE_OBJECT) { + for (int i = 0; i < field.children.length; i++) { + if (!fields.add(field.children[i].name)) { + return true; + } + if (hasDuplicateFields(field.children[i])) { + return true; + } + } + } + return false; + } + private void writeField(StringBuilder sb, DsonField field, int indent, boolean debug) { if (debug) { @@ -383,8 +417,9 @@ private void writeObject(StringBuilder sb, DsonField field, int indent, boolean indent++; Set emittedFields = new HashSet<>(); for (int i = 0; i < field.children.length; i++) { - // DD has a quirk in one file where one object pops up twice (serialized twice?) - // This is not valid JSON and removing it doesn't cause any issues, so let's just remove it here + // DD has a quirk in a few files where fields wind up twice (serialized twice?) + // This is not valid JSON and removing it doesn't cause any issues, so let's + // just remove it here if (!emittedFields.contains(field.children[i].name)) { writeField(sb, field.children[i], indent, debug); emittedFields.add(field.children[i].name); diff --git a/src/main/java/de/robojumper/ddsavereader/file/DsonTypes.java b/src/main/java/de/robojumper/ddsavereader/file/DsonTypes.java index 4fb16ac..69254f7 100644 --- a/src/main/java/de/robojumper/ddsavereader/file/DsonTypes.java +++ b/src/main/java/de/robojumper/ddsavereader/file/DsonTypes.java @@ -16,7 +16,7 @@ public enum FieldType { TYPE_TWOBOOL, // aligned, 8 bytes (only used in gameplay options??). emitted as [true, true] TYPE_STRING, // aligned, int size + null-terminated string of size (including \0) TYPE_FILE, // Actually an object, but encoded as a string (embedded DsonFile). used in - // roster.json and map.json + // roster.json and map.json TYPE_INT, // aligned, 4 byte integer // Begin hardcoded types: these types do not have enough characteristics to make // the heuristic work @@ -37,6 +37,9 @@ public enum FieldType { { "additional_mash_disabled_infestation_monster_class_ids" }, // campaign_mash.json { "party", "heroes" }, // raid.json { "skill_cooldown_keys" }, // raid.json + { "bufferedSpawningSlotsAvailable" }, // raid.json + { "curioGroups", "*", "curios" }, // raid.json + { "curioGroups", "*", "curio_table_entries" }, // raid.json { "narration_audio_event_queue_tags" }, // loading_screen.json { "dispatched_events" }, // tutorial.json }), @@ -45,11 +48,15 @@ public enum FieldType { TYPE_STRINGVECTOR(new String[][] { { "goal_ids" }, // quest.json { "roaming_dungeon_2_ids", "*", "s" }, // campaign_mash.json { "quirk_group" }, // campaign_log.json + { "backgroundNames" }, // raid.json + { "backgroundGroups", "*", "backgrounds" }, // raid.json + { "backgroundGroups", "*", "background_table_entries" }, // raid.json }), // aligned, arbitrary number of 4-byte floats. emitted as [1.0, 2.0, ...] TYPE_FLOATARRAY(new String[][] { { "map", "bounds" }, { "areas", "*", "bounds" }, { "areas", "*", "tiles", "*", "mappos" }, { "areas", "*", "tiles", "*", "sidepos" }, // map.json }), + TYPE_TWOINT(new String[][] { { "killRange" } }), // raid.json // Unknown Type TYPE_UNKNOWN; diff --git a/src/main/java/de/robojumper/ddsavereader/file/DsonWriter.java b/src/main/java/de/robojumper/ddsavereader/file/DsonWriter.java index c945d61..9a3e17b 100644 --- a/src/main/java/de/robojumper/ddsavereader/file/DsonWriter.java +++ b/src/main/java/de/robojumper/ddsavereader/file/DsonWriter.java @@ -91,7 +91,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars e2.offset = data.size(); data.write(name.getBytes(StandardCharsets.UTF_8)); data.write(0); - + try { reader.nextToken(); if (reader.getCurrentToken() == JsonToken.START_OBJECT) { @@ -115,8 +115,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars } if (reader.getCurrentToken() != JsonToken.END_OBJECT) { - throw new ParseException("Expected }", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected }", (int) reader.getCurrentLocation().getCharOffset()); } e1.numDirectChildren = numDirectChildren; @@ -134,13 +133,13 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars } else { // Now for the tricky part: Not an object, now we need to determine the type // Same as in DsonField, we first check the hardcoded types - // In order to easily use the nameStack's iterator, we temporarily push the field name + // In order to easily use the nameStack's iterator, we temporarily push the + // field name nameStack.push(name); if (DsonTypes.isA(FieldType.TYPE_FLOATARRAY, nameStack::iterator)) { align(); if (reader.getCurrentToken() != JsonToken.START_ARRAY) { - throw new ParseException("Expected [", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset()); } while (reader.nextToken() == JsonToken.VALUE_NUMBER_FLOAT) { data.write(floatBytes(reader.getFloatValue())); @@ -152,8 +151,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars } else if (DsonTypes.isA(FieldType.TYPE_INTVECTOR, nameStack::iterator)) { align(); if (reader.getCurrentToken() != JsonToken.START_ARRAY) { - throw new ParseException("Expected [", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset()); } ByteArrayOutputStream vecData = new ByteArrayOutputStream(); int numElem = 0; @@ -179,8 +177,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars } else if (DsonTypes.isA(FieldType.TYPE_STRINGVECTOR, nameStack::iterator)) { align(); if (reader.getCurrentToken() != JsonToken.START_ARRAY) { - throw new ParseException("Expected [", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset()); } ByteArrayOutputStream vecData = new ByteArrayOutputStream(); int numElem = 0; @@ -198,10 +195,23 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars } else if (DsonTypes.isA(FieldType.TYPE_FLOAT, nameStack::iterator)) { align(); if (reader.getCurrentToken() != JsonToken.VALUE_NUMBER_FLOAT) { - throw new ParseException("Expected number", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected number", (int) reader.getCurrentLocation().getCharOffset()); } data.write(floatBytes(reader.getFloatValue())); + } else if (DsonTypes.isA(FieldType.TYPE_TWOINT, nameStack::iterator)) { + align(); + if (reader.getCurrentToken() != JsonToken.START_ARRAY) { + throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset()); + } + for (int i = 0; i < 2; i++) { + if (reader.nextToken() != JsonToken.VALUE_NUMBER_INT) { + throw new ParseException("Expected int", (int) reader.getCurrentLocation().getCharOffset()); + } + data.write(intBytes(reader.getIntValue())); + } + if (reader.nextToken() != JsonToken.END_ARRAY) { + throw new ParseException("Expected ]", (int) reader.getCurrentLocation().getCharOffset()); + } } else if (DsonTypes.isA(FieldType.TYPE_CHAR, nameStack::iterator)) { if (reader.getCurrentToken() != JsonToken.VALUE_STRING) { throw new ParseException( @@ -223,13 +233,13 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars data.write(ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN) .putInt(reader.getBooleanValue() ? 1 : 0).array()); } else { - throw new ParseException("Field type not identified, only expecting\"true\" or \"false\" in arrays", + throw new ParseException( + "Field type not identified, only expecting\"true\" or \"false\" in arrays", (int) reader.getCurrentLocation().getCharOffset()); } } if (reader.nextToken() != JsonToken.END_ARRAY) { - throw new ParseException("Expected ]", - (int) reader.getCurrentLocation().getCharOffset()); + throw new ParseException("Expected ]", (int) reader.getCurrentLocation().getCharOffset()); } } else if (reader.getCurrentToken() == JsonToken.VALUE_TRUE || reader.getCurrentToken() == JsonToken.VALUE_FALSE) { diff --git a/src/test/java/de/robojumper/ddsavereader/file/ConverterTests.java b/src/test/java/de/robojumper/ddsavereader/file/ConverterTests.java index ac4b620..1097caf 100644 --- a/src/test/java/de/robojumper/ddsavereader/file/ConverterTests.java +++ b/src/test/java/de/robojumper/ddsavereader/file/ConverterTests.java @@ -10,7 +10,9 @@ import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -44,6 +46,7 @@ public void readNames() throws IOException { public void testCorrectConversion(String folderName) throws ParseException, IOException { List files = new ArrayList<>(); + Set dupeFieldFiles = new HashSet<>(); List decodedFiles = new ArrayList<>(); List reEncodedFiles = new ArrayList<>(); @@ -59,8 +62,11 @@ public void testCorrectConversion(String folderName) throws ParseException, IOEx // Every file must decode without throwing exceptions for (int i = 0; i < files.size(); i++) { try { - String file = new DsonFile(files.get(i), UnhashBehavior.POUNDUNHASH).getJSonString(0, false); - decodedFiles.add(file.getBytes(StandardCharsets.UTF_8)); + DsonFile file = new DsonFile(files.get(i), UnhashBehavior.POUNDUNHASH); + if (file.hasDuplicateFields()) { + dupeFieldFiles.add(i); + } + decodedFiles.add(file.getJSonString(0, false).getBytes(StandardCharsets.UTF_8)); } catch (Exception e) { fail(fileList.get(i) + " doesn't decode", e); } @@ -73,8 +79,9 @@ public void testCorrectConversion(String folderName) throws ParseException, IOEx } catch (Exception e) { fail(fileList.get(i) + " doesn't re-endode", e); } - // Weird quirk - if (!fileList.get(i).equals("persist.progression.json")) { + // Files with duplicate fields will not have the same size anyway. + // Filter them out here + if (!dupeFieldFiles.contains(i)) { assertEquals(reEncodedFiles.get(i).length, files.get(i).length, fileList.get(i) + " encodes to different number of bytes"); } @@ -106,6 +113,7 @@ public void testRedditProfile() throws ParseException, IOException { @Test public void testOtherFiles() throws ParseException, IOException { testCorrectConversion("otherFiles"); + testCorrectConversion("backgroundNames"); } private List getResourceFiles(String path) throws IOException { diff --git a/src/test/resources/backgroundNames/persist.raid.json b/src/test/resources/backgroundNames/persist.raid.json new file mode 100644 index 0000000..df261fa Binary files /dev/null and b/src/test/resources/backgroundNames/persist.raid.json differ