Add support for a few new fields (#10)

Add support for a few new fields

docs/dson.md

@@ -91,15 +91,15 @@ FieldInfo contains several types of information:
 -----|-----|-----|-----|-----
 numMeta2Entries|??|Field|fields|fields in canonical order 
 
-Note: The size of a field is not sepcified. A heuristic that worked for this application was to find the next biggest offset in Meta2.  
+Note: The size of a field is not specified. A heuristic that worked for this application was to find the next biggest offset in Meta2.  
 The next biggest offset is not necessarily the offset of the next Meta2 Entry. I assume that the data size isn't needed when the runtime already knows the data size of each field.
 
 ### Field
 
 **Size (bytes)**|**Type**|**Name**|**Description**
 -----|-----|-----|-----
 ??|String|name|field name including `\0` character.
-0-3|Raw|alignment|Optional empty bits for aligment depending on type
+0-3|Raw|alignment|Optional empty bits for alignment depending on type
 ??|Raw|data|field data
 
 Field data may be 4-byte aligned, depending on the type.
@@ -118,11 +118,14 @@ Int|Yes|4|4-byte Integer
 Float|Yes|4|4-byte Float
 IntVector|Yes|4+(4*n)|4-byte integer count, then [count] 4-byte integers
 StringVector|Yes|4+((?\_n)*n)|4-byte count, then [count] string length + null-terminated string
-FloatArray|Yes|4*n|arbitary number of 4-byte floats
+FloatArray|Yes|4*n|Arbitrary number of 4-byte floats
+TwoInt|Yes|8|Two 4-byte integers
 
 Notes:
 
 * Files are Strings that can be deserialized as another Dson File. They can be identified with their Magic Number (see Header).
-* Types can generally not be inferred. DsonField.java and DsonTypes.java contain an approach to efficiently identify the field type nonetheless. This approach hardcodes `FloatArray`, `StringVector`, `IntVector` and `Float` field names, and identifies the other types using a heuristic involving the data size.
+* Types can generally not be inferred. DsonField.java and DsonTypes.java contain an approach to efficiently identify the field type nonetheless. This approach hard-codes `FloatArray`, `StringVector`, `IntVector` and `Float` field names, and identifies the other types using a heuristic involving the data size.
+* Some files contain duplicate fields within the same object. This implementation ignores them, resulting in a different file size
+  when re-encoded.
 
 The object structure is defined by the order of the fields in data. Beginning with a root object, fields are read in. When an object is encountered, this object is pushed onto the stack. Parsed fields are added to the object on top of the object stack until the object on top of the stack has all its child fields, then, the elements which have all of their child fields are popped from the stack. This is similar to most other structured data formats with the exception that there is no "end object" token.

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/de/robojumper/ddsavereader/file/DsonField.java

@@ -92,7 +92,20 @@ public boolean guessType(UnhashBehavior behavior) throws ParseException {
     }
 
     private boolean parseHardcodedType(UnhashBehavior behavior) {
-        return parseFloatArray() || parseIntVector(behavior) || parseStringVector() || parseFloat();
+        return parseFloatArray() || parseIntVector(behavior) || parseStringVector() || parseFloat() || parseTwoInt();
+    }
+
+    private boolean parseTwoInt() {
+        if (DsonTypes.isA(FieldType.TYPE_TWOINT, this::nameIterator)) {
+            if (alignedSize() == 8) {
+                type = FieldType.TYPE_TWOINT;
+                byte[] tempArr = Arrays.copyOfRange(rawData, alignmentSkip(), alignmentSkip() + 8);
+                ByteBuffer buf = ByteBuffer.wrap(tempArr).order(ByteOrder.LITTLE_ENDIAN);
+                dataString = "[" + Integer.toString(buf.getInt()) + ", " + Integer.toString(buf.getInt()) + "]";
+                return true;
+            }
+        }
+        return false;
     }
 
     private boolean parseFloat() {
@@ -126,7 +139,7 @@ private boolean parseStringVector() {
                 bf.position(bf.position() + strlen);
                 if (i < arrLen - 1) {
                     // Skip for alignment, but only if we have things following
-                    bf.position(bf.position() + ((4 - (bf.position()  % 4)) % 4));
+                    bf.position(bf.position() + ((4 - (bf.position() % 4)) % 4));
                     sb.append(", ");
                 }
             }
@@ -282,10 +295,11 @@ public String getExtraComments() {
         }
         return sb.toString();
     }
-    
+
     private Iterator<String> nameIterator() {
         return new Iterator<String>() {
             private DsonField field = DsonField.this;
+
             @Override
             public boolean hasNext() {
                 return field != null;
@@ -296,6 +310,7 @@ public String next() {
                 String f = field.name;
                 field = field.parent;
                 return f;
-            }};
+            }
+        };
     }
 }

src/main/java/de/robojumper/ddsavereader/file/DsonFile.java

@@ -14,11 +14,12 @@
 import de.robojumper.ddsavereader.file.DsonFile.Meta2Block.Meta2BlockEntry;
 
 public class DsonFile {
-    
+
     public enum UnhashBehavior {
         NONE, // Don't unhash, works in all cases
         UNHASH, // Simple unhash, useful for simply looking at the files
-        POUNDUNHASH, // Unhash as ###string, useful combination: Reasonable safety against accidental collisions, still somewhat readable
+        POUNDUNHASH, // Unhash as ###string, useful combination: Reasonable safety against accidental
+                        // collisions, still somewhat readable
     };
 
     private final static char[] hexArray = "0123456789ABCDEF".toCharArray();
@@ -355,6 +356,39 @@ public String toString() {
         return getJSonString(0, false);
     }
 
+    // Whether this File has duplicate fields that will get lost when converting to
+    // string
+    // This doesn't seem to be causing any issues, but is important for test
+    // coverage because
+    // files with duplicate fields will re-encode to a different size.
+    public boolean hasDuplicateFields() {
+        Set<String> fields = new HashSet<>();
+        for (int i = 0; i < rootFields.size(); i++) {
+            if (!fields.add(rootFields.get(i).name)) {
+                return true;
+            }
+            if (hasDuplicateFields(rootFields.get(i))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean hasDuplicateFields(DsonField field) {
+        Set<String> fields = new HashSet<>();
+        if (field.type == FieldType.TYPE_OBJECT) {
+            for (int i = 0; i < field.children.length; i++) {
+                if (!fields.add(field.children[i].name)) {
+                    return true;
+                }
+                if (hasDuplicateFields(field.children[i])) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     private void writeField(StringBuilder sb, DsonField field, int indent, boolean debug) {
 
         if (debug) {
@@ -383,8 +417,9 @@ private void writeObject(StringBuilder sb, DsonField field, int indent, boolean
             indent++;
             Set<String> emittedFields = new HashSet<>();
             for (int i = 0; i < field.children.length; i++) {
-                // DD has a quirk in one file where one object pops up twice (serialized twice?)
-                // This is not valid JSON and removing it doesn't cause any issues, so let's just remove it here
+                // DD has a quirk in a few files where fields wind up twice (serialized twice?)
+                // This is not valid JSON and removing it doesn't cause any issues, so let's
+                // just remove it here
                 if (!emittedFields.contains(field.children[i].name)) {
                     writeField(sb, field.children[i], indent, debug);
                     emittedFields.add(field.children[i].name);

src/main/java/de/robojumper/ddsavereader/file/DsonTypes.java

@@ -16,7 +16,7 @@ public enum FieldType {
         TYPE_TWOBOOL, // aligned, 8 bytes (only used in gameplay options??). emitted as [true, true]
         TYPE_STRING, // aligned, int size + null-terminated string of size (including \0)
         TYPE_FILE, // Actually an object, but encoded as a string (embedded DsonFile). used in
-                   // roster.json and map.json
+                    // roster.json and map.json
         TYPE_INT, // aligned, 4 byte integer
         // Begin hardcoded types: these types do not have enough characteristics to make
         // the heuristic work
@@ -37,6 +37,9 @@ public enum FieldType {
                         { "additional_mash_disabled_infestation_monster_class_ids" }, // campaign_mash.json
                         { "party", "heroes" }, // raid.json
                         { "skill_cooldown_keys" }, // raid.json
+                        { "bufferedSpawningSlotsAvailable" }, // raid.json
+                        { "curioGroups", "*", "curios" }, // raid.json
+                        { "curioGroups", "*", "curio_table_entries" }, // raid.json
                         { "narration_audio_event_queue_tags" }, // loading_screen.json
                         { "dispatched_events" }, // tutorial.json
                 }),
@@ -45,11 +48,15 @@ public enum FieldType {
         TYPE_STRINGVECTOR(new String[][] { { "goal_ids" }, // quest.json
                 { "roaming_dungeon_2_ids", "*", "s" }, // campaign_mash.json
                 { "quirk_group" }, // campaign_log.json
+                { "backgroundNames" }, // raid.json
+                { "backgroundGroups", "*", "backgrounds" }, // raid.json
+                { "backgroundGroups", "*", "background_table_entries" }, // raid.json
         }),
         // aligned, arbitrary number of 4-byte floats. emitted as [1.0, 2.0, ...]
         TYPE_FLOATARRAY(new String[][] { { "map", "bounds" }, { "areas", "*", "bounds" },
                 { "areas", "*", "tiles", "*", "mappos" }, { "areas", "*", "tiles", "*", "sidepos" }, // map.json
         }),
+        TYPE_TWOINT(new String[][] { { "killRange" } }), // raid.json
         // Unknown Type
         TYPE_UNKNOWN;
 

src/main/java/de/robojumper/ddsavereader/file/DsonWriter.java

@@ -91,7 +91,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
         e2.offset = data.size();
         data.write(name.getBytes(StandardCharsets.UTF_8));
         data.write(0);
- 
+
         try {
             reader.nextToken();
             if (reader.getCurrentToken() == JsonToken.START_OBJECT) {
@@ -115,8 +115,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
                     }
 
                     if (reader.getCurrentToken() != JsonToken.END_OBJECT) {
-                        throw new ParseException("Expected }",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected }", (int) reader.getCurrentLocation().getCharOffset());
                     }
                     e1.numDirectChildren = numDirectChildren;
 
@@ -134,13 +133,13 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
             } else {
                 // Now for the tricky part: Not an object, now we need to determine the type
                 // Same as in DsonField, we first check the hardcoded types
-                // In order to easily use the nameStack's iterator, we temporarily push the field name
+                // In order to easily use the nameStack's iterator, we temporarily push the
+                // field name
                 nameStack.push(name);
                 if (DsonTypes.isA(FieldType.TYPE_FLOATARRAY, nameStack::iterator)) {
                     align();
                     if (reader.getCurrentToken() != JsonToken.START_ARRAY) {
-                        throw new ParseException("Expected [",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset());
                     }
                     while (reader.nextToken() == JsonToken.VALUE_NUMBER_FLOAT) {
                         data.write(floatBytes(reader.getFloatValue()));
@@ -152,8 +151,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
                 } else if (DsonTypes.isA(FieldType.TYPE_INTVECTOR, nameStack::iterator)) {
                     align();
                     if (reader.getCurrentToken() != JsonToken.START_ARRAY) {
-                        throw new ParseException("Expected [",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset());
                     }
                     ByteArrayOutputStream vecData = new ByteArrayOutputStream();
                     int numElem = 0;
@@ -179,8 +177,7 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
                 } else if (DsonTypes.isA(FieldType.TYPE_STRINGVECTOR, nameStack::iterator)) {
                     align();
                     if (reader.getCurrentToken() != JsonToken.START_ARRAY) {
-                        throw new ParseException("Expected [",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset());
                     }
                     ByteArrayOutputStream vecData = new ByteArrayOutputStream();
                     int numElem = 0;
@@ -198,10 +195,23 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
                 } else if (DsonTypes.isA(FieldType.TYPE_FLOAT, nameStack::iterator)) {
                     align();
                     if (reader.getCurrentToken() != JsonToken.VALUE_NUMBER_FLOAT) {
-                        throw new ParseException("Expected number",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected number", (int) reader.getCurrentLocation().getCharOffset());
                     }
                     data.write(floatBytes(reader.getFloatValue()));
+                } else if (DsonTypes.isA(FieldType.TYPE_TWOINT, nameStack::iterator)) {
+                    align();
+                    if (reader.getCurrentToken() != JsonToken.START_ARRAY) {
+                        throw new ParseException("Expected [", (int) reader.getCurrentLocation().getCharOffset());
+                    }
+                    for (int i = 0; i < 2; i++) {
+                        if (reader.nextToken() != JsonToken.VALUE_NUMBER_INT) {
+                            throw new ParseException("Expected int", (int) reader.getCurrentLocation().getCharOffset());
+                        }
+                        data.write(intBytes(reader.getIntValue()));
+                    }
+                    if (reader.nextToken() != JsonToken.END_ARRAY) {
+                        throw new ParseException("Expected ]", (int) reader.getCurrentLocation().getCharOffset());
+                    }
                 } else if (DsonTypes.isA(FieldType.TYPE_CHAR, nameStack::iterator)) {
                     if (reader.getCurrentToken() != JsonToken.VALUE_STRING) {
                         throw new ParseException(
@@ -223,13 +233,13 @@ private void writeField(String name, JsonParser reader) throws IOException, Pars
                             data.write(ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN)
                                     .putInt(reader.getBooleanValue() ? 1 : 0).array());
                         } else {
-                            throw new ParseException("Field type not identified, only expecting\"true\" or \"false\" in arrays",
+                            throw new ParseException(
+                                    "Field type not identified, only expecting\"true\" or \"false\" in arrays",
                                     (int) reader.getCurrentLocation().getCharOffset());
                         }
                     }
                     if (reader.nextToken() != JsonToken.END_ARRAY) {
-                        throw new ParseException("Expected ]",
-                                (int) reader.getCurrentLocation().getCharOffset());
+                        throw new ParseException("Expected ]", (int) reader.getCurrentLocation().getCharOffset());
                     }
                 } else if (reader.getCurrentToken() == JsonToken.VALUE_TRUE
                         || reader.getCurrentToken() == JsonToken.VALUE_FALSE) {

src/test/java/de/robojumper/ddsavereader/file/ConverterTests.java

@@ -10,7 +10,9 @@
 import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -44,6 +46,7 @@ public void readNames() throws IOException {
 
     public void testCorrectConversion(String folderName) throws ParseException, IOException {
         List<byte[]> files = new ArrayList<>();
+        Set<Integer> dupeFieldFiles = new HashSet<>();
         List<byte[]> decodedFiles = new ArrayList<>();
         List<byte[]> reEncodedFiles = new ArrayList<>();
 
@@ -59,8 +62,11 @@ public void testCorrectConversion(String folderName) throws ParseException, IOEx
         // Every file must decode without throwing exceptions
         for (int i = 0; i < files.size(); i++) {
             try {
-                String file = new DsonFile(files.get(i), UnhashBehavior.POUNDUNHASH).getJSonString(0, false);
-                decodedFiles.add(file.getBytes(StandardCharsets.UTF_8));
+                DsonFile file = new DsonFile(files.get(i), UnhashBehavior.POUNDUNHASH);
+                if (file.hasDuplicateFields()) {
+                    dupeFieldFiles.add(i);
+                }
+                decodedFiles.add(file.getJSonString(0, false).getBytes(StandardCharsets.UTF_8));
             } catch (Exception e) {
                 fail(fileList.get(i) + " doesn't decode", e);
             }
@@ -73,8 +79,9 @@ public void testCorrectConversion(String folderName) throws ParseException, IOEx
             } catch (Exception e) {
                 fail(fileList.get(i) + " doesn't re-endode", e);
             }
-            // Weird quirk
-            if (!fileList.get(i).equals("persist.progression.json")) {
+            // Files with duplicate fields will not have the same size anyway.
+            // Filter them out here
+            if (!dupeFieldFiles.contains(i)) {
                 assertEquals(reEncodedFiles.get(i).length, files.get(i).length,
                         fileList.get(i) + " encodes to different number of bytes");
             }
@@ -106,6 +113,7 @@ public void testRedditProfile() throws ParseException, IOException {
     @Test
     public void testOtherFiles() throws ParseException, IOException {
         testCorrectConversion("otherFiles");
+        testCorrectConversion("backgroundNames");
     }
 
     private List<String> getResourceFiles(String path) throws IOException {