Add deserialization support for new binary VDF

The new Steam beta introduced a new `appinfo.vdf` version. This
appinfo.vdf V29 introduces a new binary VDF format which does not
include field keys in binary VDF segments as-is. Instead, each key is
represented by a 32-bit integer which needs to be mapped to an actual
string using a table stored at the end of the `appinfo.vdf` file.

This also means the binary VDF segments in this case are no longer
self-contained. The developer can parse and provide the table
themselves or instead use the `ValvePython/steam` library which
contains a function to parse `appinfo.vdf` files.

Also see SteamDatabase/SteamAppInfo#56b1fec7f5ce6be961c3e44cf9baf117e363ad91

Refs ValvePython/steam#462

tests/test_binary_vdf.py

@@ -166,6 +166,12 @@ def test_raise_on_remaining_with_file(self):
             vdf.binary_load(buf, raise_on_remaining=True)
         self.assertEqual(buf.read(), b'aaaa')
 
+    def test_key_table(self):
+        test = b'\x01\x00\x00\x00\x00value\x00\x01\x02\x00\x00\x00value3\x00\x08'
+        key_table = ['key', 'key2', 'key3', 'key4']
+
+        self.assertEqual({'key': 'value', 'key3': 'value3'}, vdf.binary_loads(test, key_table=key_table))
+
     def test_vbkv_loads_empty(self):
         with self.assertRaises(ValueError):
             vdf.vbkv_loads(b'')

vdf/__init__.py

@@ -295,7 +295,7 @@ class COLOR(BASE_INT):
 BIN_INT64       = b'\x0A'
 BIN_END_ALT     = b'\x0B'
 
-def binary_loads(b, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=True):
+def binary_loads(b, mapper=dict, merge_duplicate_keys=True, alt_format=False, key_table=None, raise_on_remaining=True):
     """
     Deserialize ``b`` (``bytes`` containing a VDF in "binary form")
     to a Python object.
@@ -307,13 +307,18 @@ def binary_loads(b, mapper=dict, merge_duplicate_keys=True, alt_format=False, ra
     ``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
     same key into one instead of overwriting. You can se this to ``False`` if you are
     using ``VDFDict`` and need to preserve the duplicates.
+
+    ``key_table`` will be used to translate keys in binary VDF objects
+    which do not encode strings directly but instead store them in an out-of-band
+    table. Newer `appinfo.vdf` format stores this table the end of the file,
+    and it is needed to deserialize the binary VDF objects in that file.
     """
     if not isinstance(b, bytes):
         raise TypeError("Expected s to be bytes, got %s" % type(b))
 
-    return binary_load(BytesIO(b), mapper, merge_duplicate_keys, alt_format, raise_on_remaining)
+    return binary_load(BytesIO(b), mapper, merge_duplicate_keys, alt_format, key_table, raise_on_remaining)
 
-def binary_load(fp, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=False):
+def binary_load(fp, mapper=dict, merge_duplicate_keys=True, alt_format=False, key_table=None, raise_on_remaining=False):
     """
     Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
     binary VDF) to a Python object.
@@ -325,6 +330,11 @@ def binary_load(fp, mapper=dict, merge_duplicate_keys=True, alt_format=False, ra
     ``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
     same key into one instead of overwriting. You can se this to ``False`` if you are
     using ``VDFDict`` and need to preserve the duplicates.
+
+    ``key_table`` will be used to translate keys in binary VDF objects
+    which do not encode strings directly but instead store them in an out-of-band
+    table. Newer `appinfo.vdf` format stores this table the end of the file,
+    and it is needed to deserialize the binary VDF objects in that file.
     """
     if not hasattr(fp, 'read') or not hasattr(fp, 'tell') or not hasattr(fp, 'seek'):
         raise TypeError("Expected fp to be a file-like object with tell()/seek() and read() returning bytes")
@@ -382,7 +392,15 @@ def read_string(fp, wide=False):
                 continue
             break
 
-        key = read_string(fp)
+        if key_table:
+            # If 'key_table' was provided, each key is an int32 value that
+            # needs to be mapped to an actual field name using a key table.
+            # Newer appinfo.vdf (V29+) stores this table at the end of the file.
+            index = int32.unpack(fp.read(int32.size))[0]
+
+            key = key_table[index]
+        else:
+            key = read_string(fp)
 
         if t == BIN_NONE:
             if merge_duplicate_keys and key in stack[-1]: