Add a compressor

scripts/compressor.py

@@ -0,0 +1,68 @@
+import struct
+
+def compress(uncompressed: bytearray | bytes) -> bytearray:
+  def find_largest(uncompressed: bytearray | bytes, uncompressed_pos: int) -> tuple[int, int, int]:
+    start = max(0, uncompressed_pos - 0x10000)
+    before = uncompressed[start :]
+    after = uncompressed[uncompressed_pos : uncompressed_pos + 4]
+
+    max_len = 0
+    max_len_offset = -1
+    offset = before.find(after)
+    while offset > -1 and start + offset < uncompressed_pos:
+      this_len = 4
+      while offset + this_len < len(before) and uncompressed_pos + this_len < len(uncompressed) and this_len < 0x103 and before[offset + this_len] == uncompressed[uncompressed_pos + this_len]:
+        this_len += 1
+
+      if this_len > max_len:
+        max_len = this_len
+        max_len_offset = start + offset
+      offset = before.find(after, offset + max_len)
+
+    if max_len_offset != -1:
+      return 0, max_len_offset, max_len
+
+    if uncompressed[uncompressed_pos:uncompressed_pos + 4] == b"\0\0\0\0":
+      zero_pos = uncompressed_pos + 4
+      while zero_pos < len(uncompressed) and zero_pos < uncompressed_pos + 0x103 and uncompressed[zero_pos] == 0:
+        zero_pos += 1
+      if zero_pos < 0xffff:
+        zero_len = zero_pos - uncompressed_pos
+        return 0, 0x10000 - zero_len, zero_len
+
+    return 1, 0, 0
+
+  compressed = bytearray()
+  compressed.extend(struct.pack("<4I", 0x01da3d12, len(uncompressed), 0, 0))
+
+  uncompressed_pos = 0
+  bits = 0
+  while uncompressed_pos < len(uncompressed):
+    bits_pos = len(compressed)
+    compressed.append(0)
+    bits = 0
+    for i in range(8):
+      if uncompressed_pos >= len(uncompressed):
+        break
+
+      bits_i, max_len_offset, max_len = find_largest(uncompressed, uncompressed_pos)
+
+      if bits_i == 0:
+        real_offset = (max_len_offset + 0x10000 - 0xff - 4) & 0xffff
+        real_len = max_len - 4
+        compressed.extend(struct.pack("<HB", real_offset, real_len))
+        uncompressed_pos += max_len
+      else:
+        compressed.append(uncompressed[uncompressed_pos])
+        uncompressed_pos += 1
+
+      bits |= (bits_i << i)
+    compressed[bits_pos] = bits
+
+  if len(compressed) - 0x10 > len(uncompressed):
+    compressed = bytearray()
+    compressed.extend(struct.pack("<4I", 0x00da3d12, len(uncompressed), 0xffffffff, 0))
+    compressed.extend(uncompressed)
+  else:
+    compressed[0x08:0x0c] = struct.pack("<I", len(compressed) - 0x10)
+  return compressed

scripts/decompressor.py

@@ -1,13 +1,5 @@
 import struct
 
-def Hibit(n: int):
-  n |= (n >> 1)
-  n |= (n >> 2)
-  n |= (n >> 4)
-  n |= (n >> 8)
-  n |= (n >> 16)
-  return n - (n >> 1)
-
 # Thanks to: Jason Harley
 # Reference: https://github.com/Jas2o/KyleHyde/blob/main/KyleHyde/Formats/HotelDusk/Decompress.cs
 def decompress(compressed: bytearray | bytes) -> bytearray:
@@ -33,26 +25,13 @@ def decompress(compressed: bytearray | bytes) -> bytearray:
         compressed_pos += 1
         uncompressed_pos += 1
       else:
-        offset, = struct.unpack("<H", compressed[compressed_pos : compressed_pos + 2])
-        len = 4 + compressed[compressed_pos + 2]
+        offset, len = struct.unpack("<HB", compressed[compressed_pos : compressed_pos + 3])
+        offset = (offset + 0xff + 4) & 0xffff
+        len += 4
         compressed_pos += 3
 
-        posHi = Hibit(uncompressed_pos - 0xff - 4)
-        offHi = Hibit(offset)
-        if uncompressed_pos < 0x10000:
-          signed = offset & 0xffff
-          if signed >= 0x8000:
-            signed -= 0x10000
-          if signed < 0 and signed + 0xff + 4 >= 0:
-            offset = signed
-        elif posHi >= 0x20000 and offHi < posHi:
-          if offset + posHi < uncompressed_pos:
-            offset += posHi
-          elif offset + 0x10000 < uncompressed_pos:
-            offset += 0x10000
-        elif posHi >= 0x10000 and offHi < posHi and offset + posHi < uncompressed_pos:
-          offset += posHi
-        offset += 0xff + 4
+        while offset < uncompressed_pos - 0x10000:
+          offset += 0x10000
 
         if offset < 0 or offset + len >= sizeun:
           for x in range(len):

scripts/import_images.py

@@ -4,6 +4,7 @@
 from PIL import Image
 
 import decompressor
+import compressor
 
 os.makedirs("temp_files/pack/", exist_ok=True)
 
@@ -105,8 +106,11 @@ def get_xy_tile(width: int, height: int) -> Generator[tuple[int, int], Any, None
         decompressed[0x10 + i * 2 : 0x12 + i * 2] = struct.pack("<H", palette[i] | 0x8000)
 
       if has_header:
-        file_writer.write(b"\x12\x3d\xda\x00" + struct.pack("<I", len(decompressed)) + b"\xff\xff\xff\xff\x00\x00\x00\x00")
-      file_writer.write(decompressed)
+        compressed = compressor.compress(decompressed)
+        file_writer.write(compressed)
+        print(f"Compressed: {file_name}/{sub_name} ({len(compressed) - 0x10}/{len(decompressed)})")
+      else:
+        file_writer.write(decompressed)
       head_bin_writer.write(struct.pack("<I", len(decompressed) + 0x10))
 
     while file_writer.tell() % 0x20 != 0: