From 97690d87c208415952aeb720b5497e91c8da33ee Mon Sep 17 00:00:00 2001 From: bitbrain Date: Sun, 7 Jul 2024 17:18:12 +0100 Subject: [PATCH] Implement Godot compression spec to compress on-the-fly --- TestScene.gd | 11 +-- TestScene.tscn | 3 + addons/pandora/api.gd | 6 +- addons/pandora/plugin.gd | 4 +- addons/pandora/storage/data_storage.gd | 4 ++ .../pandora/storage/json/json_data_storage.gd | 49 ++++++------- addons/pandora/util/compression.gd | 68 +++++++++++++++++++ examples/inventory/inventory.gd | 4 +- 8 files changed, 116 insertions(+), 33 deletions(-) create mode 100644 addons/pandora/util/compression.gd diff --git a/TestScene.gd b/TestScene.gd index 519afecd..592a85b2 100644 --- a/TestScene.gd +++ b/TestScene.gd @@ -5,13 +5,16 @@ extends CenterContainer @export var entity:PandoraEntity +@onready var label = $Label + + func _ready() -> void: - print(item.get_entity_id(), " - ", item.get_entity_name()) + label.text += item.get_entity_id() + " - "+ item.get_entity_name() + '\n' var copper_ore = Pandora.get_entity(Ores.COPPER_ORE) as Item - print(copper_ore.get_entity_name()) + label.text += copper_ore.get_entity_name() + '\n' var copper_instance = copper_ore.instantiate() - print(copper_instance.get_string("Description")) + label.text += copper_instance.get_string("Description") + '\n' - print(copper_ore.get_rarity().get_rarity_color()) + label.text += str(copper_ore.get_rarity().get_rarity_color()) + '\n' diff --git a/TestScene.tscn b/TestScene.tscn index 5b24b1c8..45c69bb7 100644 --- a/TestScene.tscn +++ b/TestScene.tscn @@ -25,3 +25,6 @@ entity = SubResource("Resource_xfcj2") [node name="TextureRect" type="TextureRect" parent="."] layout_mode = 2 texture = ExtResource("1_r3xd3") + +[node name="Label" type="Label" parent="."] +layout_mode = 2 diff --git a/addons/pandora/api.gd b/addons/pandora/api.gd index 2aeaa941..4081f0ca 100644 --- a/addons/pandora/api.gd +++ b/addons/pandora/api.gd @@ -15,7 +15,7 @@ signal import_calculation_failed(reason: String) signal import_progress var _context_manager: PandoraContextManager -var _storage: PandoraJsonDataStorage +var _storage: PandoraDataStorage var _id_generator: PandoraIDGenerator var _entity_backend: PandoraEntityBackend @@ -164,7 +164,7 @@ func save_data() -> void: func calculate_import_data(path: String) -> int: - var imported_data = _storage._load_from_file(path) + var imported_data = _storage.load_from_file(path) var total_items = 0 if not imported_data.has("_entity_data"): import_calculation_failed.emit("Provided file is invalid or is corrupted.") @@ -192,7 +192,7 @@ func calculate_import_data(path: String) -> int: func import_data(path: String) -> int: - var imported_data = _storage._load_from_file(path) + var imported_data = _storage.load_from_file(path) if not imported_data.has("_entity_data"): import_failed.emit("Provided file is invalid or is corrupted.") return 0 diff --git a/addons/pandora/plugin.gd b/addons/pandora/plugin.gd index ffc5f8f7..fc8ba33f 100644 --- a/addons/pandora/plugin.gd +++ b/addons/pandora/plugin.gd @@ -4,6 +4,7 @@ extends EditorPlugin const PandoraEditor := preload("res://addons/pandora/ui/editor/pandora_editor.tscn") const PandoraIcon := preload("res://addons/pandora/icons/pandora-icon.svg") const PandoraEntityInspector = preload("res://addons/pandora/ui/editor/inspector/entity_instance_inspector.gd") +const Compression = preload("res://addons/pandora/util/compression.gd") var editor_view var entity_inspector @@ -81,7 +82,8 @@ class PandoraExportPlugin extends EditorExportPlugin: return var data:PackedByteArray = file.get_buffer(file.get_length()) if not is_debug: - data = data.compress() + var text = file.get_as_text() + data = Compression.compress(text) add_file(pandora_path, data, false) func _get_name() -> String: diff --git a/addons/pandora/storage/data_storage.gd b/addons/pandora/storage/data_storage.gd index aaa0021b..036c1cd0 100644 --- a/addons/pandora/storage/data_storage.gd +++ b/addons/pandora/storage/data_storage.gd @@ -24,3 +24,7 @@ func store_all_data(data: Dictionary, context_id: String) -> Dictionary: func get_all_data(context_id: String) -> Dictionary: return {} + + +func load_from_file(file_path: String) -> Dictionary: + return {} diff --git a/addons/pandora/storage/json/json_data_storage.gd b/addons/pandora/storage/json/json_data_storage.gd index 99720472..622e4dfa 100644 --- a/addons/pandora/storage/json/json_data_storage.gd +++ b/addons/pandora/storage/json/json_data_storage.gd @@ -25,13 +25,12 @@ func _init(data_dir: String): func store_all_data(data: Dictionary, context_id: String) -> Dictionary: var file_path = _get_file_path(context_id) var file: FileAccess - # Ensure within the Godot Engine editor, Pandora remains uncompressed - if Engine.is_editor_hint() or OS.is_debug_build(): - file = FileAccess.open(file_path, FileAccess.WRITE) - file.store_string(JSON.stringify(data, "\t")) - else: + if _should_use_compression(): file = FileAccess.open_compressed(file_path, FileAccess.WRITE) file.store_string(JSON.stringify(data)) + else: + file = FileAccess.open(file_path, FileAccess.WRITE) + file.store_string(JSON.stringify(data, "\t")) file.close() return data @@ -39,11 +38,10 @@ func store_all_data(data: Dictionary, context_id: String) -> Dictionary: func get_all_data(context_id: String) -> Dictionary: var file_path = _get_file_path(context_id) var file: FileAccess - # Ensure within the Godot Engine editor, Pandora remains uncompressed - if Engine.is_editor_hint() or OS.is_debug_build(): - file = FileAccess.open(file_path, FileAccess.READ) - else: + if _should_use_compression(): file = FileAccess.open_compressed(file_path, FileAccess.READ) + else: + file = FileAccess.open(file_path, FileAccess.READ) var json: JSON = JSON.new() if file != null: var text = file.get_as_text() @@ -70,6 +68,22 @@ func get_decompressed_data(file_path: String) -> Dictionary: return {} +func load_from_file(file_path: String) -> Dictionary: + var file: FileAccess + if _should_use_compression(): + file = FileAccess.open_compressed(file_path, FileAccess.READ) + else: + file = FileAccess.open(file_path, FileAccess.READ) + if FileAccess.file_exists(file_path) and file != null: + var content = file.get_as_text() + file.close() + var json = JSON.new() + json.parse(content) + return json.get_data() + else: + return {} + + func _get_directory_path(context_id: String) -> String: var directory_path = "" if data_directory.ends_with("//"): @@ -89,17 +103,6 @@ func _get_file_path(context_id: String) -> String: return "%s/data.pandora" % [_get_directory_path(context_id)] -func _load_from_file(file_path: String) -> Dictionary: - var file: FileAccess - if OS.is_debug_build(): - file = FileAccess.open(file_path, FileAccess.READ) - else: - file = FileAccess.open_compressed(file_path, FileAccess.READ) - if FileAccess.file_exists(file_path) and file != null: - var content = file.get_as_text() - file.close() - var json = JSON.new() - json.parse(content) - return json.get_data() - else: - return {} +func _should_use_compression() -> bool: + # Ensure within the Godot Engine editor Pandora remains uncompressed + return not Engine.is_editor_hint() and not OS.is_debug_build() diff --git a/addons/pandora/util/compression.gd b/addons/pandora/util/compression.gd new file mode 100644 index 00000000..e798ff97 --- /dev/null +++ b/addons/pandora/util/compression.gd @@ -0,0 +1,68 @@ +const BLOCK_SIZE = 4096 +const MAGIC = "GCPF" + + +## magic +## char[4] "GCPF" +## +## header +## uint32_t compression_mode (Compression::MODE_ZSTD by default) +## uint32_t block_size (4096 by default) +## uint32_t uncompressed_size +## +## block compressed sizes, number of blocks = (uncompressed_size / block_size) + 1 +## uint32_t block_sizes[] +## +## followed by compressed block data, same as calling `compress` for each source `block_size` +static func compress(text: String, compression_mode:FileAccess.CompressionMode = FileAccess.COMPRESSION_FASTLZ) -> PackedByteArray: + var data = _encode_string(text) + var uncompressed_size = data.size() + + var num_blocks = int(ceil(float(uncompressed_size) / BLOCK_SIZE)) + + var buffer = PackedByteArray() + + buffer.append_array(_encode_string(MAGIC)) + + buffer.append_array(_encode_uint32(compression_mode)) + buffer.append_array(_encode_uint32(BLOCK_SIZE)) + buffer.append_array(_encode_uint32(uncompressed_size)) + + var block_sizes = PackedByteArray() + var compressed_blocks = [] + + for i in range(num_blocks): + var start = i * BLOCK_SIZE + var end = min((i + 1) * BLOCK_SIZE, uncompressed_size) + var block_data = PackedByteArray() + var block_index = start + while block_index < end: + block_data.append(data[block_index]) + block_index += 1 + + var compressed_block = block_data.compress(compression_mode) + block_sizes.append_array(_encode_uint32(compressed_block.size())) + compressed_blocks.append(compressed_block) + + buffer.append_array(block_sizes) + + for block in compressed_blocks: + buffer.append_array(block) + + return buffer + + +static func _encode_uint32(value: int) -> PackedByteArray: + var arr = PackedByteArray() + arr.append(value & 0xFF) + arr.append((value >> 8) & 0xFF) + arr.append((value >> 16) & 0xFF) + arr.append((value >> 24) & 0xFF) + return arr + + +static func _encode_string(value: String) -> PackedByteArray: + var arr = PackedByteArray() + for char in value: + arr.append_array(char.to_ascii_buffer()) + return arr diff --git a/examples/inventory/inventory.gd b/examples/inventory/inventory.gd index 9019242c..c85f9ee5 100644 --- a/examples/inventory/inventory.gd +++ b/examples/inventory/inventory.gd @@ -3,7 +3,7 @@ class_name Inventory extends Node signal item_added(item:Item, index:int) signal item_removed(item:Item, index:int) - + # id -> ItemInstance var _slots = {} @@ -24,7 +24,7 @@ func add_item(item:Item, index:int) -> void: return existing_item.set_current_stacksize(new_stacksize) item_added.emit(existing_item, index) - + func remove_item(index:int) -> Item: if _slots.has(index):