From 6f89539dff44eb7ca30091d6f0793af71e85381e Mon Sep 17 00:00:00 2001
From: miguel <miguel-gonzalez@gmx.de>
Date: Sun, 14 Jul 2024 09:50:06 +0100
Subject: [PATCH] Ensure categegory and entity names are tokenized correctly,
 closes #186 (#192)

---
 .../util/category_id_file_generator.gd        | 25 +++++++++++--------
 .../pandora/util/entity_id_file_generator.gd  |  5 +++-
 addons/pandora/util/tokenizer.gd              | 21 ++++++++++++++++
 data.pandora                                  | 18 ++++++++++++-
 pandora/categories.gd                         |  1 +
 pandora/mock_entities.gd                      |  7 ++++++
 6 files changed, 64 insertions(+), 13 deletions(-)
 create mode 100644 addons/pandora/util/tokenizer.gd
 create mode 100644 pandora/mock_entities.gd

diff --git a/addons/pandora/util/category_id_file_generator.gd b/addons/pandora/util/category_id_file_generator.gd
index c5c1028..7697f29 100644
--- a/addons/pandora/util/category_id_file_generator.gd
+++ b/addons/pandora/util/category_id_file_generator.gd
@@ -1,3 +1,15 @@
+const Tokenizer = preload("tokenizer.gd")
+
+
+class CategoryTuple:
+	var category_name: String
+	var category_id: String
+
+	func _init(category_name: String, category_id: String) -> void:
+		self.category_name = category_name
+		self.category_id = category_id
+
+
 ## Generates a .gd file that allows for easier access
 ## of categories and subcategories of the data
 static func regenerate_category_id_file(root_categories: Array[PandoraCategory]) -> void:
@@ -53,7 +65,7 @@ static func generate_category_id_file(
 		var line = (
 			'const %s = "%s"'
 			% [
-				category_tuple.category_name.to_upper().replace(" ", "_"),
+				Tokenizer.tokenize(category_tuple.category_name),
 				category_tuple.category_id
 			]
 		)
@@ -73,7 +85,7 @@ static func generate_category_id_file(
 			line = (
 				'	const %s = "%s"'
 				% [
-					category_tuple.category_name.to_upper().replace(" ", "_"),
+					Tokenizer.tokenize(category_tuple.category_name),
 					category_tuple.category_id
 				]
 			)
@@ -81,12 +93,3 @@ static func generate_category_id_file(
 		file_access.store_line("\n")
 
 	file_access.close()
-
-
-class CategoryTuple:
-	var category_name: String
-	var category_id: String
-
-	func _init(category_name: String, category_id: String) -> void:
-		self.category_name = category_name
-		self.category_id = category_id
diff --git a/addons/pandora/util/entity_id_file_generator.gd b/addons/pandora/util/entity_id_file_generator.gd
index 8a6517a..dbe85b2 100644
--- a/addons/pandora/util/entity_id_file_generator.gd
+++ b/addons/pandora/util/entity_id_file_generator.gd
@@ -1,3 +1,6 @@
+const Tokenizer = preload("tokenizer.gd")
+
+
 ## Generates a .gd file that allows for easier access
 ## of entities
 static func regenerate_id_files(root_categories: Array[PandoraCategory]) -> void:
@@ -49,7 +52,7 @@ static func regenerate_entity_id_file(
 		file_access.store_line(
 			(
 				"const "
-				+ entity_name.to_upper().replace(" ", "_")
+				+ Tokenizer.tokenize(entity_name)
 				+ " = "
 				+ '"'
 				+ entity.get_entity_id()
diff --git a/addons/pandora/util/tokenizer.gd b/addons/pandora/util/tokenizer.gd
new file mode 100644
index 0000000..f02f7b1
--- /dev/null
+++ b/addons/pandora/util/tokenizer.gd
@@ -0,0 +1,21 @@
+static var regex:RegEx
+
+## Converts any string into a GDScript-valid token (cannot contain special characters)
+static func tokenize(value:String) -> String:
+	if regex == null:
+		regex = RegEx.new()
+		regex.compile("(^[^A-Z_])|([^A-Z0-9_])")
+	var token = value.to_upper()
+
+	var regex_matches = regex.search_all(token)
+	var offset = 0
+	for regex_match in regex_matches:
+		var start = regex_match.get_start()
+		var end = regex_match.get_end()
+		var length = end - start
+		var text = token.substr(start + offset, length)
+		var replacement = "_"
+		token = token.substr(0, start + offset) + replacement + token.substr(end + offset)
+		offset += replacement.length() - text.length()
+
+	return token
diff --git a/data.pandora b/data.pandora
index 734f478..82335a6 100644
--- a/data.pandora
+++ b/data.pandora
@@ -136,8 +136,10 @@
 			},
 			{
 				"_category_id": "",
+				"_generate_ids": true,
 				"_icon_color": "ffffff00",
 				"_id": "54",
+				"_ids_generation_class": "MockEntities",
 				"_index": 0,
 				"_name": "Mock (Required for Testing!)",
 				"_script_path": "res://mock/custom_mock_entity.gd"
@@ -227,6 +229,20 @@
 						}
 					}
 				}
+			},
+			{
+				"_category_id": "54",
+				"_icon_color": "ffffff00",
+				"_id": "7",
+				"_index": 0,
+				"_name": "1 Some Entity"
+			},
+			{
+				"_category_id": "54",
+				"_icon_color": "ffffff00",
+				"_id": "8",
+				"_index": 0,
+				"_name": "/1209I)(#)9390292084"
 			}
 		],
 		"_properties": [
@@ -396,7 +412,7 @@
 	},
 	"_id_generator": {
 		"_ids_by_context": {
-			"default": 6
+			"default": 8
 		}
 	}
 }
\ No newline at end of file
diff --git a/pandora/categories.gd b/pandora/categories.gd
index 4b2d434..dfcba78 100644
--- a/pandora/categories.gd
+++ b/pandora/categories.gd
@@ -7,6 +7,7 @@ const RARITY = "16"
 const QUESTS = "27"
 const DIALOGUES = "28"
 const SPELLS = "29"
+const MOCK__REQUIRED_FOR_TESTING__ = "54"
 const NPCS = "2"
 
 
diff --git a/pandora/mock_entities.gd b/pandora/mock_entities.gd
new file mode 100644
index 0000000..5115630
--- /dev/null
+++ b/pandora/mock_entities.gd
@@ -0,0 +1,7 @@
+# Do not modify! Auto-generated file.
+class_name MockEntities
+
+
+const MOCKENTITY = "55"
+const __SOME_ENTITY = "7"
+const _1209I____9390292084 = "8"