Ensure categegory and entity names are tokenized correctly, closes #186…

… (#192)

addons/pandora/util/category_id_file_generator.gd

@@ -1,3 +1,15 @@
+const Tokenizer = preload("tokenizer.gd")
+
+
+class CategoryTuple:
+	var category_name: String
+	var category_id: String
+
+	func _init(category_name: String, category_id: String) -> void:
+		self.category_name = category_name
+		self.category_id = category_id
+
+
 ## Generates a .gd file that allows for easier access
 ## of categories and subcategories of the data
 static func regenerate_category_id_file(root_categories: Array[PandoraCategory]) -> void:
@@ -53,7 +65,7 @@ static func generate_category_id_file(
 		var line = (
 			'const %s = "%s"'
 			% [
-				category_tuple.category_name.to_upper().replace(" ", "_"),
+				Tokenizer.tokenize(category_tuple.category_name),
 				category_tuple.category_id
 			]
 		)
@@ -73,20 +85,11 @@ static func generate_category_id_file(
 			line = (
 				'	const %s = "%s"'
 				% [
-					category_tuple.category_name.to_upper().replace(" ", "_"),
+					Tokenizer.tokenize(category_tuple.category_name),
 					category_tuple.category_id
 				]
 			)
 			file_access.store_line(line)
 		file_access.store_line("\n")
 
 	file_access.close()
-
-
-class CategoryTuple:
-	var category_name: String
-	var category_id: String
-
-	func _init(category_name: String, category_id: String) -> void:
-		self.category_name = category_name
-		self.category_id = category_id

addons/pandora/util/entity_id_file_generator.gd

@@ -1,3 +1,6 @@
+const Tokenizer = preload("tokenizer.gd")
+
+
 ## Generates a .gd file that allows for easier access
 ## of entities
 static func regenerate_id_files(root_categories: Array[PandoraCategory]) -> void:
@@ -49,7 +52,7 @@ static func regenerate_entity_id_file(
 		file_access.store_line(
 			(
 				"const "
-				+ entity_name.to_upper().replace(" ", "_")
+				+ Tokenizer.tokenize(entity_name)
 				+ " = "
 				+ '"'
 				+ entity.get_entity_id()

addons/pandora/util/tokenizer.gd

@@ -0,0 +1,21 @@
+static var regex:RegEx
+
+## Converts any string into a GDScript-valid token (cannot contain special characters)
+static func tokenize(value:String) -> String:
+	if regex == null:
+		regex = RegEx.new()
+		regex.compile("(^[^A-Z_])|([^A-Z0-9_])")
+	var token = value.to_upper()
+
+	var regex_matches = regex.search_all(token)
+	var offset = 0
+	for regex_match in regex_matches:
+		var start = regex_match.get_start()
+		var end = regex_match.get_end()
+		var length = end - start
+		var text = token.substr(start + offset, length)
+		var replacement = "_"
+		token = token.substr(0, start + offset) + replacement + token.substr(end + offset)
+		offset += replacement.length() - text.length()
+
+	return token

data.pandora

@@ -136,8 +136,10 @@
 			},
 			{
 				"_category_id": "",
+				"_generate_ids": true,
 				"_icon_color": "ffffff00",
 				"_id": "54",
+				"_ids_generation_class": "MockEntities",
 				"_index": 0,
 				"_name": "Mock (Required for Testing!)",
 				"_script_path": "res://mock/custom_mock_entity.gd"
@@ -227,6 +229,20 @@
 						}
 					}
 				}
+			},
+			{
+				"_category_id": "54",
+				"_icon_color": "ffffff00",
+				"_id": "7",
+				"_index": 0,
+				"_name": "1 Some Entity"
+			},
+			{
+				"_category_id": "54",
+				"_icon_color": "ffffff00",
+				"_id": "8",
+				"_index": 0,
+				"_name": "/1209I)(#)9390292084"
 			}
 		],
 		"_properties": [
@@ -396,7 +412,7 @@
 	},
 	"_id_generator": {
 		"_ids_by_context": {
-			"default": 6
+			"default": 8
 		}
 	}
 }

pandora/categories.gd

@@ -7,6 +7,7 @@ const RARITY = "16"
 const QUESTS = "27"
 const DIALOGUES = "28"
 const SPELLS = "29"
+const MOCK__REQUIRED_FOR_TESTING__ = "54"
 const NPCS = "2"
 
 

pandora/mock_entities.gd

@@ -0,0 +1,7 @@
+# Do not modify! Auto-generated file.
+class_name MockEntities
+
+
+const MOCKENTITY = "55"
+const __SOME_ENTITY = "7"
+const _1209I____9390292084 = "8"