add control characters

ku-nlp · May 9, 2024 · b0d7654 · b0d7654
1 parent 68ba9f4
commit b0d7654
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/kwja/utils/constants.py b/src/kwja/utils/constants.py
@@ -197,7 +197,7 @@
 # ---------- char module|text normalization ----------
 # 制御文字(\t,\nを含む)は削除
 TRANSLATION_TABLE: Dict[int, Optional[int]] = str.maketrans(
-    '"#▁', "”＃▂", "".join(chr(i) for i in [*range(32), *range(127, 160)])
+    '"#▁', "”＃▂", "".join(chr(i) for i in [*range(32), *range(127, 160), 5760, *range(8232, 8234)])
 )