Employ optional nfc2nfd Unicode transform for all descriptive analysers

giellalt · Nov 20, 2023 · 09ded13 · 09ded13
1 parent 6d27a3d
commit 09ded13
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 0 deletions.
diff --git a/am-shared/src_alt_orth-include.am b/am-shared/src_alt_orth-include.am
@@ -78,6 +78,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/$(DEFAULT_ORTH)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -94,6 +95,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"         \
 			.o. @\"filters/remove-mwe-tags.$$*\"                   \
 			.o. @\"$$<\"                                           \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"          \
 			.o. @\"orthography/$(DEFAULT_ORTH)-to-$(1).compose.$$*\" \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"         \
@@ -119,6 +121,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/raw-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -135,6 +138,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"         \
 			.o. @\"filters/remove-mwe-tags.$$*\"                   \
 			.o. @\"$$<\"                                           \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"          \
 			.o. @\"orthography/raw-to-$(1).compose.$$*\"           \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"         \

diff --git a/am-shared/src_alt_ws-include.am b/am-shared/src_alt_ws-include.am
@@ -81,6 +81,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/$(DEFAULT_WS)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -97,6 +98,7 @@ analyser-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%            \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"         \
 			.o. @\"filters/remove-mwe-tags.$$*\"                   \
 			.o. @\"$$<\"                                           \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"          \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"         \
 			.o. @\"filters/remove-word-boundary.$$*\"              \

diff --git a/am-shared/src_dictionary-include.am b/am-shared/src_dictionary-include.am
@@ -52,6 +52,7 @@ analyser-dict-gt-desc.tmp.%: analyser-raw-gt-desc.%            \
 					filters/remove-mwe-tags.%                  \
 					orthography/inituppercase.compose.%        \
 					orthography/spellrelax.compose.%           \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$(AM_V_XFST_TOOL)$(PRINTF) "read regex \
 				@\"filters/remove-area-tags.$*\"                 \
@@ -67,6 +68,7 @@ analyser-dict-gt-desc.tmp.%: analyser-raw-gt-desc.%            \
 			.o. @\"filters/remove-Use_PMatch-strings.$*\"        \
 			.o. @\"filters/remove-mwe-tags.$*\"                  \
 			.o. @\"$<\"                                          \
+			    $(GLT_NFCNFD_COMPOSE) \
 			    $(GLT_DOWNCASE_COMPOSE) \
 			.o. @\"filters/remove-hyphenation-marks.$*\"         \
 			.o. @\"filters/remove-infl_deriv-borders.$*\"        \
@@ -172,6 +174,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/$$(DEFAULT_WS)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -188,6 +191,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"        \
 			.o. @\"filters/remove-mwe-tags.$$*\"                  \
 			.o. @\"$$<\"                                          \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"         \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"        \
 			.o. @\"filters/remove-word-boundary.$$*\"             \
@@ -270,6 +274,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/raw-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -288,6 +293,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"        \
 			.o. @\"filters/remove-mwe-tags.$$*\"                  \
 			.o. @\"$$<\"                                          \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"         \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"        \
 			.o. @\"filters/remove-word-boundary.$$*\"             \
@@ -313,6 +319,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 					filters/remove-Use_minus_PMatch-tags.%     \
 					filters/remove-Use_PMatch-strings.%        \
 					filters/remove-mwe-tags.%                  \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/spellrelax.$(1).compose.%      \
 					orthography/$(DEFAULT_ORTH)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -331,6 +338,7 @@ analyser-dict-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%       \
 			.o. @\"filters/remove-Use_PMatch-strings.$$*\"        \
 			.o. @\"filters/remove-mwe-tags.$$*\"                  \
 			.o. @\"$$<\"                                          \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"         \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"        \
 			.o. @\"filters/remove-word-boundary.$$*\"             \

diff --git a/am-shared/src_disamb-include.am b/am-shared/src_disamb-include.am
@@ -62,6 +62,7 @@ analyser-disamb-gt-desc.tmp1.%: analyser-raw-gt-desc.%        \
 					filters/block-mwe-compounds.%             \
 					orthography/inituppercase.compose.%       \
 					orthography/spellrelax.compose.%          \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$(AM_V_XFST_TOOL)$(PRINTF) "read regex \
 				@\"filters/remove-variant-tags.$*\"             \
@@ -70,6 +71,7 @@ analyser-disamb-gt-desc.tmp1.%: analyser-raw-gt-desc.%        \
 			.o. @\"filters/remove-mwe-tags.$*\"                 \
 			.o. @\"filters/block-mwe-compounds.$*\"             \
 			.o. @\"$<\"                                         \
+			    $(GLT_NFCNFD_COMPOSE) \
 				$(GLT_DOWNCASE_COMPOSE) \
 			.o. @\"filters/remove-hyphenation-marks.$*\"        \
 			.o. @\"filters/remove-infl_deriv-borders.$*\"       \
@@ -104,6 +106,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%     \
 					filters/remove-variant-tags.%               \
 					filters/remove-mwe-tags.%                   \
 					filters/block-mwe-compounds.%               \
+					$(GLT_NFCNFD_FILTER)                        \
 					orthography/spellrelax.$(1).compose.%       \
 					orthography/$(DEFAULT_WS)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
@@ -113,6 +116,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%     \
 			.o. @\"filters/remove-mwe-tags.$$*\"                   \
 			.o. @\"filters/block-mwe-compounds.$$*\"               \
 			.o. @\"$$<\"                                           \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"          \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"         \
 			.o. @\"filters/remove-word-boundary.$$*\"              \
@@ -151,6 +155,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%      \
 					orthography/spellrelax.$(1).compose.%        \
 					orthography/inituppercase.compose.%          \
 					orthography/$(DEFAULT_ORTH)-to-$(1).compose.% \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
 				@\"filters/remove-orthography-tags.$$*\"             \
@@ -159,6 +164,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%      \
 			.o. @\"filters/remove-mwe-tags.$$*\"                     \
 			.o. @\"filters/block-mwe-compounds.$$*\"                 \
 			.o. @\"$$<\"                                             \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			    $(GLT_DOWNCASE_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"            \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"           \
@@ -186,6 +192,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%      \
 					orthography/spellrelax.$(1).compose.%        \
 					orthography/inituppercase.compose.%          \
 					orthography/raw-to-$(1).compose.%            \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
 				@\"filters/remove-orthography-tags.$$*\"             \
@@ -194,6 +201,7 @@ analyser-disamb-gt-desc.$(1).tmp1.%: analyser-raw-gt-desc.%      \
 			.o. @\"filters/remove-mwe-tags.$$*\"                     \
 			.o. @\"filters/block-mwe-compounds.$$*\"                 \
 			.o. @\"$$<\"                                             \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			    $(GLT_DOWNCASE_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"            \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"           \

diff --git a/am-shared/src_gramcheck-include.am b/am-shared/src_gramcheck-include.am
@@ -29,6 +29,7 @@ analyser-gramcheck-gt-desc.tmp.%: analyser-raw-gt-desc.%      \
 					filters/block-mwe-compounds.%             \
 					orthography/inituppercase.compose.%       \
 					orthography/spellrelax.compose.%          \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$(AM_V_XFST_TOOL)$(PRINTF) "read regex \
 				@\"filters/remove-orig_lang-tags.$*\"           \
@@ -40,6 +41,7 @@ analyser-gramcheck-gt-desc.tmp.%: analyser-raw-gt-desc.%      \
 			.o. @\"filters/remove-Use_PMatch-tags.$*\"          \
 			.o. @\"filters/block-mwe-compounds.$*\"             \
 			.o. @\"$<\"                                         \
+			    $(GLT_NFCNFD_COMPOSE) \
 			    $(GLT_DOWNCASE_COMPOSE) \
 			.o. @\"filters/remove-hyphenation-marks.$*\"        \
 			.o. @\"filters/remove-infl_deriv-borders.$*\"       \
@@ -127,6 +129,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%   \
 					filters/remove-Use_PMatch-tags.%            \
 					filters/block-mwe-compounds.%               \
 					orthography/spellrelax.$(1).compose.%       \
+					$(GLT_NFCNFD_FILTER)                      \
 					orthography/$(DEFAULT_WS)-to-$(1).compose.%
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
 				@\"filters/remove-orig_lang-tags.$$*\"             \
@@ -139,6 +142,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%   \
 			.o. @\"filters/remove-Use_PMatch-tags.$$*\"            \
 			.o. @\"filters/block-mwe-compounds.$$*\"               \
 			.o. @\"$$<\"                                           \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"          \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"         \
 			.o. @\"filters/remove-word-boundary.$$*\"              \
@@ -234,6 +238,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%    \
 					orthography/spellrelax.$(1).compose.%        \
 					orthography/inituppercase.compose.%          \
 					orthography/$(DEFAULT_ORTH)-to-$(1).compose.% \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
 				@\"filters/remove-orig_lang-tags.$$*\"               \
@@ -246,6 +251,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%    \
 			.o. @\"filters/remove-Use_PMatch-tags.$$*\"              \
 			.o. @\"filters/block-mwe-compounds.$$*\"                 \
 			.o. @\"$$<\"                                             \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			    $(GLT_DOWNCASE_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"            \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"           \
@@ -277,6 +283,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%    \
 					orthography/spellrelax.$(1).compose.%        \
 					orthography/inituppercase.compose.%          \
 					orthography/raw-to-$(1).compose.%            \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$$(AM_V_XFST_TOOL)$$(PRINTF) "read regex \
 				@\"filters/remove-orig_lang-tags.$$*\"               \
@@ -289,6 +296,7 @@ analyser-gramcheck-gt-desc.$(1).tmp.%: analyser-raw-gt-desc.%    \
 			.o. @\"filters/remove-Use_PMatch-tags.$$*\"              \
 			.o. @\"filters/block-mwe-compounds.$$*\"                 \
 			.o. @\"$$<\"                                             \
+			    $(GLT_NFCNFD_COMPOSE_DEF) \
 			    $(GLT_DOWNCASE_COMPOSE_DEF) \
 			.o. @\"filters/remove-hyphenation-marks.$$*\"            \
 			.o. @\"filters/remove-infl_deriv-borders.$$*\"           \

diff --git a/am-shared/src_tts-include.am b/am-shared/src_tts-include.am
@@ -152,9 +152,11 @@ analyser-tts-gt-input.tmp.%: analyser-tts-gt-input.midtmp.%          \
 					filters/remove-word-boundary.%            \
 					orthography/inituppercase.compose.%       \
 					orthography/spellrelax.compose.%          \
+					$(GLT_NFCNFD_FILTER)                      \
 					$(GLT_DOWNCASE_FILTER)
 	$(AM_V_XFST_TOOL)$(PRINTF) "read regex \
 				@\"$<\" \
+			    $(GLT_NFCNFD_COMPOSE) \
 				$(GLT_DOWNCASE_COMPOSE) \
 			.o. @\"filters/remove-hyphenation-marks.$*\"        \
 			.o. @\"filters/remove-infl_deriv-borders.$*\"       \