Improve lang validity check (#275)

* Improve lang validity check The list returned by getISOLanguages does not include deprecated language codes that are still accepted to create locales. * Use a one-liner
OpenNMT · Jan 25, 2022 · f25f034 · f25f034
1 parent 56158de
commit f25f034
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 8 deletions.
diff --git a/bindings/python/test/test.py b/bindings/python/test/test.py
@@ -56,6 +56,10 @@ def test_invalid_lang():
         pyonmttok.Tokenizer("conservative", lang="xxx")
 
 
+def test_deprecated_lang():
+    pyonmttok.Tokenizer("conservative", lang="tl")
+
+
 def test_invalid_sentencepiece_model():
     with pytest.raises(ValueError):
         pyonmttok.Tokenizer("none", sp_model_path="xxx")

diff --git a/src/unicode/Unicode.cc b/src/unicode/Unicode.cc
@@ -235,14 +235,7 @@ namespace onmt
 
     bool is_valid_language(const char* language)
     {
-      for (const char* const* available_languages = icu::Locale::getISOLanguages();
-           *available_languages;
-           ++available_languages)
-      {
-        if (strcmp(*available_languages, language) == 0)
-          return true;
-      }
-      return false;
+      return icu::Locale(language).getISO3Language()[0] != '\0';
     }
 
     // The functions below are made backward compatible with the Kangxi and Kanbun script names