adding tests

Author: MagedSaeed

test.py

@@ -1,45 +1,77 @@
 import unittest
-from tkseem.util import remove_tashkeel
-import tkseem as tk
-
 
-class TestUnit(unittest.TestCase):
-    def test_tashkeel(self):
-        self.assertEqual(
-            remove_tashkeel("مِكَرٍّ مِفَرٍّ مُقبِلٍ مُدبِرٍ مَعًا"),
-            "مكر مفر مقبل مدبر معا",
-            "Remove Tashkeel is not working",
-        )
+import tkseem as tk
 
 
 class TokenizersTestUnit(unittest.TestCase):
     sample_text = "مرحبا أيها الأصدقاء"
     token = "نص"
+    chars = False
     tokenizer = None
-    token_id = None
+
+    def print_string(self, method_name, output, method_arguments=None):
+        return f'{self.tokenizer} method {method_name}() output {f"on arguments: [{method_arguments}]" if method_arguments else ""} is: {output}'
 
     def test_tokenize(self):
         tokenized = self.tokenizer.tokenize(self.sample_text)
-        print(f"{self.tokenizer} tokenize() output:", tokenized)
+        print(self.print_string("tokenize", tokenized, method_arguments="sample_text"))
         return self.assertIsNotNone(tokenized)
 
     def test_detokenize(self):
         tokenized = self.tokenizer.tokenize(self.sample_text)
         detokenized = self.tokenizer.detokenize(tokenized)
         print(
-            f"{self.tokenizer} detokenize() output on the previously tokenized text:",
-            detokenized,
+            self.print_string(
+                "detokenize", detokenized, method_arguments="tokenized sample_text"
+            )
         )
         return self.assertIsNotNone(detokenized)
 
     def test_token_to_id(self):
-        self.token_id = self.tokenizer.token_to_id(self.token)
-        print(f"{self.tokenizer} token_to_id() output:", self.token_id)
-        return self.assertIsNotNone(self.token_id)
+        token = self.token if not self.chars else self.token[0]
+        token_id = self.tokenizer.token_to_id(token)
+        print(self.print_string("token_to_id", token_id, method_arguments=token))
+        return self.assertIsNotNone(token_id)
 
+    def test_id_to_token(self):
+        token = self.token if not self.chars else self.token[0]
+        token_id = self.tokenizer.token_to_id(token)
+        matched_token = self.tokenizer.id_to_token(token_id)
+        print(
+            self.print_string(
+                "id_to_token", matched_token, method_arguments=f"'{token}' id"
+            )
+        )
+        return self.assertEqual(matched_token, token)
+
+    def test_encode(self):
+        encoded = self.tokenizer.encode(self.sample_text)
+        print(self.print_string("encode", encoded, method_arguments="sample_text"))
+        return self.assertIsNotNone(encoded)
+
+    def test_decode(self):
+        encoded = self.tokenizer.encode(self.sample_text)
+        decoded = self.tokenizer.decode(encoded)
+        print(
+            self.print_string("decode", decoded, method_arguments="encoded sample_text")
+        )
+        return self.assertIsNotNone(decoded)
 
-word_tokenizer = tk.WordTokenizer()
-word_tokenizer.train("tasks/samples/data.txt")
-TokenizersTestUnit.tokenizer = word_tokenizer
-unittest.main(argv=["first-arg-is-ignored"], exit=False)
 
+for tokenizer in (
+    tk.SentencePieceTokenizer(),
+    tk.WordTokenizer(),
+    tk.MorphologicalTokenizer(),
+    tk.CharacterTokenizer(),
+    tk.DisjointLetterTokenizer(),
+    tk.RandomTokenizer(),
+):
+    try:
+        tokenizer.train("tasks/samples/data.txt")
+    except TypeError as type_error:
+        print(f"{tokenizer} does not need file_path to train")
+        tokenizer.train()
+    if isinstance(tokenizer, tk.CharacterTokenizer):
+        TokenizersTestUnit.chars = True
+    TokenizersTestUnit.tokenizer = tokenizer
+    unittest.main(argv=["first-arg-is-ignored"], exit=False)