ISSUE-21: Enhance parsing in no-todos

Author: AFulgens

hooks/no_todos.py

@@ -27,8 +27,17 @@ def main(argv: Sequence[str] | None = None) -> int:
     result = 0
 
     for filename in args.filenames:
-        with open(filename, encoding='utf-8') as f:
-            lines = f.readlines()
+        try:
+            with open(filename, encoding='utf-8-sig') as f:
+                lines = f.readlines()
+        except (UnicodeDecodeError, UnicodeError):
+            print(f'{filename}: cannot be read as UTF-8, trying UTF-16')
+            try:
+                with open(filename, encoding='utf-16') as f:
+                    lines = f.readlines()
+            except (UnicodeDecodeError, UnicodeError):
+                print(f'{filename}: cannot be read as UTF-16, skipping it')
+                continue
 
         for tag in disallowed:
             for line in lines:

tests/test_no_todos.py

@@ -18,7 +18,7 @@ def test_has_todo(tmpdir):
         # noinspection SpellCheckingInspection
         f.write_text('TODO: ¥eßűs, ∂éñ∂ þħïs!', encoding='utf-8')
         assert no_todos.main((str(f),)) == 1
-        assert mocked_print.call_args_list[0].args[0] \
+        assert mocked_print.call_args_list[-1].args[0] \
             .endswith('todo.txt: contains TODO')
 
 
@@ -42,7 +42,7 @@ def test_has_todo_but_excepted(tmpdir):
         # noinspection SpellCheckingInspection
         f.write_text('TODO: ฿űþ ñ∅™ þħïs!', encoding='utf-8')
         assert no_todos.main(('-e', 'todo.txt', str(f))) == 0
-        assert mocked_print.call_args_list[0].args[0] \
+        assert mocked_print.call_args_list[-1].args[0] \
             .endswith('todo.txt: contains TODO, but is on the exception list')
 
 
@@ -64,3 +64,27 @@ def test_complex_case(tmpdir):
     # noinspection SpellCheckingInspection
     f3.write_text('TODO: ¥eßűs, ∂éñ∂ þħïs!-3', encoding='utf-8')
     assert no_todos.main(('-e', 'todo_2.txt', str(f1), str(f2), str(f3))) == 2
+
+
+def test_non_utf_8(tmpdir):
+    with patch('builtins.print') as mocked_print:
+        f1 = tmpdir.join('todo_utf8_bom.txt')
+        # noinspection SpellCheckingInspection
+        f1.write_text('TODO: ¥eßűs, ∂éñ∂ þħïs!-1', encoding='utf-8-sig')
+        assert no_todos.main((str(f1),)) == 1
+        for a in mocked_print.call_args_list:
+            assert not a.args[0].endswith(
+                'todo_utf8_bom.txt: cannot be read as UTF-8, trying UTF-16'
+            )
+        f2 = tmpdir.join('todo_utf16.txt')
+        # noinspection SpellCheckingInspection
+        f2.write_text('TODO: ¥eßűs, ∂éñ∂ þħïs!-1', encoding='utf-16')
+        assert no_todos.main((str(f1),str(f2))) == 2
+        assert mocked_print.call_args_list[-2].args[0] \
+            .endswith('todo_utf16.txt: cannot be read as UTF-8, trying UTF-16')
+        f3 = tmpdir.join('todo_greek.txt')
+        # noinspection SpellCheckingInspection
+        f3.write_text('TODO: Υeσΰς, δέηδ ΤΞΪΣ!-1', encoding='ISO-8859-7')
+        assert no_todos.main((str(f1),str(f2),str(f3))) == 2
+        assert mocked_print.call_args_list[-1].args[0] \
+            .endswith('todo_greek.txt: cannot be read as UTF-16, skipping it')