Merge pull request #595 from greenbone/improve_encoding_detection

Improve encoding detection

pyproject.toml

@@ -35,6 +35,7 @@ python-magic = "^0.4.25"
 chardet = ">=4,<6"
 validators = ">=0.18.2,<0.21.0"
 gitpython = "^3.1.31"
+charset-normalizer = "^3.2.0"
 
 [tool.poetry.dev-dependencies]
 autohooks = ">=21.7.0"

tests/plugins/test_encoding.py

@@ -65,7 +65,8 @@ def test_some_invalid_characters(self):
 
             self.assertIsInstance(results[0], LinterError)
             self.assertEqual(
-                "VT uses a wrong encoding. Detected encoding is utf-8.",
+                "VT uses a wrong encoding. "
+                "Allowed encodings are ascii, latin_1.",
                 results[0].message,
             )
             self.assertEqual(
@@ -97,6 +98,7 @@ def test_invisible_whitespace(self):
             self.assertEqual(len(results), 1)
             self.assertIsInstance(results[0], LinterError)
             self.assertEqual(
-                "VT uses a wrong encoding. Detected encoding is utf-8.",
+                "VT uses a wrong encoding. "
+                "Allowed encodings are ascii, latin_1.",
                 results[0].message,
             )

troubadix/plugins/encoding.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 from typing import Iterable, Iterator
 
-import chardet
+import charset_normalizer
 
 from troubadix.plugin import LineContentPlugin, LinterError, LinterResult
 
@@ -28,6 +28,8 @@
 # Temporary only check for chars in between 7f-9f, like in the old Feed-QA...
 CHAR_SET = r"[\x7F-\x9F]"
 
+ALLOWED_ENCODINGS = ["ascii", "latin_1"]
+
 
 class CheckEncoding(LineContentPlugin):
     name = "check_encoding"
@@ -37,18 +39,14 @@ def check_lines(
         nasl_file: Path,
         lines: Iterable[str],
     ) -> Iterator[LinterResult]:
-        content = nasl_file.read_bytes()
+        match = charset_normalizer.from_path(
+            nasl_file, threshold=0.4, cp_isolation=ALLOWED_ENCODINGS
+        ).best()
 
-        detection = chardet.detect(content)
-        encoding = detection.get("encoding")
-        if encoding and encoding not in [
-            "ascii",
-            "latin1",
-            "ISO-8859-1",
-        ]:
+        if not match:
             yield LinterError(
-                f"VT uses a wrong encoding. Detected "
-                f"encoding is {encoding}.",
+                f"VT uses a wrong encoding. "
+                f"Allowed encodings are {', '.join(ALLOWED_ENCODINGS)}.",
                 file=nasl_file,
                 plugin=self.name,
             )