Merge pull request #87 from akaihola/newline-splitting

Split newlines only at Python universal newlines (LF, CRLF, CR)

CHANGES.rst

@@ -15,6 +15,8 @@ Removed
 
 Fixed
 -----
+- Only split input files at Python's universal newlines (LF, CRLF, CR), not on more
+  exotic newline sequences. This fixes some edge cases in Darker.
 
 
 2.1.0_ - 2024-11-19

src/darkgraylib/tests/test_utils.py

@@ -138,6 +138,12 @@ def test_textdocument_encoded_string(encoding, newline, expect):
     dict(
         doc=TextDocument(string="zéro\r\nun\r\n", newline="\r\n"), expect=("zéro", "un")
     ),
+    dict(
+        doc=TextDocument(
+            string="# coding: iso-8859-5\n# б\x85б\x86\n", encoding="iso-8859-5"
+        ),
+        expect=("# coding: iso-8859-5", "# б\x85б\x86"),
+    ),
 )
 def test_textdocument_lines(doc, expect):
     """TextDocument.lines is correct after parsing a string with different newlines"""

src/darkgraylib/utils.py

@@ -22,6 +22,22 @@ def detect_newline(string: str) -> str:
     return "\n"
 
 
+def normalize_newlines(string: str) -> str:
+    """Normalize newlines in a string to LF"""
+    return io.IncrementalNewlineDecoder(None, True).decode(string)
+
+
+def splitlines(string: str) -> list[str]:
+    """Split a string into lines at universal newlines."""
+    if not string:
+        return []
+    return (
+        normalize_newlines(string)  # Normalize newlines to LF
+        .rstrip("\n")  # Remove trailing newline
+        .split("\n")  # Split into lines
+    )
+
+
 class TextDocument:
     """Store & handle a multi-line text document, either as a string or list of lines"""
 
@@ -65,7 +81,7 @@ def encoded_string(self) -> bytes:
     def lines(self) -> TextLines:
         """Return the document as a list of lines converting and caching if necessary"""
         if self._lines is None:
-            self._lines = tuple((self._string or "").splitlines())
+            self._lines = tuple(splitlines(self._string or ""))
         return self._lines
 
     @property