Many 'o' fixes

.coveragerc

@@ -2,8 +2,12 @@
 branch=True
 source=unasync
 
+[paths]
+source = src/unasync
+
 [report]
 precision = 1
 exclude_lines =
   pragma: no cover
   abc.abstractmethod
+  \# PY2

.coveragerc-py2

@@ -0,0 +1,13 @@
+[run]
+branch=True
+source=unasync
+
+[paths]
+source = src/unasync
+
+[report]
+precision = 1
+exclude_lines =
+  pragma: no cover
+  abc.abstractmethod
+  \# PY3

ci/travis.sh

@@ -51,6 +51,16 @@ if [ "$USE_PYPY_RELEASE_VERSION" != "" ]; then
     source testenv/bin/activate
 fi
 
+case "${MACPYTHON:-${TRAVIS_PYTHON_VERSION:-}}" in
+	2*)
+		COVERAGE_FILE=.coveragerc-py2
+	;;
+
+	*)
+		COVERAGE_FILE=.coveragerc
+	;;
+esac
+
 pip install -U pip setuptools wheel
 
 if [ "$CHECK_FORMATTING" = "1" ]; then
@@ -91,7 +101,7 @@ else
     mkdir empty
     cd empty
 
-    pytest -ra -v --cov=unasync --cov-config=../.coveragerc --verbose ../tests
+    pytest -ra -v --cov=unasync --cov-config="../${COVERAGE_FILE}" --verbose ../tests
 
     bash <(curl -s https://codecov.io/bash)
 fi

src/unasync/__init__.py

@@ -1,9 +1,11 @@
+# -*- encoding: utf8 -*-
 """Top-level package for unasync."""
 
 from __future__ import print_function
 
 import collections
 import errno
+import io
 import os
 import sys
 import tokenize as std_tokenize
@@ -34,13 +36,34 @@
     "StopAsyncIteration": "StopIteration",
 }
 
+_TYPE_COMMENT_PREFIX = "# type: "
+
+
+if sys.version_info[0] == 2:  # PY2
+
+    def isidentifier(s):
+        return all([c.isalnum() or c == "_" for c in s])
+
+    StringIO = io.BytesIO
+else:  # PY3
+
+    def isidentifier(s):
+        return s.isidentifier()
+
+    StringIO = io.StringIO
+
+if hasattr(os, "fspath"):  # PY3
+    fspath = os.fspath
+else:  # PY2
+    fspath = str
+
 
 class Rule:
     """A single set of rules for 'unasync'ing file(s)"""
 
     def __init__(self, fromdir, todir, additional_replacements=None):
-        self.fromdir = fromdir.replace("/", os.sep)
-        self.todir = todir.replace("/", os.sep)
+        self.fromdir = fspath(fromdir).replace("/", os.sep)
+        self.todir = fspath(todir).replace("/", os.sep)
 
         # Add any additional user-defined token replacements to our list.
         self.token_replacements = _ASYNC_TO_SYNC.copy()
@@ -51,6 +74,8 @@ def _match(self, filepath):
         """Determines if a Rule matches a given filepath and if so
         returns a higher comparable value if the match is more specific.
         """
+        filepath = fspath(filepath)
+
         file_segments = [x for x in filepath.split(os.sep) if x]
         from_segments = [x for x in self.fromdir.split(os.sep) if x]
         len_from_segments = len(from_segments)
@@ -65,9 +90,10 @@ def _match(self, filepath):
         return False
 
     def _unasync_file(self, filepath):
+        filepath = fspath(filepath)
         with open(filepath, "rb") as f:
             write_kwargs = {}
-            if sys.version_info[0] >= 3:
+            if sys.version_info[0] >= 3:  # PY3  # pragma: no branch
                 encoding, _ = std_tokenize.detect_encoding(f.readline)
                 write_kwargs["encoding"] = encoding
                 f.seek(0)
@@ -82,7 +108,57 @@ def _unasync_file(self, filepath):
     def _unasync_tokens(self, tokens):
         # TODO __await__, ...?
         used_space = None
+        context = None  # Can be `None`, `"func_decl"`, `"func_name"`, `"arg_list"`, `"arg_list_end"`, `"return_type"`
+        brace_depth = 0
+        typing_ctx = False
+
         for space, toknum, tokval in tokens:
+            # Update context state tracker
+            if context is None and toknum == std_tokenize.NAME and tokval == "def":
+                context = "func_decl"
+            elif context == "func_decl" and toknum == std_tokenize.NAME:
+                context = "func_name"
+            elif context == "func_name" and toknum == std_tokenize.OP and tokval == "(":
+                context = "arg_list"
+            elif context == "arg_list":
+                if toknum == std_tokenize.OP and tokval in ("(", "["):
+                    brace_depth += 1
+                elif (
+                    toknum == std_tokenize.OP
+                    and tokval in (")", "]")
+                    and brace_depth >= 1
+                ):
+                    brace_depth -= 1
+                elif toknum == std_tokenize.OP and tokval == ")":
+                    context = "arg_list_end"
+                elif toknum == std_tokenize.OP and tokval == ":" and brace_depth < 1:
+                    typing_ctx = True
+                elif toknum == std_tokenize.OP and tokval == "," and brace_depth < 1:
+                    typing_ctx = False
+            elif (
+                context == "arg_list_end"
+                and toknum == std_tokenize.OP
+                and tokval == "->"
+            ):
+                context = "return_type"
+                typing_ctx = True
+            elif context == "return_type":
+                if toknum == std_tokenize.OP and tokval in ("(", "["):
+                    brace_depth += 1
+                elif (
+                    toknum == std_tokenize.OP
+                    and tokval in (")", "]")
+                    and brace_depth >= 1
+                ):
+                    brace_depth -= 1
+                elif toknum == std_tokenize.OP and tokval == ":":
+                    context = None
+                    typing_ctx = False
+            else:  # Something unexpected happend - reset state
+                context = None
+                brace_depth = 0
+                typing_ctx = False
+
             if tokval in ["async", "await"]:
                 # When removing async or await, we want to use the whitespace that
                 # was before async/await before the next token so that
@@ -93,8 +169,59 @@ def _unasync_tokens(self, tokens):
                 if toknum == std_tokenize.NAME:
                     tokval = self._unasync_name(tokval)
                 elif toknum == std_tokenize.STRING:
-                    left_quote, name, right_quote = tokval[0], tokval[1:-1], tokval[-1]
-                    tokval = left_quote + self._unasync_name(name) + right_quote
+                    # Strings in typing context are forward-references and should be unasyncified
+                    quote = ""
+                    prefix = ""
+                    while ord(tokval[0]) in range(ord("a"), ord("z") + 1):
+                        prefix += tokval[0]
+                        tokval = tokval[1:]
+
+                    if tokval.startswith('"""') and tokval.endswith('"""'):
+                        quote = '"""'  # Broken syntax highlighters workaround: """
+                    elif tokval.startswith("'''") and tokval.endswith("'''"):
+                        quote = "'''"  # Broken syntax highlighters wokraround: '''
+                    elif tokval.startswith('"') and tokval.endswith('"'):
+                        quote = '"'
+                    elif tokval.startswith(  # pragma: no branch
+                        "'"
+                    ) and tokval.endswith("'"):
+                        quote = "'"
+                    assert (
+                        len(quote) > 0
+                    ), "Quoting style of string {0!r} unknown".format(tokval)
+                    stringval = tokval[len(quote) : -len(quote)]
+                    if typing_ctx:
+                        stringval = _untokenize(
+                            self._unasync_tokens(_tokenize(StringIO(stringval)))
+                        )
+                    else:
+                        stringval = self._unasync_name(stringval)
+                    tokval = prefix + quote + stringval + quote
+                elif toknum == std_tokenize.COMMENT and tokval.startswith(
+                    _TYPE_COMMENT_PREFIX
+                ):
+                    type_decl, suffix = tokval[len(_TYPE_COMMENT_PREFIX) :], ""
+                    if "#" in type_decl:
+                        type_decl, suffix = type_decl.split("#", 1)
+                        suffix = "#" + suffix
+                    type_decl_stripped = type_decl.strip()
+
+                    # Do not process `type: ignore` or `type: ignore[…]` as these aren't actual identifiers
+                    is_type_ignore = type_decl_stripped == "ignore"
+                    is_type_ignore |= type_decl_stripped.startswith(
+                        "ignore"
+                    ) and not isidentifier(type_decl_stripped[0:7])
+                    if not is_type_ignore:
+                        # Preserve trailing whitespace since the tokenizer won't
+                        trailing_space_len = len(type_decl) - len(type_decl.rstrip())
+                        if trailing_space_len > 0:
+                            suffix = type_decl[-trailing_space_len:] + suffix
+                            type_decl = type_decl[:-trailing_space_len]
+                        type_decl = _untokenize(
+                            self._unasync_tokens(_tokenize(StringIO(type_decl)))
+                        )
+
+                    tokval = _TYPE_COMMENT_PREFIX + type_decl + suffix
                 if used_space is None:
                     used_space = space
                 yield (used_space, tokval)
@@ -128,12 +255,16 @@ def unasync_files(fpath_list, rules):
 
 
 def _get_tokens(f):
-    if sys.version_info[0] == 2:
+    if sys.version_info[0] == 2:  # PY2
         for tok in std_tokenize.generate_tokens(f.readline):
             type_, string, start, end, line = tok
             yield Token(type_, string, start, end, line)
-    else:
-        for tok in std_tokenize.tokenize(f.readline):
+    else:  # PY3
+        if isinstance(f, io.TextIOBase):
+            gen = std_tokenize.generate_tokens(f.readline)
+        else:
+            gen = std_tokenize.tokenize(f.readline)
+        for tok in gen:
             if tok.type == std_tokenize.ENCODING:
                 continue
             yield tok
@@ -143,13 +274,16 @@ def _tokenize(f):
     last_end = (1, 0)
     for tok in _get_tokens(f):
         if last_end[0] < tok.start[0]:
-            yield ("", std_tokenize.STRING, " \\\n")
+            # Somehow Python 3.5 and below produce the ENDMARKER in a way that
+            # causes superfluous continuation lines to be generated
+            if tok.type != std_tokenize.ENDMARKER:
+                yield (" ", std_tokenize.NEWLINE, "\\\n")
             last_end = (tok.start[0], 0)
 
         space = ""
         if tok.start > last_end:
             assert tok.start[0] == last_end[0]
-            space = " " * (tok.start[1] - last_end[1])
+            space = tok.line[last_end[1] : tok.start[1]]
         yield (space, tok.type, tok.string)
 
         last_end = tok.end

test-requirements.txt

@@ -1,2 +1,3 @@
 pytest>=4.3.0
-pytest-cov
+pytest-cov
+pathlib2 ; python_version < '3.5'

tests/data/async/tabs.py

@@ -0,0 +1,8 @@
+# fmt: off
+async def dummy():
+	await dummy2()  # This line is indented with a tab that should be preserved
+# fmt: on
+
+
+async def dummy2():
+    await dummy()  # This one uses 4 spaces and these should also be preserved

tests/data/async/typing.py

@@ -3,3 +3,26 @@
 typing.AsyncIterable[bytes]
 typing.AsyncIterator[bytes]
 typing.AsyncGenerator[bytes]
+
+# A typed function that takes the first item of an (a)sync iterator and returns it
+async def func1(a: typing.AsyncIterable[int]) -> str:
+    it: typing.AsyncIterator[int] = a.__aiter__()
+    b: int = await it.__anext__()
+    return str(b)
+
+
+# Same as the above but using old-style typings (mainly for Python 2.7 – 3.5 compatibility)
+async def func2(a):  # type: (typing.AsyncIterable[int]) -> str
+    it = a.__aiter__()  # type: typing.AsyncIterator[int]
+    b = await it.__anext__()  # type: int
+    return str(b)
+
+
+# And some funky edge cases to at least cover the relevant at all in this test
+a: int = 5
+b: str = a  # type: ignore  # This is the actual comment and the type declaration silences the warning that would otherwise happen
+c: str = a  # type: ignore2  # This is the actual comment and the declaration declares another type, both of which are wrong
+
+# fmt: off
+# And some genuine trailing whitespace (uww…)
+z = a  # type: int   

tests/data/async/typing_py3.py

@@ -0,0 +1,13 @@
+# fmt: off
+# A forward-reference typed function that returns an iterator for an (a)sync iterable
+async def aiter1(a: "typing.AsyncIterable[int]") -> 'typing.AsyncIterable[int]':
+	return a.__aiter__()
+
+# Same as the above but using tripple-quoted strings
+async def aiter2(a: """typing.AsyncIterable[int]""") -> r'''typing.AsyncIterable[int]''':
+	return a.__aiter__()
+
+# Same as the above but without forward-references
+async def aiter3(a: typing.AsyncIterable[int]) -> typing.AsyncIterable[int]:
+	return a.__aiter__()
+# fmt: on

tests/data/sync/tabs.py

@@ -0,0 +1,8 @@
+# fmt: off
+def dummy():
+	dummy2()  # This line is indented with a tab that should be preserved
+# fmt: on
+
+
+def dummy2():
+    dummy()  # This one uses 4 spaces and these should also be preserved

tests/data/sync/typing.py

@@ -3,3 +3,26 @@
 typing.Iterable[bytes]
 typing.Iterator[bytes]
 typing.Generator[bytes]
+
+# A typed function that takes the first item of an (a)sync iterator and returns it
+def func1(a: typing.Iterable[int]) -> str:
+    it: typing.Iterator[int] = a.__iter__()
+    b: int = it.__next__()
+    return str(b)
+
+
+# Same as the above but using old-style typings (mainly for Python 2.7 – 3.5 compatibility)
+def func2(a):  # type: (typing.Iterable[int]) -> str
+    it = a.__iter__()  # type: typing.Iterator[int]
+    b = it.__next__()  # type: int
+    return str(b)
+
+
+# And some funky edge cases to at least cover the relevant at all in this test
+a: int = 5
+b: str = a  # type: ignore  # This is the actual comment and the type declaration silences the warning that would otherwise happen
+c: str = a  # type: ignore2  # This is the actual comment and the declaration declares another type, both of which are wrong
+
+# fmt: off
+# And some genuine trailing whitespace (uww…)
+z = a  # type: int   

tests/data/sync/typing_py3.py

@@ -0,0 +1,13 @@
+# fmt: off
+# A forward-reference typed function that returns an iterator for an (a)sync iterable
+def aiter1(a: "typing.Iterable[int]") -> 'typing.Iterable[int]':
+	return a.__iter__()
+
+# Same as the above but using tripple-quoted strings
+def aiter2(a: """typing.Iterable[int]""") -> r'''typing.Iterable[int]''':
+	return a.__iter__()
+
+# Same as the above but without forward-references
+def aiter3(a: typing.Iterable[int]) -> typing.Iterable[int]:
+	return a.__iter__()
+# fmt: on