Merge pull request #9404 from jdufresne/mypy-fixup

Handle several mypy TODO comments and exceptions

Author: pradyunsg

news/40711960-12d9-4e58-8322-21e5975a804e.trivial.rst

@@ -1,13 +1,11 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import os
 import sys
 
 from setuptools import find_packages, setup
 
 
 def read(rel_path):
+    # type: (str) -> str
     here = os.path.abspath(os.path.dirname(__file__))
     # intentionally *not* adding an encoding option to open, See:
     #   https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
@@ -16,6 +14,7 @@ def read(rel_path):
 
 
 def get_version(rel_path):
+    # type: (str) -> str
     for line in read(rel_path).splitlines():
         if line.startswith('__version__'):
             # __version__ = "0.9"

setup.py

@@ -1,15 +1,12 @@
 """Base option parser setup"""
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import optparse
 import shutil
 import sys
 import textwrap
 from contextlib import suppress
-from typing import Any
+from typing import Any, Dict, Iterator, List, Tuple
 
 from pip._internal.cli.status_codes import UNKNOWN_ERROR
 from pip._internal.configuration import Configuration, ConfigurationError
@@ -22,16 +19,19 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter):
     """A prettier/less verbose help formatter for optparse."""
 
     def __init__(self, *args, **kwargs):
+        # type: (*Any, **Any) -> None
         # help position must be aligned with __init__.parseopts.description
         kwargs["max_help_position"] = 30
         kwargs["indent_increment"] = 1
         kwargs["width"] = shutil.get_terminal_size()[0] - 2
         super().__init__(*args, **kwargs)
 
     def format_option_strings(self, option):
+        # type: (optparse.Option) -> str
         return self._format_option_strings(option)
 
     def _format_option_strings(self, option, mvarfmt=" <{}>", optsep=", "):
+        # type: (optparse.Option, str, str) -> str
         """
         Return a comma-separated list of option strings and metavars.
 
@@ -49,17 +49,20 @@ def _format_option_strings(self, option, mvarfmt=" <{}>", optsep=", "):
             opts.insert(1, optsep)
 
         if option.takes_value():
+            assert option.dest is not None
             metavar = option.metavar or option.dest.lower()
             opts.append(mvarfmt.format(metavar.lower()))
 
         return "".join(opts)
 
     def format_heading(self, heading):
+        # type: (str) -> str
         if heading == "Options":
             return ""
         return heading + ":\n"
 
     def format_usage(self, usage):
+        # type: (str) -> str
         """
         Ensure there is only one newline between usage and the first heading
         if there is no description.
@@ -68,6 +71,7 @@ def format_usage(self, usage):
         return msg
 
     def format_description(self, description):
+        # type: (str) -> str
         # leave full control over description to us
         if description:
             if hasattr(self.parser, "main"):
@@ -86,13 +90,15 @@ def format_description(self, description):
             return ""
 
     def format_epilog(self, epilog):
+        # type: (str) -> str
         # leave full control over epilog to us
         if epilog:
             return epilog
         else:
             return ""
 
     def indent_lines(self, text, indent):
+        # type: (str, str) -> str
         new_lines = [indent + line for line in text.split("\n")]
         return "\n".join(new_lines)
 
@@ -107,9 +113,12 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter):
     """
 
     def expand_default(self, option):
+        # type: (optparse.Option) -> str
         default_values = None
         if self.parser is not None:
+            assert isinstance(self.parser, ConfigOptionParser)
             self.parser._update_defaults(self.parser.defaults)
+            assert option.dest is not None
             default_values = self.parser.defaults.get(option.dest)
         help_text = super().expand_default(option)
 
@@ -129,6 +138,7 @@ def expand_default(self, option):
 
 class CustomOptionParser(optparse.OptionParser):
     def insert_option_group(self, idx, *args, **kwargs):
+        # type: (int, Any, Any) -> optparse.OptionGroup
         """Insert an OptionGroup at a given position."""
         group = self.add_option_group(*args, **kwargs)
 
@@ -139,6 +149,7 @@ def insert_option_group(self, idx, *args, **kwargs):
 
     @property
     def option_list_all(self):
+        # type: () -> List[optparse.Option]
         """Get a list of all options, including those in option groups."""
         res = self.option_list[:]
         for i in self.option_groups:
@@ -166,18 +177,22 @@ def __init__(
         super().__init__(*args, **kwargs)
 
     def check_default(self, option, key, val):
+        # type: (optparse.Option, str, Any) -> Any
         try:
             return option.check_value(key, val)
         except optparse.OptionValueError as exc:
             print(f"An error occurred during configuration: {exc}")
             sys.exit(3)
 
     def _get_ordered_configuration_items(self):
+        # type: () -> Iterator[Tuple[str, Any]]
         # Configuration gives keys in an unordered manner. Order them.
         override_order = ["global", self.name, ":env:"]
 
         # Pool the options into different groups
-        section_items = {name: [] for name in override_order}
+        section_items = {
+            name: [] for name in override_order
+        }  # type: Dict[str, List[Tuple[str, Any]]]
         for section_key, val in self.config.items():
             # ignore empty values
             if not val:
@@ -197,6 +212,7 @@ def _get_ordered_configuration_items(self):
                 yield key, val
 
     def _update_defaults(self, defaults):
+        # type: (Dict[str, Any]) -> Dict[str, Any]
         """Updates the given defaults with values from the config files and
         the environ. Does a little special handling for certain types of
         options (lists)."""
@@ -215,6 +231,8 @@ def _update_defaults(self, defaults):
             if option is None:
                 continue
 
+            assert option.dest is not None
+
             if option.action in ("store_true", "store_false"):
                 try:
                     val = strtobool(val)
@@ -240,6 +258,7 @@ def _update_defaults(self, defaults):
                 val = val.split()
                 val = [self.check_default(option, key, v) for v in val]
             elif option.action == "callback":
+                assert option.callback is not None
                 late_eval.add(option.dest)
                 opt_str = option.get_opt_string()
                 val = option.convert_value(opt_str, val)
@@ -258,6 +277,7 @@ def _update_defaults(self, defaults):
         return defaults
 
     def get_default_values(self):
+        # type: () -> optparse.Values
         """Overriding to make updating the defaults after instantiation of
         the option parser possible, _update_defaults() does the dirty work."""
         if not self.process_default_values:
@@ -272,12 +292,14 @@ def get_default_values(self):
 
         defaults = self._update_defaults(self.defaults.copy())  # ours
         for option in self._get_all_options():
+            assert option.dest is not None
             default = defaults.get(option.dest)
             if isinstance(default, str):
                 opt_str = option.get_opt_string()
                 defaults[option.dest] = option.check_value(opt_str, default)
         return optparse.Values(defaults)
 
     def error(self, msg):
+        # type: (str) -> None
         self.print_usage(sys.stderr)
         self.exit(UNKNOWN_ERROR, f"{msg}\n")

src/pip/_internal/cli/parser.py

@@ -37,7 +37,7 @@
 
 
 def get_keyring_auth(url, username):
-    # type: (str, str) -> Optional[AuthInfo]
+    # type: (Optional[str], Optional[str]) -> Optional[AuthInfo]
     """Return the tuple auth for a given url from keyring."""
     global keyring
     if not url or not keyring:

src/pip/_internal/network/auth.py

@@ -2,8 +2,15 @@
 network request configuration and behavior.
 """
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
+# When mypy runs on Windows the call to distro.linux_distribution() is skipped
+# resulting in the failure:
+#
+#     error: unused 'type: ignore' comment
+#
+# If the upstream module adds typing, this comment should be removed. See
+# https://github.com/nir0s/distro/pull/269
+#
+# mypy: warn-unused-ignores=False
 
 import email.utils
 import ipaddress
@@ -15,13 +22,14 @@
 import sys
 import urllib.parse
 import warnings
-from typing import Any, Iterator, List, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple, Union
 
 from pip._vendor import requests, urllib3
 from pip._vendor.cachecontrol import CacheControlAdapter
 from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
-from pip._vendor.requests.models import Response
+from pip._vendor.requests.models import PreparedRequest, Response
 from pip._vendor.requests.structures import CaseInsensitiveDict
+from pip._vendor.urllib3.connectionpool import ConnectionPool
 from pip._vendor.urllib3.exceptions import InsecureRequestWarning
 
 from pip import __version__
@@ -89,6 +97,7 @@ def looks_like_ci():
 
 
 def user_agent():
+    # type: () -> str
     """
     Return a string representing the user agent.
     """
@@ -98,15 +107,14 @@ def user_agent():
         "implementation": {
             "name": platform.python_implementation(),
         },
-    }
+    }  # type: Dict[str, Any]
 
     if data["implementation"]["name"] == 'CPython':
         data["implementation"]["version"] = platform.python_version()
     elif data["implementation"]["name"] == 'PyPy':
-        if sys.pypy_version_info.releaselevel == 'final':
-            pypy_version_info = sys.pypy_version_info[:3]
-        else:
-            pypy_version_info = sys.pypy_version_info
+        pypy_version_info = sys.pypy_version_info  # type: ignore
+        if pypy_version_info.releaselevel == 'final':
+            pypy_version_info = pypy_version_info[:3]
         data["implementation"]["version"] = ".".join(
             [str(x) for x in pypy_version_info]
         )
@@ -119,9 +127,12 @@ def user_agent():
 
     if sys.platform.startswith("linux"):
         from pip._vendor import distro
+
+        # https://github.com/nir0s/distro/pull/269
+        linux_distribution = distro.linux_distribution()  # type: ignore
         distro_infos = dict(filter(
             lambda x: x[1],
-            zip(["name", "version", "id"], distro.linux_distribution()),
+            zip(["name", "version", "id"], linux_distribution),
         ))
         libc = dict(filter(
             lambda x: x[1],
@@ -170,8 +181,16 @@ def user_agent():
 
 class LocalFSAdapter(BaseAdapter):
 
-    def send(self, request, stream=None, timeout=None, verify=None, cert=None,
-             proxies=None):
+    def send(
+        self,
+        request,  # type: PreparedRequest
+        stream=False,  # type: bool
+        timeout=None,  # type: Optional[Union[float, Tuple[float, float]]]
+        verify=True,  # type: Union[bool, str]
+        cert=None,  # type: Optional[Union[str, Tuple[str, str]]]
+        proxies=None,  # type:Optional[Mapping[str, str]]
+    ):
+        # type: (...) -> Response
         pathname = url_to_path(request.url)
 
         resp = Response()
@@ -198,18 +217,33 @@ def send(self, request, stream=None, timeout=None, verify=None, cert=None,
         return resp
 
     def close(self):
+        # type: () -> None
         pass
 
 
 class InsecureHTTPAdapter(HTTPAdapter):
 
-    def cert_verify(self, conn, url, verify, cert):
+    def cert_verify(
+        self,
+        conn,  # type: ConnectionPool
+        url,  # type: str
+        verify,  # type: Union[bool, str]
+        cert,  # type: Optional[Union[str, Tuple[str, str]]]
+    ):
+        # type: (...) -> None
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
 
 class InsecureCacheControlAdapter(CacheControlAdapter):
 
-    def cert_verify(self, conn, url, verify, cert):
+    def cert_verify(
+        self,
+        conn,  # type: ConnectionPool
+        url,  # type: str
+        verify,  # type: Union[bool, str]
+        cert,  # type: Optional[Union[str, Tuple[str, str]]]
+    ):
+        # type: (...) -> None
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
 
@@ -407,6 +441,7 @@ def is_secure_origin(self, location):
         return False
 
     def request(self, method, url, *args, **kwargs):
+        # type: (str, str, *Any, **Any) -> Response
         # Allow setting a default timeout on a session
         kwargs.setdefault("timeout", self.timeout)
 

src/pip/_internal/network/session.py

@@ -12,13 +12,12 @@
 
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
-# mypy: disallow-untyped-defs=False
 
 import logging
 import sys
 from collections import defaultdict
 from itertools import chain
-from typing import DefaultDict, List, Optional, Set, Tuple
+from typing import DefaultDict, Iterable, List, Optional, Set, Tuple
 
 from pip._vendor.packaging import specifiers
 from pip._vendor.pkg_resources import Distribution
@@ -388,6 +387,7 @@ def _resolve_one(
         more_reqs = []  # type: List[InstallRequirement]
 
         def add_req(subreq, extras_requested):
+            # type: (Distribution, Iterable[str]) -> None
             sub_install_req = self._make_install_req(
                 str(subreq),
                 req_to_install,
@@ -447,6 +447,7 @@ def get_installation_order(self, req_set):
         ordered_reqs = set()  # type: Set[InstallRequirement]
 
         def schedule(req):
+            # type: (InstallRequirement) -> None
             if req.satisfied_by or req in ordered_reqs:
                 return
             if req.constraint:

src/pip/_internal/resolution/legacy/resolver.py

@@ -1,9 +1,6 @@
 """Stuff that differs in different Python versions and platform
 distributions."""
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import os
 import sys