Enable mypy, part 1 (#7442)

Summary:

This is the first PR to enable MyPy typechecker.

There are few next steps, but this can land as-is as a standalone PR.

Will keep enabling in smaller batches, via directory by directory.

Also side note, pyre-ignore can be migrated to "# type: ignore" since pyre is backwards compatible with mypy.

.github/workflows/lint.yml

@@ -20,13 +20,19 @@ jobs:
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-linter
+      submodules: 'true'
       fetch-depth: 0
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
         # The generic Linux job chooses to use base env, not the one setup by the image
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
         conda activate "${CONDA_ENV}"
+        
+        # For mypy linting, we need to first install executorch first so that
+        # it builds the python package information.
+        BUILD_TOOL="cmake"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
         CACHE_DIRECTORY="/tmp/.lintbin"
         # Try to recover the cached binaries

.gitignore

@@ -1,5 +1,6 @@
 .hypothesis
 buck-out/
+.mypy_cache/
 buck2-bin/
 cmake-out*
 .DS_Store

.lintrunner.toml

@@ -285,3 +285,49 @@ command = [
     '--',
     '@{{PATHSFILE}}',
 ]
+
+[[linter]]
+code = 'MYPY'
+include_patterns = [
+    # TODO(https://github.com/pytorch/executorch/issues/7441): Gradually start enabling all folders.
+    # 'backends/**/*.py',
+    'build/**/*.py',
+    # 'codegen/**/*.py',
+    # 'devtools/**/*.py',
+    # 'docs/**/*.py',
+    # 'examples/**/*.py',
+    # 'exir/**/*.py',
+    # 'extension/**/*.py',
+    'kernels/**/*.py',
+    # 'profiler/**/*.py',
+    'runtime/**/*.py',
+    'scripts/**/*.py',
+    # 'test/**/*.py',
+    # 'util/**/*.py',
+    '*.py',
+]
+exclude_patterns = [
+    'third-party/**',
+    '**/third-party/**',
+    'scripts/check_binary_dependencies.py',
+]
+command = [
+    'python',
+    '-m',
+    'lintrunner_adapters',
+    'run',
+    'mypy_linter',
+    '--config=.mypy.ini',
+    '--show-disable',
+    '--',
+    '@{{PATHSFILE}}'
+]
+init_command = [
+    'python',
+    '-m',
+    'lintrunner_adapters',
+    'run',
+    'pip_init',
+    '--dry-run={{DRYRUN}}',
+    '--requirement=requirements-lintrunner.txt',
+]

.mypy.ini

@@ -0,0 +1,65 @@
+[mypy]
+allow_redefinition = True
+warn_unused_configs = True
+warn_redundant_casts = True
+show_error_codes = True
+show_column_numbers = True
+disallow_untyped_decorators = True
+follow_imports = normal
+local_partial_types = True
+enable_error_code = possibly-undefined
+warn_unused_ignores = False
+
+# TODO(https://github.com/pytorch/executorch/issues/7441): Remove this
+# disable_error_code = import-untyped
+
+files =
+    backends,
+    codegen,
+    devtools
+    examples,
+    exir,
+    extension,
+    kernels,
+    profiler,
+    runtime,
+    scripts,
+    util
+
+mypy_path = executorch
+
+[mypy-executorch.codegen.*]
+follow_untyped_imports = True
+
+[mypy-executorch.extension.*]
+follow_untyped_imports = True
+
+[mypy-executorch.exir.*]
+follow_untyped_imports = True
+
+[mypy-executorch.kernels.*]
+follow_untyped_imports = True
+
+[mypy-executorch.runtime.*]
+follow_untyped_imports = True
+
+[mypy-torchgen.*]
+follow_untyped_imports = True
+
+[mypy-setuptools.*]
+ignore_missing_imports = True
+
+[mypy-buck_util]
+ignore_missing_imports = True
+
+[mypy-tomllib]
+ignore_missing_imports = True
+
+[mypy-zstd]
+ignore_missing_imports = True
+
+[mypy-yaml]
+ignore_missing_imports = True
+
+[mypy-ruamel]
+ignore_missing_imports = True

build/buck_util.py

@@ -24,7 +24,10 @@ def run(self, args: Sequence[str]) -> list[str]:
         """Runs buck2 with the given args and returns its stdout as a sequence of lines."""
         try:
             cp: subprocess.CompletedProcess = subprocess.run(
-                [self._path] + args, capture_output=True, cwd=BUCK_CWD, check=True
+                [self._path] + args,  # type: ignore[operator]
+                capture_output=True,
+                cwd=BUCK_CWD,
+                check=True,
             )
             return [line.strip().decode("utf-8") for line in cp.stdout.splitlines()]
         except subprocess.CalledProcessError as ex:

build/extract_sources.py

@@ -85,7 +85,7 @@ def __init__(
         base_dict: Optional[dict] = None,
     ) -> None:
         self._state: Target._InitState = Target._InitState.UNINITIALIZED
-        self._sources = frozenset()
+        self._sources: frozenset[str] = frozenset()
 
         self.name = name
         # Extend the base lists with the target-specific entries.

codegen/tools/gen_all_oplist.py

@@ -145,7 +145,7 @@ def main(argv: List[Any]) -> None:
 
     # Optionally check if the operators in the model file list are overlapping.
     if options.check_ops_not_overlapping:
-        ops = {}
+        ops = {}  # type: ignore[var-annotated]
         for model_dict in model_dicts:
             for op_name in model_dict["operators"]:
                 if op_name in ops:

codegen/tools/gen_oplist.py

@@ -79,7 +79,7 @@ class KernelType(IntEnum):
 
 
 def _get_operators(model_file: str) -> List[str]:
-    from executorch.codegen.tools.selective_build import (
+    from executorch.codegen.tools.selective_build import (  # type: ignore[import-not-found]
         _get_program_from_buffer,
         _get_program_operators,
     )
@@ -96,7 +96,7 @@ def _get_operators(model_file: str) -> List[str]:
 
 def _get_kernel_metadata_for_model(model_file: str) -> Dict[str, List[str]]:
 
-    from executorch.codegen.tools.selective_build import (
+    from executorch.codegen.tools.selective_build import (  # type: ignore[import-not-found]
         _get_io_metadata_for_program_operators,
         _get_program_from_buffer,
         _IOMetaData,
@@ -159,7 +159,7 @@ def _dump_yaml(
     include_all_operators: bool = False,
 ):
     # no debug info yet
-    output = {}
+    output: dict[str, Any] = {}
     operators: Dict[str, Dict[str, object]] = {}
     for op_name in op_list:
         op = SelectiveBuildOperator.from_yaml_dict(
@@ -208,7 +208,7 @@ def gen_oplist(
     assert output_path, "Need to provide output_path for dumped yaml file."
     op_set = set()
     source_name = None
-    et_kernel_metadata = {}
+    et_kernel_metadata = {}  # type: ignore[var-annotated]
     if root_ops:
         # decide delimiter
         delimiter = "," if "," in root_ops else " "

codegen/tools/gen_ops_def.py

@@ -31,7 +31,7 @@ def get_operators(model_file: str) -> List[Operator]:
 
 def dump_yaml(model_file: str, output_file: str) -> None:
     ops = get_operators(model_file)
-    m = []
+    m = []  # type: ignore[var-annotated]
     for op in ops:
         if op.name.startswith("aten::"):
             schemas = torch._C._jit_get_schemas_for_operator(op.name)

codegen/tools/merge_yaml.py

@@ -44,8 +44,7 @@ def get_canonical_opname(func: object) -> str:
         Returns:
             str: canonical name of the operator
         """
-        # pyre-ignore
-        opname = func["op"] if "op" in func else func["func"].split("(")[0]
+        opname = func["op"] if "op" in func else func["func"].split("(")[0]  # type: ignore
         if "::" not in opname:
             opname = "aten::" + opname
         return opname

codegen/tools/test/test_gen_oplist_real_model.py

@@ -13,7 +13,7 @@
     _get_operators,
 )
 
-from libfb.py import parutil
+from libfb.py import parutil  # type: ignore[import-not-found]
 
 MODEL_PATH: Final[str] = parutil.get_file_path("ModuleLinear.pte", pkg=__package__)
 

kernels/test/gen_supported_features_test.py

@@ -8,7 +8,7 @@
 import unittest
 
 import yaml
-from executorch.kernels.test.gen_supported_features import (
+from executorch.kernels.test.gen_supported_features import (  # type: ignore[import-not-found]
     generate_definition,
     generate_header,
 )

requirements-lintrunner.txt

@@ -20,3 +20,6 @@ usort==1.0.8.post1
 # Other linters
 clang-format==18.1.3
 cmakelint==1.4.1
+
+# MyPy
+mypy==1.14.1

scripts/file_size_compare.py

@@ -170,6 +170,7 @@ def main() -> int:
         )
     elif args.max_size is not None:
         return compare_against_max(args.compare_file, args.max_size)
+    return 0
 
 
 if __name__ == "__main__":

setup.py

@@ -145,7 +145,7 @@ def string(cls) -> str:
                     open(os.path.join(cls._root_dir(), "version.txt")).read().strip()
                 )
                 if cls.git_hash():
-                    version += "+" + cls.git_hash()[:7]
+                    version += "+" + cls.git_hash()[:7]  # type: ignore[index]
             cls.__string_attr = version
         return cls.__string_attr