From 69205ad8ea42018b7840627122ac72a11b4be863 Mon Sep 17 00:00:00 2001 From: tristenallen Date: Tue, 10 Oct 2023 13:51:23 -0700 Subject: [PATCH 1/3] Create small binary to deserialize pytype ASTs and print them. Adds a very small debugging tool (`parse_pickle`) which takes in a single argument (path to a pickled AST), deserializes it, prints the deserialized AST, and then finally re-prints the deserialized AST as a `pyi`-format string. Heavily borrows from `parse_pyi.py` in the same directory. #tftypes PiperOrigin-RevId: 572356461 --- pytype/pyi/CMakeLists.txt | 16 ++++++++ pytype/pyi/parse_pickle.py | 78 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 pytype/pyi/parse_pickle.py diff --git a/pytype/pyi/CMakeLists.txt b/pytype/pyi/CMakeLists.txt index 603a94034..3a7ca5ca5 100644 --- a/pytype/pyi/CMakeLists.txt +++ b/pytype/pyi/CMakeLists.txt @@ -149,6 +149,22 @@ toplevel_py_binary( pytype.pytd.pytd_for_parser ) +toplevel_py_binary( + NAME + parse_pickle + SRCS + parse_pickle.py + MAIN + parse_pickle.py + DEPS + .types + pytype.config + pytype.load_pytd + pytype.utils + pytype.imports.imports + pytype.pytd.pytd +) + py_library( NAME parser_test_base diff --git a/pytype/pyi/parse_pickle.py b/pytype/pyi/parse_pickle.py new file mode 100644 index 000000000..79d1d7265 --- /dev/null +++ b/pytype/pyi/parse_pickle.py @@ -0,0 +1,78 @@ +"""Testing code to print a pickled ast.""" + +import argparse +import sys +from typing import Optional + +from pytype import config +from pytype import load_pytd +from pytype import module_utils +from pytype import utils +from pytype.imports import pickle_utils +from pytype.pyi import types +from pytype.pytd import pytd +from pytype.pytd import pytd_utils +from pytype.pytd import serialize_ast + +_ParseError = types.ParseError + + +def _make_argument_parser() -> argparse.ArgumentParser: + """Creates and returns an argument parser.""" + + o = argparse.ArgumentParser() + o.add_argument('pytd', nargs='?', default=None, + help='Serialized AST to diagnose.') + o.add_argument('--pyi', nargs='?', default=None, + help='An optional pyi file to pickle in lieu of an existing ' + 'serialized AST.') + return o + + +def _pickle(src_path: str) -> Optional[bytes]: + """Run the serialization code on the pyi file at the given src_path.""" + + with open(src_path) as f: + src = f.read() + module_name = module_utils.path_to_module_name(src_path) + options = config.Options.create(module_name=module_name, + input_filename=src_path, + validate_version=False) + loader = load_pytd.Loader(options) + try: + ast: pytd.TypeDeclUnit = serialize_ast.SourceToExportableAst( + module_name, src, loader) + except _ParseError as e: + header = utils.COLOR_ERROR_NAME_TEMPLATE % 'ParseError:' + print(f'{header} Invalid type stub for module {module_name!r}:\n{e}', + file=sys.stderr) + return None + + return pickle_utils.StoreAst(ast) + + +def main() -> None: + args = _make_argument_parser().parse_args() + if args.pyi: + data = _pickle(args.pyi) + if not data: + sys.exit(1) + else: + with open(args.pytd, 'rb') as f: + data = f.read() + + try: + out: serialize_ast.SerializableAst = pickle_utils.LoadAst(data) + except _ParseError as e: + print(e) + sys.exit(1) + + print('-------------pytd-------------') + print(out.ast) + + print('-------------pyi--------------') + print(pytd_utils.Print(out.ast)) + + +if __name__ == '__main__': + main() From 5ba62a89648e78206cc1576c5dcfc524ea150e7f Mon Sep 17 00:00:00 2001 From: rechen Date: Tue, 10 Oct 2023 14:19:14 -0700 Subject: [PATCH 2/3] Bump pytype's maximum supported version. I'd say it's fair to call 3.11 "supported" now that pytype's own tests pass and all the pynext targets type-check cleanly. I removed a couple of the `validate_version=False` settings in tyre because, on second thought, I don't like the idea of non-test code turning off version validation. This shouldn't cause us much trouble with 3.11 -> 3.12, since it'll just cause some test failures that don't block anything else. PiperOrigin-RevId: 572365056 --- README.md | 4 ++-- docs/index.md | 4 ++-- docs/support.md | 7 ++++--- pytype/utils.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index faa6e438e..505d60278 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ merge-pyi -i .py .pytype/pyi/.pyi ## Requirements -You need a Python 3.8-3.10 interpreter to run pytype, as well as an +You need a Python 3.8-3.11 interpreter to run pytype, as well as an interpreter in `$PATH` for the Python version of the code you're analyzing -(supported: 3.8-3.10). +(supported: 3.8-3.11). Platform support: diff --git a/docs/index.md b/docs/index.md index 4a6445e0f..6b3658384 100755 --- a/docs/index.md +++ b/docs/index.md @@ -89,9 +89,9 @@ merge-pyi -i .py .pytype/pyi/.pyi ## Requirements -You need a Python 3.8-3.10 interpreter to run pytype, as well as an +You need a Python 3.8-3.11 interpreter to run pytype, as well as an interpreter in `$PATH` for the Python version of the code you're analyzing -(supported: 3.8-3.10). +(supported: 3.8-3.11). Platform support: diff --git a/docs/support.md b/docs/support.md index ecf82eb86..031664d3a 100644 --- a/docs/support.md +++ b/docs/support.md @@ -17,7 +17,7 @@ of pytype. * [Third-Party Libraries](#third-party-libraries) - + @@ -31,7 +31,7 @@ of pytype. (upcoming versions), if applicable Version | Analyzes | Runs In | Issue -:-----: | :--------: | :--------: | :------------: +:-----: | :--------: | :--------: | :----------: 2.7 | 2021.08.03 | 2020.04.01 | [#545][py27] 3.5 | 2021.09.09 | 2020.10.08 | [#677][py35] 3.6 | 2022.01.05 | 2022.01.05 | @@ -39,7 +39,8 @@ Version | Analyzes | Runs In | Issue 3.8 | ✅ | ✅ | 3.9 | ✅ | ✅ | 3.10 | ✅ | ✅ | -3.11 | ❌ | ❌ | [#1308][py311] +3.11 | ✅ | ✅ | +3.12 | ❌ | ❌ | ## Features diff --git a/pytype/utils.py b/pytype/utils.py index 643f7451f..a73928c2c 100644 --- a/pytype/utils.py +++ b/pytype/utils.py @@ -66,9 +66,9 @@ def validate_version(python_version): elif (3, 0) <= python_version <= (3, 7): raise UsageError( "Python versions 3.0 - 3.7 are not supported. Use 3.8 and higher.") - elif python_version > (3, 10) and _VALIDATE_PYTHON_VERSION_UPPER_BOUND: + elif python_version > (3, 11) and _VALIDATE_PYTHON_VERSION_UPPER_BOUND: # We have an explicit per-minor-version mapping in opcodes.py - raise UsageError("Python versions > 3.10 are not yet supported.") + raise UsageError("Python versions > 3.11 are not yet supported.") def strip_prefix(string, prefix): From 8eabac0691442881a2dec4c612d845258fd99245 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Tue, 10 Oct 2023 14:26:07 -0700 Subject: [PATCH 3/3] 3.11 preparation. --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 5 +---- requirements.txt | 2 +- setup.cfg | 3 ++- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be4a86158..07638770b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: platform: - manylinux2014_x86_64 - manylinux2014_aarch64 - pyver: ['cp38-cp38', 'cp39-cp39', 'cp310-cp310'] + pyver: ['cp38-cp38', 'cp39-cp39', 'cp310-cp310', 'cp311-cp311'] steps: - uses: actions/checkout@v4 with: @@ -58,7 +58,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python_version: ['3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5e44acee..4d387c1e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,12 +14,9 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] experimental: [false] include: - - os: ubuntu-20.04 - python-version: '3.11' - experimental: true - os: ubuntu-20.04 python-version: '3.12' experimental: true diff --git a/requirements.txt b/requirements.txt index 6484a98e4..7ae274e57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ libcst>=1.0.1 networkx<3.2 ninja>=1.10.0.post2 pybind11>=2.10.1 -pycnite>=2023.9.14 +pycnite>=2023.10.5 pydot>=1.4.2 pylint>=2.14.4 tabulate>=0.8.10 diff --git a/setup.cfg b/setup.cfg index 5f0650358..e9d664dff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython Topic :: Software Development @@ -36,7 +37,7 @@ install_requires = libcst>=1.0.1 networkx<3.2 ninja>=1.10.0.post2 - pycnite>=2023.9.14 + pycnite>=2023.10.5 pydot>=1.4.2 tabulate>=0.8.10 toml>=0.10.2