diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46a6685..1781569 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: uses: jakebailey/pyright-action@v2 with: python-version: ${{ matrix.python-version }} - verify-types: deferred + verify-types: defer-imports test: runs-on: ${{ matrix.os }} diff --git a/README.rst b/README.rst index 45593d9..6eb76f0 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,8 @@ ======== -deferred +defer-imports ======== -.. image:: https://github.com/Sachaa-Thanasius/deferred/actions/workflows/ci.yml/badge.svg - :alt: CI Status - :target: https://github.com/Sachaa-Thanasius/deferred/actions/workflows/ci.yml - -.. image:: https://img.shields.io/github/license/Sachaa-Thanasius/deferred.svg +.. image:: https://img.shields.io/github/license/Sachaa-Thanasius/defer-imports.svg :alt: License: MIT :target: https://opensource.org/licenses/MIT @@ -21,7 +17,7 @@ Installation This can be installed via pip:: - python -m pip install git@https://github.com/Sachaa-Thanasius/deferred + python -m pip install git@https://github.com/Sachaa-Thanasius/defer-imports It can also easily be vendored, as it has zero dependencies and is less than 1,000 lines of code. @@ -32,25 +28,25 @@ Usage Setup ----- -``deferred`` hooks into the Python import system with a path hook. That path hook needs to be registered before code using the import-delaying context manager, ``defer_imports_until_use``, is parsed. To do that, include the following somewhere such that it will be executed before your code: +``defer-imports`` hooks into the Python import system with a path hook. That path hook needs to be registered before code using the import-delaying context manager, ``defer_imports.until_use``, is parsed. To do that, include the following somewhere such that it will be executed before your code: .. code:: python - import deferred + import defer_imports - deferred.install_defer_import_hook() + defer_imports.install_defer_import_hook() Example ------- -Assuming the path hook has been registered, you can use the ``defer_imports_until_use`` context manager to decide which imports should be deferred. For instance: +Assuming the path hook has been registered, you can use the ``defer_imports.until_use`` context manager to decide which imports should be deferred. For instance: .. code:: python - from deferred import defer_imports_until_use + import defer_imports - with defer_imports_until_use: + with defer_imports.until_use: import inspect from typing import Final @@ -62,7 +58,7 @@ Use Cases - If imports are necessary to get symbols that are only used within annotations, but such imports would cause import chains. - - The current workaround for this is to perform the problematic imports within ``if typing.TYPE_CHECKING: ...`` blocks and then stringify the fake-imported, nonexistent symbols to prevent NameErrors at runtime; however, resulting annotations are difficult to introspect with standard library introspection tools, since they assume the symbols exist. Using ``with defer_imports_until_use: ...`` in such a circumstance would ensure that the symbols will be imported and saved in the local namespace, but only upon introspection, making the imports non-circular and almost free in most circumstances. + - The current workaround for this is to perform the problematic imports within ``if typing.TYPE_CHECKING: ...`` blocks and then stringify the fake-imported, nonexistent symbols to prevent NameErrors at runtime; however, the resulting annotations raise errors on introspection. Using ``with defer_imports.until_use: ...`` instead would ensure that the symbols will be imported and saved in the local namespace, but only upon introspection, making the imports non-circular and almost free in most circumstances. - If expensive imports are only necessary for certain code paths that won't always be taken, e.g. in subcommands in CLI tools. @@ -104,11 +100,11 @@ How? The core of this package is quite simple: when import statments are executed, the resulting values are special proxies representing the delayed import, which are then saved in the local namespace with special keys instead of normal string keys. When a user requests the normal string key corresponding to the import, the relevant import is executed and both the special key and the proxy replace themselves with the correct string key and import result. Everything stems from this. -The ``defer_imports_until_used`` context manager is what causes the proxies to be returned by the import statements: it temporarily replaces ``builtins.__import__`` with a version that will give back proxies that store the arguments needed to execute the *actual* import at a later time. +The ``defer_imports.until_use`` context manager is what causes the proxies to be returned by the import statements: it temporarily replaces ``builtins.__import__`` with a version that will give back proxies that store the arguments needed to execute the *actual* import at a later time. Those proxies don't use those stored ``__import__`` arguments themselves, though; the aforementioned special keys are what use the proxy's stored arguments to trigger the late import. These keys are aware of the namespace, the *dictionary*, they live in, are aware of the proxy they are the key for, and have overriden their ``__eq__`` and ``__hash__`` methods so that they know when they've been queried. In a sense, they're like descriptors, but instead of "owning the dot", they're "owning the brackets". Once such a key has been matched (i.e. someone uses the name of the import), it can use its corresponding proxy's stored arguments to execute the late import and *replace itself and the proxy* in the local namespace. That way, as soon as the name of the deferred import is referenced, all a user sees in the local namespace is a normal string key and the result of the resolved import. -The missing intermediate step is making sure these special proxies are stored with these special keys in the namespace. After all, Python name binding semantics only allow regular strings to be used as variable names/namespace keys; how can this be bypassed? ``deferred``'s answer is a little compile-time instrumentation. When a user calls ``deferred.install_deferred_import_hook()`` to set up the library machinery (see "Setup" above), what they are actually doing is installing an import hook that will modify the code of any given Python file that uses the ``defer_imports_until_use`` context manager. Using AST transformation, it adds a few lines of code around imports within that context manager to reassign the returned proxies to special keys in the local namespace (via ``locals()``). +The missing intermediate step is making sure these special proxies are stored with these special keys in the namespace. After all, Python name binding semantics only allow regular strings to be used as variable names/namespace keys; how can this be bypassed? ``defer-imports``'s answer is a little compile-time instrumentation. When a user calls ``defer_imports.install_deferred_import_hook()`` to set up the library machinery (see "Setup" above), what they are actually doing is installing an import hook that will modify the code of any given Python file that uses the ``defer_imports.until_use`` context manager. Using AST transformation, it adds a few lines of code around imports within that context manager to reassign the returned proxies to special keys in the local namespace (via ``locals()``). With this methodology, we can avoid using implementation-specific hacks like frame manipulation to modify the locals. We can even avoid changing the contract of ``builtins.__import__``, which specifically says it does not modify the global or local namespaces that are passed into it. We may modify and replace members of it, but at no point do we change its size while within ``__import__`` by removing or adding anything. @@ -122,8 +118,8 @@ A bit rough, but there are currently two ways of measuring activation and/or imp - To prevent bytecode caching from impacting the benchmark, run with `python -B `_, which will set ``sys.dont_write_bytecode`` to ``True`` and cause the benchmark script to purge all existing ``__pycache__`` folders in the project directory. - PyPy is excluded from the benchmark since it takes time to ramp up. - - The cost of registering ``deferred``'s import hook is ignored since that is a one-time startup cost that will hopefully be reduced in time. - - An sample run: + - The cost of registering ``defer-imports``'s import hook is ignored since that is a one-time startup cost that will hopefully be reduced in time. + - An sample run across versions using ``hatch run benchmark:bench``: (Run once with ``__pycache__`` folders removed and ``sys.dont_write_bytecode=True``): @@ -132,28 +128,28 @@ A bit rough, but there are currently two ways of measuring activation and/or imp ============== ======= ========== =================== CPython 3.9 regular 0.48585s (409.31x) CPython 3.9 slothy 0.00269s (2.27x) - CPython 3.9 deferred 0.00119s (1.00x) + CPython 3.9 defer-imports 0.00119s (1.00x) \-\- \-\- \-\- \-\- CPython 3.10 regular 0.41860s (313.20x) CPython 3.10 slothy 0.00458s (3.43x) - CPython 3.10 deferred 0.00134s (1.00x) + CPython 3.10 defer-imports 0.00134s (1.00x) \-\- \-\- \-\- \-\- CPython 3.11 regular 0.60501s (279.51x) CPython 3.11 slothy 0.00570s (2.63x) - CPython 3.11 deferred 0.00216s (1.00x) + CPython 3.11 defer-imports 0.00216s (1.00x) \-\- \-\- \-\- \-\- CPython 3.12 regular 0.53233s (374.40x) CPython 3.12 slothy 0.00552s (3.88x) - CPython 3.12 deferred 0.00142s (1.00x) + CPython 3.12 defer-imports 0.00142s (1.00x) \-\- \-\- \-\- \-\- CPython 3.13 regular 0.53704s (212.19x) CPython 3.13 slothy 0.00319s (1.26x) - CPython 3.13 deferred 0.00253s (1.00x) + CPython 3.13 defer-imports 0.00253s (1.00x) ============== ======= ========== =================== -- ``python -m timeit -n 1 -r 1 -- "import deferred"`` +- ``python -m timeit -n 1 -r 1 -- "import defer_imports"`` - - Substitute ``deferred`` with other modules, e.g. ``slothy``, to compare. + - Substitute ``defer_imports`` with other modules, e.g. ``slothy``, to compare. - This has great variance, so only value the resulting time relative to another import's time in the same process if possible. @@ -162,6 +158,6 @@ Acknowledgements - All the packages mentioned in "Why?" above, for providing inspiration. - `PEP 690 `_ and its authors, for pushing lazy imports to the point of almost being accepted as a core part of CPython's import system. -- Jelle Zijlstra, for so easily creating and sharing a `sample implementation `_ that ``slothy`` and ``deferred`` are based on. +- Jelle Zijlstra, for so easily creating and sharing a `sample implementation `_ that ``slothy`` and ``defer-imports`` are based on. - `slothy `_, for being a major reference and inspiration for this project. - Sinbad, for all his feedback. diff --git a/benchmark/bench_samples.py b/benchmark/bench_samples.py index 4be84bc..86c238b 100644 --- a/benchmark/bench_samples.py +++ b/benchmark/bench_samples.py @@ -1,6 +1,6 @@ # pyright: reportUnusedImport=none """Simple benchark script for comparing the import time of the Python standard library when using regular imports, -deferred-influence imports, and slothy-influenced imports. +defer_imports-influence imports, and slothy-influenced imports. The sample scripts being imported are generated with benchmark/generate_samples.py. """ @@ -10,7 +10,7 @@ import time from pathlib import Path -import deferred +import defer_imports class CatchTime: @@ -44,13 +44,13 @@ def bench_regular() -> float: return ct.elapsed -def bench_deferred() -> float: - deferred.install_defer_import_hook() +def bench_defer_imports() -> float: + defer_imports.install_defer_import_hook() with CatchTime() as ct: - import benchmark.sample_deferred + import benchmark.sample_defer_imports - deferred.uninstall_defer_import_hook() + defer_imports.uninstall_defer_import_hook() return ct.elapsed @@ -63,7 +63,7 @@ def bench_slothy() -> float: BENCH_FUNCS = { "regular": bench_regular, "slothy": bench_slothy, - "deferred": bench_deferred, + "defer_imports": bench_defer_imports, } diff --git a/benchmark/generate_samples.py b/benchmark/generate_samples.py index a1c9d7a..fdc9463 100644 --- a/benchmark/generate_samples.py +++ b/benchmark/generate_samples.py @@ -1,4 +1,4 @@ -"""Generate sample scripts with the same set of imports but influenced by different libraries, e.g. deferred.""" +"""Generate sample scripts with the same set of imports but influenced by different libraries, e.g. defer_imports.""" from pathlib import Path @@ -563,21 +563,21 @@ def main() -> None: regular_contents = "\n".join((PYRIGHT_IGNORE_DIRECTIVES, GENERATED_BY_COMMENT, STDLIB_IMPORTS)) regular_path.write_text(regular_contents, encoding="utf-8") - # deferred-instrumented and deferred-hooked imports - deferred_path = bench_path / "sample_deferred.py" - deferred_contents = ( + # defer_imports-instrumented and defer_imports-hooked imports + defer_imports_path = bench_path / "sample_defer_imports.py" + defer_imports_contents = ( f"{PYRIGHT_IGNORE_DIRECTIVES}\n" f"{GENERATED_BY_COMMENT}\n" - "from deferred import defer_imports_until_use\n" + "import defer_imports\n" "\n" "\n" - "with defer_imports_until_use:\n" + "with defer_imports.until_use:\n" f"{INDENTED_STDLIB_IMPORTS}" ) - deferred_path.write_text(deferred_contents, encoding="utf-8") + defer_imports_path.write_text(defer_imports_contents, encoding="utf-8") tests_path = Path().resolve() / "tests" / "stdlib_imports.py" - tests_path.write_text(deferred_contents, encoding="utf-8") + tests_path.write_text(defer_imports_contents, encoding="utf-8") # slothy-hooked imports slothy_path = bench_path / "sample_slothy.py" diff --git a/benchmark/sample_deferred.py b/benchmark/sample_defer_imports.py similarity index 99% rename from benchmark/sample_deferred.py rename to benchmark/sample_defer_imports.py index 0b781bc..2163528 100644 --- a/benchmark/sample_deferred.py +++ b/benchmark/sample_defer_imports.py @@ -1,9 +1,9 @@ # pyright: reportUnusedImport=none, reportMissingTypeStubs=none # Generated by benchmark/generate_samples.py -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import __future__ # import _bootlocale # Doesn't exist on 3.11 on Windows diff --git a/pyproject.toml b/pyproject.toml index 4f02f56..8bb00b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,12 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "deferred" +name = "defer-imports" description = "Lazy imports with regular syntax in pure Python." requires-python = ">=3.9" license = "MIT" readme = { file = "README.rst", content-type = "text/x-rst" } -authors = [ - { name = "Sachaa-Thanasius", email = "111999343+Sachaa-Thanasius@users.noreply.github.com" }, -] +authors = [{ name = "Sachaa-Thanasius" }] classifiers = [ "Development Status :: 3 - Alpha", "Natural Language :: English", @@ -31,21 +29,21 @@ classifiers = [ dynamic = ["version"] [tool.hatch.version] -path = "src/deferred/_core.py" +path = "src/defer_imports/_core.py" [project.optional-dependencies] benchmark = ["slothy"] test = ["pytest"] -cov = ["deferred[test]", "coverage", "covdefaults"] -dev = ["deferred[benchmark,cov]", "pre-commit"] +cov = ["defer-imports[test]", "coverage", "covdefaults"] +dev = ["defer-imports[benchmark,cov]", "pre-commit"] [project.urls] -Documentation = "https://github.com/Sachaa-Thanasius/deferred#readme" -Issues = "https://github.com/Sachaa-Thanasius/deferred/issues" -Source = "https://github.com/Sachaa-Thanasius/deferred" +Documentation = "https://github.com/Sachaa-Thanasius/defer-imports#readme" +Issues = "https://github.com/Sachaa-Thanasius/defer-imports/issues" +Source = "https://github.com/Sachaa-Thanasius/defer-imports" [tool.hatch.build.targets.wheel] -packages = ["src/deferred"] +packages = ["src/defer_imports"] # -------- Benchmark config @@ -77,11 +75,11 @@ addopts = [ ] [tool.coverage.paths] -deferred = ["src"] +defer_imports = ["src"] [tool.coverage.run] plugins = ["covdefaults"] -source = ["deferred", "tests"] +source = ["defer_imports", "tests"] [tool.coverage.report] # It's a work in progress. @@ -157,7 +155,7 @@ extend-ignore = [ unfixable = [ "ERA", # Prevent unlikely erroneous deletion. ] -typing-modules = ["deferred._typing"] +typing-modules = ["defer_imports._typing"] [tool.ruff.lint.isort] lines-after-imports = 2 @@ -173,10 +171,10 @@ keep-runtime-typing = true "F403", "F405", ] -"src/deferred/_core.py" = [ +"src/defer_imports/_core.py" = [ "A002", # Allow some shadowing of builtins by parameter names. ] -"src/deferred/_typing.py" = [ +"src/defer_imports/_typing.py" = [ "F822", # __all__ has names that are only provided by module-level __getattr__. ] @@ -201,7 +199,7 @@ keep-runtime-typing = true # -------- Type-checker config [tool.pyright] -include = ["src/deferred", "tests"] +include = ["src/defer_imports", "tests"] pythonVersion = "3.9" pythonPlatform = "All" typeCheckingMode = "strict" diff --git a/src/deferred/__init__.py b/src/defer_imports/__init__.py similarity index 74% rename from src/deferred/__init__.py rename to src/defer_imports/__init__.py index 196966e..9cce1a8 100644 --- a/src/deferred/__init__.py +++ b/src/defer_imports/__init__.py @@ -7,13 +7,13 @@ """ from ._console import DeferredInteractiveConsole -from ._core import __version__, defer_imports_until_use, install_defer_import_hook, uninstall_defer_import_hook +from ._core import __version__, install_defer_import_hook, uninstall_defer_import_hook, until_use __all__ = ( "__version__", - "defer_imports_until_use", "install_defer_import_hook", "uninstall_defer_import_hook", + "until_use", "DeferredInteractiveConsole", ) diff --git a/src/deferred/__main__.py b/src/defer_imports/__main__.py similarity index 100% rename from src/deferred/__main__.py rename to src/defer_imports/__main__.py diff --git a/src/deferred/_console.py b/src/defer_imports/_console.py similarity index 91% rename from src/deferred/_console.py rename to src/defer_imports/_console.py index 1151b34..22be21b 100644 --- a/src/deferred/_console.py +++ b/src/defer_imports/_console.py @@ -8,8 +8,8 @@ class DeferredInteractiveConsole(InteractiveConsole): - """An emulator of the interactive Python interpreter, but with deferred's compile-time hook baked in to ensure that - defer_imports_until_use works as intended directly in the console. + """An emulator of the interactive Python interpreter, but with defer_import's compile-time hook baked in to ensure that + defer_imports.until_use works as intended directly in the console. """ def __init__(self) -> None: diff --git a/src/deferred/_core.py b/src/defer_imports/_core.py similarity index 84% rename from src/deferred/_core.py rename to src/defer_imports/_core.py index c4a88c0..82355ba 100644 --- a/src/deferred/_core.py +++ b/src/defer_imports/_core.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT -"""The implementation details for deferred's magic.""" +"""The implementation details for defer_imports's magic.""" from __future__ import annotations @@ -28,12 +28,12 @@ SourceData: _tp.TypeAlias = "_tp.Union[_tp.ReadableBuffer, str, ast.Module, ast.Expression, ast.Interactive]" -BYTECODE_HEADER = f"deferred{__version__}".encode() -"""Custom header for deferred-instrumented bytecode files. Should be updated with every version release.""" +BYTECODE_HEADER = f"defer_imports{__version__}".encode() +"""Custom header for defer_imports-instrumented bytecode files. Should be updated with every version release.""" class DeferredInstrumenter(ast.NodeTransformer): - """AST transformer that instruments imports within "with defer_imports_until_use: ..." blocks so that their + """AST transformer that instruments imports within "with defer_imports.until_use: ..." blocks so that their results are assigned to custom keys in the global namespace. """ @@ -54,7 +54,7 @@ def instrument(self, mode: str = "exec") -> _tp.Any: return ast.fix_missing_locations(self.visit(to_visit)) def _visit_scope(self, node: ast.AST) -> ast.AST: - """Track Python scope changes. Used to determine if defer_imports_until_use usage is valid.""" + """Track Python scope changes. Used to determine if defer_imports.until_use usage is valid.""" self.scope_depth += 1 try: @@ -91,7 +91,7 @@ def _get_node_context(self, node: ast.stmt): # noqa: ANN202 # Version-dependent @staticmethod def _create_import_name_replacement(name: str) -> ast.If: - """Create an AST for changing the name of a variable in locals if the variable is a deferred proxy. + """Create an AST for changing the name of a variable in locals if the variable is a defer_imports proxy. The resulting node if unparsed is almost equivalent to the following:: @@ -177,12 +177,12 @@ def _substitute_import_keys(self, import_nodes: list[ast.stmt]) -> list[ast.stmt node = import_nodes[i] if not isinstance(node, (ast.Import, ast.ImportFrom)): - msg = "with defer_imports_until_use blocks must only contain import statements" + msg = "with defer_imports.until_use blocks must only contain import statements" raise SyntaxError(msg, self._get_node_context(node)) # noqa: TRY004 for alias in node.names: if alias.name == "*": - msg = "import * not allowed in with defer_imports_until_use blocks" + msg = "import * not allowed in with defer_imports.until_use blocks" raise SyntaxError(msg, self._get_node_context(node)) new_import_nodes.insert(i + 1, self._create_import_name_replacement(alias.asname or alias.name)) @@ -199,49 +199,42 @@ def _substitute_import_keys(self, import_nodes: list[ast.stmt]) -> list[ast.stmt @staticmethod def check_With_for_defer_usage(node: ast.With) -> bool: return len(node.items) == 1 and ( - ( - # Allow "with defer_imports_until_use". - isinstance(node.items[0].context_expr, ast.Name) - and node.items[0].context_expr.id == "defer_imports_until_use" - ) - or ( - # Allow "with deferred.defer_imports_until_use". - isinstance(node.items[0].context_expr, ast.Attribute) - and isinstance(node.items[0].context_expr.value, ast.Name) - and node.items[0].context_expr.value.id == "deferred" - and node.items[0].context_expr.attr == "defer_imports_until_use" - ) + # Allow "with defer_imports.until_use". + isinstance(node.items[0].context_expr, ast.Attribute) + and isinstance(node.items[0].context_expr.value, ast.Name) + and node.items[0].context_expr.value.id == "defer_imports" + and node.items[0].context_expr.attr == "until_use" ) def visit_With(self, node: ast.With) -> ast.AST: - """Check that "with defer_imports_until_use" blocks are valid and if so, hook all imports within. + """Check that "with defer_imports.until_use" blocks are valid and if so, hook all imports within. Raises ------ SyntaxError: If any of the following conditions are met, in order of priority: - 1. "defer_imports_until_use" is being used in a class or function scope. - 2. "defer_imports_until_use" block contains a statement that isn't an import. - 3. "defer_imports_until_use" block contains a wildcard import. + 1. "defer_imports.until_use" is being used in a class or function scope. + 2. "defer_imports.until_use" block contains a statement that isn't an import. + 3. "defer_imports.until_use" block contains a wildcard import. """ if not self.check_With_for_defer_usage(node): return self.generic_visit(node) if self.scope_depth != 0: - msg = "with defer_imports_until_use only allowed at module level" + msg = "with defer_imports.until_use only allowed at module level" raise SyntaxError(msg, self._get_node_context(node)) node.body = self._substitute_import_keys(node.body) return node def visit_Module(self, node: ast.Module) -> ast.AST: - """Insert imports necessary to make defer_imports_until_use work properly. The import is placed after the + """Insert imports necessary to make defer_imports.until_use work properly. The import is placed after the module docstring and after __future__ imports. Notes ----- - This assumes the module is not empty and "with defer_imports_until_use" is used somewhere in it. + This assumes the module is not empty and "with defer_imports.until_use" is used somewhere in it. """ expect_docstring = True @@ -266,7 +259,7 @@ def visit_Module(self, node: ast.Module) -> ast.AST: # Import key and proxy classes. defer_aliases = [ast.alias(name=name, asname=f"@{name}") for name in defer_class_names] - key_and_proxy_import = ast.ImportFrom(module="deferred._core", names=defer_aliases, level=0) + key_and_proxy_import = ast.ImportFrom(module="defer_imports._core", names=defer_aliases, level=0) node.body.insert(position, key_and_proxy_import) # Clean up the namespace. @@ -300,52 +293,45 @@ def sliding_window(iterable: _tp.Iterable[_tp.T], n: int) -> _tp.Iterable[tuple[ yield tuple(window) -class DeferredFileLoader(SourceFileLoader): - """A file loader that instruments .py files which use "with defer_imports_until_use: ...".""" +def check_source_for_defer_usage(data: _tp.Union[_tp.ReadableBuffer, str]) -> tuple[str, bool]: + """Get the encoding of the given code and also check if it uses "with defer_imports.until_use".""" - @staticmethod - def check_source_for_defer_usage(data: _tp.Union[_tp.ReadableBuffer, str]) -> tuple[str, bool]: - """Get the encoding of the given code and also check if it uses "with defer_imports_until_use".""" + tok_NAME, tok_OP = tokenize.NAME, tokenize.OP - tok_NAME, tok_OP = tokenize.NAME, tokenize.OP + if isinstance(data, str): + token_stream = tokenize.generate_tokens(io.StringIO(data).readline) + encoding = "utf-8" + else: + token_stream = tokenize.tokenize(io.BytesIO(data).readline) + encoding = next(token_stream).string + + uses_defer = any( + match_token(tok1, type=tok_NAME, string="with") + and match_token(tok2, type=tok_NAME, string="defer_imports") + and match_token(tok3, type=tok_OP, string=".") + and match_token(tok4, type=tok_NAME, string="until_use") + for tok1, tok2, tok3, tok4 in sliding_window(token_stream, 4) + ) - if isinstance(data, str): - token_stream = tokenize.generate_tokens(io.StringIO(data).readline) - encoding = "utf-8" - else: - token_stream = tokenize.tokenize(io.BytesIO(data).readline) - encoding = next(token_stream).string - - uses_defer = any( - match_token(tok1, type=tok_NAME, string="with") - and ( - ( - # Allow "with defer_imports_until_use". - match_token(tok2, type=tok_NAME, string="defer_imports_until_use") - and match_token(tok3, type=tok_OP, string=":") - ) - or ( - # Allow "with deferred.defer_imports_until_use". - match_token(tok2, type=tok_NAME, string="deferred") - and match_token(tok3, type=tok_OP, string=".") - and match_token(tok4, type=tok_NAME, string="defer_imports_until_use") - ) - ) - for tok1, tok2, tok3, tok4 in sliding_window(token_stream, 4) - ) + return encoding, uses_defer - return encoding, uses_defer - @staticmethod - def check_ast_for_defer_usage(data: ast.AST) -> tuple[str, bool]: - """Check if the given AST uses "with defer_imports_until_use". Also assume "utf-8" is the the encoding.""" +def check_ast_for_defer_usage(data: ast.AST) -> tuple[str, bool]: + """Check if the given AST uses "with defer_imports.until_use". Also assume "utf-8" is the the encoding.""" + + uses_defer = any( + isinstance(node, ast.With) and DeferredInstrumenter.check_With_for_defer_usage(node) for node in ast.walk(data) + ) + encoding = "utf-8" + return encoding, uses_defer - uses_defer = any( - isinstance(node, ast.With) and DeferredInstrumenter.check_With_for_defer_usage(node) - for node in ast.walk(data) - ) - encoding = "utf-8" - return encoding, uses_defer + +class DeferredFileLoader(SourceFileLoader): + """A file loader that instruments .py files which use "with defer_imports.until_use: ...".""" + + @staticmethod + def check_for_defer_usage(data: SourceData) -> tuple[str, bool]: + return check_ast_for_defer_usage(data) if isinstance(data, ast.AST) else check_source_for_defer_usage(data) def source_to_code( # pyright: ignore [reportIncompatibleMethodOverride] self, @@ -360,10 +346,7 @@ def source_to_code( # pyright: ignore [reportIncompatibleMethodOverride] if not data: return super().source_to_code(data, path, _optimize=_optimize) # pyright: ignore # See note above. - if isinstance(data, ast.AST): - encoding, uses_defer = self.check_ast_for_defer_usage(data) - else: - encoding, uses_defer = self.check_source_for_defer_usage(data) + encoding, uses_defer = self.check_for_defer_usage(data) if not uses_defer: return super().source_to_code(data, path, _optimize=_optimize) # pyright: ignore # See note above. @@ -376,7 +359,7 @@ def get_data(self, path: str) -> bytes: Notes ----- - If the path points to a bytecode file, check for a deferred-specific header. If the header is invalid, raise + If the path points to a bytecode file, check for a defer_imports-specific header. If the header is invalid, raise OSError to invalidate the bytecode; importlib._boostrap_external.SourceLoader.get_code expects this [1]_. Another option is to monkeypatch importlib.util.cache_from_source, as beartype [2]_ and typeguard [3]_ do, but that seems @@ -394,12 +377,12 @@ def get_data(self, path: str) -> bytes: if not path.endswith(tuple(BYTECODE_SUFFIXES)): return data - if not data.startswith(b"deferred"): - msg = '"deferred" header missing from bytecode' + if not data.startswith(b"defer_imports"): + msg = '"defer_imports" header missing from bytecode' raise OSError(msg) if not data.startswith(BYTECODE_HEADER): - msg = '"deferred" header is outdated' + msg = '"defer_imports" header is outdated' raise OSError(msg) return data[len(BYTECODE_HEADER) :] @@ -409,7 +392,7 @@ def set_data(self, path: str, data: _tp.ReadableBuffer, *, _mode: int = 0o666) - Notes ----- - If the file is a bytecode one, prepend a deferred-specific header to it. That way, instrumented bytecode can be + If the file is a bytecode one, prepend a defer_imports-specific header to it. That way, instrumented bytecode can be identified and invalidated later if necessary [1]_. References @@ -637,7 +620,6 @@ def deferred___import__( # noqa: ANN202 locals: _tp.MutableMapping[str, object], fromlist: _tp.Optional[_tp.Sequence[str]] = None, level: int = 0, - /, ): """An limited replacement for __import__ that supports deferred imports by returning proxies.""" @@ -683,7 +665,7 @@ def deferred___import__( # noqa: ANN202 def install_defer_import_hook() -> None: - """Insert deferred's path hook right before the default FileFinder one in sys.path_hooks. + """Insert defer_imports's path hook right before the default FileFinder one in sys.path_hooks. This can be called in a few places, e.g. __init__.py of a package, a .pth file in site packages, etc. """ @@ -701,7 +683,7 @@ def install_defer_import_hook() -> None: def uninstall_defer_import_hook() -> None: - """Remove deferred's path hook if it's in sys.path_hooks.""" + """Remove defer_imports's path hook if it's in sys.path_hooks.""" try: sys.path_hooks.remove(DEFERRED_PATH_HOOK) @@ -714,7 +696,7 @@ def uninstall_defer_import_hook() -> None: @_tp.final class DeferredContext: - """The type for defer_imports_until_use.""" + """The type for defer_imports.until_use.""" __slots__ = ("_import_ctx_token", "_defer_ctx_token") @@ -729,7 +711,7 @@ def __exit__(self, *exc_info: object) -> None: builtins.__import__ = original_import.get() -defer_imports_until_use: _tp.Final[DeferredContext] = DeferredContext() +until_use: _tp.Final[DeferredContext] = DeferredContext() """A context manager within which imports occur lazily. Not reentrant. This will not work correctly if install_defer_import_hook() was not called first elsewhere. @@ -737,7 +719,7 @@ def __exit__(self, *exc_info: object) -> None: Raises ------ SyntaxError - If defer_imports_until_use is used improperly, e.g.: + If defer_imports.until_use is used improperly, e.g.: 1. It is being used in a class or function scope. 2. It contains a statement that isn't an import. 3. It contains a wildcard import. diff --git a/src/deferred/_typing.py b/src/defer_imports/_typing.py similarity index 100% rename from src/deferred/_typing.py rename to src/defer_imports/_typing.py diff --git a/src/deferred/_typing.pyi b/src/defer_imports/_typing.pyi similarity index 100% rename from src/deferred/_typing.pyi rename to src/defer_imports/_typing.pyi diff --git a/src/deferred/py.typed b/src/defer_imports/py.typed similarity index 100% rename from src/deferred/py.typed rename to src/defer_imports/py.typed diff --git a/tests/stdlib_imports.py b/tests/stdlib_imports.py index 0b781bc..2163528 100644 --- a/tests/stdlib_imports.py +++ b/tests/stdlib_imports.py @@ -1,9 +1,9 @@ # pyright: reportUnusedImport=none, reportMissingTypeStubs=none # Generated by benchmark/generate_samples.py -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import __future__ # import _bootlocale # Doesn't exist on 3.11 on Windows diff --git a/tests/test_deferred.py b/tests/test_deferred.py index b17f12a..5e98df8 100644 --- a/tests/test_deferred.py +++ b/tests/test_deferred.py @@ -1,4 +1,4 @@ -"""Tests for deferred. +"""Tests for defer_imports. Notes ----- @@ -15,7 +15,7 @@ import pytest -from deferred._core import ( +from defer_imports._core import ( BYTECODE_HEADER, DEFERRED_PATH_HOOK, DeferredFileLoader, @@ -60,7 +60,7 @@ def temp_cache_module(name: str, module: ModuleType): """'''Module docstring here'''""", '''\ """Module docstring here""" -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy del @DeferredImportKey, @DeferredImportProxy ''', id="Inserts statements after module docstring", @@ -69,7 +69,7 @@ def temp_cache_module(name: str, module: ModuleType): """from __future__ import annotations""", """\ from __future__ import annotations -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy del @DeferredImportKey, @DeferredImportProxy """, id="Inserts statements after __future__ import", @@ -78,16 +78,16 @@ def temp_cache_module(name: str, module: ModuleType): """\ from contextlib import nullcontext -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use, nullcontext(): +with defer_imports.until_use, nullcontext(): import inspect """, """\ -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy from contextlib import nullcontext -from deferred import defer_imports_until_use -with defer_imports_until_use, nullcontext(): +import defer_imports +with defer_imports.until_use, nullcontext(): import inspect del @DeferredImportKey, @DeferredImportProxy """, @@ -95,15 +95,15 @@ def temp_cache_module(name: str, module: ModuleType): ), pytest.param( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import inspect """, """\ -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy -from deferred import defer_imports_until_use -with defer_imports_until_use: +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +import defer_imports +with defer_imports.until_use: @local_ns = locals() @temp_proxy = None import inspect @@ -117,17 +117,16 @@ def temp_cache_module(name: str, module: ModuleType): ), pytest.param( """\ -from deferred import defer_imports_until_use +import defer_imports - -with defer_imports_until_use: +with defer_imports.until_use: import importlib import importlib.abc """, """\ -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy -from deferred import defer_imports_until_use -with defer_imports_until_use: +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +import defer_imports +with defer_imports.until_use: @local_ns = locals() @temp_proxy = None import importlib @@ -145,15 +144,15 @@ def temp_cache_module(name: str, module: ModuleType): ), pytest.param( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from . import a """, """\ -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy -from deferred import defer_imports_until_use -with defer_imports_until_use: +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +import defer_imports +with defer_imports.until_use: @local_ns = locals() @temp_proxy = None from . import a @@ -167,15 +166,15 @@ def temp_cache_module(name: str, module: ModuleType): ), pytest.param( """\ -import deferred +import defer_imports -with deferred.defer_imports_until_use: +with defer_imports.until_use: from . import a """, """\ -from deferred._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy -import deferred -with deferred.defer_imports_until_use: +from defer_imports._core import DeferredImportKey as @DeferredImportKey, DeferredImportProxy as @DeferredImportProxy +import defer_imports +with defer_imports.until_use: @local_ns = locals() @temp_proxy = None from . import a @@ -185,12 +184,12 @@ def temp_cache_module(name: str, module: ModuleType): del @temp_proxy, @local_ns del @DeferredImportKey, @DeferredImportProxy """, - id="with deferred.defer_imports_until_use", + id="with defer_imports.until_use", ), ], ) def test_instrumentation(before: str, after: str): - """Test what code is generated by the instrumentation side of deferred.""" + """Test what code is generated by the instrumentation side of defer_imports.""" import ast import io @@ -204,7 +203,7 @@ def test_instrumentation(before: str, after: str): def test_path_hook_installation(): - """Test the API for putting/removing the deferred path hook from sys.path_hooks.""" + """Test the API for putting/removing the defer_imports path hook from sys.path_hooks.""" # It shouldn't be on there by default. assert DEFERRED_PATH_HOOK not in sys.path_hooks @@ -250,9 +249,9 @@ def test_not_deferred(tmp_path: Path): def test_regular_import(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import inspect """ @@ -274,9 +273,9 @@ def sample_func(a: int, c: float) -> float: ... def test_regular_import_with_rename(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import inspect as gin """ @@ -307,9 +306,9 @@ def sample_func(a: int, b: str) -> str: ... def test_regular_import_nested(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import importlib.abc """ @@ -329,9 +328,9 @@ def test_regular_import_nested(tmp_path: Path): def test_regular_import_nested_with_rename(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import collections.abc as xyz """ @@ -378,9 +377,9 @@ def test_regular_import_nested_with_rename(tmp_path: Path): def test_from_import(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from inspect import isfunction, signature """ @@ -409,9 +408,9 @@ def test_from_import(tmp_path: Path): def test_from_import_with_rename(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from inspect import Signature as MySignature """ @@ -435,14 +434,14 @@ def test_from_import_with_rename(tmp_path: Path): def test_deferred_header_in_instrumented_pycache(tmp_path: Path): - """Test that the deferred-specific bytecode header is being prepended to the bytecode cache files of - deferred-instrumented modules. + """Test that the defer_imports-specific bytecode header is being prepended to the bytecode cache files of + defer_imports-instrumented modules. """ source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import asyncio """ @@ -460,9 +459,9 @@ def test_deferred_header_in_instrumented_pycache(tmp_path: Path): def test_error_if_non_import(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: print("Hello world") """ @@ -480,10 +479,10 @@ def test_error_if_non_import(tmp_path: Path): def test_error_if_import_in_class(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports class Example: - with defer_imports_until_use: + with defer_imports.until_use: from inspect import signature """ @@ -497,15 +496,15 @@ class Example: assert exc_info.value.filename == str(module_path) assert exc_info.value.lineno == 4 assert exc_info.value.offset == 5 - assert exc_info.value.text == " with defer_imports_until_use:\n from inspect import signature" + assert exc_info.value.text == " with defer_imports.until_use:\n from inspect import signature" def test_error_if_import_in_function(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports def test(): - with defer_imports_until_use: + with defer_imports.until_use: import inspect return inspect.signature(test) @@ -520,14 +519,14 @@ def test(): assert exc_info.value.filename == str(module_path) assert exc_info.value.lineno == 4 assert exc_info.value.offset == 5 - assert exc_info.value.text == " with defer_imports_until_use:\n import inspect" + assert exc_info.value.text == " with defer_imports.until_use:\n import inspect" def test_error_if_wildcard_import(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from typing import * """ @@ -545,9 +544,9 @@ def test_error_if_wildcard_import(tmp_path: Path): def test_top_level_and_submodules_1(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import importlib import importlib.abc import importlib.util @@ -589,9 +588,9 @@ def test_top_level_and_submodules_2(tmp_path: Path): source = """\ from pprint import pprint -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import asyncio import asyncio.base_events import asyncio.base_futures @@ -609,9 +608,9 @@ def test_top_level_and_submodules_2(tmp_path: Path): def test_mixed_from_same_module(tmp_path: Path): source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import asyncio from asyncio import base_events from asyncio import base_futures @@ -653,7 +652,7 @@ def test_mixed_from_same_module(tmp_path: Path): def test_relative_imports(tmp_path: Path): - """Test a synthetic package that uses relative imports within defer_imports_until_use blocks. + """Test a synthetic package that uses relative imports within defer_imports.until_use blocks. The package has the following structure: . @@ -667,9 +666,9 @@ def test_relative_imports(tmp_path: Path): sample_pkg_path.mkdir() sample_pkg_path.joinpath("__init__.py").write_text( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from . import a from .a import A from .b import B @@ -736,9 +735,9 @@ def test_circular_imports(tmp_path: Path): circular_pkg_path.mkdir() circular_pkg_path.joinpath("__init__.py").write_text( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import circular_pkg.main """, encoding="utf-8", @@ -811,9 +810,9 @@ def test_thread_safety(tmp_path: Path): """ source = """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: import inspect """ @@ -880,9 +879,9 @@ def test_leaking_patch(tmp_path: Path): leaking_patch_pkg_path.joinpath("__init__.py").touch() leaking_patch_pkg_path.joinpath("a.py").write_text( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from .b import B """, encoding="utf-8", @@ -934,9 +933,9 @@ def test_type_statement_312(tmp_path: Path): type_stmt_pkg_path.mkdir() type_stmt_pkg_path.joinpath("__init__.py").write_text( """\ -from deferred import defer_imports_until_use +import defer_imports -with defer_imports_until_use: +with defer_imports.until_use: from .exp import Expensive type ManyExpensive = tuple[Expensive, ...]