From 0848c421200b2737f43c4f2c6b4d24cc5ac592eb Mon Sep 17 00:00:00 2001 From: Robert Bamler Date: Tue, 2 Mar 2021 00:43:46 +0100 Subject: [PATCH 1/3] Recurse through submodules in extension modules The original method of finding submodules by recursing through the directory structure didn't work with extension modules (aka native "C" extensions) because there is no directory structure to recurse through. This commit adds code to recurse through submodules by inspecting `obj.__all__`, i.e., using the same mechanism that is already used for finding variables, functions, and classes. This is a minimalistic first sketch of a solution. It works on a native extension that I tried it on but it introduces some duplication since there are now two strategies to search for submodules. For a native module, the first strategy (searching the directory structure) will fail but the second (new) strategy (using `obj.__all__`) will work. For regular modules, both strategies will likely work and so the second strategy will just overwrite the results from the first strategy with identical objects. This should technically work but it seems unnecessary. I just don't feel confident enough to remove the first strategy without further guidance. --- pdoc/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index 7b6c873e..a66b9e8a 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -708,6 +708,10 @@ def is_from_this_module(obj): self.doc[name] = Function(name, self, obj) elif inspect.isclass(obj): self.doc[name] = Class(name, self, obj) + elif inspect.ismodule(obj): + self.doc[name] = Module( + obj, docfilter=docfilter, supermodule=self, + context=context, skip_errors=skip_errors) elif name in var_docstrings: self.doc[name] = Variable(name, self, var_docstrings[name], obj=obj) From 4dd1ccf2427eb783b99903a1d32a2dc4067e6c89 Mon Sep 17 00:00:00 2001 From: Robert Bamler Date: Wed, 3 Mar 2021 19:25:32 +0100 Subject: [PATCH 2/3] Detect submods of native mods more conservatively - The submodule detection for extension modules (aka native "C" extensions) now runs only if the regular submodule detection didn't run. - Further, the submodule detection for extension modules runs only if submodules are explicitly exposed via the "__all__" attribute. This is is because it would otherwise be difficult or impossible to reliably detect cycles and/or reexports of external modules. --- pdoc/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index a66b9e8a..c02213b1 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -708,10 +708,6 @@ def is_from_this_module(obj): self.doc[name] = Function(name, self, obj) elif inspect.isclass(obj): self.doc[name] = Class(name, self, obj) - elif inspect.ismodule(obj): - self.doc[name] = Module( - obj, docfilter=docfilter, supermodule=self, - context=context, skip_errors=skip_errors) elif name in var_docstrings: self.doc[name] = Variable(name, self, var_docstrings[name], obj=obj) @@ -765,6 +761,24 @@ def iter_modules(paths): if m.is_namespace and not m.doc: del self.doc[root] self._context.pop(m.refname) + elif hasattr(self.obj, '__all__'): + # Python extension modules don't get recognized by `is_package` because they have no + # "__path__" attribute. We treat them here separately. We support submodules of + # extension modules only if they are explicitly exposed via the "__all__" attribute + # because otherwise it's hard to distinguish proper submodules from re-exports (i.e., + # the function `is_from_this_module` doesn't work on submodules). + for name, obj in public_objs: + if inspect.ismodule(obj) and not hasattr(obj, '__file__') and name not in self.doc: + try: + m = Module( + obj, docfilter=docfilter, supermodule=self, + context=self._context, skip_errors=skip_errors) + except Exception as ex: + if skip_errors: + warn(str(ex), Module.ImportWarning) + continue + raise + self.doc[name] = m # Apply docfilter if docfilter: From 5601d8e51a87162027c097cd5070c06978e7454c Mon Sep 17 00:00:00 2001 From: Robert Bamler Date: Wed, 3 Mar 2021 19:27:47 +0100 Subject: [PATCH 3/3] Add unit test for extension module with submodule For portability reasons, this unit test only simulates how an extension model would look like. It doesn't actually package an extension module, because then we'd have to provide a separte extension module for each platform, os, and python version. --- pdoc/test/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py index c967dc87..d95e6191 100644 --- a/pdoc/test/__init__.py +++ b/pdoc/test/__init__.py @@ -472,6 +472,35 @@ def test_module(self): self.assertEqual(sorted(m.name for m in m.submodules()), [EXAMPLE_MODULE + '.' + m for m in submodules]) + @ignore_warnings + def test_module_without_path(self): + # GH-319: https://github.com/pdoc3/pdoc/issues/319 + parent_module = ModuleType('parent_module') + child_module1 = ModuleType('child_module1') + child_module2 = ModuleType('child_module2') + grandchild_module = ModuleType('grandchild_module') + + child_module1.grandchild_module = grandchild_module + child_module1.__all__ = ['grandchild_module'] + + parent_module.child_module1 = child_module1 + parent_module.child_module2 = child_module2 + parent_module.__all__ = ['child_module1', 'child_module2'] + + assert not hasattr(parent_module, '__path__') + assert not hasattr(child_module1, '__path__') + assert not hasattr(child_module2, '__path__') + assert not hasattr(grandchild_module, '__path__') + + parent_module_pdoc = pdoc.Module(parent_module) + + children_modules_pdoc = sorted(parent_module_pdoc.submodules(), key=lambda m: m.name) + self.assertEqual( + [m.name for m in children_modules_pdoc], ['child_module1', 'child_module2']) + self.assertEqual( + [m.name for m in children_modules_pdoc[0].submodules()], ['grandchild_module']) + self.assertEqual(children_modules_pdoc[1].submodules(), []) + def test_Module_find_class(self): class A: pass