From cbfb53449019d71df0853d64024af2442d37d306 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 17 Apr 2024 18:21:13 +0200 Subject: [PATCH 01/13] Add `ipytest.autconfig(coverage=True)` --- Readme.md | 26 ++++++++++++++++---------- ipytest/_config.py | 25 +++++++++++++++++-------- ipytest/_impl.py | 20 +++++++++++++++++--- ipytest/cov.py | 3 +++ ipytest/coveragerc | 3 +++ 5 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 ipytest/coveragerc diff --git a/Readme.md b/Readme.md index 5b0ff74..5346861 100644 --- a/Readme.md +++ b/Readme.md @@ -99,22 +99,23 @@ this case, using [`ipytest.run()`][ipytest.run] and | [`ipytest.cov`](#ipytestcov) -### `ipytest.autoconfig(rewrite_asserts=, magics=, clean=, addopts=, run_in_thread=, defopts=, display_columns=, raise_on_error=)` +### `ipytest.autoconfig(rewrite_asserts=, magics=, clean=, addopts=, run_in_thread=, defopts=, display_columns=, raise_on_error=, coverage=)` -[ipytest.autoconfig]: #ipytestautoconfigrewrite_assertsdefault-magicsdefault-cleandefault-addoptsdefault-run_in_threaddefault-defoptsdefault-display_columnsdefault-raise_on_errordefault +[ipytest.autoconfig]: #ipytestautoconfigrewrite_assertsdefault-magicsdefault-cleandefault-addoptsdefault-run_in_threaddefault-defoptsdefault-display_columnsdefault-raise_on_errordefault-coveragedefault Configure `ipytest` with reasonable defaults. Specifically, it sets: -* `rewrite_asserts`: `True` -* `magics`: `True` -* `clean`: `'[Tt]est*'` * `addopts`: `('-q', '--color=yes')` -* `run_in_thread`: `False` +* `clean`: `'[Tt]est*'` +* `coverage`: `False` * `defopts`: `'auto'` * `display_columns`: `100` +* `magics`: `True` * `raise_on_error`: `False` +* `rewrite_asserts`: `True` +* `run_in_thread`: `False` See [`ipytest.config`][ipytest.config] for details. @@ -169,9 +170,9 @@ inside a CI/CD context, use `ipytest.autoconfig(raise_on_error=True)`. -### `ipytest.config(rewrite_asserts=, magics=, clean=, addopts=, run_in_thread=, defopts=, display_columns=, raise_on_error=)` +### `ipytest.config(rewrite_asserts=, magics=, clean=, addopts=, run_in_thread=, defopts=, display_columns=, raise_on_error=, coverage=)` -[ipytest.config]: #ipytestconfigrewrite_assertskeep-magicskeep-cleankeep-addoptskeep-run_in_threadkeep-defoptskeep-display_columnskeep-raise_on_errorkeep +[ipytest.config]: #ipytestconfigrewrite_assertskeep-magicskeep-cleankeep-addoptskeep-run_in_threadkeep-defoptskeep-display_columnskeep-raise_on_errorkeep-coveragedefault Configure `ipytest` @@ -186,6 +187,11 @@ The following settings are supported: * `rewrite_asserts` (default: `False`): enable ipython AST transforms globally to rewrite asserts * `magics` (default: `False`): if set to `True` register the ipytest magics +* `coverage` (default: `False`): if `True` configure `pytest` to collect + coverage information. This functionality requires the `pytest-cov` package + to be installed. It adds `--cov --cov-config={CONFIG}` to the arguments + when invoking `pytest`. **WARNING**: this option will hide existing + coverage configuration files * `clean` (default: `[Tt]est*`): the pattern used to clean variables * `addopts` (default: `()`): pytest command line arguments to prepend to every pytest invocation. For example setting @@ -217,9 +223,9 @@ The following settings are supported: The return code of the last pytest invocation. -### `ipytest.run(*args, module=None, plugins=(), run_in_thread=, raise_on_error=, addopts=, defopts=, display_columns=)` +### `ipytest.run(*args, module=None, plugins=(), run_in_thread=, raise_on_error=, addopts=, defopts=, display_columns=, coverage=)` -[ipytest.run]: #ipytestrunargs-modulenone-plugins-run_in_threaddefault-raise_on_errordefault-addoptsdefault-defoptsdefault-display_columnsdefault +[ipytest.run]: #ipytestrunargs-modulenone-plugins-run_in_threaddefault-raise_on_errordefault-addoptsdefault-defoptsdefault-display_columnsdefault-coveragedefault Execute all tests in the passed module (defaults to `__main__`) with pytest. diff --git a/ipytest/_config.py b/ipytest/_config.py index b82ee27..16b531e 100644 --- a/ipytest/_config.py +++ b/ipytest/_config.py @@ -5,25 +5,27 @@ default_clean = "[Tt]est*" defaults = { - "rewrite_asserts": True, - "magics": True, - "clean": default_clean, "addopts": ("-q", "--color=yes"), - "run_in_thread": False, + "clean": default_clean, + "coverage": False, "defopts": "auto", "display_columns": 100, + "magics": True, "raise_on_error": False, + "rewrite_asserts": True, + "run_in_thread": False, } current_config = { - "rewrite_asserts": False, - "magics": False, - "clean": default_clean, "addopts": (), - "run_in_thread": False, + "clean": default_clean, + "coverage": False, "defopts": "auto", "display_columns": 100, + "magics": False, "raise_on_error": False, + "rewrite_asserts": False, + "run_in_thread": False, } _rewrite_transformer = None @@ -67,6 +69,7 @@ def autoconfig( defopts=default, display_columns=default, raise_on_error=default, + coverage=default, ): """Configure `ipytest` with reasonable defaults. @@ -91,6 +94,7 @@ def config( defopts=keep, display_columns=keep, raise_on_error=keep, + coverage=default, ): """Configure `ipytest` @@ -105,6 +109,11 @@ def config( * `rewrite_asserts` (default: `False`): enable ipython AST transforms globally to rewrite asserts * `magics` (default: `False`): if set to `True` register the ipytest magics + * `coverage` (default: `False`): if `True` configure `pytest` to collect + coverage information. This functionality requires the `pytest-cov` package + to be installed. It adds `--cov --cov-config={CONFIG}` to the arguments + when invoking `pytest`. **WARNING**: this option will hide existing + coverage configuration files * `clean` (default: `[Tt]est*`): the pattern used to clean variables * `addopts` (default: `()`): pytest command line arguments to prepend to every pytest invocation. For example setting diff --git a/ipytest/_impl.py b/ipytest/_impl.py index 0d93d8c..a42d11c 100644 --- a/ipytest/_impl.py +++ b/ipytest/_impl.py @@ -28,6 +28,7 @@ def run( addopts=default, defopts=default, display_columns=default, + coverage=default, ): """Execute all tests in the passed module (defaults to `__main__`) with pytest. @@ -64,6 +65,7 @@ def run( addopts = default.unwrap(addopts, current_config["addopts"]) defopts = default.unwrap(defopts, current_config["defopts"]) display_columns = default.unwrap(display_columns, current_config["display_columns"]) + coverage = default.unwrap(coverage, current_config["coverage"]) if module is None: import __main__ as module @@ -77,6 +79,7 @@ def run( addopts=addopts, defopts=defopts, display_columns=display_columns, + coverage=coverage, ) ipytest.exit_code = exit_code @@ -250,13 +253,15 @@ def force_reload(*include: str, modules: Optional[Dict[str, ModuleType]] = None) modules.pop(name, None) -def _run_impl(*args, module, plugins, addopts, defopts, display_columns): +def _run_impl(*args, module, plugins, addopts, defopts, display_columns, coverage): with _prepared_env(module, display_columns=display_columns) as filename: - full_args = _build_full_args(args, filename, addopts=addopts, defopts=defopts) + full_args = _build_full_args( + args, filename, addopts=addopts, defopts=defopts, coverage=coverage + ) return pytest.main(full_args, plugins=[*plugins, FixProgramNamePlugin()]) -def _build_full_args(args, filename, *, addopts, defopts): +def _build_full_args(args, filename, *, addopts, defopts, coverage): arg_mapping = ArgMapping( # use basename to ensure --deselect works # (see also: https://github.com/pytest-dev/pytest/issues/6751) @@ -266,7 +271,16 @@ def _build_full_args(args, filename, *, addopts, defopts): def _fmt(arg): return arg.format_map(arg_mapping) + if coverage: + import ipytest.cov + + coverage_args = ("--cov", f"--cov-config={ipytest.cov.config_path}") + + else: + coverage_args = () + all_args = [ + *coverage_args, *(_fmt(arg) for arg in addopts), *(_fmt(arg) for arg in args), ] diff --git a/ipytest/cov.py b/ipytest/cov.py index 5590b25..4164791 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -36,6 +36,9 @@ def test(): __all__ = [] +config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coveragerc") + + def coverage_init(reg, options): reg.add_file_tracer(IPythonPlugin()) diff --git a/ipytest/coveragerc b/ipytest/coveragerc new file mode 100644 index 0000000..35f472c --- /dev/null +++ b/ipytest/coveragerc @@ -0,0 +1,3 @@ +[run] +plugins = + ipytest.cov From 2218e2b154206d3458c5d0e606f3e638cebf91f6 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:28:05 +0200 Subject: [PATCH 02/13] Start to implement check for potential config files --- ipytest/cov.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ipytest/cov.py b/ipytest/cov.py index 4164791..114f26f 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -39,6 +39,15 @@ def test(): config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coveragerc") +def find_coverage_configs(): + cands = [".coveragerc", "setup.cfg", "tox.ini", "pyproject.toml"] + + # setup.cfg, tox.ini: [coverage: + # pyproject.toml: [tool.coverage + + pass + + def coverage_init(reg, options): reg.add_file_tracer(IPythonPlugin()) From 655eabd3e0fe8c1b4359568dff8e9923721a1704 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:40:17 +0200 Subject: [PATCH 03/13] Move coverage tests into submodule --- tests/{ => TestCoverageWithConfig}/.coveragerc | 0 tests/{ => TestCoverageWithConfig}/TestCoverage.ipynb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{ => TestCoverageWithConfig}/.coveragerc (100%) rename tests/{ => TestCoverageWithConfig}/TestCoverage.ipynb (100%) diff --git a/tests/.coveragerc b/tests/TestCoverageWithConfig/.coveragerc similarity index 100% rename from tests/.coveragerc rename to tests/TestCoverageWithConfig/.coveragerc diff --git a/tests/TestCoverage.ipynb b/tests/TestCoverageWithConfig/TestCoverage.ipynb similarity index 100% rename from tests/TestCoverage.ipynb rename to tests/TestCoverageWithConfig/TestCoverage.ipynb From 06893060eb5285306b7096ad6bf37617da7b0e2c Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:41:03 +0200 Subject: [PATCH 04/13] Fix branch coverage in notebooks --- Readme.md | 5 +++++ ipytest/cov.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 5346861..15ce819 100644 --- a/Readme.md +++ b/Readme.md @@ -335,6 +335,11 @@ With this config file, the coverage can be collected using def test(): ... ``` +There are some known issues of `ipytest.cov` + +- Each notebook cell is reported as an individual file +- Marking code to be excluded in branch coverage is currently not supported + (incl. coveragepy pragmas) [coverage-py-config-docs]: https://coverage.readthedocs.io/en/latest/config.html [ipytest-cov-pytest-cov]: https://pytest-cov.readthedocs.io/en/latest/config.html diff --git a/ipytest/cov.py b/ipytest/cov.py index 114f26f..4f1f7e4 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -19,6 +19,11 @@ def test(): ... ``` +There are some known issues of `ipytest.cov` + +- Each notebook cell is reported as an individual file +- Marking code to be excluded in branch coverage is currently not supported + (incl. coveragepy pragmas) [coverage-py-config-docs]: https://coverage.readthedocs.io/en/latest/config.html [ipytest-cov-pytest-cov]: https://pytest-cov.readthedocs.io/en/latest/config.html @@ -103,6 +108,11 @@ def source_filename(self): class IPythonFileReporter(coverage.python.PythonFileReporter): + # TODO: implement fully from scratch to be independent from PythonFileReporter impl + + def __repr__(self) -> str: + return f"" + @property def parser(self): if self._parser is None: @@ -113,6 +123,10 @@ def parser(self): def source(self): if self.filename not in linecache.cache: - raise ValueError() + raise RuntimeError(f"Could not lookup source for {self.filename!r}") return "".join(linecache.cache[self.filename][2]) + + def no_branch_lines(self): + # TODO: figure out how to implement this (require coverage config) + return set() From 87d4e14cbeb728271bd53fc54ae4ba1d2e5760ae Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:43:16 +0200 Subject: [PATCH 05/13] Test branch coverage --- .../TestCoverageWithConfig/TestCoverage.ipynb | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/tests/TestCoverageWithConfig/TestCoverage.ipynb b/tests/TestCoverageWithConfig/TestCoverage.ipynb index 40fef2b..7ee21fb 100644 --- a/tests/TestCoverageWithConfig/TestCoverage.ipynb +++ b/tests/TestCoverageWithConfig/TestCoverage.ipynb @@ -45,6 +45,27 @@ " ]\n", "\n", "\n", + "def check_coverage():\n", + " assert Path(\".coverage\").exists()\n", + " assert Path(\"coverage.json\").exists()\n", + " \n", + " with open(\"coverage.json\", \"rt\") as fobj:\n", + " data = json.load(fobj)\n", + " \n", + " assert get_executed_lines(data, func.__code__.co_filename) == [\n", + " \"if x % 2 == 0:\", \n", + " \"return \\\"even\\\"\", \n", + " \"return \\\"odd\\\"\",\n", + " ]\n", + " \n", + " assert get_executed_lines(data, test.__code__.co_filename) == [\n", + " \"assert func(0) == \\\"even\\\"\",\n", + " \"assert func(1) == \\\"odd\\\"\",\n", + " \"assert func(2) == \\\"even\\\"\",\n", + " \"assert func(3) == \\\"odd\\\"\",\n", + " ]\n", + "\n", + "\n", "def func(x):\n", " if x % 2 == 0:\n", " return \"even\"\n", @@ -81,30 +102,47 @@ "metadata": {}, "outputs": [], "source": [ - "assert Path(\".coverage\").exists()\n", - "assert Path(\"coverage.json\").exists()\n", - "\n", - "with open(\"coverage.json\", \"rt\") as fobj:\n", - " data = json.load(fobj)\n", - "\n", - "assert get_executed_lines(data, func.__code__.co_filename) == [\n", - " \"if x % 2 == 0:\", \n", - " \"return \\\"even\\\"\", \n", - " \"return \\\"odd\\\"\",\n", - "]\n", + "check_coverage()" + ] + }, + { + "cell_type": "markdown", + "id": "8f29c4bd-7bc4-4e72-a27a-e5b2fd52b8af", + "metadata": {}, + "source": [ + "# Branch coverage also works " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f063c773-a9c9-4c65-9bed-75095d3f1891", + "metadata": {}, + "outputs": [], + "source": [ + "%%ipytest --cov --cov-branch --cov-report=json\n", "\n", - "assert get_executed_lines(data, test.__code__.co_filename) == [\n", - " \"assert func(0) == \\\"even\\\"\",\n", - " \"assert func(1) == \\\"odd\\\"\",\n", - " \"assert func(2) == \\\"even\\\"\",\n", - " \"assert func(3) == \\\"odd\\\"\",\n", - "]" + "def test():\n", + " assert func(0) == \"even\"\n", + " assert func(1) == \"odd\"\n", + " assert func(2) == \"even\"\n", + " assert func(3) == \"odd\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a01bbcd-8569-4e49-b89f-82e575824e8a", + "metadata": {}, + "outputs": [], + "source": [ + "check_coverage()" ] }, { "cell_type": "code", "execution_count": null, - "id": "db0a0b1c-d7bc-4550-8033-f0f569b8cdad", + "id": "8cd681fc-745d-438f-8cbf-968543d3afc9", "metadata": {}, "outputs": [], "source": [] From 40faa862ad1a0a6116b098ce2982a7652862c529 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:51:42 +0200 Subject: [PATCH 06/13] Add tests for ipytest.autoconfig(coverage=True) --- tests/TestCoverage.ipynb | 185 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/TestCoverage.ipynb diff --git a/tests/TestCoverage.ipynb b/tests/TestCoverage.ipynb new file mode 100644 index 0000000..006fb72 --- /dev/null +++ b/tests/TestCoverage.ipynb @@ -0,0 +1,185 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d32b9292-2a37-47b7-8451-c24b4c6cf65a", + "metadata": {}, + "outputs": [], + "source": [ + "import ipytest\n", + "ipytest.autoconfig(coverage=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ffcb1df-d1fd-42f0-a0d4-c227e3be17ac", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import linecache\n", + "\n", + "from pathlib import Path\n", + "\n", + "\n", + "def get_executed_lines(data, filename):\n", + " return [\n", + " linecache.getline(filename, line).strip()\n", + " for line in sorted(data[\"files\"][filename][\"executed_lines\"])\n", + " ]\n", + "\n", + "\n", + "def check_coverage():\n", + " assert Path(\".coverage\").exists()\n", + " assert Path(\"coverage.json\").exists()\n", + " \n", + " with open(\"coverage.json\", \"rt\") as fobj:\n", + " data = json.load(fobj)\n", + " \n", + " assert get_executed_lines(data, func.__code__.co_filename) == [\n", + " \"if x % 2 == 0:\", \n", + " \"return \\\"even\\\"\", \n", + " \"return \\\"odd\\\"\",\n", + " ]\n", + " \n", + " assert get_executed_lines(data, test.__code__.co_filename) == [\n", + " \"assert func(0) == \\\"even\\\"\",\n", + " \"assert func(1) == \\\"odd\\\"\",\n", + " \"assert func(2) == \\\"even\\\"\",\n", + " \"assert func(3) == \\\"odd\\\"\",\n", + " ]\n", + "\n", + "\n", + "def delete_generated_files():\n", + " Path(\".coverage\").unlink(missing_ok=True)\n", + " Path(\"coverage.json\").unlink(missing_ok=True)\n", + "\n", + "def func(x):\n", + " if x % 2 == 0:\n", + " return \"even\"\n", + "\n", + " else:\n", + " return \"odd\"" + ] + }, + { + "cell_type": "markdown", + "id": "84a35ab0-24c0-492b-8338-598029e0d7e1", + "metadata": {}, + "source": [ + "# without branch coverage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5083c7e5-05a5-4e5e-8149-8d2897180bbf", + "metadata": {}, + "outputs": [], + "source": [ + "delete_generated_files()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6003ecec-0ee8-4a95-bec2-c12cfa4ceb31", + "metadata": {}, + "outputs": [], + "source": [ + "%%ipytest --cov --cov-report=json\n", + "\n", + "def test():\n", + " assert func(0) == \"even\"\n", + " assert func(1) == \"odd\"\n", + " assert func(2) == \"even\"\n", + " assert func(3) == \"odd\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b4e226a-8170-4830-88b3-90fe8fe4fd2a", + "metadata": {}, + "outputs": [], + "source": [ + "check_coverage()" + ] + }, + { + "cell_type": "markdown", + "id": "52973cee-3ae9-498b-bd83-10fd053331ce", + "metadata": {}, + "source": [ + "# with branch coverage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d91378c5-ff72-41f7-a457-1517e3c6fb87", + "metadata": {}, + "outputs": [], + "source": [ + "delete_generated_files()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d46f99a2-97e7-4493-899d-13e66ef82d34", + "metadata": {}, + "outputs": [], + "source": [ + "%%ipytest --cov --cov-branch --cov-report=json\n", + "\n", + "def test():\n", + " assert func(0) == \"even\"\n", + " assert func(1) == \"odd\"\n", + " assert func(2) == \"even\"\n", + " assert func(3) == \"odd\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e411051-a6b7-4b65-b2fc-1d6ddc9874f8", + "metadata": {}, + "outputs": [], + "source": [ + "check_coverage()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d74022a8-4fae-45e3-8684-be4bd8093fde", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 56aa7185f9f61e9f1e685c313d639e4f20de3f75 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 13:58:59 +0200 Subject: [PATCH 07/13] Add cross ref between docs --- Readme.md | 7 ++++--- ipytest/_config.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Readme.md b/Readme.md index 15ce819..8cb405d 100644 --- a/Readme.md +++ b/Readme.md @@ -189,9 +189,10 @@ The following settings are supported: * `magics` (default: `False`): if set to `True` register the ipytest magics * `coverage` (default: `False`): if `True` configure `pytest` to collect coverage information. This functionality requires the `pytest-cov` package - to be installed. It adds `--cov --cov-config={CONFIG}` to the arguments - when invoking `pytest`. **WARNING**: this option will hide existing - coverage configuration files + to be installed. It adds `--cov --cov-config={GENERATED_CONFIG}` to the + arguments when invoking `pytest`. **WARNING**: this option will hide + existing coverage configuration files. See [`ipytest.cov`](#ipytestcov) + for details * `clean` (default: `[Tt]est*`): the pattern used to clean variables * `addopts` (default: `()`): pytest command line arguments to prepend to every pytest invocation. For example setting diff --git a/ipytest/_config.py b/ipytest/_config.py index 16b531e..6aa6c0e 100644 --- a/ipytest/_config.py +++ b/ipytest/_config.py @@ -111,9 +111,10 @@ def config( * `magics` (default: `False`): if set to `True` register the ipytest magics * `coverage` (default: `False`): if `True` configure `pytest` to collect coverage information. This functionality requires the `pytest-cov` package - to be installed. It adds `--cov --cov-config={CONFIG}` to the arguments - when invoking `pytest`. **WARNING**: this option will hide existing - coverage configuration files + to be installed. It adds `--cov --cov-config={GENERATED_CONFIG}` to the + arguments when invoking `pytest`. **WARNING**: this option will hide + existing coverage configuration files. See [`ipytest.cov`](#ipytestcov) + for details * `clean` (default: `[Tt]est*`): the pattern used to clean variables * `addopts` (default: `()`): pytest command line arguments to prepend to every pytest invocation. For example setting From 4b1ee74485a658f4fe65b0966fc336622634ccb6 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 14:00:12 +0200 Subject: [PATCH 08/13] Fix linter --- ipytest/cov.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipytest/cov.py b/ipytest/cov.py index 4f1f7e4..7057c87 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -45,7 +45,7 @@ def test(): def find_coverage_configs(): - cands = [".coveragerc", "setup.cfg", "tox.ini", "pyproject.toml"] + _cands = [".coveragerc", "setup.cfg", "tox.ini", "pyproject.toml"] # setup.cfg, tox.ini: [coverage: # pyproject.toml: [tool.coverage From c4b938787b6e19bba744b7af00fb78c19eaae266 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 14:06:31 +0200 Subject: [PATCH 09/13] Update ipytest.cov docs --- Readme.md | 7 ++++++- ipytest/cov.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 8cb405d..0ca5565 100644 --- a/Readme.md +++ b/Readme.md @@ -327,7 +327,7 @@ plugins = ipytest.cov ``` -With this config file, the coverage can be collected using +With this config file, coverage information can be collected using [pytest-cov][ipytest-cov-pytest-cov] with ```python @@ -336,6 +336,11 @@ With this config file, the coverage can be collected using def test(): ... ``` + +`ipytest.autoconfig(coverage=True)` automatically adds the `--cov` flag and the +path of a generated config file to the Pytest invocation. In this case no +further configuration is required. + There are some known issues of `ipytest.cov` - Each notebook cell is reported as an individual file diff --git a/ipytest/cov.py b/ipytest/cov.py index 7057c87..8bac48e 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -10,7 +10,7 @@ ipytest.cov ``` -With this config file, the coverage can be collected using +With this config file, coverage information can be collected using [pytest-cov][ipytest-cov-pytest-cov] with ```python @@ -19,6 +19,11 @@ def test(): ... ``` + +`ipytest.autoconfig(coverage=True)` automatically adds the `--cov` flag and the +path of a generated config file to the Pytest invocation. In this case no +further configuration is required. + There are some known issues of `ipytest.cov` - Each notebook cell is reported as an individual file From 3690a805404d4eb788b3d53a48fbc78919d70dc9 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 14:28:43 +0200 Subject: [PATCH 10/13] Add helper to detect potential coveragepy config files --- ipytest/cov.py | 30 +++++++++++++++++++++++++----- pyproject.toml | 4 +++- tests/test_ipytest_cov.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tests/test_ipytest_cov.py diff --git a/ipytest/cov.py b/ipytest/cov.py index 8bac48e..15ad688 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -37,6 +37,7 @@ def test(): import os import os.path import re +from pathlib import Path import coverage.parser import coverage.plugin @@ -49,13 +50,32 @@ def test(): config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coveragerc") -def find_coverage_configs(): - _cands = [".coveragerc", "setup.cfg", "tox.ini", "pyproject.toml"] +def find_coverage_configs(root): + root = Path(root) - # setup.cfg, tox.ini: [coverage: - # pyproject.toml: [tool.coverage + result = [] + if (p := root.joinpath(".coveragerc")).exists(): + result.append(p) - pass + result += _find_files_with_lines(root, ["setup.cfg", "tox.ini"], r"^\[coverage:.*$") + result += _find_files_with_lines(root, ["pyproject.toml"], r"^\[tool\.coverage.*$") + + return result + + +def _find_files_with_lines(root, paths, pat): + for path in paths: + path = root.joinpath(path) + if path.exists(): + try: + with open(path, "rt") as fobj: + for line in fobj: + if re.match(pat, line) is not None: + yield path + break + + except Exception: + pass def coverage_init(reg, options): diff --git a/pyproject.toml b/pyproject.toml index b388464..c6da236 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,10 +60,12 @@ select = [ ] ignore = [ - "E501", + "E501", "SIM117", # poetry uses a non-standard pyproject.toml format "RUF200", # trailing comma rule may conflict with the formatter "COM812", + # wrong types in pytest.mark.parametrize + "PT006", ] diff --git a/tests/test_ipytest_cov.py b/tests/test_ipytest_cov.py new file mode 100644 index 0000000..8d4058b --- /dev/null +++ b/tests/test_ipytest_cov.py @@ -0,0 +1,31 @@ +import pytest + +from ipytest.cov import find_coverage_configs + + +@pytest.mark.parametrize( + "files, expected", + [ + pytest.param({".coveragerc": ""}, [".coveragerc"]), + pytest.param({"setup.cfg": ""}, []), + pytest.param({"setup.cfg": "[coverage:"}, ["setup.cfg"]), + pytest.param({"tox.ini": ""}, []), + pytest.param({"tox.ini": "[coverage:"}, ["tox.ini"]), + pytest.param({"pyproject.toml": ""}, []), + pytest.param({"pyproject.toml": "[tool.coverage"}, ["pyproject.toml"]), + pytest.param( + { + ".coveragerc": "", + "setup.cfg": "[coverage:", + "pyproject.toml": "[tool.coverage", + "tox.ini": "[coverage:", + }, + [".coveragerc", "setup.cfg", "tox.ini", "pyproject.toml"], + ), + ], +) +def test_find_coverage_configs(tmp_path, files, expected): + for name, content in files.items(): + tmp_path.joinpath(name).write_text(content) + + assert [p.name for p in find_coverage_configs(tmp_path)] == expected From 6cb694b26bcb5bb9d62b617eec8e5140100baeba Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 17:13:11 +0200 Subject: [PATCH 11/13] Add experimental filename translation for coverage --- Readme.md | 16 +++++++++ ipytest/cov.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index 0ca5565..4307b2d 100644 --- a/Readme.md +++ b/Readme.md @@ -344,12 +344,28 @@ further configuration is required. There are some known issues of `ipytest.cov` - Each notebook cell is reported as an individual file +- Lines that are executed at import time may not be encountered in tracing and + may be reported as not-covered (One example is the line of a function + definition) - Marking code to be excluded in branch coverage is currently not supported (incl. coveragepy pragmas) [coverage-py-config-docs]: https://coverage.readthedocs.io/en/latest/config.html [ipytest-cov-pytest-cov]: https://pytest-cov.readthedocs.io/en/latest/config.html +#### `ipytest.cov.translate_cell_filenames(enabled=True)` + +[ipytest.cov.translate_cell_filenames]: #ipytestcovtranslate_cell_filenamesenabledtrue + +Translate the filenames of notebook cells in coverage information. + +If enabled, `ipytest.cov` will translate the temporary file names generated +by ipykernel (e.g, `ipykernel_24768/3920661193.py`) to their cell names +(e.g., `In[6]`). + +**Warning**: this is an experimental feature and not subject to any +stability guarantees. + ## Development diff --git a/ipytest/cov.py b/ipytest/cov.py index 15ad688..52de87a 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -27,6 +27,9 @@ def test(): There are some known issues of `ipytest.cov` - Each notebook cell is reported as an individual file +- Lines that are executed at import time may not be encountered in tracing and + may be reported as not-covered (One example is the line of a function + definition) - Marking code to be excluded in branch coverage is currently not supported (incl. coveragepy pragmas) @@ -37,16 +40,17 @@ def test(): import os import os.path import re + from pathlib import Path +from typing import Optional import coverage.parser import coverage.plugin import coverage.python -# prevent the definitions from being documented in the readme -__all__ = [] - +__all__ = ["translate_cell_filenames"] +_cell_filenames_tracker = None config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coveragerc") @@ -78,6 +82,29 @@ def _find_files_with_lines(root, paths, pat): pass +def translate_cell_filenames(enabled=True): + """Translate the filenames of notebook cells in coverage information. + + If enabled, `ipytest.cov` will translate the temporary file names generated + by ipykernel (e.g, `ipykernel_24768/3920661193.py`) to their cell names + (e.g., `In[6]`). + + **Warning**: this is an experimental feature and not subject to any + stability guarantees. + """ + global _cell_filenames_tracker + + from IPython import get_ipython + + if enabled and _cell_filenames_tracker is None: + _cell_filenames_tracker = CellFilenamesTracker() + _cell_filenames_tracker.register(get_ipython()) + + elif not enabled and _cell_filenames_tracker is not None: + _cell_filenames_tracker.unregister() + _cell_filenames_tracker = None + + def coverage_init(reg, options): reg.add_file_tracer(IPythonPlugin()) @@ -155,3 +182,67 @@ def source(self): def no_branch_lines(self): # TODO: figure out how to implement this (require coverage config) return set() + + def relative_filename(self) -> str: + if _cell_filenames_tracker is None: + return self.filename + + return _cell_filenames_tracker.translate_filename(self.filename) + + +class CellFilenamesTracker: + """An IPython plugin to map temporary filenames to cells""" + + def __init__(self): + self._info = {} + self._execution_count_counts = {} + self._shell = None + + def register(self, shell): + if self._shell is not None: + self.unregister() + + shell.events.register("post_run_cell", self.on_post_run_cell) + self._shell = shell + + def unregister(self): + if self._shell is not None: + self._shell.events.unregister("post_run_cell", self.on_post_run_cell) + self._shell = None + + def on_post_run_cell(self, result): + if self._shell is None: + return + + try: + filename = self._shell.compile.get_code_name( + result.info.raw_cell, + None, + None, + ) + except Exception as _exc: + # TODO: log exception + return + + # NOTE: inside magic cells, the cell may be executed without storing the + # history, e.g., inside the `%%ipytest` cell magic. In that case the + # `execution_count` is `None`. Use the shell's execution count. However, + # now it may be found multiple times. Therefore use an increasing + # counter to avoid collisions + execution_count = ( + result.execution_count + if result.execution_count is not None + else self._shell.execution_count + ) + if execution_count in self._execution_count_counts: + self._execution_count_counts[execution_count] += 1 + self._info[ + filename + ] = f"In[{execution_count}/{self._execution_count_counts[execution_count]}]" + + else: + self._execution_count_counts[execution_count] = 0 + self._info[filename] = f"In[{execution_count}]" + + def translate_filename(self, filename: str) -> Optional[int]: + return self._info.get(filename, filename) From dc1d3ca48bb29405e9b0f845652a5b4e0bc6c437 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 17:26:01 +0200 Subject: [PATCH 12/13] Update changelog --- Changes.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Changes.md b/Changes.md index f50e4aa..cf271f4 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,14 @@ Note: development is tracked on the [`develop` branch](https://github.com/chmp/ipytest/tree/develop). +## `0.14.2` + +- Support collecting branch coverage in notebooks (e.g., via `--cov--branch`) +- Add `ipytest.autoconfig(coverage=True)` to simplify using `pytest-cov` inside + notebooks +- Add experimental `ipytest.cov.translate_cell_filenames()` to simplify + interpretation of collected coverage information + ## `0.14.1` - Add a [Coverage.py](https://coverage.readthedocs.io/en/latest/index.html) From 13d7e5af7fbdd8d9ae7c4133e011a8d9490da1a0 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 21 Apr 2024 17:42:14 +0200 Subject: [PATCH 13/13] Add a warning when using `ipytest.autoconfig(coverage=True)` and existing configs --- ipytest/_impl.py | 45 +++++++++++++ ipytest/cov.py | 30 --------- .../TestCoverageWarning.ipynb | 65 +++++++++++++++++++ tests/test_ipytest_cov.py | 2 +- 4 files changed, 111 insertions(+), 31 deletions(-) create mode 100644 tests/TestCoverageWithConfig/TestCoverageWarning.ipynb diff --git a/ipytest/_impl.py b/ipytest/_impl.py index a42d11c..1d874f6 100644 --- a/ipytest/_impl.py +++ b/ipytest/_impl.py @@ -4,6 +4,7 @@ import importlib import os import pathlib +import re import shlex import sys import threading @@ -258,6 +259,9 @@ def _run_impl(*args, module, plugins, addopts, defopts, display_columns, coverag full_args = _build_full_args( args, filename, addopts=addopts, defopts=defopts, coverage=coverage ) + if coverage: + warn_for_existing_coverage_configs() + return pytest.main(full_args, plugins=[*plugins, FixProgramNamePlugin()]) @@ -535,3 +539,44 @@ def is_notebook_node_id(prev: Optional[str], arg: str) -> bool: return all( not is_notebook_node_id(prev, arg) for prev, arg in zip([None, *args], args) ) + + +def warn_for_existing_coverage_configs(): + if configs := find_coverage_configs("."): + print( + "Warning: found existing coverage.py configuration in " + f"{[p.name for p in configs]}. " + "These config files are ignored when using " + "`ipytest.autoconfig(coverage=True)`." + "Consider adding the `ipytest.cov` plugin directly to the config " + "files and adding `--cov` to the `%%ipytest` invocation.", + file=sys.stderr, + ) + + +def find_coverage_configs(root): + root = pathlib.Path(root) + + result = [] + if (p := root.joinpath(".coveragerc")).exists(): + result.append(p) + + result += _find_files_with_lines(root, ["setup.cfg", "tox.ini"], r"^\[coverage:.*$") + result += _find_files_with_lines(root, ["pyproject.toml"], r"^\[tool\.coverage.*$") + + return result + + +def _find_files_with_lines(root, paths, pat): + for path in paths: + path = root.joinpath(path) + if path.exists(): + try: + with open(path, "rt") as fobj: + for line in fobj: + if re.match(pat, line) is not None: + yield path + break + + except Exception: + pass diff --git a/ipytest/cov.py b/ipytest/cov.py index 52de87a..5b9c327 100644 --- a/ipytest/cov.py +++ b/ipytest/cov.py @@ -40,8 +40,6 @@ def test(): import os import os.path import re - -from pathlib import Path from typing import Optional import coverage.parser @@ -54,34 +52,6 @@ def test(): config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "coveragerc") -def find_coverage_configs(root): - root = Path(root) - - result = [] - if (p := root.joinpath(".coveragerc")).exists(): - result.append(p) - - result += _find_files_with_lines(root, ["setup.cfg", "tox.ini"], r"^\[coverage:.*$") - result += _find_files_with_lines(root, ["pyproject.toml"], r"^\[tool\.coverage.*$") - - return result - - -def _find_files_with_lines(root, paths, pat): - for path in paths: - path = root.joinpath(path) - if path.exists(): - try: - with open(path, "rt") as fobj: - for line in fobj: - if re.match(pat, line) is not None: - yield path - break - - except Exception: - pass - - def translate_cell_filenames(enabled=True): """Translate the filenames of notebook cells in coverage information. diff --git a/tests/TestCoverageWithConfig/TestCoverageWarning.ipynb b/tests/TestCoverageWithConfig/TestCoverageWarning.ipynb new file mode 100644 index 0000000..fd56fd5 --- /dev/null +++ b/tests/TestCoverageWithConfig/TestCoverageWarning.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "12d3268b-7d27-4ac9-b21a-fdd6f825d0bf", + "metadata": {}, + "source": [ + "**Note:** This notebook only tests that the warning with `coverage=True` and an existing config files runs without issue" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdf258d0-b8b7-4fbb-96c7-664d120c4abc", + "metadata": {}, + "outputs": [], + "source": [ + "import ipytest\n", + "ipytest.autoconfig(coverage=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f82fbdcb-2460-4f14-9ee5-c69571b4acfd", + "metadata": {}, + "outputs": [], + "source": [ + "%%ipytest\n", + "\n", + "def test():\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd681fc-745d-438f-8cbf-968543d3afc9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_ipytest_cov.py b/tests/test_ipytest_cov.py index 8d4058b..eb58334 100644 --- a/tests/test_ipytest_cov.py +++ b/tests/test_ipytest_cov.py @@ -1,6 +1,6 @@ import pytest -from ipytest.cov import find_coverage_configs +from ipytest._impl import find_coverage_configs @pytest.mark.parametrize(