diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cee98dc33d..19c73c10b8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -36,8 +36,12 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.10"] - session: ["tests", "doctest", "gallery", "linkcheck"] + session: ["doctest", "gallery", "linkcheck"] include: + - os: "ubuntu-latest" + python-version: "3.10" + session: "tests" + coverage: True - os: "ubuntu-latest" python-version: "3.9" session: "tests" @@ -132,5 +136,10 @@ jobs: - name: "iris ${{ matrix.session }}" env: PY_VER: ${{ matrix.python-version }} + COVERAGE: ${{ matrix.coverage }} run: | nox --session ${{ matrix.session }} -- --verbose + + - name: Upload coverage report + uses: codecov/codecov-action@v3 + if: ${{ matrix.coverage }} \ No newline at end of file diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index a38e426e6a..51de44c8d2 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -108,6 +108,12 @@ This document explains the changes made to Iris for this release #. `@rcomer`_ removed some old infrastructure that printed test timings. (:pull:`5101`) +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This can + be enabled by setting the environment variable ``COVERAGE=True`` when running + the tests with nox, i.e. ``nox --session tests``. (:pull:`4765`) + +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests`` + option from Iris' test runner. (:pull:`4765`) .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py index bfb2cc2402..7f9439d4b6 100644 --- a/lib/iris/tests/runner/_runner.py +++ b/lib/iris/tests/runner/_runner.py @@ -35,18 +35,13 @@ class TestRunner: ("system-tests", "s", "Run the limited subset of system tests."), ("gallery-tests", "e", "Run the gallery code tests."), ("default-tests", "d", "Run the default tests."), - ( - "coding-tests", - "c", - "Run the coding standards tests. (These are a " - "subset of the default tests.)", - ), ( "num-processors=", "p", "The number of processors used for running " "the tests.", ), ("create-missing", "m", "Create missing test result files."), + ("coverage", "c", "Enable coverage testing"), ] boolean_options = [ "no-data", @@ -54,8 +49,8 @@ class TestRunner: "stop", "gallery-tests", "default-tests", - "coding-tests", "create-missing", + "coverage", ] def initialize_options(self): @@ -64,9 +59,9 @@ def initialize_options(self): self.system_tests = False self.gallery_tests = False self.default_tests = False - self.coding_tests = False self.num_processors = None self.create_missing = False + self.coverage = False def finalize_options(self): # These environment variables will be propagated to all the @@ -84,8 +79,6 @@ def finalize_options(self): tests.append("system") if self.default_tests: tests.append("default") - if self.coding_tests: - tests.append("coding") if self.gallery_tests: tests.append("gallery") if not tests: @@ -109,8 +102,6 @@ def run(self): tests.append("lib/iris/tests/system_test.py") if self.default_tests: tests.append("lib/iris/tests") - if self.coding_tests: - tests.append("lib/iris/tests/test_coding_standards.py") if self.gallery_tests: import iris.config @@ -136,6 +127,9 @@ def run(self): if self.stop: args.append("-x") + if self.coverage: + args.extend(["--cov=lib/iris", "--cov-report=xml"]) + result = True for test in tests: args[0] = test diff --git a/noxfile.py b/noxfile.py index 8aabf862fb..85806ab558 100755 --- a/noxfile.py +++ b/noxfile.py @@ -28,6 +28,7 @@ #: Cirrus-CI environment variable hook. PY_VER = os.environ.get("PY_VER", _PY_VERSIONS_ALL) +COVERAGE = os.environ.get("COVERAGE", False) #: Default cartopy cache directory. CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") @@ -176,6 +177,9 @@ def tests(session: nox.sessions.Session): """ Perform iris system, integration and unit tests. + Coverage testing is enabled if the COVERAGE environment variable is set to + True. + Parameters ---------- session: object @@ -185,13 +189,15 @@ def tests(session: nox.sessions.Session): prepare_venv(session) session.install("--no-deps", "--editable", ".") session.env.update(ENV) - session.run( + run_args = [ "python", "-m", "iris.tests.runner", "--default-tests", - "--system-tests", - ) + ] + if COVERAGE: + run_args.append("--coverage") + session.run(*run_args) @nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda") diff --git a/pyproject.toml b/pyproject.toml index bdb8a431e5..fdbeff7809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,20 @@ verbose = "False" [tool.pytest.ini_options] addopts = "-ra" testpaths = "lib/iris" + +[tool.coverage.run] +branch = true +source = [ + "lib/iris", +] +omit = [ + "lib/iris/tests/*", + "lib/iris/etc/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if __name__ == .__main__.:" +] \ No newline at end of file diff --git a/requirements/ci/nox.lock/py310-linux-64.lock b/requirements/ci/nox.lock/py310-linux-64.lock index 910a390493..ba2974fc06 100644 --- a/requirements/ci/nox.lock/py310-linux-64.lock +++ b/requirements/ci/nox.lock/py310-linux-64.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 234b47d943728b5abe70fba0fd74c6adc10e4f1e2a14b919344f8a693b5b3e6f +# input_hash: f8af5f4aafcb766f463a1a897d3dab9e04f05f1494bced5931d78175ca0c66df @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080 @@ -150,7 +150,7 @@ https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda#c2566c2ea5f153ddd6bf4acaf7547d97 https://conda.anaconda.org/conda-forge/linux-64/libclang13-15.0.7-default_h3e3d535_1.conda#a3a0f7a6f0885f5e1e0ec691566afb77 https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h36d4200_3.conda#c9f4416a34bc91e0eb029f912c68f81f -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.0-hdc1c0ab_0.conda#c44acb3847ff118c068b662aff858afd +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.1-hdc1c0ab_0.conda#81eaeb3b35163c8e90e57532bc93754d https://conda.anaconda.org/conda-forge/linux-64/libpq-15.2-hb675445_0.conda#4654b17eccaba55b8581d6b9c77f53cc https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-252-h2a991cd_0.tar.bz2#3c5ae9f61f663b3d5e1bf7f7da0c85f5 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h1daa5a0_1.conda#77003f63d1763c1e6569a02c1742c9f4 @@ -191,14 +191,15 @@ https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h166bdaf_0.tar.bz2#c9b568bd804cb2903c6be6f5f68182e4 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2#536cc5db4d0a3ba0630541aec064b5e4 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2#f59c1242cc1dd93e72c2ee2b360979eb -https://conda.anaconda.org/conda-forge/noarch/zipp-3.13.0-pyhd8ed1ab_0.conda#41b09d997939e83b231c4557a90c3b13 +https://conda.anaconda.org/conda-forge/noarch/zipp-3.14.0-pyhd8ed1ab_0.conda#01ea04980fa39d7b6dbdd6c67016d177 https://conda.anaconda.org/conda-forge/noarch/babel-2.11.0-pyhd8ed1ab_0.tar.bz2#2ea70fde8d581ba9425a761609eed6ba https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.11.2-pyha770c72_0.conda#88b59f6989f0ed5ab3433af0b82555e1 https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2#d1a88f3ed5b52e1024b80d4bcd26a7a0 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda#800596144bb613cd7ac58b80900ce835 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar.bz2#94ce7a76b0c912279f6958e0b6b21d2b https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.7-py310hdf3cbec_0.conda#7bf9d8c765b6b04882c719509652c6d6 -https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.0-hdc1c0ab_0.conda#5d9ac94ee84305ada32c3d287d0ec602 +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.1.0-py310h1fa729e_0.conda#da7c45dbe780f5e162011a3af44e5009 +https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.1-hdc1c0ab_0.conda#1968e4fef727858ac04746560e820928 https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2#12ebe92a8a578bc903bd844744f4d040 https://conda.anaconda.org/conda-forge/linux-64/glib-2.74.1-h6239696_1.tar.bz2#f3220a9e9d3abcbfca43419a219df7e4 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_1.conda#811c4d55cf17b42336ffa314239717b0 @@ -234,6 +235,7 @@ https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.3-py310h9b08913_0.con https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.0.0-pyhd8ed1ab_0.conda#c34694044915d7f291ef257029f2e2af https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310h15e2413_1.conda#5be35366687def87437d210fd673100c https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.11.0-py310heca2aa9_3.conda#3b1946b676534472ce65181dda0b9554 +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.2.0-pyhd8ed1ab_0.conda#70ab87b96126f35d1e68de2ad9fb6423 https://conda.anaconda.org/conda-forge/noarch/setuptools-scm-7.1.0-pyhd8ed1ab_0.conda#6613dbb3b25cc648a107f33ca9f80fc1 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-napoleon-0.7-py_0.tar.bz2#0bc25ff6f2e34af63ded59692df5f749 @@ -242,8 +244,8 @@ https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.22.0-h4243ec0 https://conda.anaconda.org/conda-forge/noarch/identify-2.5.18-pyhd8ed1ab_0.conda#e07a5691c27e65d8d3d9278c578c7771 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_h1e13492_2.conda#d4ed7704f0fa589e4d7656780fa87557 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h0a86a1f_103.conda#7f69695b684f2595d9ba1ce26d693b7d -https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-hd33c08f_1.conda#667dc93c913f0156e1237032e3a22046 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.2-nompi_py310h55e1e36_100.tar.bz2#4dd7aa28fb7d9a6de061c9579a30e7dd +https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.13-hd33c08f_0.conda#e3b13445b8ee9d6a3d53a714f89ccd76 https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.5.10-mpi_mpich_h862c5c2_100.conda#56e43c5226670aa0943fae9a2628a934 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda#d41957700e83bbb925928764cb7f8878 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.19.0-pyhd8ed1ab_0.conda#afaa9bf6992f67a82d75fad47a93ec84 diff --git a/requirements/ci/py310.yml b/requirements/ci/py310.yml index d79015c055..3d15063baf 100644 --- a/requirements/ci/py310.yml +++ b/requirements/ci/py310.yml @@ -39,6 +39,7 @@ dependencies: - pre-commit - psutil - pytest + - pytest-cov - pytest-xdist - requests