From 5ad93c4109fe441f0994e2ccf340b1c6ffa8ea3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 10 Sep 2024 10:14:56 +0200 Subject: [PATCH 1/2] DEP: move dev-only dependencies to requirements files --- .github/dependabot.yml | 9 +++ .github/workflows/type-checking.yaml | 2 +- MANIFEST.in | 1 + doc/source/developing/building_the_docs.rst | 2 +- pyproject.toml | 37 +---------- requirements/docs.txt | 12 ++++ requirements/minimal_env.txt | 69 +++++++++++++++++++++ requirements/typecheck.txt | 5 ++ tests/ci_install.sh | 2 +- 9 files changed, 101 insertions(+), 38 deletions(-) create mode 100644 requirements/docs.txt create mode 100644 requirements/minimal_env.txt create mode 100644 requirements/typecheck.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e9767227a12..2ed6f7807c4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,14 @@ version: 2 updates: +- package-ecosystem: pip + directory: /requirements + schedule: + interval: monthly + groups: + actions: + patterns: + - '*' + - package-ecosystem: github-actions directory: /.github/workflows schedule: diff --git a/.github/workflows/type-checking.yaml b/.github/workflows/type-checking.yaml index 6931f4a45e4..3c7f7307d2f 100644 --- a/.github/workflows/type-checking.yaml +++ b/.github/workflows/type-checking.yaml @@ -37,7 +37,7 @@ jobs: - name: Build run: | python3 -m pip install --upgrade pip - python3 -m pip install -e .[typecheck] + python3 -m pip install -e . -r requirements/typecheck.txt - run: python -m pip list diff --git a/MANIFEST.in b/MANIFEST.in index e1e1eb29d52..442aa58d9e3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -70,5 +70,6 @@ include yt/default.mplstyle prune yt/frontends/_skeleton recursive-include yt/frontends/amrvac *.par +recursive-exclude requirements *.txt exclude .codecov.yml .coveragerc .git-blame-ignore-revs .gitmodules .hgchurn .mailmap exclude .pre-commit-config.yaml clean.sh nose_answer.cfg nose_unit.cfg nose_ignores.txt diff --git a/doc/source/developing/building_the_docs.rst b/doc/source/developing/building_the_docs.rst index e1b56298026..ef8d08b8c60 100644 --- a/doc/source/developing/building_the_docs.rst +++ b/doc/source/developing/building_the_docs.rst @@ -90,7 +90,7 @@ the top level of a local copy, run .. code-block:: bash - $ python -m pip install -e ".[doc]" + $ python -m pip install -e . -r requirements/docs.txt Quick versus Full Documentation Builds ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/pyproject.toml b/pyproject.toml index 86bdc5de23f..85f9189d2f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ keywords = [ "astronomy astrophysics visualization amr adaptivemeshrefinement", ] requires-python = ">=3.9.2" + +# keep in sync with requirements/minimal_env.txt dependencies = [ "cmyt>=1.1.2", "ewah-bool-utils>=1.2.0", @@ -190,37 +192,9 @@ full = [ ] # dev-only extra targets -doc = [ - "alabaster>=0.7.13", - "bottle>=0.12.25", - "ipykernel>=6.29.4", - "jinja2<3.1.0", # see https://github.com/readthedocs/readthedocs.org/issues/9037 - "jupyter-client>=8.3.1", - "nbsphinx>=0.9.3", - "nose~=1.3.7; python_version < '3.10'", - "pytest>=6.1", - "pyx>=0.15", - "sphinx>=7.2.5", - "sphinx-bootstrap-theme>=0.8.1", - "sphinx-rtd-theme>=1.3.0", -] mapserver = [ "bottle", ] -minimal = [ - "cmyt==1.1.2", - "ewah-bool-utils==1.2.0", - "matplotlib==3.5", - "more-itertools==8.4", - "numpy==1.19.3", - "packaging==20.9", - "pillow==8.0.0", - "tomli-w==0.4.0", - "tqdm==3.4.0", - "unyt==2.9.2", - "tomli==1.2.3;python_version < '3.11'", - "typing-extensions==4.4.0;python_version < '3.12'", -] test = [ "pyaml>=17.10.0", "pytest>=6.1", @@ -231,13 +205,6 @@ test = [ "nose-timer~=1.0.0; python_version < '3.10'", "imageio!=2.35.0", # see https://github.com/yt-project/yt/issues/4966 ] -typecheck = [ - "mypy==1.8.0", - "types-PyYAML==6.0.12.12", - "types-chardet==5.0.4.6", - "types-requests==2.31.0.20240125", - "typing-extensions==4.4.0; python_version < '3.12'", -] [project.scripts] yt = "yt.utilities.command_line:run_main" diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 00000000000..e577a5b968d --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,12 @@ +alabaster>=0.7.13 +bottle>=0.12.25 +ipykernel>=6.29.4 +jinja2<3.1.0 # see https://github.com/readthedocs/readthedocs.org/issues/9037 +jupyter-client>=8.3.1 +nbsphinx>=0.9.3 +nose~=1.3.7; python_version < '3.10' +pytest>=6.1 +pyx>=0.15 +sphinx>=7.2.5 +sphinx-bootstrap-theme>=0.8.1 +sphinx-rtd-theme>=1.3.0 diff --git a/requirements/minimal_env.txt b/requirements/minimal_env.txt new file mode 100644 index 00000000000..086d0076d2c --- /dev/null +++ b/requirements/minimal_env.txt @@ -0,0 +1,69 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --python=3.9 --python-platform=x86_64-unknown-linux-gnu --resolution=lowest-direct --no-build +cmyt==1.1.2 + # via yt (pyproject.toml) +colorspacious==1.1.2 + # via cmyt +cycler==0.12.1 + # via matplotlib +ewah-bool-utils==1.2.0 + # via yt (pyproject.toml) +fonttools==4.53.1 + # via matplotlib +kiwisolver==1.4.7 + # via matplotlib +matplotlib==3.5.0 + # via + # yt (pyproject.toml) + # cmyt +more-itertools==8.4.0 + # via + # yt (pyproject.toml) + # cmyt +mpmath==1.3.0 + # via sympy +numpy==1.19.3 + # via + # yt (pyproject.toml) + # cmyt + # colorspacious + # ewah-bool-utils + # matplotlib + # unyt +packaging==20.9 + # via + # yt (pyproject.toml) + # matplotlib + # setuptools-scm +pillow==8.0.0 + # via + # yt (pyproject.toml) + # matplotlib +pyparsing==3.1.4 + # via + # matplotlib + # packaging +python-dateutil==2.9.0.post0 + # via matplotlib +setuptools==74.1.2 + # via setuptools-scm +setuptools-scm==8.1.0 + # via matplotlib +six==1.16.0 + # via python-dateutil +sympy==1.13.2 + # via unyt +tomli==1.2.3 + # via + # yt (pyproject.toml) + # setuptools-scm +tomli-w==0.4.0 + # via yt (pyproject.toml) +tqdm==3.4.0 + # via yt (pyproject.toml) +typing-extensions==4.4.0 + # via + # yt (pyproject.toml) + # setuptools-scm +unyt==2.9.2 + # via yt (pyproject.toml) diff --git a/requirements/typecheck.txt b/requirements/typecheck.txt new file mode 100644 index 00000000000..8d0e0ceaffa --- /dev/null +++ b/requirements/typecheck.txt @@ -0,0 +1,5 @@ +mypy==1.8.0 +types-PyYAML==6.0.12.12 +types-chardet==5.0.4.6 +types-requests==2.31.0.20240125 +typing-extensions==4.4.0; python_version < '3.12' diff --git a/tests/ci_install.sh b/tests/ci_install.sh index b73c90f739b..a29d9d53341 100644 --- a/tests/ci_install.sh +++ b/tests/ci_install.sh @@ -32,7 +32,7 @@ fi # but the primary intention is to embed this script in CI jobs if [[ ${dependencies} == "minimal" ]]; then # test with minimal versions of runtime dependencies - python -m pip install -e ".[test,minimal]" + python -m pip install -e ".[test]" -r requirements/minimal_env.txt elif [[ ${dependencies} == "cartopy" ]]; then python -m pip install 'cartopy>=0.22' # scipy is an optional dependency to cartopy From 342340c728ebea72e079f32181eb894cabeb1574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 11 Sep 2024 17:16:56 +0200 Subject: [PATCH 2/2] DOC: update developer guide on Handling Dependencies --- doc/source/developing/testing.rst | 55 ++++++++++++++----------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 96ddfad8424..eab8270fe58 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -441,24 +441,26 @@ Handling yt Dependencies ------------------------ Our dependencies are specified in ``pyproject.toml``. Hard dependencies are found in -``options.install_requires``, while optional dependencies are specified in -``options.extras_require``. The ``full`` target contains the specs to run our +``project.dependencies``, while optional dependencies are specified in +``project.optional-dependencies``. The ``full`` target contains the specs to run our test suite, which are intended to be as modern as possible (we don't set upper -limits to versions unless we need to). The ``minimal`` target is used to check -that we don't break backward compatibility with old versions of upstream -projects by accident. It is intended to pin strictly our minimal supported -versions. The ``test`` target specifies the tools needed to run the tests, but +limits to versions unless we need to). + +The ``test`` target specifies the tools needed to run the tests, but not needed by yt itself. +Documentation and typechecking requirements are found in ``requirements/``, +and used in ``tests/ci_install.sh``. + **Python version support.** -When a new Python version is released, it takes about -a month or two for yt to support it, since we're dependent on bigger projects -like numpy and matplotlib. We vow to follow numpy's deprecation plan regarding -our supported versions for Python and numpy, defined formally in `NEP 29 -`_. However, we try to -avoid bumping our minimal requirements shortly before a yt release. +We vow to follow numpy's deprecation plan regarding our supported versions for Python +and numpy, defined formally in +`NEP 29 `_, but generally +support larger version intervals than recommended in this document. **Third party dependencies.** +We attempt to make yt compatible with a wide variety of upstream software +versions. However, sometimes a specific version of a project that yt depends on causes some breakage and must be blacklisted in the tests or a more experimental project that yt depends on optionally might change sufficiently @@ -466,29 +468,20 @@ that the yt community decides not to support an old version of that project. **Note.** Some of our optional dependencies are not trivial to install and their support -may vary across platforms. To manage such issue, we currently use requirement -files in additions to ``pyproject.toml``. They are found in -``tests/*requirements.txt`` and used in ``tests/ci_install.sh``. - -We attempt to make yt compatible with a wide variety of upstream software -versions. However, sometimes a specific version of a project that yt depends on -causes some breakage and must be blacklisted in the tests or a more -experimental project that yt depends on optionally might change sufficiently -that the yt community decides not to support an old version of that project. - -To handle cases like this, the versions of upstream software projects installed -on the machines running the yt test suite are pinned to specific version -numbers that must be updated manually. This prevents breaking the yt tests when -a new version of an upstream dependency is released and allows us to manage -updates in upstream projects at our pace. +may vary across platforms. If you would like to add a new dependency for yt (even an optional dependency) or would like to update a version of a yt dependency, you must edit the -``tests/test_requirements.txt`` file, this path is relative to the root of the -repository. This file contains an enumerated list of direct dependencies and -pinned version numbers. For new dependencies, simply append the name of the new +``pyproject.toml`` file. For new dependencies, simply append the name of the new dependency to the end of the file, along with a pin to the latest version number of the package. To update a package's version, simply update the version number in the entry for that package. -Finally, we also run a set of tests with "minimal" dependencies installed. When adding tests that depend on an optional dependency, you can wrap the test with the ``yt.testing.requires_module decorator`` to ensure it does not run during the minimal dependency tests (see yt/frontends/amrvac/tests/test_read_amrvac_namelist.py for a good example). If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. +Finally, we also run a set of tests with "minimal" dependencies installed. +When adding tests that depend on an optional dependency, you can wrap the test +with the ``yt.testing.requires_module decorator`` to ensure it does not run +during the minimal dependency tests (see +``yt/frontends/amrvac/tests/test_read_amrvac_namelist.py`` for a good example). +If for some reason you need to update the listing of packages that are installed +for the "minimal" dependency tests, you will need to update +``requirements/minimal_env.txt``.