From cc2c11923f840f14b3aa9e735e3898d8e0a79eaa Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Sun, 1 Oct 2023 15:30:06 -0400 Subject: [PATCH] split `requirements-dev.txt` into first and third party dev dependencies (#7849) Co-authored-by: Nadia Dencheva --- .github/workflows/ci.yml | 1 + JenkinsfileRT | 11 +++++------ JenkinsfileRT_dev | 3 +-- requirements-dev-st.txt | 12 ++++++++++++ ...s-dev.txt => requirements-dev-thirdparty.txt | 17 ++++++++--------- tox.ini | 9 +++++---- 6 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 requirements-dev-st.txt rename requirements-dev.txt => requirements-dev-thirdparty.txt (51%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbadab05fa..acf175acf0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,7 @@ jobs: - linux: py311-xdist - macos: py311-xdist pytest-results-summary: true + - linux: py311-stdevdeps-xdist - linux: py311-devdeps-xdist - linux: py311-xdist-cov coverage: codecov diff --git a/JenkinsfileRT b/JenkinsfileRT index d46da0e938..e98cd61119 100644 --- a/JenkinsfileRT +++ b/JenkinsfileRT @@ -74,15 +74,11 @@ bc0.conda_ver = '22.11.1' bc0.conda_packages = [ "python=${python_version}", ] -bc0.pip_reqs_files = ['requirements-sdp.txt', 'requirements-max.txt'] +bc0.pip_reqs_files = ['requirements-sdp.txt', 'requirements-max.txt', 'requirements-dev-st.txt'] bc0.build_cmds = [ "pip install certifi -U --force-reinstall", "pip install -e .[test,sdp] --no-cache-dir", "pip install pytest-xdist", - // these latest-commit installs are overridden by `requirements-sdp.txt` if it is populated - "pip install git+https://github.com/spacetelescope/stdatamodels.git@master git+https://github.com/spacetelescope/stcal.git@main", - "pip install -r requirements-max.txt", - "pip install -r requirements-sdp.txt", ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"] bc0.test_cmds = [ "pytest --cov-report=xml --cov=./ -r sxf --bigdata --slow \ @@ -107,5 +103,8 @@ bc1.name = 'macos-stable-deps' bc1.test_cmds = [] bc1.test_configs = [] -utils.run([jobconfig, bc0, bc1]) +bc2 = utils.copy(bc0) +bc2.pip_reqs_files = ['requirements-dev-st.txt'] + +utils.run([jobconfig, bc0, bc1, bc2]) } // withCredentials diff --git a/JenkinsfileRT_dev b/JenkinsfileRT_dev index 9dec77686e..22bfcdeb4b 100644 --- a/JenkinsfileRT_dev +++ b/JenkinsfileRT_dev @@ -74,12 +74,11 @@ bc0.conda_ver = '22.11.1' bc0.conda_packages = [ "python=${python_version}", ] -bc0.pip_reqs_files = ['requirements-dev.txt'] +bc0.pip_reqs_files = ['requirements-dev-st.txt', 'requirements-dev-thirdparty.txt'] bc0.build_cmds = [ "pip install certifi -U --force-reinstall", "pip install -e .[test,sdp] --no-cache-dir", "pip install pytest-xdist", - "pip install -r requirements-dev.txt", ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"] bc0.test_cmds = [ "pytest -r sxf --bigdata --slow \ diff --git a/requirements-dev-st.txt b/requirements-dev-st.txt new file mode 100644 index 0000000000..d2b0ff70ea --- /dev/null +++ b/requirements-dev-st.txt @@ -0,0 +1,12 @@ +# JWST upstream packages +git+https://github.com/spacetelescope/stdatamodels + +# shared upstream packages +git+https://github.com/spacetelescope/stcal +git+https://github.com/spacetelescope/stpipe + +# Other important upstream packages +git+https://github.com/spacetelescope/crds +git+https://github.com/spacetelescope/gwcs +git+https://github.com/spacetelescope/tweakwcs +git+https://github.com/spacetelescope/wiimatch diff --git a/requirements-dev.txt b/requirements-dev-thirdparty.txt similarity index 51% rename from requirements-dev.txt rename to requirements-dev-thirdparty.txt index 6ba6c67c6e..43c31ff8bd 100644 --- a/requirements-dev.txt +++ b/requirements-dev-thirdparty.txt @@ -1,17 +1,16 @@ +# ASDF upstream packages git+https://github.com/asdf-format/asdf-standard git+https://github.com/asdf-format/asdf git+https://github.com/asdf-format/asdf-transform-schemas git+https://github.com/asdf-format/asdf-coordinates-schemas git+https://github.com/asdf-format/asdf-wcs-schemas -git+https://github.com/astropy/astropy + +# Use weekly astropy dev build git+https://github.com/astropy/asdf-astropy -git+https://github.com/spacetelescope/crds -git+https://github.com/spacetelescope/gwcs -numpy>=0.0.dev0 git+https://github.com/astropy/photutils +--extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre + +# Use Bi-weekly numpy/scipy dev builds +--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple +numpy>=0.0.dev0 scipy>=0.0.dev0 -git+https://github.com/spacetelescope/stdatamodels -git+https://github.com/spacetelescope/stpipe -git+https://github.com/spacetelescope/stcal -git+https://github.com/spacetelescope/tweakwcs -git+https://github.com/spacetelescope/wiimatch diff --git a/tox.ini b/tox.ini index 6b4c6f451b..b764984a4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = check-{style,security,dependencies} - test{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov}-xdist + test{,-oldestdeps,-stdevdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov}-xdist build-{docs,dist} # tox environments are constructed with so-called 'factors' (or terms) @@ -41,7 +41,8 @@ commands = [testenv] description = run tests - devdeps: with the latest developer version of key dependencies + stdevdeps: with the latest developer version of upstream spacetelescope dependencies + devdeps: with the latest developer version of upstream third-party dependencies oldestdeps: with the oldest supported version of key dependencies sdpdeps: with the recent STScI DMS release pinned dependencies pyargs: with --pyargs on installed package @@ -60,7 +61,6 @@ pass_env = VALIDATE_ON_ASSIGNMENT TEST_BIGDATA set_env = - devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple sdpdeps,regtests: CRDS_CONTEXT = jwst-edit extras = test @@ -73,7 +73,8 @@ package = commands_pre = oldestdeps: minimum_dependencies jwst --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt - devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager + stdevdeps: pip install -r requirements-dev-st.txt -U --upgrade-strategy eager + devdeps: pip install -r requirements-dev-thirdparty.txt -U --upgrade-strategy eager sdpdeps: pip install -r requirements-sdp.txt pip freeze commands =