From d137d274699e40131310a62d07ebf06de7cbecf4 Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Mon, 18 Nov 2024 18:04:09 +0000 Subject: [PATCH 1/6] tox: fix tox integration with conda use setup-miniconda github action and specify more packages in conda_deps for each testenv --- .github/workflows/mac-test.yml | 44 +++++++++++++++++++++++++----- tox.ini | 50 +++++++++++++++------------------- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/.github/workflows/mac-test.yml b/.github/workflows/mac-test.yml index 2a26882d7ae..0f179352655 100644 --- a/.github/workflows/mac-test.yml +++ b/.github/workflows/mac-test.yml @@ -13,16 +13,46 @@ jobs: max-parallel: 4 matrix: os: [macos-latest] - python-version: ['3.10', '3.11', '3.12'] + python-version: + - '3.10' + - '3.11' + - '3.12' + + # this is needed for conda environments to activate automatically + defaults: + run: + shell: bash -el {0} + steps: - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + + - name: Cache conda packages + uses: actions/cache@v4 + env: + # increment to reset cache + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ matrix.python-version}}-${{ env.CACHE_NUMBER }} + + - name: Configure conda + uses: conda-incubator/setup-miniconda@v3 with: + activate-environment: test + channels: conda-forge + miniforge-version: latest python-version: ${{ matrix.python-version }} - - run: | - brew install fftw openssl gsl - pip install --upgrade pip setuptools "tox<4.0.0" - - name: run basic pycbc test suite + + - name: Conda info + run: conda info --all + + - name: Install tox + run: | + conda install \ + pip \ + setuptools \ + "tox<4.0.0" + + - name: Run basic pycbc test suite run: | tox -e py-unittest diff --git a/tox.ini b/tox.ini index d3d96b2039b..c9bca64cd27 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,9 @@ deps = mkl;'arm' not in platform_machine [testenv] -allowlist_externals = bash +allowlist_externals = + bash + conda passenv=LAL_DATA_PATH conda_deps=openssl conda_channels=conda-forge @@ -28,20 +30,17 @@ deps = ; Needed for `BBHx` package to work with PyCBC git+https://github.com/titodalcanton/BBHx.git@py39-and-cleanup; sys_platform == 'linux' git+https://github.com/gwastro/BBHX-waveform-model.git; sys_platform == 'linux' -conda_deps= - mysqlclient - lin: gcc_linux-64>=12.2.0 - lin: gxx_linux-64>=12.2.0 - ; mac doesn't work, need fix - ; mac: clang_osx-64 - ; mac: clangxx_osx-64 +conda_deps = + c-compiler + cxx-compiler gsl - lapack==3.6.1 -conda_channels=conda-forge -setenv = - ; Tell the linker to look for shared libs inside the temporary Conda env. - ; Needed to build BBHx's wheel, whick links to LAPACK. - LIBRARY_PATH={envdir}/lib:{env:LIBRARY_PATH:} + mysqlclient + ; for BBHx + liblapacke + openblas +conda_channels = conda-forge +commands_pre = + conda list commands = pytest # The following are long running or may require @@ -73,21 +72,16 @@ deps = ; Needed for `BBHx` package to work with PyCBC git+https://github.com/titodalcanton/BBHx.git@py39-and-cleanup; sys_platform == 'linux' git+https://github.com/gwastro/BBHX-waveform-model.git; sys_platform == 'linux' -conda_deps= +conda_deps = + c-compiler + cxx-compiler mysqlclient - lin: gcc_linux-64>=12.2.0 - lin: gxx_linux-64>=12.2.0 - ; mac doesn't work, need fix - ; mac: clang_osx-64 - ; mac: clangxx_osx-64 - binutils_linux-64>=2.39 - gsl - lapack==3.6.1 - openmpi -conda_channels=conda-forge + ; for BBHx + liblapacke + openblas +conda_channels = conda-forge setenv = PYCBC_TEST_TYPE=docs - ; Tell the linker to look for shared libs inside the temporary Conda env. - ; Needed to build BBHx's wheel, whick links to LAPACK. - LIBRARY_PATH={envdir}/lib:{env:LIBRARY_PATH:} +commands_pre = + conda list commands = bash tools/pycbc_test_suite.sh From 6b656e0d767f51aae410250bd926138fbe1e28ec Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Mon, 18 Nov 2024 18:04:48 +0000 Subject: [PATCH 2/6] tox: install ligo-segments and python-ligo-lw with conda these packages don't install cleanly with pypi, but the conda packages have patches --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index c9bca64cd27..0a70549180f 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,9 @@ conda_deps = cxx-compiler gsl mysqlclient + ; these packages don't install cleanly with pip, conda has patches + ligo-segments + python-ligo-lw ; for BBHx liblapacke openblas @@ -76,6 +79,9 @@ conda_deps = c-compiler cxx-compiler mysqlclient + ; these packages don't install cleanly with pip, conda has patches + ligo-segments + python-ligo-lw ; for BBHx liblapacke openblas From 0d1c9bc56744bdce55b9767596d7ec210ea7fcba Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Wed, 20 Nov 2024 10:20:28 +0000 Subject: [PATCH 3/6] test: use numpy.longdouble instead of float128 float128 isn't available on macOS ARM64 --- test/validation_code/old_coinc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/validation_code/old_coinc.py b/test/validation_code/old_coinc.py index 4d150d64a31..a8eb869f8a5 100644 --- a/test/validation_code/old_coinc.py +++ b/test/validation_code/old_coinc.py @@ -361,8 +361,8 @@ def cluster_coincs(stat, time1, time2, timeslide_id, slide, window, argmax=numpy else: time = 0.5 * (time2 + time1) - tslide = timeslide_id.astype(numpy.float128) - time = time.astype(numpy.float128) + tslide = timeslide_id.astype(numpy.longdouble) + time = time.astype(numpy.longdouble) span = (time.max() - time.min()) + window * 10 time = time + span * tslide @@ -411,8 +411,8 @@ def cluster_coincs_multiifo(stat, time_coincs, timeslide_id, slide, window, argm nifos_minusone = (num_ifos - numpy.ones_like(num_ifos)) time_avg = time_avg + (nifos_minusone * timeslide_id * slide)/num_ifos - tslide = timeslide_id.astype(numpy.float128) - time_avg = time_avg.astype(numpy.float128) + tslide = timeslide_id.astype(numpy.longdouble) + time_avg = time_avg.astype(numpy.longdouble) span = (time_avg.max() - time_avg.min()) + window * 10 time_avg = time_avg + span * tslide From b341640d5914885d5d7395c37b512e323f622cfd Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Wed, 20 Nov 2024 14:08:55 +0000 Subject: [PATCH 4/6] tox: clean up duplicate package lists --- tox.ini | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/tox.ini b/tox.ini index 0a70549180f..0660e52710b 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,22 @@ deps = :preinstall: -rrequirements.txt -rcompanion.txt mkl;'arm' not in platform_machine +conda_deps = + c-compiler + cxx-compiler + gsl + mysqlclient + ; these packages don't install cleanly with pip, conda has patches + ligo-segments + python-ligo-lw + +[bbhx] +deps = + git+https://github.com/titodalcanton/BBHx.git@py39-and-cleanup; sys_platform == 'linux' + git+https://github.com/gwastro/BBHX-waveform-model.git; sys_platform == 'linux' +conda_deps = + liblapacke + openblas [testenv] allowlist_externals = @@ -26,22 +42,12 @@ platform = lin: linux [testenv:py-unittest] deps = {[base]deps} + {[bbhx]deps} pytest - ; Needed for `BBHx` package to work with PyCBC - git+https://github.com/titodalcanton/BBHx.git@py39-and-cleanup; sys_platform == 'linux' - git+https://github.com/gwastro/BBHX-waveform-model.git; sys_platform == 'linux' -conda_deps = - c-compiler - cxx-compiler - gsl - mysqlclient - ; these packages don't install cleanly with pip, conda has patches - ligo-segments - python-ligo-lw - ; for BBHx - liblapacke - openblas conda_channels = conda-forge +conda_deps = + {[base]conda_deps} + {[bbhx]conda_deps} commands_pre = conda list commands = pytest @@ -72,20 +78,11 @@ commands = bash tools/pycbc_test_suite.sh [testenv:py-docs] deps = {[base]deps} - ; Needed for `BBHx` package to work with PyCBC - git+https://github.com/titodalcanton/BBHx.git@py39-and-cleanup; sys_platform == 'linux' - git+https://github.com/gwastro/BBHX-waveform-model.git; sys_platform == 'linux' -conda_deps = - c-compiler - cxx-compiler - mysqlclient - ; these packages don't install cleanly with pip, conda has patches - ligo-segments - python-ligo-lw - ; for BBHx - liblapacke - openblas + {[bbhx]deps} conda_channels = conda-forge +conda_deps = + {[base]conda_deps} + {[bbhx]conda_deps} setenv = PYCBC_TEST_TYPE=docs commands_pre = From 3f41ac1abc7d5e66a9d9d7be61a5ac6ffff03e5a Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Wed, 20 Nov 2024 14:09:39 +0000 Subject: [PATCH 5/6] ci: unpin tox --- .github/workflows/basic-tests.yml | 2 +- .github/workflows/mac-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 54d5ee2c9b8..45cd7248d07 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -25,7 +25,7 @@ jobs: run: | sudo apt-get -o Acquire::Retries=3 update sudo apt-get -o Acquire::Retries=3 install *fftw3* mpi intel-mkl* git-lfs graphviz - pip install "tox<4.0.0" pip setuptools --upgrade + pip install tox pip setuptools --upgrade - name: installing auxiliary data files run: | GIT_LFS_SKIP_SMUDGE=1 git clone https://git.ligo.org/lscsoft/lalsuite-extra diff --git a/.github/workflows/mac-test.yml b/.github/workflows/mac-test.yml index 0f179352655..8062a81fe6b 100644 --- a/.github/workflows/mac-test.yml +++ b/.github/workflows/mac-test.yml @@ -51,7 +51,7 @@ jobs: conda install \ pip \ setuptools \ - "tox<4.0.0" + tox - name: Run basic pycbc test suite run: | From 6885854c3e1e10251bfd7bba7d7fae85f5d6bbe9 Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Wed, 20 Nov 2024 14:20:33 +0000 Subject: [PATCH 6/6] tox: further simplify duplicate configuration --- tox.ini | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/tox.ini b/tox.ini index 0660e52710b..4c160e9c846 100644 --- a/tox.ini +++ b/tox.ini @@ -32,25 +32,24 @@ conda_deps = allowlist_externals = bash conda -passenv=LAL_DATA_PATH -conda_deps=openssl conda_channels=conda-forge -platform = lin: linux - mac: darwin - -# This test should run on almost anybody's environment -[testenv:py-unittest] -deps = - {[base]deps} - {[bbhx]deps} - pytest -conda_channels = conda-forge conda_deps = {[base]conda_deps} {[bbhx]conda_deps} commands_pre = conda list commands = pytest +deps = + {[base]deps} + {[bbhx]deps} + pytest +passenv=LAL_DATA_PATH +platform = + lin: linux + mac: darwin + +# This test should run on almost anybody's environment +[testenv:py-unittest] # The following are long running or may require # some extra system-level dependencies or static files. @@ -76,15 +75,6 @@ setenv = PYCBC_TEST_TYPE=inference commands = bash tools/pycbc_test_suite.sh [testenv:py-docs] -deps = - {[base]deps} - {[bbhx]deps} -conda_channels = conda-forge -conda_deps = - {[base]conda_deps} - {[bbhx]conda_deps} setenv = PYCBC_TEST_TYPE=docs -commands_pre = - conda list commands = bash tools/pycbc_test_suite.sh