From 5902c24102770ccde6ccf921191d5333418e1ba1 Mon Sep 17 00:00:00 2001 From: Matt Cieslak Date: Thu, 29 Aug 2024 15:17:06 -0400 Subject: [PATCH] [CI] Update CI to be more efficient (#814) * try smaller image * use home * use more cpus * don't be implicit * actually increase cpus * remove recon packages --- pyproject.toml | 2 -- qsiprep/data/boilerplate.bib | 30 -------------------------- qsiprep/tests/test_cli.py | 42 +++++++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index beb7d173..e09f3e7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ requires-python = ">=3.10" dependencies = [ 'importlib_resources; python_version < "3.11"', "dipy>=1.8.0,<1.9.0", - "dmri-amico == 1.5.4", "fury", "indexed_gzip <= 1.8.7", "jinja2 < 3.1", @@ -33,7 +32,6 @@ dependencies = [ "numpy <= 1.26.3", "pandas < 2.0.0", "psutil <= 5.9.8", - "pyAFQ == 1.3.2", "pybids", "pyyaml", "scikit-image", diff --git a/qsiprep/data/boilerplate.bib b/qsiprep/data/boilerplate.bib index e5da0e10..06b8062a 100644 --- a/qsiprep/data/boilerplate.bib +++ b/qsiprep/data/boilerplate.bib @@ -25,26 +25,6 @@ @article{dipy publisher={Frontiers} } - -@article{pyafq, - title = {Evaluating the reliability of human brain white matter tractometry}, - author = {Kruper, J. and Yeatman, J.D. and Richie-Halford, A. and Bloom, D. and Grotheer, M. and Caffarra, S. and Kiar, G. and Karipidis, I.I. and Roy, E. and Chandio, B.Q. and Garyfallidis, E. and Rokem, A.}, - journal = {Aperture Neuro}, - volume = {1}, - pages = {1-25}, - year = {2021}, -} - -@article{pyafq2, - title = {Tract profiles of White Matter Properties: Automating fiber-tract quantification}, - volume = {7}, - doi = {10.1371/journal.pone.0049790}, - number = {11}, - journal = {PLoS ONE}, - author = {Yeatman, Jason D. and Dougherty, Robert F. and Myall, Nathaniel J. and Wandell, Brian A. and Feldman, Heidi M.}, - year = {2012} -} - @article{nipype1, author = {Gorgolewski, K. and Burns, C. D. and Madison, C. and Clark, D. and Halchenko, Y. O. and Waskom, M. L. and Ghosh, S.}, doi = {10.3389/fninf.2011.00013}, @@ -556,16 +536,6 @@ @article{noddi publisher={Elsevier} } -@article{amico, - title={Accelerated microstructure imaging via convex optimization (AMICO) from diffusion MRI data}, - author={Daducci, Alessandro and Canales-Rodr{\'\i}guez, Erick J and Zhang, Hui and Dyrby, Tim B and Alexander, Daniel C and Thiran, Jean-Philippe}, - journal={NeuroImage}, - volume={105}, - pages={32--44}, - year={2015}, - publisher={Elsevier} -} - @article{drbuddi, title={DR-BUDDI (Diffeomorphic Registration for Blip-Up blip-Down Diffusion Imaging) method for correcting echo planar imaging distortions}, author={Irfanoglu, M Okan and Modi, Pooja and Nayak, Amritha and Hutchinson, Elizabeth B and Sarlls, Joelle and Pierpaoli, Carlo}, diff --git a/qsiprep/tests/test_cli.py b/qsiprep/tests/test_cli.py index 1be64e83..26795175 100644 --- a/qsiprep/tests/test_cli.py +++ b/qsiprep/tests/test_cli.py @@ -2,7 +2,7 @@ import os import sys -from unittest.mock import patch +from unittest.mock import DEFAULT, patch import pytest from nipype import config as nipype_config @@ -20,6 +20,8 @@ nipype_config.enable_debug_mode() +DEFAULT_NUM_CPUS = 4 + @pytest.mark.integration @pytest.mark.dsdti_fmap @@ -181,7 +183,6 @@ def test_drbuddi_rpe(data_dir, output_dir, working_dir): "participant", f"-w={work_dir}", "--sloppy", - "--nthreads=4", "--anat-modality=none", "--denoise-method=none", "--b1_biascorrect_stage=none", @@ -224,7 +225,6 @@ def test_drbuddi_shoreline_epi(data_dir, output_dir, working_dir): "--hmc-model=none", "--output-resolution=2", "--shoreline-iters=1", - "--nthreads=1", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -259,9 +259,8 @@ def test_drbuddi_tensorline_epi(data_dir, output_dir, working_dir): "--b1-biascorrect-stage=none", "--pepolar-method=DRBUDDI", "--hmc-model=tensor", - "--output-resolution=2", + "--output-resolution=5", "--shoreline-iters=1", - "--nthreads=1", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -307,7 +306,6 @@ def test_dscsdsi(data_dir, output_dir, working_dir): "--hmc-transform=Rigid", "--output-resolution=5", "--shoreline-iters=1", - "--nthreads=1", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -350,7 +348,6 @@ def test_dsdti_nofmap(data_dir, output_dir, working_dir): "--unringing-method=rpg", "--b1-biascorrect-stage=none", "--output-resolution=5", - "--nthreads=1", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -393,7 +390,6 @@ def test_dsdti_synfmap(data_dir, output_dir, working_dir): "--force-syn", "--b1-biascorrect-stage=final", "--output-resolution=5", - "--nthreads=1", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -505,8 +501,6 @@ def test_maternal_brain_project(data_dir, output_dir, working_dir): "--output-resolution=5", "--hmc-model=3dSHORE", f"--bids-filter-file={bids_filter}", - "--nthreads=4", - "--omp-nthreads=4", ] _run_and_generate(TEST_NAME, parameters, test_main=True) @@ -541,13 +535,34 @@ def test_forrest_gump(data_dir, output_dir, working_dir): "--write-graph", "--output-resolution=5", f"--bids-filter-file={bids_filter}", - "--nthreads=4", - "--omp-nthreads=4", ] _run_and_generate(TEST_NAME, parameters, test_main=True) +def _check_arg_specified(argname, arglist): + for arg in arglist: + if arg.startswith(argname): + return True + return False + + +def _update_resources(parameters): + """We should use all the available CPUs for testing. + + Sometimes a test will set a specific amount of cpus. In that + case, the number should be kept. Otherwise, try to read the + env variable (specified in each job in config.yml). If + this variable doesn't work, just set it to 4. + """ + nthreads = int(os.environ.get("CIRCLECPUS", DEFAULT_NUM_CPUS)) + if not _check_arg_specified("--nthreads", parameters): + parameters.append(f"--nthreads={nthreads}") + if not _check_arg_specified("--omp-nthreads", parameters): + parameters.append(f"--omp-nthreads={nthreads}") + return parameters + + def _run_and_generate(test_name, parameters, test_main=True): from qsiprep import config @@ -556,6 +571,9 @@ def _run_and_generate(test_name, parameters, test_main=True): parameters.append("--notrack") parameters.append("-vv") + # Update resource parameters + parameters = _update_resources(parameters) + if test_main: # This runs, but for some reason doesn't count toward coverage. argv = ["qsiprep"] + parameters