From 55bb5617a742b808e73c51724f9d768700cef45c Mon Sep 17 00:00:00 2001 From: Matt Cieslak Date: Thu, 18 Apr 2024 21:07:49 -0400 Subject: [PATCH] [FIX] Update Everything (#724) --- .circleci/CUDATest.sh | 43 +++++++++++++ .circleci/config.yml | 8 +-- .circleci/get_data.sh | 62 ++++++++++++++++--- qsiprep/interfaces/dsi_studio.py | 4 +- qsiprep/interfaces/freesurfer.py | 6 +- qsiprep/interfaces/tortoise.py | 24 +++---- qsiprep/workflows/dwi/util.py | 2 +- wrapper/qsiprep_container/qsiprep_docker.py | 2 +- .../qsiprep_container/qsiprep_singularity.py | 2 +- 9 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 .circleci/CUDATest.sh diff --git a/.circleci/CUDATest.sh b/.circleci/CUDATest.sh new file mode 100644 index 00000000..3970feb2 --- /dev/null +++ b/.circleci/CUDATest.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +cat << DOC + +Test paired DWI series with DRBUDDI +=================================== + +This tests the following features: + - Eddy is run on a CPU + - DRBUDDI is run with two DWI series + +DOC + +set +e +source ./get_data.sh +TESTDIR=${PWD} +TESTNAME=DRBUDDI_RPE +get_config_data ${TESTDIR} +get_bids_data ${TESTDIR} drbuddi_rpe_series +CFG=${TESTDIR}/data/nipype.cfg +EDDY_CFG=${TESTDIR}/data/eddy_cuda_config.json + +# For the run +setup_dir ${TESTDIR}/${TESTNAME} +TEMPDIR=${TESTDIR}/${TESTNAME}/work +OUTPUT_DIR=${TESTDIR}/${TESTNAME}/derivatives +BIDS_INPUT_DIR=${TESTDIR}/data/tinytensor_rpe_series +export FS_LICENSE=${TESTDIR}/data/license.txt +QSIPREP_CMD=$(run_qsiprep_cmd ${BIDS_INPUT_DIR} ${OUTPUT_DIR}) + +# Do the anatomical run on its own +${QSIPREP_CMD} \ + -w ${TEMPDIR} \ + --sloppy \ + --anat-modality none \ + --denoise-method none \ + --b1_biascorrect_stage none \ + --pepolar-method DRBUDDI \ + --eddy_config ${EDDY_CFG} \ + --output-resolution 5 \ + -vv --stop-on-first-crash + + diff --git a/.circleci/config.yml b/.circleci/config.yml index d114b604..72520e63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,7 +109,7 @@ jobs: - run: *runinstall - run: name: Run TENSORLine with epi fmaps and DRBUDDI - no_output_timeout: 1h + no_output_timeout: 2h command: | cd .circleci bash DRBUDDI_TENSORLine_epi.sh @@ -124,7 +124,7 @@ jobs: - run: *runinstall - run: name: Run SHORELine with epi fmaps and DRBUDDI - no_output_timeout: 1h + no_output_timeout: 2h command: | cd .circleci bash DRBUDDI_SHORELine_epi.sh @@ -139,7 +139,7 @@ jobs: - run: *runinstall - run: name: Run Eddy with rpe series fmaps and DRBUDDI - no_output_timeout: 1h + no_output_timeout: 3h command: | cd .circleci bash DRBUDDI_eddy_rpe_series.sh @@ -154,7 +154,7 @@ jobs: - run: *runinstall - run: name: Test the intramodal template workflow - no_output_timeout: 1h + no_output_timeout: 2h command: | cd .circleci bash IntramodalTemplate.sh diff --git a/.circleci/get_data.sh b/.circleci/get_data.sh index 198942f3..692c764d 100644 --- a/.circleci/get_data.sh +++ b/.circleci/get_data.sh @@ -1,3 +1,7 @@ +IMAGE=pennbbl/qsiprep:unstable + +# Set this to be comfortable on the testing machine +MAX_CPUS=18 if [[ "$SHELL" =~ zsh ]]; then setopt SH_WORD_SPLIT @@ -5,8 +9,14 @@ fi # Edit these for project-wide testing WGET="wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q" -LOCAL_PATCH=/Users/mcieslak/projects/qsiprep/qsiprep -IMAGE=pennbbl/qsiprep:unstable + +# Patch in a local copy of qsiprep? +LOCAL_PATCH="" +if [[ -d /Users/mcieslak/projects/qsiprep/qsiprep ]]; then + LOCAL_PATCH=/Users/mcieslak/projects/qsiprep/qsiprep +elif [[ -d /home/matt/projects/qsiprep ]]; then + LOCAL_PATCH=/home/matt/projects/qsiprep/qsiprep +fi # Determine if we're in a CI test if [[ "${CIRCLECI}" = "true" ]]; then @@ -19,9 +29,21 @@ if [[ "${CIRCLECI}" = "true" ]]; then fi else IN_CI="false" - NTHREADS=1 - OMP_NTHREADS=1 + N_CPUS="" + which nproc && N_CPUS=$(nproc) + if [[ -z "${N_CPUS}" ]]; then + N_CPUS=$(sysctl -n hw.logicalcpu) + fi + [[ -z "${N_CPUS}" ]] && N_CPUS=1 + + NTHREADS=$(( $N_CPUS < $MAX_CPUS ? $N_CPUS : $MAX_CPUS )) + OMP_NTHREADS=$NTHREADS fi + +# Determine if we have a gpu +GPU_FLAG="" +which nvidia-smi && GPU_FLAG="--gpus all" + export IN_CI NTHREADS OMP_NTHREADS run_qsiprep_cmd () { @@ -34,7 +56,7 @@ run_qsiprep_cmd () { QSIPREP_RUN="/opt/conda/envs/qsiprep/bin/qsiprep ${bids_dir} ${output_dir} participant" else # Otherwise we're going to use docker from the outside - QSIPREP_RUN="qsiprep-docker ${bids_dir} ${output_dir} participant -e qsiprep_DEV 1 -u $(id -u) -i ${IMAGE}" + QSIPREP_RUN="qsiprep-docker ${GPU_FLAG} ${bids_dir} ${output_dir} participant -e qsiprep_DEV 1 -u $(id -u) -i ${IMAGE}" CFG=$(printenv NIPYPE_CONFIG) if [[ -n "${CFG}" ]]; then QSIPREP_RUN="${QSIPREP_RUN} --config ${CFG}" @@ -42,7 +64,7 @@ run_qsiprep_cmd () { if [[ -n "${LOCAL_PATCH}" ]]; then #echo "Using qsiprep patch: ${LOCAL_PATCH}" - QSIPREP_RUN="${QSIPREP_RUN} --patch-qsiprep ${LOCAL_PATCH} --patch-nipype /Users/mcieslak/projects/Nipype/nipype" + QSIPREP_RUN="${QSIPREP_RUN} --patch-qsiprep ${LOCAL_PATCH}" fi fi echo "${QSIPREP_RUN} --nthreads ${NTHREADS} --omp-nthreads ${OMP_NTHREADS}" @@ -88,7 +110,7 @@ get_config_data() { cat > ${WORKDIR}/data/eddy_config.json << "EOT" { "flm": "linear", - "slm": "linear", + "slm": "none", "fep": false, "interp": "spline", "nvoxhp": 100, @@ -106,8 +128,34 @@ get_config_data() { "output_type": "NIFTI_GZ", "args": "" } +EOT + + # Get an eddy config. It's used for some tests + cat > ${WORKDIR}/data/eddy_cuda_config.json << "EOT" +{ + "flm": "quadratic", + "slm": "none", + "fep": false, + "interp": "spline", + "nvoxhp": 100, + "fudge_factor": 10, + "dont_sep_offs_move": false, + "dont_peas": false, + "niter": 2, + "method": "jac", + "repol": true, + "num_threads": 1, + "is_shelled": true, + "use_cuda": true, + "mporder": 1, + "cnr_maps": true, + "residuals": false, + "output_type": "NIFTI_GZ", + "args": "" +} EOT chmod a+r ${WORKDIR}/data/eddy_config.json + chmod a+r ${WORKDIR}/data/eddy_cuda_config.json # We always need a freesurfer license echo "cHJpbnRmICJtYXR0aGV3LmNpZXNsYWtAcHN5Y2gudWNzYi5lZHVcbjIwNzA2XG4gKkNmZVZkSDVVVDhyWVxuIEZTQllaLlVrZVRJQ3dcbiIgPiBsaWNlbnNlLnR4dAo=" | base64 -d | sh diff --git a/qsiprep/interfaces/dsi_studio.py b/qsiprep/interfaces/dsi_studio.py index 29f264c2..9c98aea6 100644 --- a/qsiprep/interfaces/dsi_studio.py +++ b/qsiprep/interfaces/dsi_studio.py @@ -178,7 +178,7 @@ class DSIStudioReconstructionInputSpec(DSIStudioCommandLineInputSpec): grad_dev = File( desc="Gradient deviation file", exists=True, copyfile=True, position=-1, argstr="#%s" ) - thread_count = traits.Int(1, usedefault=True, argstr="--thread_count=%d") + thread_count = traits.Int(1, usedefault=True, argstr="--thread_count=%d", nohash=True) dti_no_high_b = traits.Bool( True, @@ -349,7 +349,7 @@ class DSIStudioConnectivityMatrixInputSpec(DSIStudioCommandLineInputSpec): smoothing = traits.CFloat(argstr="--smoothing=%.2f") min_length = traits.CInt(argstr="--min_length=%d") max_length = traits.CInt(argstr="--max_length=%d") - thread_count = traits.Int(1, argstr="--thread_count=%d", usedefault=True) + thread_count = traits.Int(1, argstr="--thread_count=%d", usedefault=True, nohash=True) class DSIStudioConnectivityMatrixOutputSpec(TraitedSpec): diff --git a/qsiprep/interfaces/freesurfer.py b/qsiprep/interfaces/freesurfer.py index 91b93059..b7726221 100644 --- a/qsiprep/interfaces/freesurfer.py +++ b/qsiprep/interfaces/freesurfer.py @@ -36,7 +36,7 @@ isdefined, traits, ) -from nipype.interfaces.freesurfer.base import FSCommandOpenMP, FSTraitedSpecOpenMP +from nipype.interfaces.freesurfer.base import FSCommandOpenMP, FSTraitedSpec from nipype.interfaces.freesurfer.utils import LTAConvert from nipype.utils.filemanip import copyfile, filename_to_list, fname_presuffix from scipy.ndimage.morphology import binary_fill_holes @@ -45,6 +45,10 @@ from ..niworkflows.interfaces.utils import _copyxform +class FSTraitedSpecOpenMP(FSTraitedSpec): + num_threads = traits.Int(desc="allows for specifying more threads", nohash=True) + + class StructuralReference(fs.RobustTemplate): """Variation on RobustTemplate that simply copies the source if a single volume is provided. diff --git a/qsiprep/interfaces/tortoise.py b/qsiprep/interfaces/tortoise.py index 335a3882..8a9cdb4b 100644 --- a/qsiprep/interfaces/tortoise.py +++ b/qsiprep/interfaces/tortoise.py @@ -42,23 +42,8 @@ SLOPPY_DRBUDDI = ( "--DRBUDDI_stage " - "\[learning_rate=\{0.1\},cfs=\{100:8:4\},field_smoothing=\{9:0\}," + "\[learning_rate=\{0.3\},cfs=\{100:8:4\},field_smoothing=\{9:0\}," "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] " - "--DRBUDDI_stage " - "\[learning_rate=\{0.25\},cfs=\{100:6:3\},field_smoothing=\{8:0\}," - "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] " - "--DRBUDDI_stage " - "\[learning_rate=\{0.5\},cfs=\{100:4:2\},field_smoothing=\{7:0\}," - "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] " - "--DRBUDDI_stage " - "\[learning_rate=\{1.25\},cfs=\{100:2:1\},field_smoothing=\{6:0\}," - "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] " - "--DRBUDDI_stage " - "\[learning_rate=\{1.\},cfs=\{100:1:0\},field_smoothing=\{5:0\}," - "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] " - "--DRBUDDI_stage " - "\[learning_rate=\{1.\},cfs=\{20:1:0\},field_smoothing=\{4:0\}," - "metrics=\{MSJac:CC\},restrict_constrain=\{0:0\}\]" ) @@ -219,6 +204,12 @@ def write_dummy_bmtxt(nii_file): class _DRBUDDIInputSpec(TORTOISEInputSpec): + num_threads = traits.Int( + desc="number of OMP threads", + argstr="--ncores %d", + help="Number of cores to use in the CPU version. The default is 50% of system cores.", + nohash=True, + ) blip_up_image = File( exists=True, help="Full path to the input UP NIFTI file to be corrected.", @@ -292,6 +283,7 @@ class _DRBUDDIInputSpec(TORTOISEInputSpec): sloppy = traits.Bool( False, argstr=SLOPPY_DRBUDDI, desc="use underpowered (sloppy) registration for speed" ) + disable_itk_threads = traits.Bool(True, usedefault=True, argstr="--disable_itk_threads") class _DRBUDDIOutputSpec(TraitedSpec): diff --git a/qsiprep/workflows/dwi/util.py b/qsiprep/workflows/dwi/util.py index ffb18988..772be578 100644 --- a/qsiprep/workflows/dwi/util.py +++ b/qsiprep/workflows/dwi/util.py @@ -128,7 +128,7 @@ def init_dwi_reference_wf( register_t1_to_raw = pe.Node( ants.Registration(from_file=affine_transform), name="register_t1_to_raw", - n_proces=omp_nthreads, + n_procs=omp_nthreads, ) t1_mask_to_b0 = pe.Node( ants.ApplyTransforms(interpolation="MultiLabel", invert_transform_flags=[True]), diff --git a/wrapper/qsiprep_container/qsiprep_docker.py b/wrapper/qsiprep_container/qsiprep_docker.py index 12c27b21..be5f4f59 100644 --- a/wrapper/qsiprep_container/qsiprep_docker.py +++ b/wrapper/qsiprep_container/qsiprep_docker.py @@ -58,7 +58,7 @@ MISSING = """ Image '{}' is missing Would you like to download? [Y/n] """ -PKG_PATH = "/usr/local/miniconda/lib/python3.10/site-packages" +PKG_PATH = "/opt/conda/envs/qsiprep/lib/python3.10/site-packages" # Monkey-patch Py2 subprocess if not hasattr(subprocess, "DEVNULL"): diff --git a/wrapper/qsiprep_container/qsiprep_singularity.py b/wrapper/qsiprep_container/qsiprep_singularity.py index 2e0c5fc9..40028182 100644 --- a/wrapper/qsiprep_container/qsiprep_singularity.py +++ b/wrapper/qsiprep_container/qsiprep_singularity.py @@ -57,7 +57,7 @@ MISSING = """ Image '{}' is missing Would you like to download? [Y/n] """ -PKG_PATH = "/usr/local/miniconda/lib/python3.10/site-packages" +PKG_PATH = "/opt/conda/envs/qsiprep/lib/python3.10/site-packages" # Monkey-patch Py2 subprocess if not hasattr(subprocess, "DEVNULL"):