[FIX] Update Everything (#724)

.circleci/CUDATest.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+cat << DOC
+
+Test paired DWI series with DRBUDDI
+===================================
+
+This tests the following features:
+ - Eddy is run on a CPU
+ - DRBUDDI is run with two DWI series
+
+DOC
+
+set +e
+source ./get_data.sh
+TESTDIR=${PWD}
+TESTNAME=DRBUDDI_RPE
+get_config_data ${TESTDIR}
+get_bids_data ${TESTDIR} drbuddi_rpe_series
+CFG=${TESTDIR}/data/nipype.cfg
+EDDY_CFG=${TESTDIR}/data/eddy_cuda_config.json
+
+# For the run
+setup_dir ${TESTDIR}/${TESTNAME}
+TEMPDIR=${TESTDIR}/${TESTNAME}/work
+OUTPUT_DIR=${TESTDIR}/${TESTNAME}/derivatives
+BIDS_INPUT_DIR=${TESTDIR}/data/tinytensor_rpe_series
+export FS_LICENSE=${TESTDIR}/data/license.txt
+QSIPREP_CMD=$(run_qsiprep_cmd ${BIDS_INPUT_DIR} ${OUTPUT_DIR})
+
+# Do the anatomical run on its own
+${QSIPREP_CMD} \
+	-w ${TEMPDIR} \
+	--sloppy \
+	--anat-modality none \
+	--denoise-method none \
+	--b1_biascorrect_stage none \
+	--pepolar-method DRBUDDI \
+	--eddy_config ${EDDY_CFG} \
+	--output-resolution 5 \
+    -vv --stop-on-first-crash
+
+

.circleci/config.yml

@@ -109,7 +109,7 @@ jobs:
       - run: *runinstall
       - run:
           name: Run TENSORLine with epi fmaps and DRBUDDI
-          no_output_timeout: 1h
+          no_output_timeout: 2h
           command: |
             cd .circleci
             bash DRBUDDI_TENSORLine_epi.sh
@@ -124,7 +124,7 @@ jobs:
       - run: *runinstall
       - run:
           name: Run SHORELine with epi fmaps and DRBUDDI
-          no_output_timeout: 1h
+          no_output_timeout: 2h
           command: |
             cd .circleci
             bash DRBUDDI_SHORELine_epi.sh
@@ -139,7 +139,7 @@ jobs:
       - run: *runinstall
       - run:
           name: Run Eddy with rpe series fmaps and DRBUDDI
-          no_output_timeout: 1h
+          no_output_timeout: 3h
           command: |
             cd .circleci
             bash DRBUDDI_eddy_rpe_series.sh
@@ -154,7 +154,7 @@ jobs:
       - run: *runinstall
       - run:
           name: Test the intramodal template workflow
-          no_output_timeout: 1h
+          no_output_timeout: 2h
           command: |
             cd .circleci
             bash IntramodalTemplate.sh

.circleci/get_data.sh

@@ -1,12 +1,22 @@
+IMAGE=pennbbl/qsiprep:unstable
+
+# Set this to be comfortable on the testing machine
+MAX_CPUS=18
 
 if [[ "$SHELL" =~ zsh ]]; then
   setopt SH_WORD_SPLIT
 fi
 
 # Edit these for project-wide testing
 WGET="wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q"
-LOCAL_PATCH=/Users/mcieslak/projects/qsiprep/qsiprep
-IMAGE=pennbbl/qsiprep:unstable
+
+# Patch in a local copy of qsiprep?
+LOCAL_PATCH=""
+if [[ -d /Users/mcieslak/projects/qsiprep/qsiprep ]]; then
+    LOCAL_PATCH=/Users/mcieslak/projects/qsiprep/qsiprep
+elif [[ -d /home/matt/projects/qsiprep ]]; then
+    LOCAL_PATCH=/home/matt/projects/qsiprep/qsiprep
+fi
 
 # Determine if we're in a CI test
 if [[ "${CIRCLECI}" = "true" ]]; then
@@ -19,9 +29,21 @@ if [[ "${CIRCLECI}" = "true" ]]; then
   fi
 else
   IN_CI="false"
-  NTHREADS=1
-  OMP_NTHREADS=1
+  N_CPUS=""
+  which nproc && N_CPUS=$(nproc)
+  if [[ -z "${N_CPUS}" ]]; then
+    N_CPUS=$(sysctl -n hw.logicalcpu)
+  fi
+  [[ -z "${N_CPUS}" ]] && N_CPUS=1
+
+  NTHREADS=$(( $N_CPUS < $MAX_CPUS ? $N_CPUS : $MAX_CPUS ))
+  OMP_NTHREADS=$NTHREADS
 fi
+
+# Determine if we have a gpu
+GPU_FLAG=""
+which nvidia-smi && GPU_FLAG="--gpus all"
+
 export IN_CI NTHREADS OMP_NTHREADS
 
 run_qsiprep_cmd () {
@@ -34,15 +56,15 @@ run_qsiprep_cmd () {
     QSIPREP_RUN="/opt/conda/envs/qsiprep/bin/qsiprep ${bids_dir} ${output_dir} participant"
   else
     # Otherwise we're going to use docker from the outside
-    QSIPREP_RUN="qsiprep-docker ${bids_dir} ${output_dir} participant -e qsiprep_DEV 1 -u $(id -u) -i ${IMAGE}"
+    QSIPREP_RUN="qsiprep-docker ${GPU_FLAG} ${bids_dir} ${output_dir} participant -e qsiprep_DEV 1 -u $(id -u) -i ${IMAGE}"
     CFG=$(printenv NIPYPE_CONFIG)
     if [[ -n "${CFG}" ]]; then
         QSIPREP_RUN="${QSIPREP_RUN} --config ${CFG}"
     fi
 
     if [[ -n "${LOCAL_PATCH}" ]]; then
       #echo "Using qsiprep patch: ${LOCAL_PATCH}"
-      QSIPREP_RUN="${QSIPREP_RUN} --patch-qsiprep ${LOCAL_PATCH} --patch-nipype /Users/mcieslak/projects/Nipype/nipype"
+      QSIPREP_RUN="${QSIPREP_RUN} --patch-qsiprep ${LOCAL_PATCH}"
     fi
   fi
   echo "${QSIPREP_RUN} --nthreads ${NTHREADS} --omp-nthreads ${OMP_NTHREADS}"
@@ -88,7 +110,7 @@ get_config_data() {
     cat > ${WORKDIR}/data/eddy_config.json << "EOT"
 {
   "flm": "linear",
-  "slm": "linear",
+  "slm": "none",
   "fep": false,
   "interp": "spline",
   "nvoxhp": 100,
@@ -106,8 +128,34 @@ get_config_data() {
   "output_type": "NIFTI_GZ",
   "args": ""
 }
+EOT
+
+    # Get an eddy config. It's used for some tests
+    cat > ${WORKDIR}/data/eddy_cuda_config.json << "EOT"
+{
+  "flm": "quadratic",
+  "slm": "none",
+  "fep": false,
+  "interp": "spline",
+  "nvoxhp": 100,
+  "fudge_factor": 10,
+  "dont_sep_offs_move": false,
+  "dont_peas": false,
+  "niter": 2,
+  "method": "jac",
+  "repol": true,
+  "num_threads": 1,
+  "is_shelled": true,
+  "use_cuda": true,
+  "mporder": 1,
+  "cnr_maps": true,
+  "residuals": false,
+  "output_type": "NIFTI_GZ",
+  "args": ""
+}
 EOT
     chmod a+r ${WORKDIR}/data/eddy_config.json
+    chmod a+r ${WORKDIR}/data/eddy_cuda_config.json
 
     # We always need a freesurfer license
     echo "cHJpbnRmICJtYXR0aGV3LmNpZXNsYWtAcHN5Y2gudWNzYi5lZHVcbjIwNzA2XG4gKkNmZVZkSDVVVDhyWVxuIEZTQllaLlVrZVRJQ3dcbiIgPiBsaWNlbnNlLnR4dAo=" | base64 -d | sh

qsiprep/interfaces/dsi_studio.py

@@ -178,7 +178,7 @@ class DSIStudioReconstructionInputSpec(DSIStudioCommandLineInputSpec):
     grad_dev = File(
         desc="Gradient deviation file", exists=True, copyfile=True, position=-1, argstr="#%s"
     )
-    thread_count = traits.Int(1, usedefault=True, argstr="--thread_count=%d")
+    thread_count = traits.Int(1, usedefault=True, argstr="--thread_count=%d", nohash=True)
 
     dti_no_high_b = traits.Bool(
         True,
@@ -349,7 +349,7 @@ class DSIStudioConnectivityMatrixInputSpec(DSIStudioCommandLineInputSpec):
     smoothing = traits.CFloat(argstr="--smoothing=%.2f")
     min_length = traits.CInt(argstr="--min_length=%d")
     max_length = traits.CInt(argstr="--max_length=%d")
-    thread_count = traits.Int(1, argstr="--thread_count=%d", usedefault=True)
+    thread_count = traits.Int(1, argstr="--thread_count=%d", usedefault=True, nohash=True)
 
 
 class DSIStudioConnectivityMatrixOutputSpec(TraitedSpec):

qsiprep/interfaces/freesurfer.py

@@ -36,7 +36,7 @@
     isdefined,
     traits,
 )
-from nipype.interfaces.freesurfer.base import FSCommandOpenMP, FSTraitedSpecOpenMP
+from nipype.interfaces.freesurfer.base import FSCommandOpenMP, FSTraitedSpec
 from nipype.interfaces.freesurfer.utils import LTAConvert
 from nipype.utils.filemanip import copyfile, filename_to_list, fname_presuffix
 from scipy.ndimage.morphology import binary_fill_holes
@@ -45,6 +45,10 @@
 from ..niworkflows.interfaces.utils import _copyxform
 
 
+class FSTraitedSpecOpenMP(FSTraitedSpec):
+    num_threads = traits.Int(desc="allows for specifying more threads", nohash=True)
+
+
 class StructuralReference(fs.RobustTemplate):
     """Variation on RobustTemplate that simply copies the source if a single
     volume is provided.

qsiprep/interfaces/tortoise.py

@@ -42,23 +42,8 @@
 
 SLOPPY_DRBUDDI = (
     "--DRBUDDI_stage "
-    "\[learning_rate=\{0.1\},cfs=\{100:8:4\},field_smoothing=\{9:0\},"
+    "\[learning_rate=\{0.3\},cfs=\{100:8:4\},field_smoothing=\{9:0\},"
     "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] "
-    "--DRBUDDI_stage "
-    "\[learning_rate=\{0.25\},cfs=\{100:6:3\},field_smoothing=\{8:0\},"
-    "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] "
-    "--DRBUDDI_stage "
-    "\[learning_rate=\{0.5\},cfs=\{100:4:2\},field_smoothing=\{7:0\},"
-    "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] "
-    "--DRBUDDI_stage "
-    "\[learning_rate=\{1.25\},cfs=\{100:2:1\},field_smoothing=\{6:0\},"
-    "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] "
-    "--DRBUDDI_stage "
-    "\[learning_rate=\{1.\},cfs=\{100:1:0\},field_smoothing=\{5:0\},"
-    "metrics=\{MSJac:CC\},restrict_constrain=\{1:1\}\] "
-    "--DRBUDDI_stage "
-    "\[learning_rate=\{1.\},cfs=\{20:1:0\},field_smoothing=\{4:0\},"
-    "metrics=\{MSJac:CC\},restrict_constrain=\{0:0\}\]"
 )
 
 
@@ -219,6 +204,12 @@ def write_dummy_bmtxt(nii_file):
 
 
 class _DRBUDDIInputSpec(TORTOISEInputSpec):
+    num_threads = traits.Int(
+        desc="number of OMP threads",
+        argstr="--ncores %d",
+        help="Number of cores to use in the CPU version. The default is 50% of system cores.",
+        nohash=True,
+    )
     blip_up_image = File(
         exists=True,
         help="Full path to the input UP NIFTI file to be corrected.",
@@ -292,6 +283,7 @@ class _DRBUDDIInputSpec(TORTOISEInputSpec):
     sloppy = traits.Bool(
         False, argstr=SLOPPY_DRBUDDI, desc="use underpowered (sloppy) registration for speed"
     )
+    disable_itk_threads = traits.Bool(True, usedefault=True, argstr="--disable_itk_threads")
 
 
 class _DRBUDDIOutputSpec(TraitedSpec):

qsiprep/workflows/dwi/util.py

@@ -128,7 +128,7 @@ def init_dwi_reference_wf(
         register_t1_to_raw = pe.Node(
             ants.Registration(from_file=affine_transform),
             name="register_t1_to_raw",
-            n_proces=omp_nthreads,
+            n_procs=omp_nthreads,
         )
         t1_mask_to_b0 = pe.Node(
             ants.ApplyTransforms(interpolation="MultiLabel", invert_transform_flags=[True]),

wrapper/qsiprep_container/qsiprep_docker.py

@@ -58,7 +58,7 @@
 MISSING = """
 Image '{}' is missing
 Would you like to download? [Y/n] """
-PKG_PATH = "/usr/local/miniconda/lib/python3.10/site-packages"
+PKG_PATH = "/opt/conda/envs/qsiprep/lib/python3.10/site-packages"
 
 # Monkey-patch Py2 subprocess
 if not hasattr(subprocess, "DEVNULL"):

wrapper/qsiprep_container/qsiprep_singularity.py

@@ -57,7 +57,7 @@
 MISSING = """
 Image '{}' is missing
 Would you like to download? [Y/n] """
-PKG_PATH = "/usr/local/miniconda/lib/python3.10/site-packages"
+PKG_PATH = "/opt/conda/envs/qsiprep/lib/python3.10/site-packages"
 
 # Monkey-patch Py2 subprocess
 if not hasattr(subprocess, "DEVNULL"):