From 78533ef630e75c333972b21b65df1d48506a8f53 Mon Sep 17 00:00:00 2001 From: Anthony Gagnon Date: Wed, 8 Jan 2025 15:28:11 +0000 Subject: [PATCH 1/3] change freesurfer/fastsurfer output directory location (alongside nf-pediatric output folder) + fix atlas naming. --- conf/modules.config | 44 ++++++++++++++++++++++++++--------- tests/chained.nf.test | 13 +++++++++-- tests/chained.nf.test.snap | 14 +++++++---- tests/freesurfer.nf.test | 13 +++++++++-- tests/freesurfer.nf.test.snap | 18 +++++++++----- 5 files changed, 77 insertions(+), 25 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 97c9c54..a4c0aa6 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -16,30 +16,52 @@ process { withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:FASTSURFER' { ext.acq3T = params.acq3T publishDir = [ - path: { "${params.outdir}/${meta.id}/anat/"}, + [ + path: { "${params.outdir}/../fastsurfer-v2.3.3/" }, + mode: params.publish_dir_mode, + saveAs: { + filename -> + if ( filename.contains("fastsurfer") ) { "${meta.id}" } + else if ( filename.contains("versions.yml") ) { null } + else { params.lean_output ? null : filename } + } + ], + [ + path: { "${params.outdir}/${meta.id}/anat/"}, + mode: params.publish_dir_mode, + saveAs: { + filename -> + if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" } + else if ( filename.contains("versions.yml") ) { null } + else { params.lean_output ? null : filename } + } + ] + ] + } + + withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:RECONALL' { + publishDir = [ + [ + path: { "${params.outdir}/../freesurfer-7.4.1/" }, mode: params.publish_dir_mode, saveAs: { filename -> - if ( filename.contains("fastsurfer") ) { "${meta.id}_fastsurfer" } - else if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" } + if ( filename.contains("recon_all") ) { "${meta.id}" } else if ( filename.contains("versions.yml") ) { null } else { params.lean_output ? null : filename } } - ] - } - - withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:RECONALL' { - publishDir = [ + ], + [ path: { "${params.outdir}/${meta.id}/anat/" }, mode: params.publish_dir_mode, saveAs: { filename -> - if ( filename.contains("recon_all") ) { "${meta.id}_reconall"} - else if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" } + if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" } else if ( filename.contains("versions.yml") ) { null } else { params.lean_output ? null : filename } } ] + ] } withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:BRAINNETOMECHILD' { @@ -729,7 +751,7 @@ process { mode: params.publish_dir_mode, saveAs: { filename -> - if ( filename.contains("_warped.nii.gz") ) { "${meta.id}_space-DWI_seg-BrainnetomeChild_dseg.nii.gz" } + if ( filename.contains("_warped.nii.gz") ) { "${meta.id}_space-DWI_seg-${params.atlas_name}_dseg.nii.gz" } else if ( filename.contains("versions.yml") ) { null } else { params.lean_output ? null : filename } } diff --git a/tests/chained.nf.test b/tests/chained.nf.test index 6d15029..899de03 100644 --- a/tests/chained.nf.test +++ b/tests/chained.nf.test @@ -12,6 +12,7 @@ nextflow_pipeline { params.input = "$projectDir/tests/data/samplesheet_testconn.csv" params.outdir = "$outputDir" + use_fastsurfer = true params.connectomics = true params.freesurfer = true @@ -23,6 +24,8 @@ nextflow_pipeline { then { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // All fastsurfer output. + def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( @@ -31,7 +34,9 @@ nextflow_pipeline { // Remove the nextflow version from the versions.yml because we test it using different nextflow versions. removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"), // All stable name. - stable_name + stable_name, + // All fastsurfer output. + fastsurfer_output ).match() } } @@ -88,6 +93,8 @@ nextflow_pipeline { then { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // All fastsurfer output. + def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( @@ -96,7 +103,9 @@ nextflow_pipeline { // Remove the nextflow version from the versions.yml because we test it using different nextflow versions. removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"), // All stable name. - stable_name + stable_name, + // All fastsurfer output. + fastsurfer_output ).match() } } diff --git a/tests/chained.nf.test.snap b/tests/chained.nf.test.snap index 0f9caca..d0bf453 100644 --- a/tests/chained.nf.test.snap +++ b/tests/chained.nf.test.snap @@ -351,7 +351,6 @@ "sub-test", "sub-test/anat", "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz", - "sub-test/anat/sub-test_fastsurfer", "sub-test/anat/sub-test_from-T1w_to-dwi_affine.mat", "sub-test/anat/sub-test_from-T1w_to-dwi_warp.nii.gz", "sub-test/anat/sub-test_from-dwi_to-T1w_warp.nii.gz", @@ -429,13 +428,17 @@ "sub-test/multiqc/sub-test_multiqc_plots", "sub-test/multiqc/sub-test_multiqc_report.html", "subcortical_volumes.tsv" + ], + [ + "", + "sub-test" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.1" }, - "timestamp": "2025-01-03T09:26:58.519601" + "timestamp": "2025-01-08T10:10:09.573439" }, "Freesurfer + connectomics profiles - should run successfully": { "content": [ @@ -482,7 +485,6 @@ "sub-test", "sub-test/anat", "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz", - "sub-test/anat/sub-test_fastsurfer", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt", "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz", @@ -512,13 +514,17 @@ "sub-test/multiqc/sub-test_multiqc_plots", "sub-test/multiqc/sub-test_multiqc_report.html", "subcortical_volumes.tsv" + ], + [ + "", + "sub-test" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.1" }, - "timestamp": "2025-01-03T09:22:33.330752" + "timestamp": "2025-01-08T10:05:54.243678" }, "Connectomics + tracking profiles - should run successfully": { "content": [ diff --git a/tests/freesurfer.nf.test b/tests/freesurfer.nf.test index 83b2f96..90e20c4 100644 --- a/tests/freesurfer.nf.test +++ b/tests/freesurfer.nf.test @@ -13,6 +13,7 @@ nextflow_pipeline { input = "$projectDir/tests/data/samplesheet_testtracking.csv" outdir = "$outputDir" + use_fastsurfer = false freesurfer = true fs_license = "https://www.dropbox.com/scl/fi/0s8lp6lydyd0rxawxb4jm/license.txt?rlkey=hz54oc0d4sor69avqphtrjvgn&st=9e0yij97&dl=0" @@ -22,6 +23,8 @@ nextflow_pipeline { then { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // All freesurfer output. + def freesurfer_output = getAllFilesFromDir("$outputDir/../freesurfer-7.4.1/", relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( @@ -30,7 +33,9 @@ nextflow_pipeline { // Remove the nextflow version from the versions.yml because we test it using different nextflow versions. removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"), // All stable name. - stable_name + stable_name, + // All freesurfer output. + freesurfer_output ).match() } } @@ -54,6 +59,8 @@ nextflow_pipeline { then { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // All fastsurfer output. + def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( @@ -62,7 +69,9 @@ nextflow_pipeline { // Remove the nextflow version from the versions.yml because we test it using different nextflow versions. removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"), // All stable name. - stable_name + stable_name, + // All fastsurfer output. + fastsurfer_output ).match() } } diff --git a/tests/freesurfer.nf.test.snap b/tests/freesurfer.nf.test.snap index 891364c..a16d5dd 100644 --- a/tests/freesurfer.nf.test.snap +++ b/tests/freesurfer.nf.test.snap @@ -26,7 +26,6 @@ "sub-test", "sub-test/anat", "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz", - "sub-test/anat/sub-test_fastsurfer", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt", "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz", @@ -45,20 +44,24 @@ "sub-test/multiqc/sub-test_multiqc_plots", "sub-test/multiqc/sub-test_multiqc_report.html", "subcortical_volumes.tsv" + ], + [ + "", + "sub-test" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.1" }, - "timestamp": "2025-01-02T23:01:18.627052" + "timestamp": "2025-01-08T09:57:43.80135" }, "Freesurfer profile - freesurfer - should run successfully": { "content": [ 6, { - "FASTSURFER": { - "fastsurfer": "2.3.3+0000000" + "RECONALL": { + "freesurfer": "7.4.1" }, "Workflow": { "nf/pediatric": "v1.0dev" @@ -80,7 +83,6 @@ "sub-test", "sub-test/anat", "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz", - "sub-test/anat/sub-test_fastsurfer", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json", "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt", "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz", @@ -99,12 +101,16 @@ "sub-test/multiqc/sub-test_multiqc_plots", "sub-test/multiqc/sub-test_multiqc_report.html", "subcortical_volumes.tsv" + ], + [ + "", + "sub-test" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.1" }, - "timestamp": "2025-01-02T22:41:26.12873" + "timestamp": "2025-01-08T10:01:23.448734" } } \ No newline at end of file From 6f5960387380cc3d267f295e0fb2866f0db403aa Mon Sep 17 00:00:00 2001 From: Anthony Gagnon Date: Wed, 8 Jan 2025 16:05:54 +0000 Subject: [PATCH 2/3] set parameter for user-selected freesurfer output dir + docs --- CHANGELOG.md | 6 ++++++ conf/modules.config | 4 ++-- docs/output.md | 14 ++++++++++++++ nextflow.config | 1 + nextflow_schema.json | 8 ++++++++ tests/freesurfer.nf.test | 3 ++- 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dcb35a..8090e47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - [2025-01-08] + +### `Changed` + +- Fastsurfer and freesurfer outputs are now in their own dedicated output folder. + ## [Unreleased] - [2024-12-23] ### `Added` diff --git a/conf/modules.config b/conf/modules.config index a4c0aa6..331f68e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -17,7 +17,7 @@ process { ext.acq3T = params.acq3T publishDir = [ [ - path: { "${params.outdir}/../fastsurfer-v2.3.3/" }, + path: { params.fs_output_dir ?: "${params.outdir}/../fastsurfer-v2.3.3/" }, mode: params.publish_dir_mode, saveAs: { filename -> @@ -42,7 +42,7 @@ process { withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:RECONALL' { publishDir = [ [ - path: { "${params.outdir}/../freesurfer-7.4.1/" }, + path: { params.fs_output_dir ?: "${params.outdir}/../freesurfer-7.4.1/" }, mode: params.publish_dir_mode, saveAs: { filename -> diff --git a/docs/output.md b/docs/output.md index b75a797..21f9f07 100644 --- a/docs/output.md +++ b/docs/output.md @@ -24,6 +24,20 @@ The directories listed below will be created in the results directory after the The pipeline will output only the final preprocessed files by default. This behavior is used to limit the number of files generated by the pipeline by omitting the publishing of intermediate files. It is particularly useful when running the pipeline on clusters where file quota are often rapidly met. To opt-out from the lean output version, set the `--lean_output` parameter to false when launching the pipeline (`--lean_output false`). +If you decided to run FreeSurfer or FastSurfer as part of `nf-pediatric`, your output will be located (by default) alongside the `nf-pediatric` output folder. You can select the destination folder by using the `--fs_output_dir` parameter. If you do not specify and output folder destination, it will look like this: + +```bash + + |-- multiqc + |-- sub-0001 + |-- sub-0002 + <...> +fastsurfer-v2.3.3/ + |-- sub-0001 + |-- sub-0002 + <...> +``` + ## Pipeline overview The pipeline is built using [Nextflow](https://www.nextflow.io/) and data processing steps can be grouped. The final output files are listed below for each of those steps. If you used `--lean_output false`, you will find additional files than the ones described here. diff --git a/nextflow.config b/nextflow.config index 28599d8..dd302d2 100644 --- a/nextflow.config +++ b/nextflow.config @@ -17,6 +17,7 @@ params { use_fastsurfer = true fs_license = null acq3T = true + fs_output_dir = null // Atlas options utils_folder = "${projectDir}/assets/FS_BN_GL_SF_utils/" diff --git a/nextflow_schema.json b/nextflow_schema.json index c6b490c..0f666b7 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -76,6 +76,14 @@ "fa_icon": "fas fa-magnet", "default": true, "hidden": true + }, + "fs_output_dir": { + "type": "string", + "description": "Path to FreeSurfer output directory.", + "fa_icon": "fas fa-folder-open", + "help_text": "Path to FreeSurfer output directory. This is where FreeSurfer will save its output files.", + "format": "directory-path", + "hidden": false } } }, diff --git a/tests/freesurfer.nf.test b/tests/freesurfer.nf.test index 90e20c4..493a0e0 100644 --- a/tests/freesurfer.nf.test +++ b/tests/freesurfer.nf.test @@ -48,6 +48,7 @@ nextflow_pipeline { params.input = "$projectDir/tests/data/samplesheet_testtracking.csv" params.outdir = "$outputDir" + params.fs_output_dir = "$outputDir/../onedirdown/fastsurfer-v2.3.3" params.freesurfer = true params.use_fastsurfer = true @@ -60,7 +61,7 @@ nextflow_pipeline { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) // All fastsurfer output. - def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true) + def fastsurfer_output = getAllFilesFromDir(params.fs_output_dir, relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( From b7e1d1d1496121903af40117fd3f5923a4e301ba Mon Sep 17 00:00:00 2001 From: Anthony Gagnon Date: Wed, 8 Jan 2025 18:54:02 +0000 Subject: [PATCH 3/3] fix tests with multisubjects --- tests/multisubjects.nf.test | 7 ++++++- tests/multisubjects.nf.test.snap | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/multisubjects.nf.test b/tests/multisubjects.nf.test index ab49399..8ba466d 100644 --- a/tests/multisubjects.nf.test +++ b/tests/multisubjects.nf.test @@ -12,6 +12,7 @@ nextflow_pipeline { params.input = "$projectDir/tests/data/samplesheet_multisubject.csv" params.outdir = "$outputDir" + params.use_fastsurfer = true params.connectomics = true params.tracking = true @@ -24,6 +25,8 @@ nextflow_pipeline { then { // stable name: All files + folders in ${params.outdir}/ with a stable name. def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // All fastsurfer output. + def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true) assertAll { assert workflow.success assert snapshot( @@ -32,7 +35,9 @@ nextflow_pipeline { // Remove the nextflow version from the versions.yml because we test it using different nextflow versions. removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"), // All stable name. - stable_name + stable_name, + // All fastsurfer output. + fastsurfer_output ).match() } } diff --git a/tests/multisubjects.nf.test.snap b/tests/multisubjects.nf.test.snap index 1b030e7..6be5c56 100644 --- a/tests/multisubjects.nf.test.snap +++ b/tests/multisubjects.nf.test.snap @@ -273,7 +273,6 @@ "sub-test1", "sub-test1/anat", "sub-test1/anat/sub-test1_desc-preproc_T1w.nii.gz", - "sub-test1/anat/sub-test1_fastsurfer", "sub-test1/anat/sub-test1_from-T1w_to-dwi_affine.mat", "sub-test1/anat/sub-test1_from-T1w_to-dwi_warp.nii.gz", "sub-test1/anat/sub-test1_from-dwi_to-T1w_warp.nii.gz", @@ -353,7 +352,6 @@ "sub-test2", "sub-test2/anat", "sub-test2/anat/sub-test2_desc-preproc_T1w.nii.gz", - "sub-test2/anat/sub-test2_fastsurfer", "sub-test2/anat/sub-test2_from-T1w_to-dwi_affine.mat", "sub-test2/anat/sub-test2_from-T1w_to-dwi_warp.nii.gz", "sub-test2/anat/sub-test2_from-dwi_to-T1w_warp.nii.gz", @@ -431,12 +429,17 @@ "sub-test2/multiqc/sub-test2_multiqc_plots", "sub-test2/multiqc/sub-test2_multiqc_report.html", "subcortical_volumes.tsv" + ], + [ + "", + "sub-test1", + "sub-test2" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.1" }, - "timestamp": "2025-01-03T09:38:36.893058" + "timestamp": "2025-01-08T13:42:46.737348" } } \ No newline at end of file