From 7407281a3a82d6fa9ec3b3871054020319dcbf6c Mon Sep 17 00:00:00 2001 From: Artem Sokolov Date: Sun, 25 Feb 2024 18:56:34 -0500 Subject: [PATCH] Added support for multiple samples in raw/ (#540) * Function to infer sample name from raw filename * File staging for illumination and registration * Full prototype * Updated CHANGES --- CHANGES.md | 71 +++++++++++++++++++++++++++++++++++++++++ lib/mcmicro/Util.groovy | 8 +++++ main.nf | 17 +++++++--- modules/illumination.nf | 8 ++--- modules/registration.nf | 31 +++++++++--------- 5 files changed, 110 insertions(+), 25 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 62ab9c56..67416de4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,74 @@ +### 2024-02-25 + +* Added support for multiple samples in `raw/` subdirectory. If multiple images share the same markers and should be processed with the same set of parameters, they can be placed as subdirectories of `raw/`. In other words, instead of structuring the data as multiple projects: + +``` +image1/ + markers.csv + params.yml + raw/ + tile1.rcpnl + tile2.rcpnl + +image2/ + markers.csv + params.yml + raw/ + tile1.rcpnl + tile2.rcpnl +``` + +they can be consolidated under the same project folder: + +``` +myproject/ + markers.csv + params.yml + raw/ + image1/ + tile1.rcpnl + tile2.rcpnl + image2/ + tile1.rcpnl + tile2.rcpnl +``` + +### 2024-01-31 + +* Added support for ASHLAR's [fileseries/filepattern](https://forum.image.sc/t/ashlar-how-to-pass-multiple-images-to-be-stitched/49864) feature. The patterns can be provided directly via ASHLAR's options: + +``` yaml +options: + ashlar: fileseries|... +``` + +### 2023-12-21 + +* Added a config parameter controlling how intermediates are published to project directory. The behavior can be controlling by adding the following to `custom.config`: + +``` +params.publish_dir_mode = 'copy' +``` + +and providing it to the pipeline with + +``` +nextflow run labsyspharm/mcmicro --in exemplar-001 -c custom.config +``` + +The valid values for `publish_dir_mode` can be found in [Nextflow documentation](https://nextflow.io/docs/latest/process.html#publishdir) of `publishDir`, argument `mode`. + +### 2023-08-29 + +* Added ImageJ rolling ball background subtraction module + +The new module can be selected by adding the following to `params.yml`: +``` yaml +workflow: + background: true + background-method: imagej-rolling-ball +``` + ### 2023-06-16 * If `--membrane-channel` is provided to Mesmer options, MCMICRO will automatically pass the input image both as `--nuclear-image` and as `--membrane-image` to the Mesmer CLI. diff --git a/lib/mcmicro/Util.groovy b/lib/mcmicro/Util.groovy index 7d0a708e..66f32ab3 100644 --- a/lib/mcmicro/Util.groovy +++ b/lib/mcmicro/Util.groovy @@ -1,5 +1,13 @@ package mcmicro +static def getSampleName(f, rawdir) { + // Resolve paths relative to the input project directory + String rel = rawdir.relativize(f).toString() + rel.contains('/') ? rel.split('/').head() : + rawdir.parent.getName() +} + + /** * Extracts a file ID as the first token before delim in the filename * diff --git a/main.nf b/main.nf index 6b6f7cdd..507f68d9 100644 --- a/main.nf +++ b/main.nf @@ -24,7 +24,7 @@ mcp = Opts.parseParams( "$projectDir/config/defaults.yml" ) -// Separate out workflow parameters (wfp) and module specs to simplify code +// Separate out workflow parameters (wfp) to simplify code wfp = mcp.workflow // Identify relevant precomputed intermediates @@ -71,12 +71,19 @@ rawFiles = findFiles('raw', "**${formatPattern}", // the normal way that paths are passed to channels which handles this escaping // automatically. raw = rawFiles - .map{ tuple(formatType == "single" ? it : it.parent, it) } - .map{ toStage, relPath -> tuple(toStage, toStage.parent.relativize(relPath).toString()) } + .map{ tuple( + Util.getSampleName(it, file("${params.in}/raw")), + formatType == "single" ? it : it.parent, + it + )} + .map{ sampleName, toStage, relPath -> + tuple(sampleName, toStage, toStage.parent.relativize(relPath).toString()) } // Find precomputed intermediates -pre_dfp = findFiles0('illumination', "*-dfp.tif") -pre_ffp = findFiles0('illumination', "*-ffp.tif") +pre_dfp = findFiles0('illumination', "**-dfp.tif") + .map{ tuple(Util.getSampleName(it, file("${params.in}/illumination")), it) } +pre_ffp = findFiles0('illumination', "**-ffp.tif") + .map{ tuple(Util.getSampleName(it, file("${params.in}/illumination")), it) } pre_img = findFiles('registration', "*.{ome.tiff,ome.tif,tif,tiff,btf}", {error "No pre-stitched image in ${params.in}/registration"}) pre_bsub = findFiles('background', "*.ome.tif", diff --git a/modules/illumination.nf b/modules/illumination.nf index 67ea18a5..4622fa16 100644 --- a/modules/illumination.nf +++ b/modules/illumination.nf @@ -12,7 +12,7 @@ process illumination { container "${params.contPfx}${module.container}:${module.version}" // Output profiles - publishDir "${params.in}/illumination", mode: "${params.publish_dir_mode}", + publishDir "${params.in}/illumination/${sname}", mode: "${params.publish_dir_mode}", pattern: '*.tif' // Provenance @@ -23,10 +23,10 @@ process illumination { input: val wfp val module - tuple path(raw), val(relPath) // raw is only for staging, use relPath for paths + tuple val(sname), path(raw), val(relPath) // raw is only for staging, use relPath for paths output: - path '*-dfp.tif', emit: dfp - path '*-ffp.tif', emit: ffp + tuple val(sname), path('*-dfp.tif'), emit: dfp + tuple val(sname), path('*-ffp.tif'), emit: ffp tuple path('.command.sh'), path('.command.log') when: Flow.doirun('illumination', wfp) diff --git a/modules/registration.nf b/modules/registration.nf index fefdbcd0..bfaa9276 100644 --- a/modules/registration.nf +++ b/modules/registration.nf @@ -13,11 +13,7 @@ process ashlar { input: val mcp val module - val sampleName - path lraw // Only for staging - val lrelPath // Use this for paths - path lffp - path ldfp + tuple val(sampleName), path(lraw), val(lrelPath), path(lffp), path(ldfp) output: path "*.ome.tif", emit: img @@ -50,18 +46,21 @@ workflow registration { dfp // dark-field profiles main: - rawst = raw.toSortedList{a, b -> a[0] <=> b[0]}.transpose() - sampleName = file(params.in).name - ashlar( - mcp, - mcp.modules['registration'], - sampleName, - rawst.first(), - rawst.last(), - ffp.toSortedList{a, b -> a.getName() <=> b.getName()}, - dfp.toSortedList{a, b -> a.getName() <=> b.getName()} - ) + srt = {a, b -> file(a).getName() <=> file(b).getName()} + + rawg = raw.groupTuple(sort: srt) + ffpg = ffp.groupTuple(sort: srt) + dfpg = dfp.groupTuple(sort: srt) + + inputs = rawg.join(ffpg, remainder:true).join(dfpg, remainder:true) + .map{tuple( + it[0], it[1], it[2], + it[3] == null ? [] : it[3], // Convert null to empty list + it[4] == null ? [] : it[4] // Ditto + )} + + ashlar(mcp, mcp.modules['registration'], inputs) emit: ashlar.out.img