Added support for multiple samples in raw/ (#540)

* Function to infer sample name from raw filename

* File staging for illumination and registration

* Full prototype

* Updated CHANGES

CHANGES.md

@@ -1,3 +1,74 @@
+### 2024-02-25
+
+* Added support for multiple samples in `raw/` subdirectory. If multiple images share the same markers and should be processed with the same set of parameters, they can be placed as subdirectories of `raw/`. In other words, instead of structuring the data as multiple projects:
+
+```
+image1/
+  markers.csv
+  params.yml
+  raw/
+    tile1.rcpnl
+    tile2.rcpnl
+
+image2/
+  markers.csv
+  params.yml
+  raw/
+    tile1.rcpnl
+    tile2.rcpnl
+```
+
+they can be consolidated under the same project folder:
+
+```
+myproject/
+  markers.csv
+  params.yml
+  raw/
+    image1/
+      tile1.rcpnl
+      tile2.rcpnl
+    image2/
+      tile1.rcpnl
+      tile2.rcpnl
+```
+
+### 2024-01-31
+
+* Added support for ASHLAR's [fileseries/filepattern](https://forum.image.sc/t/ashlar-how-to-pass-multiple-images-to-be-stitched/49864) feature. The patterns can be provided directly via ASHLAR's options:
+
+``` yaml
+options:
+  ashlar: fileseries|...
+```
+
+### 2023-12-21
+
+* Added a config parameter controlling how intermediates are published to project directory. The behavior can be controlling by adding the following to `custom.config`:
+
+```
+params.publish_dir_mode = 'copy'
+```
+
+and providing it to the pipeline with
+
+```
+nextflow run labsyspharm/mcmicro --in exemplar-001 -c custom.config
+```
+
+The valid values for `publish_dir_mode` can be found in [Nextflow documentation](https://nextflow.io/docs/latest/process.html#publishdir) of `publishDir`, argument `mode`.
+
+### 2023-08-29
+
+* Added ImageJ rolling ball background subtraction module
+
+The new module can be selected by adding the following to `params.yml`:
+``` yaml
+workflow:
+  background: true
+  background-method: imagej-rolling-ball
+```
+
 ### 2023-06-16
 
 * If `--membrane-channel` is provided to Mesmer options, MCMICRO will automatically pass the input image both as `--nuclear-image` and as `--membrane-image` to the Mesmer CLI.

lib/mcmicro/Util.groovy

@@ -1,5 +1,13 @@
 package mcmicro
 
+static def getSampleName(f, rawdir) {
+    // Resolve paths relative to the input project directory
+    String rel = rawdir.relativize(f).toString()
+    rel.contains('/') ? rel.split('/').head() : 
+        rawdir.parent.getName()
+}
+
+
 /**
  * Extracts a file ID as the first token before delim in the filename
  *

main.nf

@@ -24,7 +24,7 @@ mcp = Opts.parseParams(
     "$projectDir/config/defaults.yml"
 )
 
-// Separate out workflow parameters (wfp) and module specs to simplify code
+// Separate out workflow parameters (wfp) to simplify code
 wfp = mcp.workflow
 
 // Identify relevant precomputed intermediates
@@ -71,12 +71,19 @@ rawFiles = findFiles('raw', "**${formatPattern}",
 // the normal way that paths are passed to channels which handles this escaping
 // automatically.
 raw = rawFiles
-    .map{ tuple(formatType == "single" ? it : it.parent, it) }
-    .map{ toStage, relPath -> tuple(toStage, toStage.parent.relativize(relPath).toString()) }
+    .map{ tuple(
+        Util.getSampleName(it, file("${params.in}/raw")),
+        formatType == "single" ? it : it.parent, 
+        it
+    )}
+    .map{ sampleName, toStage, relPath -> 
+        tuple(sampleName, toStage, toStage.parent.relativize(relPath).toString()) }
 
 // Find precomputed intermediates
-pre_dfp   = findFiles0('illumination', "*-dfp.tif")
-pre_ffp   = findFiles0('illumination', "*-ffp.tif")
+pre_dfp   = findFiles0('illumination', "**-dfp.tif")
+    .map{ tuple(Util.getSampleName(it, file("${params.in}/illumination")), it) }
+pre_ffp   = findFiles0('illumination', "**-ffp.tif")
+    .map{ tuple(Util.getSampleName(it, file("${params.in}/illumination")), it) }
 pre_img   = findFiles('registration', "*.{ome.tiff,ome.tif,tif,tiff,btf}",
     {error "No pre-stitched image in ${params.in}/registration"})
 pre_bsub  = findFiles('background', "*.ome.tif",

modules/illumination.nf

@@ -12,7 +12,7 @@ process illumination {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output profiles
-    publishDir "${params.in}/illumination", mode: "${params.publish_dir_mode}",
+    publishDir "${params.in}/illumination/${sname}", mode: "${params.publish_dir_mode}",
       pattern: '*.tif'
 
     // Provenance
@@ -23,10 +23,10 @@ process illumination {
     input:
       val wfp
       val module
-      tuple path(raw), val(relPath) // raw is only for staging, use relPath for paths
+      tuple val(sname), path(raw), val(relPath) // raw is only for staging, use relPath for paths
     output:
-      path '*-dfp.tif', emit: dfp
-      path '*-ffp.tif', emit: ffp
+      tuple val(sname), path('*-dfp.tif'), emit: dfp
+      tuple val(sname), path('*-ffp.tif'), emit: ffp
       tuple path('.command.sh'), path('.command.log')
 
     when: Flow.doirun('illumination', wfp)

modules/registration.nf

@@ -13,11 +13,7 @@ process ashlar {
     input:
       val mcp
       val module
-      val sampleName
-      path lraw       // Only for staging
-      val lrelPath    // Use this for paths
-      path lffp
-      path ldfp
+      tuple val(sampleName), path(lraw), val(lrelPath), path(lffp), path(ldfp)
 
     output:
       path "*.ome.tif", emit: img
@@ -50,18 +46,21 @@ workflow registration {
       dfp     // dark-field profiles
 
     main:
-      rawst = raw.toSortedList{a, b -> a[0] <=> b[0]}.transpose()
-      sampleName  = file(params.in).name
 
-      ashlar(
-        mcp,
-        mcp.modules['registration'],
-        sampleName,
-        rawst.first(),
-        rawst.last(),
-        ffp.toSortedList{a, b -> a.getName() <=> b.getName()},
-        dfp.toSortedList{a, b -> a.getName() <=> b.getName()}
-      )
+    srt = {a, b -> file(a).getName() <=> file(b).getName()}
+
+    rawg = raw.groupTuple(sort: srt)
+    ffpg = ffp.groupTuple(sort: srt)
+    dfpg = dfp.groupTuple(sort: srt)
+
+    inputs = rawg.join(ffpg, remainder:true).join(dfpg, remainder:true)
+      .map{tuple(
+        it[0], it[1], it[2],
+        it[3] == null ? [] : it[3],    // Convert null to empty list
+        it[4] == null ? [] : it[4]     // Ditto
+      )}
+
+    ashlar(mcp, mcp.modules['registration'], inputs)
 
     emit:
       ashlar.out.img