Deploy for release 0.1.1 from f4bc99b

CHANGELOG.md

@@ -1,8 +1,14 @@
+# utilities 0.1.1
+
+* `mapping/process_10x_auto`: Allow grouping input fastqs per folder or via a custom regex statement.
+
+* `mapping/process_10x_auto`: Add more documentation to the arguments to clarify regex usage.
+
 # utilities 0.1.0
 
 First release of the `utilities` as a Viash+Nextflow pipeline.
 
 * `mapping/process_10x`: Mapping 10x samples
-* `mapping/process_10x (auto)`: Map all 10x Fastq files in a directory.
+* `mapping/process_10x_auto`: Map all 10x Fastq files in a directory.
 * `mapping/process_smartseq2`: Mapping smartseq2 samples
-* `mapping/process_smartseq2 (auto)`: Map all smartseq2 Fastq files in a directory.
+* `mapping/process_smartseq2_auto`: Map all smartseq2 Fastq files in a directory.

scripts/process_10x

@@ -30,7 +30,7 @@ export NXF_WORK="`pwd`/work"
 # do run
 nextflow run \
   "czbiohub/utilities" \
-  -r "0.1.0" \
+  -r "0.1.1" \
   -main-script "src/mapping/process_10x/main.nf" \
   -entry "auto" \
   -resume \

scripts/process_smartseq2

@@ -30,7 +30,7 @@ export NXF_WORK="`pwd`/work"
 # do run
 nextflow run \
   "czbiohub/utilities" \
-  -r "0.1.0" \
+  -r "0.1.1" \
   -main-script "src/mapping/process_smartseq2/main.nf" \
   -entry "auto" \
   -resume \

src/mapping/process_smartseq2/auto.vsh.yaml

@@ -40,21 +40,36 @@ functionality:
           description: A directory containing one or more smartseq2 plates.
         - type: string
           name: --fastq_regex
-          default: "(.*)_R[12](_001)?.fastq.gz"
+          default: "(.*/)?(.*)(_[A-P][0-9]{1,2})(_.*)?_R[12](_001)?.fastq.gz"
           description: |
             Only files matching this regular expression will be retained.
             For more details on the notation of regular expression, see the 
             Java Docs on [Regular expressions](https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html).
           info:
             hidden: true
         - type: string
-          name: --cell_id_replacement
+          name: --sample_id_replacement
           default: "$1"
+          description: |
+            Which groups from the fastq regex to use as the sample ID. 
+            Assuming a fastq file is detected at `dir1/dir2/sample_id_A1_foo_R1.fastq.gz`,
+            the outputs of several regexes is listed:
+
+              * `$1` -> `dir1/dir2`
+              * `$1$2` -> `dir1/dir2/sample_id`
+              * `$2` -> `sample_id`
+          info:
+            hidden: true
+        - type: string
+          name: --cell_id_replacement
+          default: "$1$2$3"
           description: |
             Which groups from the fastq regex to use as the cell ID. 
-            For example, if the `fastq_regex` has two groups which are relevant 
-            for the constructing the sample ID, the value for this parameter should
-            probably be `$1$2`.
+            Assuming a fastq file is detected at `dir1/dir2/sample_id_A1_foo_R1.fastq.gz`,
+            the outputs of several regexes is listed:
+
+              * `$1$2$3` -> `dir1/dir2/sample_id_A1`
+              * `$1$2$3$4` -> `dir1/dir2/sample_id_A1_foo`
           info:
             hidden: true
     - name: Reference

src/mapping/process_smartseq2/main.nf

@@ -192,28 +192,42 @@ workflow auto {
     .findAll{it.toString()
     .matches(auto_params.fastq_regex)}
 
-  // create output list
-
-  input_r1 = fastq_files.findAll{it.toString().matches(".*_R1[_.].*")}.sort()
-  input_r2 = fastq_files.findAll{it.toString().matches(".*_R2[_.].*")}.sort()
-  input_id = input_r1.collect{ fastq_file ->
+  // group by sample id
+  // use regex to search for the sample id
+  fastq_grouped = fastq_files.groupBy{ fastq_file ->
     fastq_file.toString()
       .replace("${auto_params.input_dir}/", "") // remove root directory
-      .replaceAll(auto_params.fastq_regex, auto_params.cell_id_replacement) // extract sample id using regex
+      .replaceAll(auto_params.fastq_regex, auto_params.sample_id_replacement) // extract sample id using regex
   }
 
-  assert input_r1.size() == input_r2.size(): "The number of R1 files (${input_r1.size()}) should equal the number of R2 files (${input_r2.size()})."
-
-  param_list = [[
-    id: auto_params.id,
-    input_id: input_id,
-    input_r1: input_r1,
-    input_r2: input_r2,
-    reference_index: auto_params.reference_index,
-    reference_gtf: auto_params.reference_gtf,
-    output_raw: "${auto_params.id}_raw",
-    output_h5mu: "${auto_params.id}.h5mu"
-  ]]
+  // create output list
+  param_list = fastq_grouped.collectMany{ sample_id, inputs ->
+
+    def input_r1 = inputs.findAll{it.toString().matches(".*_R1[_.].*")}.sort()
+    def input_r2 = inputs.findAll{it.toString().matches(".*_R2[_.].*")}.sort()
+    def input_id = input_r1.collect{ fastq_file ->
+      fastq_file.toString()
+        .replace("${auto_params.input_dir}/", "") // remove root directory
+        .replaceAll(auto_params.fastq_regex, auto_params.cell_id_replacement) // extract sample id using regex
+        .replaceAll('/$', "") // strip trailing /
+    }
+
+    if (input_r1.size() != input_r2.size()) {
+      println("Skipping sample '$sample_id' because the number of R1 files (${input_r1.size()}) does not equal the number of R2 files (${input_r2.size()}).")
+      return []
+    }
+
+    [[
+      id: sample_id,
+      input_id: input_id,
+      input_r1: input_r1,
+      input_r2: input_r2,
+      reference_index: auto_params.reference_index,
+      reference_gtf: auto_params.reference_gtf,
+      output_raw: "${sample_id}_raw",
+      output_h5mu: "${sample_id}.h5mu"
+    ]]
+  }
 
   // Log params file to output dir
   param_file = file("${getPublishDir()}/param_list.yaml")

target/docker/demux/spaceranger_mkfastq/.config.vsh.yaml

@@ -1,7 +1,7 @@
 functionality:
   name: "spaceranger_mkfastq"
   namespace: "demux"
-  version: "0.1.0"
+  version: "0.1.1"
   arguments:
   - type: "file"
     name: "--input"
@@ -79,5 +79,5 @@ info:
   output: "/home/runner/work/utilities/utilities/target/docker/demux/spaceranger_mkfastq"
   executable: "/home/runner/work/utilities/utilities/target/docker/demux/spaceranger_mkfastq/spaceranger_mkfastq"
   viash_version: "0.7.3"
-  git_commit: "fe9d13b5313a883550b35341ae3e01a48c5921c9"
+  git_commit: "f4bc99bf82a915a68541bcfd58b146064c8dba2b"
   git_remote: "https://github.com/czbiohub/utilities"