Process afterscript (#164)

* add methods.setup_process_afterscript() * remove publish dir rules and log outputs * Update CHANGELOG.md * Autofix Nextflow configuration regression tests --------- Co-authored-by: Eeman <mmootor@ip-0A125967.rhxrlfvjyzbupc03cc22jkch3c.xx.internal.cloudapp.net> Co-authored-by: Faizal-Eeman <[email protected]>
uclahs-cds · Jun 28, 2024 · d9eb189 · d9eb189
1 parent b0b9d3f
commit d9eb189
Show file tree

Hide file tree

Showing 9 changed files with 24 additions and 53 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@ All notable changes to the call-gSV pipeline.
 
 ## [Unreleased]
 ### Changed
+- Use `methods.setup_process_afterscript()` for process logs
 - Increase CPU allocation for Manta
 
 ---

diff --git a/config/methods.config b/config/methods.config
@@ -83,6 +83,7 @@ methods {
         methods.set_log_output_dir()
         methods.set_pipeline_logs()
         methods.set_process()
+        methods.setup_process_afterscript()
         retry.setup_retry()
         methods.setup_docker_cpus()
         }

diff --git a/module/bcftools.nf b/module/bcftools.nf
@@ -16,19 +16,13 @@ process convert_BCF2VCF_BCFtools {
         pattern: "*.vcf",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         path bcf_file
         val bam_sample_name
         val variant_type
 
     output:
         path "DELLY-${params.delly_version}_${variant_type}_${params.dataset_id}_${bam_sample_name}.vcf", emit: vcf_file
-        path ".command.*"
 
     """
     set -euo pipefail

diff --git a/module/delly.nf b/module/delly.nf
@@ -15,11 +15,6 @@ process call_gSV_Delly {
         pattern: "*.bcf*",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         tuple val(bam_sample_name), path(input_bam), path(input_bam_bai)
         path(reference_fasta)
@@ -29,7 +24,6 @@ process call_gSV_Delly {
     output:
         path "${params.output_filename}_${params.GSV}.bcf", emit: bcf_sv_file
         path "${params.output_filename}_${params.GSV}.bcf.csi", emit: bcf_sv_file_csi
-        path ".command.*"
         val bam_sample_name, emit: bam_sample_name
 
     script:
@@ -52,11 +46,6 @@ process regenotype_gSV_Delly {
         pattern: "*.bcf*",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         tuple val(bam_sample_name), path(input_bam), path(input_bam_bai)
         path(reference_fasta)
@@ -67,7 +56,6 @@ process regenotype_gSV_Delly {
     output:
         path "${params.output_filename}_${params.RGSV}.bcf", emit: regenotyped_sv_bcf
         path "${params.output_filename}_${params.RGSV}.bcf.csi", emit: regenotyped_sv_bcf_csi
-        path ".command.*"
 
     script:
     """
@@ -90,11 +78,6 @@ process call_gCNV_Delly {
         pattern: "*.bcf*",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         tuple val(bam_sample_name), path(input_bam), path(input_bam_bai)
         path(delly_sv_file)
@@ -105,7 +88,6 @@ process call_gCNV_Delly {
     output:
         path "${params.output_filename}_${params.GCNV}.bcf", emit: bcf_cnv_file
         path "${params.output_filename}_${params.GCNV}.bcf.csi", emit: bcf_cnv_file_csi
-        path ".command.*"
         val bam_sample_name, emit: bam_sample_name
 
     script:
@@ -128,11 +110,6 @@ process regenotype_gCNV_Delly {
         pattern: "*.bcf*",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         tuple val(bam_sample_name), path(input_bam), path(input_bam_bai)
         path(reference_fasta)
@@ -143,7 +120,6 @@ process regenotype_gCNV_Delly {
     output:
         path "${params.output_filename}_${params.RGCNV}.bcf", emit: regenotyped_cnv_bcf
         path "${params.output_filename}_${params.RGCNV}.bcf.csi", emit: regenotyped_cnv_bcf_csi
-        path ".command.*"
 
     script:
     """

diff --git a/module/manta.nf b/module/manta.nf
@@ -20,11 +20,6 @@ process call_gSV_Manta {
         mode: "copy",
         saveAs: { "${params.output_filename}_${file(it).getName()}" }
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         tuple val(bam_sample_name), path(input_bam), path(input_bam_bai)
         path(reference_fasta)
@@ -38,7 +33,6 @@ process call_gSV_Manta {
         path("${params.output_filename}_candidateSV.vcf.gz"), emit: vcf_candidate_sv_file
         path("${params.output_filename}_candidateSV.vcf.gz.tbi"), emit: vcf_candidate_sv_tbi
         path "*Stats*"
-        path ".command.*"
         val bam_sample_name, emit: bam_sample_name
 
     script:

diff --git a/module/rtgtools.nf b/module/rtgtools.nf
@@ -16,19 +16,13 @@ process run_vcfstats_RTGTools {
         pattern: "*_stats.txt",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         path vcf_sv_file
         val bam_sample_name
         val variant_type
 
     output:
         path "DELLY-${params.delly_version}_${variant_type}_${params.dataset_id}_${bam_sample_name}_stats.txt"
-        path ".command.*"
 
     """
     set -euo pipefail

diff --git a/module/sha512.nf b/module/sha512.nf
@@ -15,17 +15,13 @@ process run_sha512sum {
         pattern: "*.sha512",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process}/${task.process}-${task.index}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "/${task.process.split(':')[-1]}-${task.index}" }
 
     input:
         path input_checksum_file
 
     output:
         path "${input_checksum_file}.sha512"
-        path ".command.*"
 
     """
         set -euo pipefail

diff --git a/module/vcftools.nf b/module/vcftools.nf
@@ -15,19 +15,13 @@ process run_vcf_validator_VCFtools {
         pattern: "*_validation.txt",
         mode: "copy"
 
-    publishDir "${params.workflow_log_dir}",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
-
     input:
         path vcf_sv_file
         val bam_sample_name
         val variant_type
 
     output:
         path "DELLY-${params.delly_version}_${variant_type}_${params.dataset_id}_${bam_sample_name}_validation.txt"
-        path ".command.*"
 
     """
     set -euo pipefail

diff --git a/test/configtest-F16.json b/test/configtest-F16.json
@@ -213,6 +213,12 @@
       "work_dir": "/scratch/300935"
     },
     "process": {
+      "afterScript": {
+        "1": "",
+        "2": "",
+        "3": "",
+        "closure": ""
+      },
       "cache": false,
       "commonRetryCodes": [
         "104",
@@ -242,6 +248,21 @@
         "closure": "terminate"
       },
       "executor": "local",
+      "ext": {
+        "capture_logs": true,
+        "commonAfterScript": {
+          "1": "",
+          "2": "",
+          "3": "",
+          "closure": ""
+        },
+        "log_dir": {
+          "1": "ext",
+          "2": "ext",
+          "3": "ext",
+          "closure": "ext"
+        }
+      },
       "maxRetries": "1",
       "withLabel:process_high": {
         "cpus": {