Add NFtest (#126)

* add nftest config, yaml and assert script

* add nftest.yaml

* add nftest config and yaml input

* remove set_env in methods.config

* comment line in test config

* give execution permission to script

* Update CHANGELOG.md

* add delly and manta only test configs

* add delly only and manta only tests

* set remove_temp to true in nftest.yaml

* Update CHANGELOG.md

---------

Co-authored-by: Mootor <mmootor@ip-0A125257.rhxrlfvjyzbupc03cc22jkch3c.xx.internal.cloudapp.net>

CHANGELOG.md

@@ -8,10 +8,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 ---
 
 ## [Unreleased]
+### Added
+- Add NFTest
+
 ### Changed
 - Update DELLY `v1.1.3` to `v1.2.6`
 - Update submodules
 
+### Removed
+- Remove `set_env` method code in `default/methods.config`
+
 ---
 
 ## [6.0.0-rc.1] - 2023-08-02

config/methods.config

@@ -172,30 +172,6 @@ methods {
             }
         }
 
-    // Location of Nextflow temp directories
-    set_env = {
-        if (params.ucla_cds) {
-            /**
-             * By default, if the /scratch directory exists, set it as the Nextflow working directory
-             * If config file specified work_dir, set it as the Nextflow working directory
-             *
-             * WARNING: changing this directory can lead to high server latency and
-             * potential disk space limitations. Change with caution! The 'workDir'
-             * in Nextflow determines the location of intermediate and temporary files.
-             */
-            params.work_dir = (params.containsKey("work_dir") && params.work_dir) ? params.work_dir : "/scratch"
-            if (methods.check_workdir_permissions(params.work_dir)) {
-                workDir = params.work_dir
-                }
-            }
-        else {
-            // If work_dir was specified as a param and exists or can be created, set workDir. Otherwise, let Nextflow's default behavior dictate workDir
-            if (params.containsKey("work_dir") && params.work_dir && methods.check_workdir_permissions(params.work_dir)) {
-                workDir = params.work_dir
-                }
-            }
-        }
-
     set_pipeline_logs = {
         trace.enabled = true
         trace.file = "${params.log_output_dir}/nextflow-log/trace.txt"

nftest.yaml

@@ -0,0 +1,71 @@
+---
+global:
+  temp_dir: ./test/work
+  nf_config: test/global.config
+  remove_temp: true
+  clean_logs: true
+
+cases:
+  - name: ssv-all-tools-std-input
+    message: test development branch using all tools and standard paired tumor and normal a-mini input
+    nf_script: ./main.nf
+    nf_config: ./test/config/ssv-all-tools.config
+    params_file: ./test/yaml/ssv_test-std-input.yaml
+    skip: false
+    verbose: true
+    asserts:
+      - actual: call-sSV-*/S2_v1.1.5/DELLY-1.2.6/output/DELLY-1.2.6_TWGSAMIN000001_S2-v1.1.5.bcf
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/DELLY-1.2.6/DELLY-1.2.6_TWGSAMIN000001_S2-v1.1.5.bcf
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSmallIndels.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSmallIndels.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSV.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_diploidSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_diploidSV.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_somaticSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_somaticSV.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+  - name: ssv-delly-std-input
+    message: test development branch using DELLY and standard paired tumor and normal a-mini input
+    nf_script: ./main.nf
+    nf_config: ./test/config/ssv-delly.config
+    params_file: ./test/yaml/ssv_test-std-input.yaml
+    skip: false
+    verbose: true
+    asserts:
+      - actual: call-sSV-*/S2_v1.1.5/DELLY-1.2.6/output/DELLY-1.2.6_TWGSAMIN000001_S2-v1.1.5.bcf
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/DELLY-1.2.6/DELLY-1.2.6_TWGSAMIN000001_S2-v1.1.5.bcf
+        script: test/assert_bcf_vcf.sh
+
+  - name: ssv-manta-std-input
+    message: test development branch using Manta and standard paired tumor and normal a-mini input
+    nf_script: ./main.nf
+    nf_config: ./test/config/ssv-manta.config
+    params_file: ./test/yaml/ssv_test-std-input.yaml
+    skip: false
+    verbose: true
+    asserts:
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSmallIndels.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSmallIndels.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_candidateSV.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_diploidSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_diploidSV.vcf.gz
+        script: test/assert_bcf_vcf.sh
+
+      - actual: call-sSV-*/S2_v1.1.5/Manta-1.6.0/output/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_somaticSV.vcf.gz
+        expect: /hot/software/pipeline/pipeline-call-sSV/Nextflow/development/test-output/ssv-all-tools-std-input/S2-v1.1.5/Manta-1.6.0/Manta-1.6.0_TWGSAMIN000001_S2-v1.1.5_somaticSV.vcf.gz
+        script: test/assert_bcf_vcf.sh

test/assert_bcf_vcf.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+function md5_bcf_vcf {
+    bcftools view "$1" | grep -v '^##' | md5sum | cut -f 1 -d ' '
+}
+
+if ! which bcftools &> /dev/null
+then
+    if ! module load bcftools &> /dev/null
+    then
+        echo "bcftools command not found! Comparison failed by default." >&2
+        exit 1
+    fi
+fi
+
+received=$(md5_bcf_vcf "$1")
+expected=$(md5_bcf_vcf "$2")
+
+if [ "$received" == "$expected" ]; then
+    echo "BCF files are equal"
+    exit 0
+else
+    echo "BCF files are not equal" >&2
+    exit 1
+fi

test/config/ssv-all-tools.config

@@ -0,0 +1,42 @@
+// EXECUTION SETTINGS AND GLOBAL DEFAULTS
+
+// External config files import. DO NOT MODIFY THIS LINE!
+includeConfig "${projectDir}/config/default.config"
+includeConfig "${projectDir}/config/methods.config"
+includeConfig "${projectDir}/nextflow.config"
+
+// Inputs/parameters of the pipeline
+params {
+    dataset_id = "TWGSAMIN000001"
+
+    blcds_registered_dataset = false
+
+    reference_fasta = "/hot/ref/reference/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta"
+
+    exclusion_file = "/hot/ref/tool-specific-input/Delly/hg38/human.hg38.excl.tsv"
+
+    // select the tool(s) to run
+    algorithm = ['delly', 'manta']
+
+    save_intermediate_files = false
+
+    verbose = false
+
+    /**
+    * Set up the Delly filtering parameters
+    * The below default values are recommended to reduce runtimes.
+    * See - https://github.com/dellytools/delly 'Delly is running too slowly what can I do?' for more
+    */
+    map_qual = 20
+    min_clique_size = 5
+    mad_cutoff = 15
+
+    /**
+    * The filter condition used by the filter_BCF_BCFtools process.
+    * See http://samtools.github.io/bcftools/bcftools.html#expressions
+    * Note, put single quotes inside double quotes.
+    */
+    filter_condition = "FILTER=='PASS'"
+    }
+
+methods.setup()

test/config/ssv-delly.config

@@ -0,0 +1,42 @@
+// EXECUTION SETTINGS AND GLOBAL DEFAULTS
+
+// External config files import. DO NOT MODIFY THIS LINE!
+includeConfig "${projectDir}/config/default.config"
+includeConfig "${projectDir}/config/methods.config"
+includeConfig "${projectDir}/nextflow.config"
+
+// Inputs/parameters of the pipeline
+params {
+    dataset_id = "TWGSAMIN000001"
+
+    blcds_registered_dataset = false
+
+    reference_fasta = "/hot/ref/reference/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta"
+
+    exclusion_file = "/hot/ref/tool-specific-input/Delly/hg38/human.hg38.excl.tsv"
+
+    // select the tool(s) to run
+    algorithm = ['delly']
+
+    save_intermediate_files = false
+
+    verbose = false
+
+    /**
+    * Set up the Delly filtering parameters
+    * The below default values are recommended to reduce runtimes.
+    * See - https://github.com/dellytools/delly 'Delly is running too slowly what can I do?' for more
+    */
+    map_qual = 20
+    min_clique_size = 5
+    mad_cutoff = 15
+
+    /**
+    * The filter condition used by the filter_BCF_BCFtools process.
+    * See http://samtools.github.io/bcftools/bcftools.html#expressions
+    * Note, put single quotes inside double quotes.
+    */
+    filter_condition = "FILTER=='PASS'"
+    }
+
+methods.setup()

test/config/ssv-manta.config

@@ -0,0 +1,42 @@
+// EXECUTION SETTINGS AND GLOBAL DEFAULTS
+
+// External config files import. DO NOT MODIFY THIS LINE!
+includeConfig "${projectDir}/config/default.config"
+includeConfig "${projectDir}/config/methods.config"
+includeConfig "${projectDir}/nextflow.config"
+
+// Inputs/parameters of the pipeline
+params {
+    dataset_id = "TWGSAMIN000001"
+
+    blcds_registered_dataset = false
+
+    reference_fasta = "/hot/ref/reference/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta"
+
+    exclusion_file = "/hot/ref/tool-specific-input/Delly/hg38/human.hg38.excl.tsv"
+
+    // select the tool(s) to run
+    algorithm = ['manta']
+
+    save_intermediate_files = false
+
+    verbose = false
+
+    /**
+    * Set up the Delly filtering parameters
+    * The below default values are recommended to reduce runtimes.
+    * See - https://github.com/dellytools/delly 'Delly is running too slowly what can I do?' for more
+    */
+    map_qual = 20
+    min_clique_size = 5
+    mad_cutoff = 15
+
+    /**
+    * The filter condition used by the filter_BCF_BCFtools process.
+    * See http://samtools.github.io/bcftools/bcftools.html#expressions
+    * Note, put single quotes inside double quotes.
+    */
+    filter_condition = "FILTER=='PASS'"
+    }
+
+methods.setup()

test/global.config

@@ -0,0 +1,9 @@
+docker {
+    enabled = true
+
+    // Pass user's UID/GID and group IDs to Docker
+    uid_and_gid = "-u \$(id -u):\$(id -g)"
+    all_group_ids = "\$(for i in `id --real --groups`; do echo -n \"--group-add=\$i \"; done)"
+
+    runOptions = "${uid_and_gid} ${all_group_ids}"
+}

test/yaml/ssv_test-std-input.yaml

@@ -0,0 +1,8 @@
+---
+sample_id: "TWGSAMIN000001"
+input:
+  BAM:
+    normal:
+      - "/hot/software/pipeline/pipeline-call-sSV/Nextflow/development/input/data/TWGSAMIN000001-N003-S03-F.bam"
+    tumor:
+      - "/hot/software/pipeline/pipeline-call-sSV/Nextflow/development/input/data/TWGSAMIN000001-T003-S03-F.bam"