From 6ef3c2851e2ae0662bb503a436b0c885159f5f80 Mon Sep 17 00:00:00 2001 From: Mohammed Faizal Eeman Mootor <68152431+Faizal-Eeman@users.noreply.github.com> Date: Thu, 8 Dec 2022 16:20:02 -0800 Subject: [PATCH] Update PipeVal (#81) * add sub-module pipeline-Nextflow-module * add pipeval 3.0.0 in default.config * add pipeval submodule from pipeline-Nextflow-module and modify input validation channel in main.nf * Update CHANGELOG.md * update pipeval version logging in main.nf * udpate validation output dir naming convention in main.nf * udpate validation output dir naming convention in main.nf * Update CHANGELOG.md * update main.nf * update sub-module pipeline-Nextflow-module * fix sha512sum command in sha512.nf * change run_validate to run_validate_PipeVal in node configs Co-authored-by: Mootor --- .gitmodules | 3 +++ CHANGELOG.md | 8 ++++++- config/F16.config | 2 +- config/F2.config | 2 +- config/F32.config | 2 +- config/F72.config | 2 +- config/M64.config | 2 +- config/default.config | 5 ++--- external/pipeline-Nextflow-module | 1 + main.nf | 27 ++++++++++++++++++++---- module/sha512.nf | 2 +- module/validation.nf | 35 ------------------------------- 12 files changed, 42 insertions(+), 49 deletions(-) create mode 100644 .gitmodules create mode 160000 external/pipeline-Nextflow-module delete mode 100644 module/validation.nf diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..95e4a194 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/pipeline-Nextflow-module"] + path = external/pipeline-Nextflow-module + url = git@github.com:uclahs-cds/pipeline-Nextflow-module.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 87bfcfc0..a4748c62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,16 @@ All notable changes to the call-gSV pipeline. --- ## [Unreleased] -## Changed +### Changed - Parameterize Docker registry - Use `ghcr.io/uclahs-cds` as default registry +### Added +- Add PipeVal using `pipeline-Nextflow-module` to validate the input CSV file + +### Removed +- Remove `module/validation.nf` as PipeVal sub-module is used + --- ## [4.0.0] - 2022-08-17 diff --git a/config/F16.config b/config/F16.config index 7cf2b783..df14c520 100644 --- a/config/F16.config +++ b/config/F16.config @@ -1,5 +1,5 @@ process { - withName: run_validate { + withName: run_validate_PipeVal { cpus = 1 memory = 1.GB } diff --git a/config/F2.config b/config/F2.config index b70a6597..9ca6978e 100644 --- a/config/F2.config +++ b/config/F2.config @@ -1,5 +1,5 @@ process { - withName: run_validate { + withName: run_validate_PipeVal { cpus = 1 memory = 1.GB } diff --git a/config/F32.config b/config/F32.config index 49ca693e..f9cebd4d 100644 --- a/config/F32.config +++ b/config/F32.config @@ -1,5 +1,5 @@ process { - withName: run_validate { + withName: run_validate_PipeVal { cpus = 1 memory = 1.GB } diff --git a/config/F72.config b/config/F72.config index 566303b1..dc0a6edb 100644 --- a/config/F72.config +++ b/config/F72.config @@ -1,5 +1,5 @@ process { - withName: run_validate { + withName: run_validate_PipeVal { cpus = 1 memory = 1.GB } diff --git a/config/M64.config b/config/M64.config index ed4d5b0e..87b82c1c 100644 --- a/config/M64.config +++ b/config/M64.config @@ -1,5 +1,5 @@ process { - withName: run_validate { + withName: run_validate_PipeVal { cpus = 1 memory = 1.GB } diff --git a/config/default.config b/config/default.config index 8f03ce5b..b094b431 100644 --- a/config/default.config +++ b/config/default.config @@ -25,7 +25,7 @@ params { bcftools_version = "1.15.1" vcftools_version = "0.1.16" rtgtools_version = "3.12" - validate_version = "2.1.5" + pipeval_version = "3.0.0" // Docker tool versions docker_image_delly = "${-> params.docker_container_registry}/delly:${params.delly_version}" @@ -33,8 +33,7 @@ params { docker_image_manta = "${-> params.docker_container_registry}/manta:${params.manta_version}" docker_image_rtgtools = "${-> params.docker_container_registry}/rtg-tools:${params.rtgtools_version}" docker_image_vcftools = "${-> params.docker_container_registry}/vcftools:${params.vcftools_version}" - docker_image_validate = "${-> params.docker_container_registry}/validate:${params.validate_version}" - + docker_image_validate = "${-> params.docker_container_registry}/pipeval:${params.pipeval_version}" } process { diff --git a/external/pipeline-Nextflow-module b/external/pipeline-Nextflow-module new file mode 160000 index 00000000..5e315ab0 --- /dev/null +++ b/external/pipeline-Nextflow-module @@ -0,0 +1 @@ +Subproject commit 5e315ab002aae48df64d1ecb0689f290744aae22 diff --git a/main.nf b/main.nf index 122d999d..65f8acce 100644 --- a/main.nf +++ b/main.nf @@ -39,7 +39,7 @@ Current Configuration: bcftools: ${params.bcftools_version} vcftools: ${params.vcftools_version} rtgtools: ${params.rtgtools_version} - validation tool: ${params.validate_version} + validation tool: ${params.pipeval_version} ------------------------------------ Starting workflow... @@ -47,7 +47,9 @@ Starting workflow... """ .stripIndent() -include { run_validate } from './module/validation' +include { run_validate_PipeVal } from './external/pipeline-Nextflow-module/modules/PipeVal/validate/main.nf' addParams( + options: [ docker_image_version: params.pipeval_version ] + ) include { call_gSV_Delly; call_gCNV_Delly; regenotype_gSV_Delly; regenotype_gCNV_Delly } from './module/delly' include { call_gSV_Manta } from './module/manta' include { convert_BCF2VCF_BCFtools as convert_gSV_BCF2VCF_BCFtools; convert_BCF2VCF_BCFtools as convert_gCNV_BCF2VCF_BCFtools } from './module/bcftools' @@ -101,14 +103,31 @@ if (!params.run_delly && !params.run_manta) { reference_fasta_index = "${params.reference_fasta}.fai" -validation_channel = Channel +/** +* Create input_validation to validate the input bams +*/ +validation_mode = Channel.of("file-input") + +input_files = Channel .fromPath(params.input_csv, checkIfExists:true) .splitCsv(header:true) .map{ row -> [row.input_bam, "${row.input_bam}.bai"]} .flatten() +validation_mode + .combine(input_files) + .set { input_validation } + workflow { - run_validate(validation_channel) + /** + * Validate the input bams + */ + run_validate_PipeVal(input_validation) + // Collect and store input validation output + run_validate_PipeVal.out.validation_result.collectFile( + name: 'input_validation.txt', + storeDir: "${params.output_dir}/validation/run_validate_PipeVal" + ) if (params.run_discovery) { if (params.run_manta) { diff --git a/module/sha512.nf b/module/sha512.nf index 400d9074..f62b4897 100644 --- a/module/sha512.nf +++ b/module/sha512.nf @@ -29,6 +29,6 @@ process run_sha512sum { """ set -euo pipefail - python -m validate -t sha512-gen $input_checksum_file > ${input_checksum_file}.sha512 + sha512sum ${input_checksum_file} > ${input_checksum_file}.sha512 """ } diff --git a/module/validation.nf b/module/validation.nf deleted file mode 100644 index c5120ea6..00000000 --- a/module/validation.nf +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env nextflow - -log.info """\ ------------------------------------- - V A L I D A T I O N ------------------------------------- -Docker Images: -- docker_image_validate: ${params.docker_image_validate} -""" - -process run_validate { - container params.docker_image_validate - - publishDir "${params.output_dir}/${params.docker_image_delly.split("/")[-1].replace(':', '-').toUpperCase()}/validation/${task.process.replace(':', '/')}", - pattern: "input-validation.txt", - mode: "copy" - - publishDir "${params.log_output_dir}/process-log/${params.docker_image_delly.split("/")[-1].replace(':', '-').toUpperCase()}/validation", - pattern: ".command.*", - mode: "copy", - saveAs: { "${task.process.replace(':', '/')}-${task.index}/log${file(it).getName()}" } - - input: - path(file_to_validate) - - output: - path ".command.*" - path "input-validation.txt", emit: val_file - - script: - """ - set -euo pipefail - python -m validate -t file-input ${file_to_validate} > 'input-validation.txt' - """ - }