From fbc9aca1bd5f6220d0d69a6272caadac8e02e3a3 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 10:50:52 -0700 Subject: [PATCH 1/8] Add cmopression and indexing for expanded BED ffile --- main.nf | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/main.nf b/main.nf index 052c9a7..01dba10 100644 --- a/main.nf +++ b/main.nf @@ -17,14 +17,18 @@ include { run_depth_filter } from './module/filter_off_target_depth.nf' include { merge_bedfiles_BEDtools } from './module/merge_bedfiles_bedtools.nf' include { generate_checksum_PipeVal as generate_sha512sum } from './external/pipeline-Nextflow-module/modules/PipeVal/generate-checksum/main.nf' addParams( - options: [ - output_dir: "${params.workflow_output_dir}/output/", - log_output_dir: "${params.log_output_dir}/process-log/", - docker_image_version: params.pipeval_version, - main_process: "./", - checksum_alg: "sha512" - ] - ) + options: [ + output_dir: "${params.workflow_output_dir}/output/", + log_output_dir: "${params.log_output_dir}/process-log/", + docker_image_version: params.pipeval_version, + main_process: "./", + checksum_alg: "sha512" + ]) +include { compress_index_VCF as compress_index_BED } from './external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( + options: [ + output_dir: "${params.workflow_output_dir}/", + log_output_dir: "${params.log_output_dir}/process-log/" + ]) // Log info here log.info """\ @@ -217,9 +221,14 @@ workflow { params.target_bed, run_slop_BEDtools_expand_dbSNP.out.bed ) - - checksum_ch = checksum_ch.mix(merge_bedfiles_BEDtools.out.bed.flatten()) + compress_index_BED( + merge_bedfiles_BEDtools.out.bed + .map{ it -> [params.sample_id, it] } + ) + + checksum_ch = checksum_ch.mix(merge_bedfiles_BEDtools.out.bed.flatten()) + checksum_ch = checksum_ch.mix(compress_index_BED.out.index_out.map{ it -> [it[1], it[2]]}) } } From a0d276ee1642ecf1c7bce75a0a67099321f9afcd Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 10:50:58 -0700 Subject: [PATCH 2/8] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94adce3..49c0ec3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ A directed acyclic graph of your pipeline. | ------------ | ------------------------ | | `output_dir` | Location where generated output should be saved. | | `*target-with-enriched-off-target-intervals.bed` | New target file including original target intervals and intervals encompassing coverage-enriched off-target dbSNP sites. | +| `*target-with-enriched-off-target-intervals.bed.gz` | New compressed target file including original target intervals and intervals encompassing coverage-enriched off-target dbSNP sites. | |`*off-target-dbSNP-depth-per-base.bed`|Per-base read depth at dbSNP loci outside of targeted regions.| | `*collapsed_coverage.bed` | Per-base read depth at specified target intervals, collapsed by interval. (OPTIONAL) Set `target_depth` in config file. | |`*target-depth-per-base.bed`|Per-base read depth at target intervals (not collapsed). (OPTIONAL) set `save_raw_target_bed` in config file.| From 35e64224ea03e641e8984b388fbc5fcc00eed578 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 10:51:10 -0700 Subject: [PATCH 3/8] Updat test to include compressed BED --- nftest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nftest.yml b/nftest.yml index 656373d..1b5f194 100644 --- a/nftest.yml +++ b/nftest.yml @@ -21,3 +21,6 @@ cases: - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed method: md5 + - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz + expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz + method: md5 From d1a0cbd45426ea30bd890fb723e028d947166db1 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 10:57:20 -0700 Subject: [PATCH 4/8] Update version --- CHANGELOG.md | 6 ++++++ nextflow.config | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25cf54e..11a0c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm --- +## [v1.1.0] - 2024-03-28 +### Added +- Compressed output for the expanded set of target intervals + +--- + ## [v1.0.0] - 2024-03-27 ### Added - Add workflow to generate SVG images from embedded PlantUML source diff --git a/nextflow.config b/nextflow.config index 4d0056e..0350dee 100644 --- a/nextflow.config +++ b/nextflow.config @@ -3,5 +3,5 @@ manifest { name = 'calculate-targeted-coverage' author = 'Nicole Zeltser' description = 'Nextflow pipeline for calculating read-depth related statistics for targeted sequencing experiments' - version = '1.0.0' + version = '1.1.0' } From 4c050f6fe90d683903a3b59050cff50e04bad582 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 11:10:37 -0700 Subject: [PATCH 5/8] Update version in tests --- test/configtest-F16.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/configtest-F16.json b/test/configtest-F16.json index b4b386e..a3eb2c3 100644 --- a/test/configtest-F16.json +++ b/test/configtest-F16.json @@ -58,7 +58,7 @@ "input": { "bam": "/hot/resource/SMC-HET/tumours/A-mini/bams/n1/output/S2.T-n1.bam" }, - "log_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z", + "log_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z", "max_cpus": "16", "max_memory": "31 GB", "merge_operation": "collapse", @@ -71,7 +71,7 @@ "off_target_depth": true, "off_target_slop": "500", "output_dir": "/tmp/nf-config-test-outputs", - "output_dir_base": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F", + "output_dir_base": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F", "output_enriched_target_file": true, "patient": "null", "picard_CollectHsMetrics_extra_args": "", @@ -92,7 +92,7 @@ "target_interval_list": "", "ucla_cds": true, "work_dir": "/scratch/4674821", - "workflow_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/SAMtools-1.16.1" + "workflow_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/SAMtools-1.16.1" }, "params_schema": { "bait_bed": { @@ -320,15 +320,15 @@ }, "report": { "enabled": true, - "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/report.html" + "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/report.html" }, "timeline": { "enabled": true, - "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/timeline.html" + "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/timeline.html" }, "trace": { "enabled": true, - "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/trace.txt" + "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/trace.txt" }, "tz": "sun.util.calendar.ZoneInfo[id=\"UTC\",offset=0,dstSavings=0,useDaylight=false,transitions=0,lastRule=null]", "workDir": "/scratch/4674821", From 8270b3e42d47d5f0edeb9bdd04016c5010a4b62b Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 12:07:10 -0700 Subject: [PATCH 6/8] Update version --- test/configtest-F16.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/configtest-F16.json b/test/configtest-F16.json index a3eb2c3..e09f825 100644 --- a/test/configtest-F16.json +++ b/test/configtest-F16.json @@ -33,7 +33,7 @@ "author": "Nicole Zeltser", "description": "Nextflow pipeline for calculating read-depth related statistics for targeted sequencing experiments", "name": "calculate-targeted-coverage", - "version": "1.0.0" + "version": "1.1.0" }, "params": { "bait_bed": "", From ad3ac3190c324fbb1e90cc75d3f4d51365a44ea2 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 17:28:09 -0700 Subject: [PATCH 7/8] Make uncompressed BED intermediate --- main.nf | 1 - module/merge_bedfiles_bedtools.nf | 5 +++-- nftest.yml | 3 --- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/main.nf b/main.nf index 01dba10..eb454e6 100644 --- a/main.nf +++ b/main.nf @@ -227,7 +227,6 @@ workflow { .map{ it -> [params.sample_id, it] } ) - checksum_ch = checksum_ch.mix(merge_bedfiles_BEDtools.out.bed.flatten()) checksum_ch = checksum_ch.mix(compress_index_BED.out.index_out.map{ it -> [it[1], it[2]]}) } diff --git a/module/merge_bedfiles_bedtools.nf b/module/merge_bedfiles_bedtools.nf index 5818b0f..9ff609d 100644 --- a/module/merge_bedfiles_bedtools.nf +++ b/module/merge_bedfiles_bedtools.nf @@ -9,9 +9,10 @@ include { generate_standard_filename } from '../external/pipeline-Nextflow-modul process merge_bedfiles_BEDtools { container params.docker_image_bedtools - publishDir path: "${params.workflow_output_dir}/output/", + publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.replace(':','/')}", pattern: "*.bed", - mode: "copy" + mode: "copy", + enabled: params.save_intermediate_files publishDir path: "${params.log_output_dir}/process-log/", pattern: ".command.*", diff --git a/nftest.yml b/nftest.yml index 1b5f194..41c1616 100644 --- a/nftest.yml +++ b/nftest.yml @@ -18,9 +18,6 @@ cases: - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_off-target-dbSNP-depth-per-base.bed expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.off-target-dbSNP_depth-per-base.bed method: md5 - - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed - expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed - method: md5 - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz method: md5 From 723987a8f430f540fa5a2426f5ab56e7b309da2e Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 28 Mar 2024 20:21:05 -0700 Subject: [PATCH 8/8] Add assertion for index file --- nftest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nftest.yml b/nftest.yml index 41c1616..476d3e6 100644 --- a/nftest.yml +++ b/nftest.yml @@ -21,3 +21,6 @@ cases: - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz method: md5 + - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz.tbi + expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz.tbi + method: md5