Merge pull request #254 from UPHL-BioNGS/update-2021-11-15

Update 2021 11 15

bin/.tests.sh

@@ -2,6 +2,7 @@
 # nextflow run ~/sandbox/Cecret -profile singularity --reads /home/eriny/sandbox/test_files/cecret/reads --outdir tests -with-tower -resume
 # nextflow run ~/sandbox/Cecret -profile singularity,mpx --reads /home/eriny/sandbox/test_files/cecret/mpx --outdir tests --cleaner 'fastp' -with-tower -resume
 # nextflow run ~/sandbox/Cecret -profile singularity --nanopore /home/eriny/sandbox/test_files/cecret/nanopore --outdir tests -with-tower -resume
+# nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret -profile singularity --reads /home/eriny/sandbox/test_files/cecret/reads --outdir tests -with-tower -resume
 
 echo "usage : bash .test.sh {small,primers,else}"
 
@@ -13,15 +14,15 @@ if [ "$test" == "small" ]
 then
 
   # sample sheet
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity \
     --sample_sheet /home/eriny/sandbox/test_files/cecret/sample_sheet.csv \
     --outdir singularity_defaults_sample_sheet \
     -resume \
     -with-tower
 
   # using included nextclade data
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity \
     --sample_sheet /home/eriny/sandbox/test_files/cecret/sample_sheet.csv \
     --outdir singularity_defaults_sample_sheet_included_nextclade \
@@ -34,15 +35,15 @@ then
   for option in ${options[@]}
   do
     # defaults
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile singularity,artic_V3 \
       --$option /home/eriny/sandbox/test_files/cecret/$option \
       --outdir singularity_defaults_$option \
       -resume \
       -with-tower
 
     # removed test for bamsnap and rename because of lack of interest
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile singularity,artic_V3 \
       --$option /home/eriny/sandbox/test_files/cecret/$option \
       --outdir all_on_$option \
@@ -51,7 +52,7 @@ then
       -resume
 
     # removing primer trimming
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile singularity,artic_V3 \
       --$option /home/eriny/sandbox/test_files/cecret/$option \
       --outdir nontrimmed_$option \
@@ -60,7 +61,7 @@ then
       -resume
 
     # changing the cleaner, aligner, and trimmer
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile singularity,artic_V3 \
       --$option /home/eriny/sandbox/test_files/cecret/$option \
       --outdir toggled_$option \
@@ -72,7 +73,7 @@ then
       -resume
 
     # with UPHL's config
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile uphl,artic_V3 \
       --$option /home/eriny/sandbox/test_files/cecret/$option \
       --outdir uphl_$option \
@@ -81,7 +82,7 @@ then
   done
 
   # multifasta
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,artic_V3 \
     --reads /home/eriny/sandbox/test_files/cecret/reads \
     --single-reads /home/eriny/sandbox/test_files/cecret/single-reads \
@@ -91,7 +92,7 @@ then
     -with-tower
 
   # empty
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,artic_V3 \
     --reads doesntexit \
     --single-reads willnotexist \
@@ -105,7 +106,7 @@ then
   nanopore_primers=("midnight_idt_V1" "midnight_ont_V1" "midnight_ont_V2" "midnight_ont_V3")
   for primer in ${nanopore_primers[@]}
   do
-    nextflow run ~/sandbox/Cecret \
+    nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
       -profile singularity \
       --nanopore /home/eriny/sandbox/test_files/cecret/nanopore \
       --outdir primer_${primer}_artic \
@@ -130,7 +131,7 @@ then
     illumina_primers=("ncov_V3" "ncov_V4" "ncov_V4.1" "ncov_V5.3.2")
     for primer in ${illumina_primers[@]}
     do
-      nextflow run ~/sandbox/Cecret \
+      nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
         -profile singularity \
         --reads /home/eriny/sandbox/test_files/cecret/reads \
         --outdir primer_${primer}_${trimmer} \
@@ -154,7 +155,7 @@ then
     mpx_primers=("mpx_primalseq" "mpx_idt")
     for primer in ${mpx_primers[@]}
     do
-      nextflow run ~/sandbox/Cecret \
+      nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
         -profile singularity \
         --reads /home/eriny/sandbox/test_files/cecret/mpx_idt \
         --outdir primer_${primer}_${trimmer} \
@@ -178,14 +179,14 @@ then
 
 else
   # CDC's test data with relatedness
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,artic_V3 \
     --reads /home/eriny/sandbox/sars-cov-2-datasets/reads \
     --outdir default_datasets \
     --relatedness true  \
     -with-tower
 
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile uphl,artic_V3 \
     --reads /home/eriny/sandbox/sars-cov-2-datasets/reads \
     --outdir uphl_datasets \
@@ -194,7 +195,7 @@ else
     --relatedness true
 
   # CDC's test data with relatedness using nextalign
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,artic_V3 \
     --reads /home/eriny/sandbox/sars-cov-2-datasets/reads \
     --outdir toggled_datasets \
@@ -208,7 +209,7 @@ else
     -resume
 
   # MPX
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,mpx \
     --reads /home/eriny/sandbox/test_files/cecret/mpx \
     --outdir mpx \
@@ -219,7 +220,7 @@ else
     -resume
 
   # MPX with idt primers
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity,mpx_idt \
     --reads /home/eriny/sandbox/test_files/cecret/mpx_idt \
     --outdir mpx_idt \
@@ -230,7 +231,7 @@ else
     -resume
 
   # other
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile singularity \
     --reads /home/eriny/sandbox/test_files/cecret/mpx \
     --outdir mpx \
@@ -246,7 +247,7 @@ else
     -resume
 
   # MPX with kraken
-  nextflow run ~/sandbox/Cecret \
+  nextflow run /Volumes/IDGenomics_NAS/Bioinformatics/eriny/Cecret \
     -profile uphl,mpx \
     --reads /home/eriny/sandbox/test_files/cecret/mpx \
     --outdir uphl_mpx \

configs/cecret_config_template.config

@@ -162,6 +162,18 @@
 //# For process aci
 //params.aci = true
 
+//# For process heatcluster
+//params.heatcluster = true
+//params.heatcluster_options = "-t png"
+
+//# For process igv_reports
+//params.igv_reports = true
+//params.igv_reports_options = "--flanking 1000"
+
+//# For process phytreeviz
+//params.phytreeviz = true
+//params.phytreeviz_options = ""
+
 //# For processes artic and artic_read_filtering
 //params.artic_options = "--normalise 200 --skip-nanopolish --medaka --medaka-model r941_min_high_g360"
 //params.artic_read_filtering_options = "--min-length 400 --max-length 700"

configs/singularity.config

@@ -1,2 +1,3 @@
 singularity.enabled = true
 singularity.autoMounts = true
+singularity.runOptions = "--no-home"

main.nf

@@ -117,6 +117,7 @@ params.msa                                  = 'mafft'
 params.aci                                  = true
 params.bcftools_variants                    = true
 params.fastqc                               = true
+params.igv_reports                          = true
 params.ivar_variants                        = true
 params.samtools_stats                       = true
 params.samtools_coverage                    = true
@@ -132,6 +133,8 @@ params.multiqc                              = true
 params.relatedness                          = false
 params.snpdists                             = true
 params.iqtree2                              = true
+params.heatcluster                          = true
+params.phytreeviz                           = true
 
 //# parameters for processes with their default values
 params.artic_options                        = '--normalise 200 --skip-nanopolish --medaka --medaka-model r941_min_high_g360'
@@ -140,6 +143,8 @@ params.bcftools_variants_options            = ''
 params.fastp_options                        = ''
 params.fastqc_options                       = ''
 params.filter_options                       = ''
+params.heatcluster_options                  = '-t png'
+params.igv_reports_options                  = '--flanking 1000'
 params.iqtree2_options                      = '-ninit 2 -n 2 -me 0.05 -m GTR'
 params.ivar_consensus_options               = '-q 20 -t 0.6 -n N'
 params.ivar_trim_options                    = ''
@@ -149,6 +154,7 @@ params.minimum_depth                        = 100
 params.mpileup_depth                        = 8000
 params.multiqc_options                      = ''
 params.mafft_options                        = '--maxambiguous 0.5'
+params.phytreeviz_options                   = ''
 params.samtools_ampliconclip_options        = ''
 params.samtools_coverage_options            = ''
 params.samtools_flagstat_options            = ''
@@ -538,6 +544,8 @@ workflow CECRET {
       alignment = msa.out.msa
       matrix    = msa.out.matrix
 
+      ch_for_multiqc = ch_for_multiqc.mix(msa.out.for_multiqc)
+
     } else {
       tree      = Channel.empty()
       alignment = Channel.empty()

modules/bcftools.nf

@@ -18,6 +18,7 @@ process bcftools_variants {
   tuple val(sample), file(bam), file(reference_genome)
 
   output:
+  tuple val(sample), file(bam), file(reference_genome), file("bcftools_variants/${sample}.vcf"), emit: vcf
   path "bcftools_variants/${sample}.vcf", emit: bcftools_variants_file
   path "logs/${task.process}/${sample}.${workflow.sessionId}.log"
 

modules/freyja.nf

@@ -3,7 +3,7 @@ process freyja_variants {
   label         "process_medium"
   //errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
   publishDir    "${params.outdir}", mode: 'copy'
-  container     'quay.io/uphl/freyja:1.4.7-2023-11-07'
+  container     'quay.io/uphl/freyja:1.4.7-2023-11-14'
 
   //#UPHLICA maxForks 10
   //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
@@ -44,7 +44,7 @@ process freyja_demix {
   label         "process_medium"
   //errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
   publishDir    "${params.outdir}", mode: 'copy'
-  container     'quay.io/uphl/freyja:1.4.7-2023-11-07'
+  container     'quay.io/uphl/freyja:1.4.7-2023-11-14'
 
   //#UPHLICA maxForks 10
   //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
@@ -85,7 +85,7 @@ process freyja_boot {
   label         "process_medium"
   //errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
   publishDir    "${params.outdir}", mode: 'copy'
-  container     'quay.io/uphl/freyja:1.4.7-2023-11-07'
+  container     'quay.io/uphl/freyja:1.4.7-2023-11-14'
 
   //#UPHLICA maxForks 10
   //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
@@ -124,7 +124,7 @@ process freyja_aggregate {
   tag        "Aggregating results from freyja"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'quay.io/uphl/freyja:1.4.7-2023-11-07'
+  container  'quay.io/uphl/freyja:1.4.7-2023-11-14'
 
   //#UPHLICA maxForks 10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}

modules/heatcluster.nf

@@ -0,0 +1,42 @@
+process heatcluster {
+  tag           "HeatCluster"
+  publishDir    params.outdir, mode: 'copy'
+  container     'quay.io/uphl/heatcluster:0.4.12-2023-11-15'
+  maxForks      10
+  //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
+  //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-medium'
+  //#UPHLICA memory 1.GB
+  //#UPHLICA cpus 3
+  //#UPHLICA time '10m'
+
+  when:
+  params.heatcluster
+
+  input:
+  file(matrix)
+
+  output:
+  path "heatcluster/heatcluster*"       , optional : true
+  path "heatcluster/heatcluster_mqc.png", optional : true              , emit: for_multiqc
+  path "logs/${task.process}/${task.process}.${workflow.sessionId}.log", emit: log_files
+
+  shell:
+  '''
+    mkdir -p heatcluster logs/!{task.process}
+    log_file=logs/!{task.process}/!{task.process}.!{workflow.sessionId}.log
+
+    # time stamp + capturing tool versions
+    date > $log_file
+    heatcluster.py -v >> $log_file
+    echo "container : !{task.container}" >> $log_file
+    echo "Nextflow command : " >> $log_file
+    cat .command.sh >> $log_file
+
+    heatcluster.py !{params.heatcluster_options} \
+        -i !{matrix} \
+        -o heatcluster/heatcluster \
+        | tee -a $log_file
+
+    if [ -f "heatcluster/heatcluster.png" ] ; then cp heatcluster/heatcluster.png heatcluster/heatcluster_mqc.png ; fi
+  '''
+}

modules/igvreports.nf

@@ -0,0 +1,40 @@
+process igv_reports {
+  tag         "${sample}"
+  label       "process_high"
+  publishDir  path: "${params.outdir}", mode: 'copy'
+  container   'quay.io/biocontainers/igv-reports:1.9.1--pyh7cba7a3_0'
+
+  //#UPHLICA maxForks 10
+  //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
+  //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
+  //#UPHLICA memory 60.GB
+  //#UPHLICA cpus 14
+  //#UPHLICA time '45m'
+
+  when:
+  params.igv_reports
+
+  input:
+  tuple val(sample), file(bam), file(reference_genome), file(vcf)
+
+  output:
+    path "igv_reports/${sample}/igvjs_viewer.html"
+    path "logs/${task.process}/${sample}.${workflow.sessionId}.log"
+
+  shell:
+  '''
+    mkdir -p igv_reports/!{sample} logs/!{task.process}
+    log=logs/!{task.process}/!{sample}.!{workflow.sessionId}.log
+
+    # time stamp + capturing tool versions
+    date > $log
+    create_report -v >> $log
+
+    create_report !{params.igv_reports_options} \
+        --fasta !{reference_genome} \
+        --tracks !{vcf} !{bam} \
+        --output igv_reports/!{sample}/igvjs_viewer.html \
+        !{vcf} \
+        | tee -a $log
+  '''
+}

modules/iqtree2.nf

@@ -19,6 +19,7 @@ process iqtree2 {
 
   output:
   path "iqtree2/iqtree2.{iqtree,treefile*,mldist,log}", emit: tree
+  path "iqtree2/iqtree2.treefile.nwk"                 , emit: newick, optional: true
   path "logs/${task.process}/${task.process}.${workflow.sessionId}.log"
 
   shell:

modules/pangolin.nf

@@ -24,7 +24,7 @@ process pangolin {
 
   shell:
   '''
-    mkdir -p pangolin logs/!{task.process}
+    mkdir -p temp pangolin logs/!{task.process}
     log=logs/!{task.process}/!{task.process}.!{workflow.sessionId}.log
 
     date > $log
@@ -38,6 +38,8 @@ process pangolin {
     pangolin !{params.pangolin_options} \
       --threads !{task.cpus} \
       --outdir pangolin \
+      --tempdir temp \
+      --verbose \
       ultimate_fasta.fasta \
       | tee -a $log
     cp ultimate_fasta.fasta pangolin/combined.fasta