Merge remote-tracking branch 'origin/master' into dp-multiqc-hotfix

broadinstitute · May 3, 2020 · 59191a3 · 59191a3
2 parents 1b960c6 + 10b5656
commit 59191a3
Show file tree

Hide file tree

Showing 13 changed files with 135 additions and 58 deletions.
diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl
@@ -16,7 +16,7 @@ task assemble {
       String   sample_name = basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt")
 
       Int?     machine_mem_gb
-      String?  docker="quay.io/broadinstitute/viral-assemble"
+      String   docker="quay.io/broadinstitute/viral-assemble"
     }
 
     command {
@@ -114,7 +114,7 @@ task scaffold {
       Float?       scaffold_min_pct_contig_aligned
 
       Int?         machine_mem_gb
-      String?      docker="quay.io/broadinstitute/viral-assemble"
+      String       docker="quay.io/broadinstitute/viral-assemble"
 
       # do this in multiple steps in case the input doesn't actually have "assembly1-x" in the name
       String       sample_name = basename(basename(basename(contigs_fasta, ".fasta"), ".assembly1-trinity"), ".assembly1-spades")
@@ -201,7 +201,7 @@ task ivar_trim {
       Int?    min_quality
 
       Int?    machine_mem_gb
-      String? docker="andersenlabapps/ivar:1.2.1"
+      String  docker="andersenlabapps/ivar:1.2.1"
     }
 
     String  bam_basename=basename(aligned_bam, ".bam")
@@ -255,8 +255,7 @@ task align_reads {
     String?  aligner_options
     Boolean? skip_mark_dupes=false
 
-    Int?     machine_mem_gb
-    String?  docker="quay.io/broadinstitute/viral-core"
+    String   docker="quay.io/broadinstitute/viral-core"
 
     String   sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean")
   }
@@ -336,7 +335,7 @@ task align_reads {
 
   runtime {
     docker: "${docker}"
-    memory: select_first([machine_mem_gb, 7]) + " GB"
+    memory: "7 GB"
     cpu: 8
     disks: "local-disk 375 LOCAL"
     dx_instance_type: "mem1_ssd1_v2_x8"
@@ -359,7 +358,7 @@ task refine_assembly_with_aligned_reads {
       Int?     min_coverage=2
 
       Int?     machine_mem_gb
-      String?  docker="quay.io/broadinstitute/viral-assemble"
+      String   docker="quay.io/broadinstitute/viral-assemble"
     }
 
     command {
@@ -434,7 +433,7 @@ task refine {
       Int?    min_coverage=1
 
       Int?    machine_mem_gb
-      String? docker="quay.io/broadinstitute/viral-assemble"
+      String  docker="quay.io/broadinstitute/viral-assemble"
 
       String  assembly_basename=basename(basename(assembly_fasta, ".fasta"), ".scaffold")
     }
@@ -504,7 +503,7 @@ task refine_2x_and_plot {
       String? plot_coverage_novoalign_options="-r Random -l 40 -g 40 -x 20 -t 100 -k"
 
       Int?    machine_mem_gb
-      String? docker="quay.io/broadinstitute/viral-assemble"
+      String  docker="quay.io/broadinstitute/viral-assemble"
 
       # do this in two steps in case the input doesn't actually have "cleaned" in the name
       String  sample_name = basename(basename(reads_unmapped_bam, ".bam"), ".cleaned")

diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl
@@ -6,7 +6,7 @@ task merge_tarballs {
     String        out_filename
 
     Int?          machine_mem_gb
-    String?       docker="quay.io/broadinstitute/viral-core"
+    String        docker="quay.io/broadinstitute/viral-core"
   }
 
   command {
@@ -60,7 +60,7 @@ task illumina_demux {
     Boolean? forceGC=true
 
     Int?    machine_mem_gb
-    String? docker="quay.io/broadinstitute/viral-core"
+    String  docker="quay.io/broadinstitute/viral-core"
   }
 
   command {

diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl
@@ -10,7 +10,7 @@ task multi_align_mafft_ref {
     Float?         mafft_gapOpeningPenalty
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-phylo"
+    String         docker="quay.io/broadinstitute/viral-phylo"
   }
 
   command {
@@ -51,7 +51,7 @@ task multi_align_mafft {
     Float?         mafft_gapOpeningPenalty
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-phylo"
+    String         docker="quay.io/broadinstitute/viral-phylo"
   }
 
   command {
@@ -89,7 +89,7 @@ task index_ref {
     File?    novocraft_license
 
     Int?     machine_mem_gb
-    String?  docker="quay.io/broadinstitute/viral-core"
+    String   docker="quay.io/broadinstitute/viral-core"
   }
 
   command {
@@ -119,7 +119,7 @@ task trimal_clean_msa {
     File     in_aligned_fasta
 
     Int?     machine_mem_gb
-    String?  docker="quay.io/biocontainers/trimal:1.4.1--h6bb024c_3"
+    String   docker="quay.io/biocontainers/trimal:1.4.1--h6bb024c_3"
 
     String   input_basename = basename(basename(in_aligned_fasta, ".fasta"), ".fa")
   }

diff --git a/pipes/WDL/tasks/tasks_intrahost.wdl b/pipes/WDL/tasks/tasks_intrahost.wdl
@@ -10,7 +10,7 @@ task isnvs_per_sample {
     Int?    maxBias
 
     Int?    machine_mem_gb
-    String? docker="quay.io/broadinstitute/viral-phylo"
+    String  docker="quay.io/broadinstitute/viral-phylo"
 
     String  sample_name = basename(basename(basename(mapped_bam, ".bam"), ".all"), ".mapped")
   }
@@ -51,7 +51,7 @@ task isnvs_vcf {
     Boolean        naiveFilter=false
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-phylo"
+    String         docker="quay.io/broadinstitute/viral-phylo"
   }
 
   parameter_meta {

diff --git a/pipes/WDL/tasks/tasks_metagenomics.wdl b/pipes/WDL/tasks/tasks_metagenomics.wdl
@@ -7,7 +7,7 @@ task krakenuniq {
     File        krona_taxonomy_db_tgz  # taxonomy.tab
 
     Int?        machine_mem_gb
-    String?     docker="quay.io/broadinstitute/viral-classify"
+    String      docker="quay.io/broadinstitute/viral-classify"
   }
 
   command {
@@ -104,7 +104,7 @@ task build_krakenuniq_db {
     Int?        maxDbSize
 
     Int?        machine_mem_gb
-    String?     docker="quay.io/broadinstitute/viral-classify"
+    String      docker="quay.io/broadinstitute/viral-classify"
   }
 
   command {
@@ -164,7 +164,7 @@ task kraken {
     File        krona_taxonomy_db_tgz  # taxonomy.tab
 
     Int?        machine_mem_gb
-    String?     docker="quay.io/broadinstitute/viral-classify"
+    String      docker="quay.io/broadinstitute/viral-classify"
   }
 
   command {
@@ -247,7 +247,7 @@ task build_kraken_db {
     Int?        maxDbSize
 
     Int?        machine_mem_gb
-    String?     docker="quay.io/broadinstitute/viral-classify"
+    String      docker="quay.io/broadinstitute/viral-classify"
   }
 
   command {
@@ -312,7 +312,7 @@ task krona {
     Int?     magnitude_column
 
     Int?     machine_mem_gb
-    String?  docker="quay.io/broadinstitute/viral-classify"
+    String   docker="quay.io/broadinstitute/viral-classify"
   }
 
   String  input_basename = basename(classified_reads_txt_gz, ".txt.gz")
@@ -362,7 +362,7 @@ task krona_merge {
     String       out_basename
 
     Int?         machine_mem_gb
-    String?      docker="biocontainers/krona:v2.7.1_cv1"
+    String       docker="biocontainers/krona:v2.7.1_cv1"
   }
 
   command {
@@ -395,7 +395,7 @@ task filter_bam_to_taxa {
     Boolean?       withoutChildren=false
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-classify"
+    String         docker="quay.io/broadinstitute/viral-classify"
   }
 
   String         input_basename = basename(classified_bam, ".bam")
@@ -456,7 +456,7 @@ task kaiju {
     File     krona_taxonomy_db_tgz  # taxonomy/taxonomy.tab
 
     Int?     machine_mem_gb
-    String?  docker="quay.io/broadinstitute/viral-classify"
+    String   docker="quay.io/broadinstitute/viral-classify"
   }
 
   String   input_basename = basename(reads_unmapped_bam, ".bam")

diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl
@@ -7,7 +7,7 @@ task download_fasta {
     String         emailAddress
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-phylo"
+    String         docker="quay.io/broadinstitute/viral-phylo"
   }
 
   command {
@@ -39,7 +39,7 @@ task download_annotations {
     String         combined_out_prefix
 
     Int?           machine_mem_gb
-    String?        docker="quay.io/broadinstitute/viral-phylo"
+    String         docker="quay.io/broadinstitute/viral-phylo"
   }
 
   command {
@@ -80,7 +80,7 @@ task annot_transfer {
     Array[File]+ reference_feature_table
 
     Int?         machine_mem_gb
-    String?      docker="quay.io/broadinstitute/viral-phylo"
+    String       docker="quay.io/broadinstitute/viral-phylo"
   }
 
   Array[Int]   chr_nums=range(length(multi_aln_fasta))
@@ -136,7 +136,7 @@ task prepare_genbank {
     String       molType = "cRNA"
 
     Int?         machine_mem_gb
-    String?      docker="quay.io/broadinstitute/viral-phylo"
+    String       docker="quay.io/broadinstitute/viral-phylo"
   }
 
   command {

diff --git a/pipes/WDL/tasks/tasks_read_utils.wdl b/pipes/WDL/tasks/tasks_read_utils.wdl
@@ -11,8 +11,7 @@ task merge_and_reheader_bams {
       File?           reheader_table
       String          out_basename
 
-      Int?            machine_mem_gb
-      String?         docker="quay.io/broadinstitute/viral-core"
+      String          docker="quay.io/broadinstitute/viral-core"
     }
 
     command {
@@ -54,8 +53,8 @@ task merge_and_reheader_bams {
 
     runtime {
         docker: "${docker}"
-        memory: select_first([machine_mem_gb, 3]) + " GB"
-        cpu: 4
+        memory: "3 GB"
+        cpu: 2
         disks: "local-disk 750 LOCAL"
         dx_instance_type: "mem1_ssd2_v2_x4"
         preemptible: 0
@@ -126,10 +125,15 @@ task downsample_bams {
     Boolean?     deduplicateAfter=false
 
     Int?         machine_mem_gb
-    String?      docker="quay.io/broadinstitute/viral-core"
+    String       docker="quay.io/broadinstitute/viral-core"
   }
 
   command {
+    set -ex -o pipefail
+
+    # find 90% memory
+    mem_in_mb=`/opt/viral-ngs/source/docker/calc_mem.py mb 90`
+
     if [[ "${deduplicateBefore}" == "true" ]]; then
       DEDUP_OPTION="--deduplicateBefore"
     elif [[ "${deduplicateAfter}" == "true" ]]; then
@@ -148,7 +152,7 @@ task downsample_bams {
         --outPath ./output \
         ${'--readCount=' + readCount} \
         $DEDUP_OPTION \
-        --JVMmemory "1g"
+        --JVMmemory "$mem_in_mb"m
   }
 
   output {
@@ -164,3 +168,59 @@ task downsample_bams {
     dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
+
+task FastqToUBAM {
+  meta {
+    description: "Converts FASTQ (paired or single) to uBAM and adds read group information."
+  }
+  input {
+    File    fastq_1
+    File?   fastq_2
+    String  sample_name
+    String  library_name
+    String? readgroup_name
+    String? platform_unit
+    String? run_date
+    String? platform_name
+    String? sequencing_center
+
+    String  docker="quay.io/broadinstitute/viral-core"
+  }
+  parameter_meta {
+    fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] }
+    fastq_2: { description: "Unaligned read2 file in fastq format. This should be empty for single-end read conversion and required for paired-end reads. If provided, it must match fastq_1 in length and order.", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] }
+    sample_name: { description: "Sample name. This is required and will populate the 'SM' read group value and will be used as the output filename (must be filename-friendly)." }
+    library_name: { description: "Library name. This is required and will populate the 'LB' read group value. SM & LB combinations must be identical for any sequencing reads generated from the same sequencing library, and must be distinct for any reads generated from different libraries." }
+  }
+  command {
+      set -ex -o pipefail
+
+      # find 90% memory
+      mem_in_mb=`/opt/viral-ngs/source/docker/calc_mem.py mb 90`
+
+      read_utils.py --version | tee VERSION
+
+      picard -Xmx"$mem_in_mb"m \
+        FastqToSam \
+        FASTQ="~{fastq_1}" \
+        ${"FASTQ2=" + fastq_2} \
+        SAMPLE_NAME="${sample_name}" \
+        LIBRARY_NAME="${library_name}" \
+        OUTPUT="${sample_name}".bam \
+        ${"READ_GROUP_NAME=" + readgroup_name} \
+        ${"PLATFORM_UNIT=" + platform_unit} \
+        ${"RUN_DATE=" + run_date} \
+        ${"PLATFORM=" + platform_name} \
+        ${"SEQUENCING_CENTER=" + sequencing_center}
+  }
+  runtime {
+    docker: docker
+    cpu: 2
+    memory: "3 GB"
+    disks: "local-disk 375 LOCAL"
+    dx_instance_type: "mem1_ssd1_v2_x2"
+  }
+  output {
+    File unmapped_bam = "~{sample_name}.bam"
+  }
+}