Merge pull request #229 from UPHL-BioNGS/update-20230915

Update 20230915
UPHL-BioNGS · Sep 19, 2023 · 1b2e231 · 1b2e231
2 parents d44b41b + d18f802
commit 1b2e231
Show file tree

Hide file tree

Showing 27 changed files with 1,134 additions and 642 deletions.
diff --git a/.github/workflows/test_primers.yml b/.github/workflows/test_primers.yml
@@ -0,0 +1,72 @@
+name: Test SARS-CoV-2 primers
+
+on: [pull_request, workflow_dispatch]
+
+run-name: primers
+
+jobs:
+
+  test:
+    runs-on: ubuntu-20.04
+    strategy:
+        matrix:
+            primer: [
+                'midnight_idt_V1', 
+                'midnight_ont_V1', 
+                'midnight_ont_V2', 
+                'midnight_ont_V3', 
+                'ncov_V3', 
+                'ncov_V4', 
+                'ncov_V4.1', 
+                'ncov_V5.3.2', 
+                'mpx_primalseq', 
+                'mpx_idt'
+                ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        run: |
+            wget -qO- get.nextflow.io | bash
+            sudo mv nextflow /usr/local/bin/
+        
+      - name: Download reads
+        run: |
+            mkdir nanopore
+            cd nanopore
+            wget -q ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR224/050/SRR22452250/SRR22452250_1.fastq.gz
+            wget -q ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR224/031/SRR22452231/SRR22452231_1.fastq.gz
+            cd ../
+
+      - name: Run Cecret
+        run: |
+            nextflow run . -profile docker -c .github/workflows/github_actions.config \
+                --maxcpus 2 \
+                --medcpus 2 \
+                --primer_set ${{ matrix.primer }} \
+                --bcftools_variants false \
+                --fastqc false \
+                --ivar_variants false \
+                --samtools_stats false \
+                --samtools_coverage false \
+                --samtools_depth false \
+                --samtools_flagstat false \
+                --kraken2 false \
+                --nextclade false \
+                --pangolin false \
+                --freyja false \
+                --vadr false \
+                --relatedness false \
+                --snpdists false \
+                --iqtree2 false \
+                --bamsnap false \
+                --rename false \
+                --filter false \
+                --multiqc false
+
+            ls cecret*
+
+      - name: Clean
+        run: rm -rf work .nextflow*
diff --git a/.github/workflows/test_profile.yml b/.github/workflows/test_profile.yml
@@ -24,7 +24,7 @@ jobs:
           
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,test -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200
+          nextflow run . -profile docker,test -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --vadr false
 
           ls cecret*
             

diff --git a/.github/workflows/test_profile1.yml b/.github/workflows/test_profile1.yml
@@ -24,7 +24,7 @@ jobs:
           
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,test1 -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200
+          nextflow run . -profile docker,test1 -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --vadr false
 
           ls cecret*
             

diff --git a/.github/workflows/test_profile2.yml b/.github/workflows/test_profile2.yml
@@ -24,7 +24,7 @@ jobs:
           
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,test -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200
+          nextflow run . -profile docker,test2 -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --vadr false
 
           ls cecret*
             

diff --git a/README.md b/README.md
@@ -155,21 +155,6 @@ nextflow run UPHL-BioNGS/Cecret -profile docker --sample_sheet SampleSheet.csv
 ## Determining primer and amplicon bedfiles
 The default primer scheme of the 'Cecret' workflow is the 'V4' primer scheme developed by [artic network for SARS-CoV-2](https://artic.network/ncov-2019). Releases prior to and including '2.2.20211221' used the 'V3' primer scheme as the default. As many public health laboratories are still using 'V3', the 'V3' files are still in this repo, but now the 'V4', 'V4.1' ('V4' with a spike-in of additional primers), and 'V5.3.2' are also included. The original primer and amplicon bedfiles can be found at [artic's github repo](https://github.com/artic-network/artic-ncov2019/tree/master/primer_schemes/nCoV-2019). 
 
-Setting primers sets is with the corresponding profile.
-```
-# using artic V3 primers
-nextflow run UPHL-BioNGS/Cecret -profile singularity,artic_V3
-
-# using artic V4 primers
-nextflow run UPHL-BioNGS/Cecret -profile singularity,artic_V4
-
-# using artic V4.1 primers
-nextflow run UPHL-BioNGS/Cecret -profile singularity,artic_V4_1
-
-# using artic V5.3.2 primers
-nextflow run UPHL-BioNGS/Cecret -profile singularity,artic_V5_3_2
-```
-
 Setting primers with a parameter on the command line (these can also be defined in a config file)
 ```
 # using artic V3 primers
@@ -185,6 +170,8 @@ nextflow run UPHL-BioNGS/Cecret -profile singularity --primer_set 'ncov_V4.1'
 nextflow run UPHL-BioNGS/Cecret -profile singularity --primer_set 'ncov_V5.3.2'
 ```
 
+Some "Midnight" primers are also included and can be set with `midnight_idt_V1`, `midnight_ont_V1`, `midnight_ont_V2`, `midnight_ont_V3`.
+
 It is still possible to set 'params.primer_bed' and 'params.amplicon_bed' via the command line or in a config file with the path to the corresponding file.
 
 ## Using the included nextclade dataset

diff --git a/configs/mpx_idt.config b/configs/mpx_idt.config
@@ -1,6 +1,5 @@
 params.species                         = 'mpx'
 params.primer_set                      = 'mpx_idt'
-params.samtools_ampliconstats_options  = '--max-amplicons 1900'  
 params.nextclade_dataset               = 'hMPXV'
 params.vadr_options                    = '--split --glsearch -s -r --nomisc --r_lowsimok --r_lowsimxd 100 --r_lowsimxl 2000 --alt_pass discontn,dupregin'
 params.vadr_reference                  = 'mpxv'

diff --git a/configs/mpx_primalseq.config b/configs/mpx_primalseq.config
@@ -1,6 +1,5 @@
 params.species                         = 'mpx'
 params.primer_set                      = 'mpx_primalseq'
-params.samtools_ampliconstats_options  = '--max-amplicon-length 2500'
 params.nextclade_dataset               = 'hMPXV'
 params.vadr_options                    = '--split --glsearch -s -r --nomisc --r_lowsimok --r_lowsimxd 100 --r_lowsimxl 2000 --alt_pass discontn,dupregin'
 params.vadr_reference                  = 'mpxv'

diff --git a/main.nf b/main.nf
@@ -94,8 +94,8 @@ if ( params.fastas == params.multifastas ) {
 
 //# outdir params
 params.outdir                               = workflow.launchDir + '/cecret'
-println('The files and directory for results is ' + params.outdir)
 params.species                              = 'sarscov2'
+println('The files and directory for results is ' + params.outdir)
 println('The species used to determine default variables and subworkflows is ' + params.species)
 
 //# roughly grouping cpu usage
@@ -150,7 +150,7 @@ params.samtools_coverage_options            = ''
 params.samtools_flagstat_options            = ''
 params.samtools_depth_options               = ''
 params.samtools_stats_options               = ''
-params.samtools_ampliconstats_options       = ''
+params.samtools_ampliconstats_options       = '--max-amplicon-length 3000 --max-amplicons 3000'
 params.samtools_plot_ampliconstats_options  = '-size 1200,900 -size2 1200,900 -size3 1200,900'
 params.samtools_markdup_options             = ''
 params.samtools_fixmate_options             = ''
@@ -331,11 +331,17 @@ if ( params.ivar_variants ) {
       ch_gff_file = Channel.empty()
     } 
   }
+} else {
+  ch_gff_file = Channel.empty()
 }
 ch_gff_file.view { "GFF file : $it"}
 
 //# channels of included files
 included_primers     = [
+  workflow.projectDir + '/schema/midnight_idt_V1_SARS-CoV-2.primer.bed',
+  workflow.projectDir + '/schema/midnight_ont_V1_SARS-CoV-2.primer.bed',
+  workflow.projectDir + '/schema/midnight_ont_V2_SARS-CoV-2.primer.bed',
+  workflow.projectDir + '/schema/midnight_ont_V3_SARS-CoV-2.primer.bed',
   workflow.projectDir + '/schema/ncov_V3_nCoV-2019.primer.bed',
   workflow.projectDir + '/schema/ncov_V4_SARS-CoV-2.primer.bed',
   workflow.projectDir + '/schema/ncov_V4.1_SARS-CoV-2.primer.bed',
@@ -344,6 +350,10 @@ included_primers     = [
   workflow.projectDir + '/schema/mpx_primalseq_primer.bed'
   ]
 included_amplicons = [
+  workflow.projectDir + '/schema/midnight_idt_V1_SARS-CoV-2.insert.bed',
+  workflow.projectDir + '/schema/midnight_ont_V1_SARS-CoV-2.insert.bed',
+  workflow.projectDir + '/schema/midnight_ont_V2_SARS-CoV-2.insert.bed',
+  workflow.projectDir + '/schema/midnight_ont_V3_SARS-CoV-2.insert.bed',
   workflow.projectDir + '/schema/ncov_V3_nCoV-2019.insert.bed',
   workflow.projectDir + '/schema/ncov_V4_SARS-CoV-2.insert.bed',
   workflow.projectDir + '/schema/ncov_V4.1_SARS-CoV-2.insert.bed',
@@ -355,7 +365,19 @@ included_amplicons = [
 ch_primers                = Channel.fromPath(included_primers,   type: 'file')
 ch_amplicons              = Channel.fromPath(included_amplicons, type: 'file')
 
-available_primer_sets = ['ncov_V3', 'ncov_V4', 'ncov_V4.1', 'ncov_V5.3.2', 'mpx_primalseq', 'mpx_idt']
+available_primer_sets = [
+  'midnight_idt_V1', 
+  'midnight_ont_V1', 
+  'midnight_ont_V2', 
+  'midnight_ont_V3', 
+  'ncov_V3', 
+  'ncov_V4', 
+  'ncov_V4.1', 
+  'ncov_V5.3.2', 
+  'mpx_primalseq', 
+  'mpx_idt'
+  ]
+
 if ( params.trimmer != 'none' ) {
   //# Getting the primer file
   if (params.primer_bed) {
@@ -376,7 +398,7 @@ if ( params.trimmer != 'none' ) {
       .set { ch_primer_bed } 
   } else {
     println("No primers were found!")
-    println("Set primer schema with 'params.primer_bed'")
+    println("Set primer schema with 'params.primer_bed' or specify to 'none' if primers were not used")
     println("Or use included primer set by setting 'params.primer_set' to one of $available_primer_sets")
     exit 1
     ch_primer_bed = Channel.empty()

diff --git a/modules/artic.nf b/modules/artic.nf
@@ -2,7 +2,7 @@ process artic {
     tag        "${sample}"
     label      "process_high"
     publishDir "${params.outdir}", mode: 'copy'
-    container  'quay.io/uphl/artic:1.2.4-1.9.1'
+    container  'quay.io/uphl/artic:1.2.4-1.9.4-1'
 
     //#UPHLICA maxForks      10
     //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -61,7 +61,7 @@ process artic {
 process artic_read_filtering {
     tag        "${sample}"
     publishDir "${params.outdir}", mode: 'copy'
-    container  'quay.io/uphl/artic:1.2.4-1.9.1'
+    container  'quay.io/uphl/artic:1.2.4-1.9.4-1'
     label      "process_single"
 
     //#UPHLICA maxForks      10

diff --git a/modules/freyja.nf b/modules/freyja.nf
@@ -3,7 +3,7 @@ process freyja {
   label         "process_medium"
   errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
   publishDir    "${params.outdir}", mode: 'copy'
-  container     'quay.io/uphl/freyja:1.4.5-20230831'
+  container     'quay.io/uphl/freyja:1.4.7-20230915'
 
   //#UPHLICA maxForks 10
   //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
@@ -57,7 +57,7 @@ process freyja_aggregate {
   tag        "Aggregating results from freyja"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'quay.io/uphl/freyja:1.4.5-20230831'
+  container  'quay.io/uphl/freyja:1.4.7-20230915'
 
   //#UPHLICA maxForks 10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}

diff --git a/modules/samtools.nf b/modules/samtools.nf
@@ -2,7 +2,7 @@ process samtools_stats {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -37,7 +37,7 @@ process samtools_coverage {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -74,7 +74,7 @@ process samtools_flagstat {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -111,7 +111,7 @@ process samtools_depth {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -147,7 +147,7 @@ process samtools_ampliconstats {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -185,7 +185,7 @@ process samtools_plot_ampliconstats {
   label      "process_single"
   errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
   publishDir    "${params.outdir}", mode: 'copy'
-  container     'staphb/samtools:1.18'
+  container     'staphb/samtools:1.17'
 
   //#UPHLICA maxForks 10
   //#UPHLICA pod annotation: 'scheduler.illumina.com/presetSize', value: 'standard-xlarge'
@@ -221,7 +221,7 @@ process samtools_sort {
   tag        "${sample}"
   label      "process_high"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -258,7 +258,7 @@ process samtools_filter {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -300,7 +300,7 @@ process samtools_ampliconclip {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}
@@ -315,7 +315,7 @@ process samtools_ampliconclip {
   output:
   tuple val(sample), file("ampliconclip/${sample}.primertrim.sorted.bam"), file("ampliconclip/${sample}.primertrim.sorted.bam.bai"), emit: bam_bai
   path "logs/${task.process}/${sample}.${workflow.sessionId}.log"                                                         
-  tuple val(sample), env(trimmer_version),                                                                                           emit: trimmer_version
+  tuple val("samtools ampliconclip"), env(trimmer_version),                                                                          emit: trimmer_version
 
   shell:
   '''
@@ -340,7 +340,7 @@ process samtools_markdup {
   tag        "${sample}"
   label      "process_single"
   publishDir "${params.outdir}", mode: 'copy'
-  container  'staphb/samtools:1.18'
+  container  'staphb/samtools:1.17'
 
   //#UPHLICA maxForks      10
   //#UPHLICA errorStrategy { task.attempt < 2 ? 'retry' : 'ignore'}

diff --git a/nextflow.config b/nextflow.config
@@ -5,7 +5,7 @@ manifest {
   name              = 'Cecret'
   author            = 'Erin Young'
   homePage          = 'https://github.com/UPHL-BioNGS/Cecret'
-  version           = 'v3.8.20230907'
+  version           = 'v3.8.20230915'
   defaultBranch     = 'master'
   recurseSubmodules = false
   description       = 'Reference-based consensus creation'

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -333,10 +333,15 @@
             "default": "ncov_V4",
             "description": "Specifies a primer set included in repo",
             "enum": [
+                "midnight_idt_V1", 
+                "midnight_ont_V1", 
+                "midnight_ont_V2", 
+                "midnight_ont_V3", 
                 "ncov_V3", 
                 "ncov_V4", 
                 "ncov_V4.1", 
                 "ncov_V5.3.2", 
+                "mpx_primalseq", 
                 "mpx_idt"
             ]
         },
@@ -373,6 +378,7 @@
         "samtools_ampliconstats_options": { 
             "type": "string",
             "hidden": true,
+            "default": "--max-amplicon-length 3000 --max-amplicons 3000",
             "description": "Options for process"
         },
         "samtools_coverage": {