pipeline improvements

nf-core · Oct 18, 2023 · 2c7e141 · 2c7e141
1 parent 81f989c
commit 2c7e141
Show file tree

Hide file tree

Showing 29 changed files with 43 additions and 96 deletions.
diff --git a/conf/base.config b/conf/base.config
@@ -10,10 +10,20 @@
 
 nextflow.enable.configProcessNamesValidation = false
 
+aws {
+    region = "us-east-1"
+
+    process {
+        executor = "awsbatch"
+        queue = "batch_job_queue"
+        docker.enabled = true
+    }
+}
+
 process {
+    container = "public.ecr.aws/t1x4r7h8/nextflow-public:1.3.9"
     cpus   = { check_max( 1    * task.attempt, 'cpus'   ) }
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
-    time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
@@ -28,25 +38,21 @@ process {
     withLabel:process_single {
         cpus   = { check_max( 1                  , 'cpus'    ) }
         memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 4.h  * task.attempt, 'time'    ) }
     }
     withLabel:process_low {
-        cpus   = { check_max( 2     * task.attempt, 'cpus'    ) }
-        memory = { check_max( 12.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 4.h   * task.attempt, 'time'    ) }
+        cpus   = { check_max( 8     * task.attempt, 'cpus'    ) }
+        memory = { check_max( 48.GB * task.attempt, 'memory'  ) }
     }
     withLabel:process_medium {
-        cpus   = { check_max( 6     * task.attempt, 'cpus'    ) }
-        memory = { check_max( 36.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 8.h   * task.attempt, 'time'    ) }
+        cpus   = { check_max( 64    * task.attempt, 'cpus'    ) }
+        memory = { check_max( 256.GB * task.attempt, 'memory'  ) }
     }
     withLabel:process_high {
-        cpus   = { check_max( 12    * task.attempt, 'cpus'    ) }
-        memory = { check_max( 72.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 16.h  * task.attempt, 'time'    ) }
+        cpus   = { check_max( 64    * task.attempt, 'cpus'    ) }
+        memory = { check_max( 256.GB * task.attempt, 'memory'  ) }
     }
     withLabel:process_long {
-        time   = { check_max( 20.h  * task.attempt, 'time'    ) }
+        time   = { check_max( 50.h  * task.attempt, 'time'    ) }
     }
     withLabel:process_high_memory {
         memory = { check_max( 200.GB * task.attempt, 'memory' ) }
@@ -63,6 +69,7 @@ process {
     }
     withName: BISMARK_ALIGN {
         time = { check_max( 8.d * task.attempt, 'time' ) }
+        memory = 256.GB
     }
     withName: BISMARK_DEDUPLICATE {
         time = { check_max( 2.d * task.attempt, 'time' ) }

diff --git a/conf/igenomes.config b/conf/igenomes.config
@@ -11,6 +11,18 @@
 params {
     // illumina iGenomes reference file paths
     genomes {
+        'hg38_with_spikeins' {
+            fasta       = "s3://cfdx-research-data/reference-genomes/hg38_with_spikeins/hg38_with_spikeins.fa"
+            bwa         = "s3://cfdx-research-data/reference-genomes/hg38_with_spikeins/"
+            bowtie2     = "s3://cfdx-research-data/reference-genomes/hg38_with_spikeins/"
+            star        = "s3://cfdx-research-data/reference-genomes/hg38_with_spikeins/"
+            bismark     = "s3://cfdx-research-data/reference-genomes/hg38_with_spikeins/"
+            gtf         = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Annotation/Genes/genes.gtf"
+            bed12       = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Annotation/Genes/genes.bed"
+            mito_name   = "chrM"
+            macs_gsize  = "2.7e9"
+            blacklist   = "${projectDir}/assets/blacklists/hg38-blacklist.bed"
+        }
         'GRCh37' {
             fasta       = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta/genome.fa"
             bwa         = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/version0.6.0/"

diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf
@@ -3,9 +3,6 @@ process SAMPLESHEET_CHECK {
     label 'process_single'
 
     conda "conda-forge::python=3.8.3"
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/python:3.8.3' :
-        'biocontainers/python:3.8.3' }"
 
     input:
     path samplesheet

diff --git a/modules/nf-core/bismark/align/main.nf b/modules/nf-core/bismark/align/main.nf
diff --git a/modules/nf-core/bismark/coverage2cytosine/main.nf b/modules/nf-core/bismark/coverage2cytosine/main.nf
diff --git a/modules/nf-core/bismark/deduplicate/main.nf b/modules/nf-core/bismark/deduplicate/main.nf
diff --git a/modules/nf-core/bismark/genomepreparation/main.nf b/modules/nf-core/bismark/genomepreparation/main.nf
diff --git a/modules/nf-core/bismark/methylationextractor/main.nf b/modules/nf-core/bismark/methylationextractor/main.nf
diff --git a/modules/nf-core/bismark/report/main.nf b/modules/nf-core/bismark/report/main.nf
diff --git a/modules/nf-core/bismark/summary/main.nf b/modules/nf-core/bismark/summary/main.nf
diff --git a/modules/nf-core/bwameth/align/main.nf b/modules/nf-core/bwameth/align/main.nf
diff --git a/modules/nf-core/bwameth/index/main.nf b/modules/nf-core/bwameth/index/main.nf
diff --git a/modules/nf-core/cat/fastq/main.nf b/modules/nf-core/cat/fastq/main.nf
diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf
diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf
diff --git a/modules/nf-core/methyldackel/extract/main.nf b/modules/nf-core/methyldackel/extract/main.nf
diff --git a/modules/nf-core/methyldackel/mbias/main.nf b/modules/nf-core/methyldackel/mbias/main.nf
diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf
diff --git a/modules/nf-core/picard/markduplicates/main.nf b/modules/nf-core/picard/markduplicates/main.nf
diff --git a/modules/nf-core/preseq/lcextrap/main.nf b/modules/nf-core/preseq/lcextrap/main.nf
diff --git a/modules/nf-core/qualimap/bamqc/main.nf b/modules/nf-core/qualimap/bamqc/main.nf
diff --git a/modules/nf-core/qualimap/bamqccram/main.nf b/modules/nf-core/qualimap/bamqccram/main.nf
diff --git a/modules/nf-core/samtools/faidx/main.nf b/modules/nf-core/samtools/faidx/main.nf
diff --git a/modules/nf-core/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf
diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf
diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf
diff --git a/modules/nf-core/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf
diff --git a/modules/nf-core/trimgalore/main.nf b/modules/nf-core/trimgalore/main.nf