Merge pull request #2 from EpiDiverse/dev

merge dev for v1.0 release with admin priveleges

CHANGELOG.md

@@ -2,6 +2,10 @@
 ---
 # Releases
 
+## v1.0 - 
+* Update to Nextflow 20.07.1 syntax
+* Restructure output directories
+
 ---
 # Prereleases
 ## v0.9.1 - 

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 EpiDiverse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,16 @@
 [<img width="200" align="right" src="docs/images/euflagbetter.jpg">](https://ec.europa.eu/programmes/horizon2020/en)
 [<img width="200" align="right" src="docs/images/epidiverse-logo.jpg">](https://epidiverse.eu)
-[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.09.0-brightgreen.svg)](https://www.nextflow.io/)
-[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/)
-[![Docker](https://img.shields.io/docker/automated/epidiverse/dmr.svg)](https://hub.docker.com/r/epidiverse/dmr)
 
 EpiDiverse-DMR Pipeline
 ========================
 
+[![Nextflow](https://img.shields.io/badge/nextflow-20.07.1-45818e.svg)](https://www.nextflow.io/)
+[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-45818e.svg)](http://bioconda.github.io/)
+[![Docker](https://img.shields.io/docker/cloud/automated/epidiverse/dmr.svg)](https://hub.docker.com/r/epidiverse/dmr)
+[![Release](https://img.shields.io/github/v/release/epidiverse/dmr.svg?colorB=45818e)](https://github.com/EpiDiverse/dmr/releases/latest)
+[![Publication](https://img.shields.io/badge/Published-Bioinformatics-45818e.svg?colorB=45818e&style=popout)](https://academic.oup.com/bioinformatics/article-abstract/36/11/3314/5809142)
+[![Twitter](https://img.shields.io/twitter/follow/epidiverse?style=social)](https://twitter.com/intent/follow?screen_name=epidiverse)
+
 **EpiDiverse/dmr** is a bioinformatics analysis pipeline for calling differentially methylated positions or regions from non-model plant species.
 
 The workflow processes raw methylation data from bedGraphs resulting from the [EpiDiverse/wgbs](https://github.com/epidiverse/wgbs/) pipeline, which are then grouped for analysis with [bedtools unionbedg](https://github.com/arq5x/bedtools2). Each pairwise comparison between groups is performed with [metilene](https://www.bioinf.uni-leipzig.de/Software/metilene/), and downstream visualisation is carried out with R-packages [ggplot2]() and [gplots]() to produce distribution plots and heatmaps.
@@ -17,21 +21,21 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool
 
 ## Quick Start
 
-i. Install [`nextflow`](https://www.nextflow.io/)
+1. Install [`nextflow`](https://www.nextflow.io/)
 
-ii. Install one of [`docker`](https://docs.docker.com/engine/installation/), [`singularity`](https://www.sylabs.io/guides/3.0/user-guide/) or [`conda`](https://conda.io/miniconda.html)
+2. Install one of [`docker`](https://docs.docker.com/engine/installation/), [`singularity`](https://www.sylabs.io/guides/3.0/user-guide/) or [`conda`](https://conda.io/miniconda.html)
 
-iii. Download the pipeline and test it on a minimal dataset with a single command
+3. Download the pipeline and test it on a minimal dataset with a single command
 
 ```bash
-nextflow run epidiverse/dmr -profile test,<docker|singularity|conda>
+NXF_VER=20.07.1 nextflow run epidiverse/dmr -profile test,<docker|singularity|conda>
 ```
 
-iv. Start running your own analysis!
+4. Start running your own analysis!
 
 ```bash
-nextflow run epidiverse/dmr -profile <docker|singularity|conda> \
---input /path/to/wgbs/directory --samples /path/to/samples.tsv
+NXF_VER=20.07.1 nextflow run epidiverse/dmr -profile <docker|singularity|conda> \
+--input /path/to/wgbs/bam --samples /path/to/samples.tsv
 ```
 
 > See the [usage documentation](docs/usage.md) for all of the available options when running the pipeline.

assets/custom.config

@@ -16,6 +16,17 @@ singularity.enabled = false
 docker.enabled = false
 
 
+executor {
+
+	$pbspro {
+		jobName = { "${task.name}".replaceAll("\\s","") }
+		submitRateLimit = '20 min'
+		queueSize = 20
+		pollInterval = '30 sec'
+	}
+}
+
+
 process {
 
 	executor = 'pbspro'
@@ -27,15 +38,98 @@ process {
 	// with docker/singularity
 	container = "epidiverse/dmr"
 
-}
 
+	// top-level configuration groups
+	withLabel: "mid" {
+    	time = { 20.h * task.attempt }
+    	memory = { 8.GB * task.attempt }
+    	cpus = 2
+	}
 
-executor {
+	withLabel: "low" {
+    	time = { 8.h * task.attempt }
+    	memory = { 2.GB * task.attempt }
+    	cpus = 2
+	}
 
-	$pbspro {
-		jobName = { "${task.name}".replaceAll("\\s","") }
-		submitRateLimit = '20 min'
-		queueSize = 20
-		pollInterval = '30 sec'
+	// label processes which should kill the pipeline if they fail
+	withLabel: "finish" {
+		errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'finish' : 'retry') }
+		maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+	// label processes which can be safely ignored if they fail
+	withLabel: "ignore" {
+		errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'ignore' : 'retry') }
+		maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
 	}
-}
+
+
+	// configuration for preprocessing
+	withName: "preprocessing" {
+
+		// label 'low'
+    	//time = { 8.h * task.attempt }
+    	//memory = { 2.GB * task.attempt }
+    	//cpus = 2
+
+		// label 'finish'
+		//errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'finish' : 'retry') }
+		//maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+	// configuration for preprocessing
+	withName: "bedtools_unionbedg" {
+
+		// label 'low'
+    	//time = { 8.h * task.attempt }
+    	//memory = { 2.GB * task.attempt }
+    	//cpus = 2
+
+		// label 'finish'
+		//errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'finish' : 'retry') }
+		//maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+	// configuration for running metilene
+	withName: "metilene" {
+
+		// label 'mid'
+    	time = { 1.d * task.attempt }
+    	memory = { 8.GB * task.attempt }
+    	cpus = { 4 * task.attempt }
+
+		// label 'finish'
+		//errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'finish' : 'retry') }
+		//maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+	// configuration for visualisation of distributions
+	withName: "distributions" {
+
+		// label 'mid'		
+    	//time = { 20.h * task.attempt }
+    	//memory = { 8.GB * task.attempt }
+    	//cpus = 2
+
+		// label 'ignore'
+		//errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'ignore' : 'retry') }
+		//maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+	// configuration for visualisation of heatmaps
+	withName: "heatmaps" {
+
+		// label 'mid'
+    	//time = { 20.h * task.attempt }
+    	//memory = { 8.GB * task.attempt }
+    	//cpus = 2
+
+		// label 'ignore'
+		//errorStrategy = { sleep(Math.pow(2, task.attempt) * 100 as long); (task.attempt == process.maxRetries ? 'ignore' : 'retry') }
+		//maxRetries = { task.exitStatus in [140,141,143,137,104,134,139] ? 3 : 2 }
+	}
+
+}
+
+

config/base.config

@@ -6,64 +6,54 @@ process {
 
 	// top-level configuration groups
 	withLabel: "mid" {
-    	time = { 20.h * task.attempt }
+    	time = { 3.h * Math.pow(2, task.attempt) }
     	memory = { 8.GB * task.attempt }
     	cpus = 2
 	}
 
 	withLabel: "low" {
-    	time = { 8.h * task.attempt }
+    	time = { 3.h * Math.pow(2, task.attempt) }
     	memory = { 2.GB * task.attempt }
     	cpus = 2
 	}
 
-	// configuration for preprocessing
-	withName: "preprocessing" {
 
-		// label 'low'
-
-		errorStrategy = { task.exitStatus in [140,141,143,137,104,134,139] ? 'retry' : 'finish' }
-    	maxRetries = 1
+	// label processes which should kill the pipeline if they fail
+	withLabel: "finish" {
+		maxRetries = 3
+        errorStrategy = { ->
+			if( (task.exitStatus in [140,141,143,137,104,134,139] && task.attempt > 3) || (!(task.exitStatus in [140,141,143,137,104,134,139]) && task.attempt > 1 ) ){
+				return 'finish'
+			} else if(task.exitStatus in [140,141,143,137,104,134,139]){
+				return 'retry'
+			} else {
+				sleep(Math.pow(2, task.attempt) * 1000 as long);
+				return 'retry'
+			}
+		}
 	}
 
-	// configuration for preprocessing
-	withName: "bedtools_unionbedg" {
-
-		// label 'low'
-
-		errorStrategy = { task.exitStatus in [140,141,143,137,104,134,139] ? 'retry' : 'finish' }
-    	maxRetries = 1
+	// label processes which can be safely ignored if they fail
+	withLabel: "ignore" {
+		maxRetries = 3
+        errorStrategy = { ->
+			if( (task.exitStatus in [140,141,143,137,104,134,139] && task.attempt > 3) || (!(task.exitStatus in [140,141,143,137,104,134,139]) && task.attempt > 1 ) ){
+				return 'ignore'
+			} else if(task.exitStatus in [140,141,143,137,104,134,139]){
+				return 'retry'
+			} else {
+				sleep(Math.pow(2, task.attempt) * 1000 as long);
+				return 'retry'
+			}
+		}
 	}
 
+
 	// configuration for running metilene
 	withName: "metilene" {
 
-		// label 'mid'
-
-    	time = { 1.d * task.attempt }
-    	memory = { 8.GB * task.attempt }
+    	memory = { 4.GB * Math.pow(2, task.attempt) }
     	cpus = { 4 * task.attempt }
-
-		errorStrategy = { task.exitStatus in [140,141,143,137,104,134,139] ? 'retry' : 'finish' }
-    	maxRetries = 2
-	}
-
-	// configuration for visualisation of distributions
-	withName: "distributions" {
-
-		// label 'mid'		
-
-    	errorStrategy = { task.attempt == 1 ? 'retry' : 'ignore' }
-    	maxRetries = 1
-	}
-
-	// configuration for visualisation of heatmaps
-	withName: "heatmaps" {
-
-		// label 'mid'
-
-    	errorStrategy = { task.attempt == 1 ? 'retry' : 'ignore' }
-    	maxRetries = 1
 	}
 
 }

config/test.config

@@ -14,6 +14,7 @@ params {
     noCHH = true
 
 	// samples file
+    input = "test profile"
 	samples = "https://github.com/epidiverse/datasets/raw/dmr/samples.tsv"
 
     // set CpGPaths parameter (only available in test profile)