From c7aea974f8823ef0231eb24326175cc792a09ccf Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 5 Mar 2020 17:20:08 +0100 Subject: [PATCH 01/22] Set up GHA to install awscli with conda --- .github/workflows/aws.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/aws.yml diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml new file mode 100644 index 0000000000..34a6a99bc0 --- /dev/null +++ b/.github/workflows/aws.yml @@ -0,0 +1,23 @@ + +name: AWS Megatests + +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + +jobs: + install-conda: + name: Install miniconda + runs-on: ubuntu-latest + steps: + - uses: goanpeca/setup-miniconda@v1 + with: + auto-update-conda: true + python-version: 3.7 + - name: Conda info + run: conda install -c conda-forge awscli + From dcab9413816e1da107bf7471fad28c3a5a7628a8 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 5 Mar 2020 17:24:14 +0100 Subject: [PATCH 02/22] Update naming --- .github/workflows/aws.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 34a6a99bc0..d21b33ac37 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -10,8 +10,8 @@ on: - '*' jobs: - install-conda: - name: Install miniconda + install-awscli: + name: Install awscli runs-on: ubuntu-latest steps: - uses: goanpeca/setup-miniconda@v1 @@ -20,4 +20,4 @@ jobs: python-version: 3.7 - name: Conda info run: conda install -c conda-forge awscli - + From 4ffb22c0fbce9e659abf92c62781e374f0b22f60 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 5 Mar 2020 17:34:41 +0100 Subject: [PATCH 03/22] Update naming --- .github/workflows/aws.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index d21b33ac37..06483a0ba6 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -14,10 +14,7 @@ jobs: name: Install awscli runs-on: ubuntu-latest steps: - - uses: goanpeca/setup-miniconda@v1 - with: - auto-update-conda: true + - uses: actions/checkout@v2 python-version: 3.7 - - name: Conda info - run: conda install -c conda-forge awscli - + - name: Conda install awscli + run: conda install -c conda-forge awscli \ No newline at end of file From 67bd7bcb46726215a1a3e9f5c23334db4960a762 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 9 Mar 2020 11:31:02 +0100 Subject: [PATCH 04/22] update VEP version in Dockerfile --- containers/vep/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/vep/Dockerfile b/containers/vep/Dockerfile index 4bf89db555..e771071cb0 100644 --- a/containers/vep/Dockerfile +++ b/containers/vep/Dockerfile @@ -17,7 +17,7 @@ RUN conda env export --name nf-core-sarek-vep-dev > nf-core-sarek-vep-dev.yml # Setup default ARG variables ARG GENOME=GRCh38 ARG SPECIES=homo_sapiens -ARG VEP_VERSION=95 +ARG VEP_VERSION=99 # Download Genome RUN vep_install \ From 9e45bc26436858c314567c2d235a9c65acc7c9df Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 9 Mar 2020 11:31:20 +0100 Subject: [PATCH 05/22] code polishing --- main.nf | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/main.nf b/main.nf index 556fb1f29e..a64e49c415 100644 --- a/main.nf +++ b/main.nf @@ -57,20 +57,24 @@ def helpMessage() { snpEff, VEP, merge Default: None --skip_qc [str] Specify which QC tools to skip when running Sarek - Available: all, bamQC, BaseRecalibrator, BCFtools, Documentation, FastQC, MultiQC, samtools, vcftools, versions + Available: all, bamQC, BaseRecalibrator, BCFtools, Documentation + FastQC, MultiQC, samtools, vcftools, versions Default: None --annotate_tools [str] Specify from which tools Sarek will look for VCF files to annotate, only for step annotate Available: HaplotypeCaller, Manta, Mutect2, Strelka, TIDDIT Default: None --sentieon [bool] If sentieon is available, will enable it for preprocessing, and variant calling - Adds the following tools for --tools: DNAseq, DNAscope and TNscope + Adds the following options for --tools: DNAseq, DNAscope and TNscope --annotation_cache [bool] Enable the use of cache for annotation, to be used with --snpeff_cache and/or --vep_cache --snpeff_cache [file] Specity the path to snpEff cache, to be used with --annotation_cache --vep_cache [file] Specity the path to VEP cache, to be used with --annotation_cache - --pon [file] Panel-of-normals VCF (bgzipped, indexed). See: https://software.broadinstitute.org/gatk/documentation/tooldocs/current/org_broadinstitute_hellbender_tools_walkers_mutect_CreateSomaticPanelOfNormals.php + --pon [file] Panel-of-normals VCF (bgzipped, indexed) + See: https://software.broadinstitute.org/gatk/documentation/tooldocs/current/org_broadinstitute_hellbender_tools_walkers_mutect_CreateSomaticPanelOfNormals.php --pon_index [file] Index of pon panel-of-normals VCF - --ascat_ploidy [int] Use this parameter together with to overwrite default behavior from ASCAT regarding ploidy. Note: Also requires that --ascat_purity is set. - --ascat_purity [int] Use this parameter to overwrite default behavior from ASCAT regarding purity. Note: Also requires that --ascat_ploidy is set. + --ascat_ploidy [int] Use this parameter to overwrite default behavior from ASCAT regarding ploidy + Requires that --ascat_purity is set + --ascat_purity [int] Use this parameter to overwrite default behavior from ASCAT regarding purity + Requires that --ascat_ploidy is set Trimming: --trim_fastq [bool] Run Trim Galore @@ -105,7 +109,7 @@ def helpMessage() { If none provided, will be generated automatically if a knownIndels file is provided --species [str] Species for VEP --snpeff_db [str] snpEff Database version - --vep_cache_version [str] VEP Cache version + --vep_cache_version [int] VEP Cache version Other options: --outdir [file] The output directory where the results will be saved @@ -120,9 +124,9 @@ def helpMessage() { -name [str] Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic AWSBatch options: - --awsqueue [str] The AWSBatch JobQueue that needs to be set when running on AWSBatch - --awsregion [str] The AWS Region for your AWS Batch job to run on - --awscli [str] Path to the AWS CLI tool + --awsqueue [str] The AWSBatch JobQueue that needs to be set when running on AWSBatch + --awsregion [str] The AWS Region for your AWS Batch job to run on + --awscli [str] Path to the AWS CLI tool """.stripIndent() } @@ -471,7 +475,6 @@ ch_cadd_wg_snvs_tbi = params.cadd_wg_snvs_tbi ? Channel.value(file(params.cadd_w ch_pon = params.pon ? Channel.value(file(params.pon)) : "null" ch_target_bed = params.target_bed ? Channel.value(file(params.target_bed)) : "null" - /* ================================================================================ PRINTING SUMMARY From a6c9a87675d2d8cba21bd523f139171e09825ce8 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 9 Mar 2020 11:31:34 +0100 Subject: [PATCH 06/22] remove Jenkinsfile --- Jenkinsfile | 63 ----------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 2a26fd3d5f..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,63 +0,0 @@ -pipeline { - agent any - - environment { - JENKINS_API = credentials('api') - } - - stages { - stage('Docker setup') { - steps { - sh "docker pull nfcore/sarek:dev" - sh "docker tag nfcore/sarek:dev nfcore/sarek:dev" - sh "docker pull nfcore/sareksnpeff:dev.GRCh37" - sh "docker tag nfcore/sareksnpeff:dev.GRCh37 nfcore/sareksnpeff:dev.GRCh37" - sh "docker pull nfcore/sarekvep:dev.GRCh37" - sh "docker tag nfcore/sarekvep:dev.GRCh37 nfcore/sarekvep:dev.GRCh37" - } - } - stage('Annotation') { - steps { - sh "nextflow run . -profile test_annotation,kraken --verbose --tools snpeff,vep,merge" - } - } - stage('Germline') { - steps { - sh "rm -rf data/" - sh "git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data" - sh "nextflow run . -profile test,kraken --input data/testdata/tiny/normal" - sh "nextflow run . -profile test,kraken --input=false --step recalibrate -resume" - sh "nextflow run . -profile test,kraken --input=false --step variantCalling" - sh "rm -rf data/" - } - } - stage('Minimal') { - steps { - sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome smallerGRCh37 --no_intervals --tools Manta,mpileup,Strelka" - sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome smallerGRCh37 --tools Manta,mpileup,Strelka" - sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome minimalGRCh37 --no_intervals --tools Manta,mpileup,Strelka" - sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome minimalGRCh37 --tools Manta,mpileup,Strelka" - } - } - stage('Profile') { - steps { - sh "nextflow run . -profile test_splitfastq,kraken --verbose" - sh "nextflow run . -profile test_targeted,kraken --verbose" - } - } - stage('Tools') { - steps { - sh "nextflow run . -profile test_tool,kraken --verbose --tools Haplotypecaller,Freebayes,Manta,mpileup,Mutect2,Strelka" - } - } - } - - post { - failure { - script { - def response = sh(script: "curl -u ${JENKINS_API_USR}:${JENKINS_API_PSW} ${BUILD_URL}/consoleText", returnStdout: true).trim().replace('\n', '
') - def comment = pullRequest.comment("## :rotating_light: Buil log output:
${response}
") - } - } - } -} From f59264c1f7b40e6ebaa0298c5f606e41ac042584 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 9 Mar 2020 11:39:31 +0100 Subject: [PATCH 07/22] update citation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c73bcd706..77d7156b60 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ For further information or help, don't hesitate to get in touch on [Slack](https ## Citation If you use `nf-core/sarek` for your analysis, please cite the `Sarek` pre-print as follows: -> Garcia M, Juhos S, Larsson M et al. **Sarek: A portable workflow for whole-genome sequencing analysis of germline and somatic variants [version 1; peer review: awaiting peer review].**. *F1000Research* 2020, 9:63. [doi: 10.12688/f1000research.16665.1](https://f1000research.com/articles/9-63/v1). +> Garcia M, Juhos S, Larsson M et al. **Sarek: A portable workflow for whole-genome sequencing analysis of germline and somatic variants [version 1; peer review: 1 approved]** *F1000Research* 2020, 9:63 [doi: 10.12688/f1000research.16665.1](https://f1000research.com/articles/9-63/v1). You can cite the sarek zenodo record for a specific version using the following [doi: 10.5281/zenodo.3476426](https://zenodo.org/badge/latestdoi/184289291) From 17dab24b139d087e4e7c1baea86141fe2ff89d3c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 10 Mar 2020 10:24:43 +0100 Subject: [PATCH 08/22] update documentation --- docs/annotation.md | 28 +++++++++++++++------- download_cache.nf | 60 ++++++++++++++++++++++------------------------ main.nf | 1 + 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/docs/annotation.md b/docs/annotation.md index 8500a6d59e..c34b04a63c 100644 --- a/docs/annotation.md +++ b/docs/annotation.md @@ -16,16 +16,26 @@ With Sarek, annotation is done using `snpEff`, `VEP`, or even both consecutively VCF produced by Sarek will be annotated if `snpEff` or `VEP` are specified with the `--tools` command. As Sarek will use `bgzip` and `tabix` to compress and index VCF files annotated, it expects VCF files to be sorted. -In these examples, all command lines will be launched starting with step `annotate`. +In these examples, all command lines will be launched starting with `--step annotate`. It can of course be started directly from any other step instead. ## Using genome specific containers -Sarek has already designed containers with `snpEff` and `VEP` files for `GRCh37`, `GRCh38` and `GRCm38`. +Sarek has already designed containers with `snpEff` and `VEP` files for Human (`GRCh37`, `GRCh38`) Mouse (`GRCm38`), Dog (`CanFam3.1`) and Roundworm (`WBcel235`). Default settings will run using these containers. The main Sarek container has also `snpEff` and `VEP` installed, but without the cache files that can be downloaded separately. +## Download cache + +An Nextflow helper script has been designed to help downloading `snpEff` and `VEP` cache. +Such files are meant to be share between multiple users, so this script is mainly meant for people administrating servers, clusters and advanced users. + +```bash +nextflow run download_cache.nf --snpeff_cache --snpeff_db --genome +nextflow run download_cache.nf --vep_cache --species --vep_cache_version --genome +``` + ## Using downloaded cache Both `snpEff` and `VEP` enable usage of cache. @@ -36,8 +46,8 @@ The cache will only be used when `--annotation_cache` and cache directories are Example: ```bash -nextflow run nf-core/sarek --tools snpEff --step annotate --sample file.vcf.gz --snpEff_cache /Path/To/snpEffCache --annotation_cache -nextflow run nf-core/sarek --tools VEP --step annotate --sample file.vcf.gz --vep_cache /Path/To/vepCache --annotation_cache +nextflow run nf-core/sarek --tools snpEff --step annotate --sample file.vcf.gz --snpEff_cache --annotation_cache +nextflow run nf-core/sarek --tools VEP --step annotate --sample file.vcf.gz --vep_cache --annotation_cache ``` ## Using VEP CADD plugin @@ -52,10 +62,10 @@ Example: ```bash nextflow run nf-core/sarek --step annotate --tools VEP --sample file.vcf.gz --cadd_cache \ - --cadd_InDels /PathToCADD/InDels.tsv.gz \ - --cadd_InDels_tbi /PathToCADD/InDels.tsv.gz.tbi \ - --cadd_WG_SNVs /PathToCADD/whole_genome_SNVs.tsv.gz \ - --cadd_WG_SNVs_tbi /PathToCADD/whole_genome_SNVs.tsv.gz.tbi + --cadd_InDels \ + --cadd_InDels_tbi \ + --cadd_WG_SNVs \ + --cadd_WG_SNVs_tbi ``` ### Downloading CADD files @@ -64,7 +74,7 @@ An helper script has been designed to help downloading CADD files. Such files are meant to be share between multiple users, so this script is mainly meant for people administrating servers, clusters and advanced users. ```bash -nextflow run downloadcache.nf --cadd_cache /Path/To/CADDcache --cadd_version --genome +nextflow run download_cache.nf --cadd_cache --cadd_version --genome ``` ## Using VEP GeneSplicer plugin diff --git a/download_cache.nf b/download_cache.nf index f5be2032cd..988b098d58 100644 --- a/download_cache.nf +++ b/download_cache.nf @@ -23,29 +23,27 @@ def helpMessage() { log.info nfcoreHeader() log.info""" -Usage: - --help - you're reading it - - nextflow run download_cache.nf [--snpeff_cache ] [--vep_cache ] - [--cadd_cache --cadd_version ] - --snpeff_cache - Specify path to snpEff cache - If none, will use snpEff version specified in configuration - Will use snpEff cache version for ${params.genome}: ${params.genomes[params.genome].snpeff_db} in igenomes configuration file: - Change with --genome or in configuration files - --vep_cache - Specify path to VEP cache - If none, will use VEP version specified in configuration - Will from th iGenomes configuration file for ${params.genome}: - VEP cache version: ${params.genomes[params.genome].vep_cache_version} - and species ${params.genomes[params.genome].species} - Change with --genome or in configuration files - --cadd_cache - Specify path to CADD cache - Will use CADD version specified - --cadd_version - Will specify which CADD version to download + Usage: + + The typical command for running the pipeline is as follows: + + nextflow run nf-core/sarek/download_cache.nf -profile docker --genome --help + [--snpeff_cache --snpeff_db_version ] + [--vep_cache --vep_cache_version --species ] + [--cadd_cache --cadd_version ] + + Options: + --help [bool] You're reading it + --snpeff_cache [file] Path to snpEff cache + --snpeff_db_version [str] snpEff DB version + Default: ${params.genomes[params.genome].snpeff_db} + --vep_cache [file] Path to VEP cache + --vep_cache_version [int] VEP cache version + Default: ${params.genomes[params.genome].vep_cache_version} + --species [str] Species + Default: ${params.genomes[params.genome].species} + --cadd_cache [file] Path to CADD cache + --cadd_version [str] CADD version to download """.stripIndent() } @@ -216,20 +214,18 @@ process DownloadCADD { cadd_files = cadd_files.dump(tag: 'cadd_files') -def nfcoreHeader(){ +def nfcoreHeader() { // Log colors ANSI codes - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_dim = params.monochrome_logs ? '' : "\033[2m"; c_black = params.monochrome_logs ? '' : "\033[0;30m"; - c_red = params.monochrome_logs ? '' : "\033[0;31m"; - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; c_blue = params.monochrome_logs ? '' : "\033[0;34m"; + c_dim = params.monochrome_logs ? '' : "\033[2m"; + c_green = params.monochrome_logs ? '' : "\033[0;32m"; c_purple = params.monochrome_logs ? '' : "\033[0;35m"; - c_cyan = params.monochrome_logs ? '' : "\033[0;36m"; + c_reset = params.monochrome_logs ? '' : "\033[0m"; c_white = params.monochrome_logs ? '' : "\033[0;37m"; + c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; - return """ ${c_dim}----------------------------------------------------${c_reset} + return """ -${c_dim}--------------------------------------------------${c_reset}- ${c_green},--.${c_black}/${c_green},-.${c_reset} ${c_blue} ___ __ __ __ ___ ${c_green}/,-._.--~\'${c_reset} ${c_blue} |\\ | |__ __ / ` / \\ |__) |__ ${c_yellow}} {${c_reset} @@ -243,7 +239,7 @@ def nfcoreHeader(){ ${c_white}`${c_green}|${c_reset}____${c_green}\\${c_reset}´${c_reset} ${c_purple} nf-core/sarek v${workflow.manifest.version}${c_reset} - ${c_dim}----------------------------------------------------${c_reset} + -${c_dim}--------------------------------------------------${c_reset}- """.stripIndent() } diff --git a/main.nf b/main.nf index a64e49c415..08e9876c22 100644 --- a/main.nf +++ b/main.nf @@ -44,6 +44,7 @@ def helpMessage() { Default: Mapping Options: + --help [bool] You're reading it --no_gvcf [bool] No g.vcf output from HaplotypeCaller --no_strelka_bp [bool] Will not use Manta candidateSmallIndels for Strelka as Best Practice --no_intervals [bool] Disable usage of intervals From e4fb71a01f1aa25d8398482a5837f544754cf7a5 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 10:48:48 +0100 Subject: [PATCH 09/22] Add variables for access tokens --- .github/workflows/aws.yml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 06483a0ba6..7661ba1da7 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -2,19 +2,30 @@ name: AWS Megatests on: - push: - branches: - - '*' pull_request: branches: - '*' + push: + branches: + - '*' + release: + types: [published] jobs: - install-awscli: - name: Install awscli + run-awstest: + name: Run AWS test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: goanpeca/setup-miniconda@v1 + with: + auto-update-conda: true python-version: 3.7 - - name: Conda install awscli - run: conda install -c conda-forge awscli \ No newline at end of file + - name: awscli install + run: conda install -c conda-forge awscli + - name: Start AWS batch job + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} + TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} + run: | + aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" From 717da50d795b5947d55677ba43e28d1af126fd64 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 10:54:21 +0100 Subject: [PATCH 10/22] Test awscli install separatly --- .github/workflows/aws.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 7661ba1da7..8358e55749 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -16,16 +16,7 @@ jobs: name: Run AWS test runs-on: ubuntu-latest steps: - - uses: goanpeca/setup-miniconda@v1 - with: - auto-update-conda: true - python-version: 3.7 - - name: awscli install - run: conda install -c conda-forge awscli - - name: Start AWS batch job - env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} - TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} - run: | - aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" + - uses: actions/checkout@v2 + python-version: 3.7 + - name: Conda install awscli + run: conda install -c conda-forge awscli From 8aa31bbdb6812f41fd275db089e5fef3533a43ec Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 10:59:00 +0100 Subject: [PATCH 11/22] Add with: keyword --- .github/workflows/aws.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 8358e55749..7c680d92e8 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: python-version: 3.7 - name: Conda install awscli run: conda install -c conda-forge awscli From 3f52deef8b599fcaab3770d7e53f55b8b44c7488 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 11:24:21 +0100 Subject: [PATCH 12/22] Change action to goanpeca and test aws install --- .github/workflows/aws.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 7c680d92e8..8c15ec4220 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -16,8 +16,11 @@ jobs: name: Run AWS test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1.0.2 with: - python-version: 3.7 - - name: Conda install awscli - run: conda install -c conda-forge awscli + auto-update-conda: true + python-version: 3.7 + - name: Install awscli + run: conda install -c conda-forge awscli + \ No newline at end of file From a8a19e4f89b4de818d50178f793e892ab905c985 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 11:28:38 +0100 Subject: [PATCH 13/22] Add aws run back in --- .github/workflows/aws.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 8c15ec4220..7038e3452b 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -23,4 +23,11 @@ jobs: python-version: 3.7 - name: Install awscli run: conda install -c conda-forge awscli + - name: Start AWS batch job + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} + TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} + run: | + aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" \ No newline at end of file From 58200c28e245de4933965d307f75b03944837ffb Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 12:22:12 +0100 Subject: [PATCH 14/22] set to only be run on PR to master or release --- .github/workflows/aws.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 7038e3452b..111f9e2b8c 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -4,10 +4,7 @@ name: AWS Megatests on: pull_request: branches: - - '*' - push: - branches: - - '*' + - 'master' release: types: [published] From 61da7db6542efaae4207a64ac669b5ab8b32653b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 16:48:57 +0100 Subject: [PATCH 15/22] update CHANGELOG + contributor list --- CHANGELOG.md | 6 +++++- README.md | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3306980fe6..7145a21e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#115](https://github.com/nf-core/sarek/pull/115) - Add [@szilvajuhos](https://github.com/szilvajuhos) abstract for ESHG2020 - [#117](https://github.com/nf-core/sarek/pull/117) - Add `Trim Galore` possibilities to Sarek - [#141](https://github.com/nf-core/sarek/pull/141) - Add containers for `WBcel235` -- GitHub actions AWS megatests +- [#154](https://github.com/nf-core/sarek/pull/154) - Add AWS mega test GitHub Actions ### `Changed` @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#141](https://github.com/nf-core/sarek/pull/141) - Update `VEP` databases to `99` - [#143](https://github.com/nf-core/sarek/pull/143) - Revert `snpEff` cache version to `75` for `GRCh37` - [#143](https://github.com/nf-core/sarek/pull/143) - Revert `snpEff` cache version to `86` for `GRCh38` +- [#152](https://github.com/nf-core/sarek/pull/152) - Update docs ### `Fixed` @@ -52,9 +53,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#129](https://github.com/nf-core/sarek/pull/129) - Apply @drpatelh fix for `mardown_to_html.py` compatibility with Python 2 - [#129](https://github.com/nf-core/sarek/pull/129) - Removed `Python` `3.7.3` from conda environment due to incompatibility - [#129](https://github.com/nf-core/sarek/pull/129) - Change ascii characters that were not supported from the `output.md` docs +- [#140](https://github.com/nf-core/sarek/pull/140) - Fix extra T/N combinations for `ASCAT` cf [#136](https://github.com/nf-core/sarek/issues/136) - [#141](https://github.com/nf-core/sarek/pull/141) - Fix `download_cache.nf` script to download cache for `snpEff` and `VEP` - [#143](https://github.com/nf-core/sarek/pull/143) - Fix annotation CI testing with `snpEff` and `VEP` - [#144](https://github.com/nf-core/sarek/pull/144) - Fix CircleCI for building `VEP` containers +- [#146](https://github.com/nf-core/sarek/pull/146) - Fix `--no_intervals` for `Mutect2` cf [#135](https://github.com/nf-core/sarek/issues/135) ### `Deprecated` @@ -93,6 +96,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#107](https://github.com/nf-core/sarek/pull/107) - `--singleCPUMem` is now removed, use `--single_cpu_mem` instead - [#107](https://github.com/nf-core/sarek/pull/107) - `--snpeffDb` is now removed, use `--snpeff_db` instead - [#107](https://github.com/nf-core/sarek/pull/107) - `--vepCacheVersion` is now removed, use `--vep_cache_version` instead +- [#152](https://github.com/nf-core/sarek/pull/152) - Removed `Jenkinsfile` ## [2.5.2] - Jåkkåtjkaskajekna diff --git a/README.md b/README.md index 77d7156b60..4fa84f9aea 100644 --- a/README.md +++ b/README.md @@ -82,19 +82,28 @@ Main authors: Helpful contributors: -* [Johannes Alneberg](https://github.com/alneberg) -* [Phil Ewels](https://github.com/ewels) +* [Adrian Lärkeryd](https://github.com/adrlar) +* [Alexander Peltzer](https://github.com/apeltzer) +* [Chela James](https://github.com/chelauk) +* [Francesco L](https://github.com/nibscles) +* [FriederikeHanssen](https://github.com/FriederikeHanssen) +* [Gisela Gabernet](https://github.com/ggabernet) * [Jesper Eisfeldt](https://github.com/J35P312) +* [Johannes Alneberg](https://github.com/alneberg) +* [KochTobi](https://github.com/KochTobi) +* [Lucia Conde](https://github.com/lconde-ucl) * [Malin Larsson](https://github.com/malinlarsson) * [Marcel Martin](https://github.com/marcelm) -* [Alexander Peltzer](https://github.com/apeltzer) * [Nilesh Tawari](https://github.com/nilesh-tawari) +* [Phil Ewels](https://github.com/ewels) +* [Sabrina Krakau](https://github.com/skrakau) +* [Sebastian-D](https://github.com/Sebastian-D) +* [Winni Kretzschmar](https://github.com/winni2k) * [arontommi](https://github.com/arontommi) * [bjornnystedt](https://github.com/bjornnystedt) +* [cgpu](https://github.com/cgpu) * [gulfshores](https://github.com/gulfshores) -* [KochTobi](https://github.com/KochTobi) * [pallolason](https://github.com/pallolason) -* [Sebastian-D](https://github.com/Sebastian-D) * [silviamorins](https://github.com/silviamorins) ## Contributions & Support From 0f5132267305ce07d3df265177aeafcca4320eed Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 17:04:02 +0100 Subject: [PATCH 16/22] Move changes from aws.yml to awstest.yml --- .github/workflows/aws.yml | 30 ------------------------ .github/workflows/awstest.yml | 43 +++++++++++++++++------------------ 2 files changed, 21 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/aws.yml diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml deleted file mode 100644 index 111f9e2b8c..0000000000 --- a/.github/workflows/aws.yml +++ /dev/null @@ -1,30 +0,0 @@ - -name: AWS Megatests - -on: - pull_request: - branches: - - 'master' - release: - types: [published] - -jobs: - run-awstest: - name: Run AWS test - runs-on: ubuntu-latest - steps: - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.0.2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} - TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} - run: | - aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" - \ No newline at end of file diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index d9b0ce4d18..801d68ec8f 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -1,31 +1,30 @@ -name: nf-core aws test +name: AWS Megatests # This workflow is triggered on PRs to the master branch. # It runs the -profile 'test' on AWS batch + on: pull_request: branches: - - master + - 'master' release: types: [published] jobs: - Awstest: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ['ubuntu-latest'] - python-version: ['3.7'] - steps: - - uses: goanpeca/setup-miniconda@v1 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - - name: Install aws client - run: conda install -c conda-forge awscli - - name: Start AWS batch job - env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} - TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} - run: | - aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" + run-awstest: + name: Run AWS test + runs-on: ubuntu-latest + steps: + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1.0.2 + with: + auto-update-conda: true + python-version: 3.7 + - name: Install awscli + run: conda install -c conda-forge awscli + - name: Start AWS batch job + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} + TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} + run: | + aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" From f1c14152364acd1bcadb775f622c7f3de6936b97 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Wed, 11 Mar 2020 22:44:20 +0100 Subject: [PATCH 17/22] Add nextflow tower properly as suggested --- .github/workflows/awstest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index 801d68ec8f..a8a379412e 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -5,7 +5,7 @@ name: AWS Megatests on: pull_request: branches: - - 'master' + - master release: types: [published] @@ -27,4 +27,4 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}} TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}} run: | - aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides command=nf-core/sarek,"-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA}" + aws batch submit-job --region eu-west-1 --job-name nf-core-sarek --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow --container-overrides '{"command": ["nf-core/sarek", "-r ${GITHUB_SHA} -profile test --outdir s3://nf-core-awsmegatests/sarek/results-${GITHUB_SHA} -w s3://nf-core-awsmegatests/sarek/work-${GITHUB_SHA} -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' From 9c372bef3c676cf0e7d10e9d87c98ad34d7c4134 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 12 Mar 2020 09:33:02 +0100 Subject: [PATCH 18/22] Update docs/annotation.md Co-Authored-By: Gisela Gabernet --- docs/annotation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/annotation.md b/docs/annotation.md index c34b04a63c..e7df1f4287 100644 --- a/docs/annotation.md +++ b/docs/annotation.md @@ -28,7 +28,7 @@ The main Sarek container has also `snpEff` and `VEP` installed, but without the ## Download cache -An Nextflow helper script has been designed to help downloading `snpEff` and `VEP` cache. +A Nextflow helper script has been designed to help downloading `snpEff` and `VEP` cache. Such files are meant to be share between multiple users, so this script is mainly meant for people administrating servers, clusters and advanced users. ```bash From f4fd830d21f0d04061cfcfb56afe1a3b191b0ef9 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 12 Mar 2020 09:33:11 +0100 Subject: [PATCH 19/22] Update docs/annotation.md Co-Authored-By: Gisela Gabernet --- docs/annotation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/annotation.md b/docs/annotation.md index e7df1f4287..48521947cb 100644 --- a/docs/annotation.md +++ b/docs/annotation.md @@ -29,7 +29,7 @@ The main Sarek container has also `snpEff` and `VEP` installed, but without the ## Download cache A Nextflow helper script has been designed to help downloading `snpEff` and `VEP` cache. -Such files are meant to be share between multiple users, so this script is mainly meant for people administrating servers, clusters and advanced users. +Such files are meant to be shared between multiple users, so this script is mainly meant for people administrating servers, clusters and advanced users. ```bash nextflow run download_cache.nf --snpeff_cache --snpeff_db --genome From ba9b481ac23835aaa704ac10618b91628161bf6e Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 12 Mar 2020 10:23:13 +0100 Subject: [PATCH 20/22] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7145a21e0c..b16d6abfb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#115](https://github.com/nf-core/sarek/pull/115) - Add [@szilvajuhos](https://github.com/szilvajuhos) abstract for ESHG2020 - [#117](https://github.com/nf-core/sarek/pull/117) - Add `Trim Galore` possibilities to Sarek - [#141](https://github.com/nf-core/sarek/pull/141) - Add containers for `WBcel235` -- [#154](https://github.com/nf-core/sarek/pull/154) - Add AWS mega test GitHub Actions +- [#150](https://github.com/nf-core/sarek/pull/150), [#151](https://github.com/nf-core/sarek/pull/151), [#154](https://github.com/nf-core/sarek/pull/154) - Add AWS mega test GitHub Actions ### `Changed` From 1f4f7cfb57665c927f3c963d4168535b11a3c039 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 12 Mar 2020 17:54:59 +0100 Subject: [PATCH 21/22] fix typo + add sentieon to condition for dbsnp channels creation --- main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.nf b/main.nf index 08e9876c22..788301b250 100644 --- a/main.nf +++ b/main.nf @@ -285,8 +285,8 @@ if (params.chrDir) exit 1, "The params `--chrDir` has been removed.\n\tPlease ch params.chrLength = null if (params.chrLength) exit 1, "The params `--chrLength` has been removed.\n\tPlease check: https://nf-co.re/sarek/docs/usage.md#--chr_length" -params.dnsnpIndex = null -if (params.dnsnpIndex) exit 1, "The params `--dnsnpIndex` has been removed.\n\tPlease check: https://nf-co.re/sarek/docs/usage.md#--dnsnp_index" +params.dbsnpIndex = null +if (params.dbsnpIndex) exit 1, "The params `--dbsnpIndex` has been removed.\n\tPlease check: https://nf-co.re/sarek/docs/usage.md#--dbsnp_index" params.fastaFai = null if (params.fastaFai) exit 1, "The params `--fastaFai` has been removed.\n\tPlease check: https://nf-co.re/sarek/docs/usage.md#--fasta_fai" @@ -438,7 +438,7 @@ params.ac_loci_gc = params.genome && 'ascat' in tools ? params.genomes[params.ge params.bwa = params.genome && params.fasta && 'mapping' in step ? params.genomes[params.genome].bwa ?: null : null params.chr_dir = params.genome && 'controlfreec' in tools ? params.genomes[params.genome].chr_dir ?: null : null params.chr_length = params.genome && 'controlfreec' in tools ? params.genomes[params.genome].chr_length ?: null : null -params.dbsnp = params.genome && ('mapping' in step || 'controlfreec' in tools || 'haplotypecaller' in tools || 'mutect2' in tools) ? params.genomes[params.genome].dbsnp ?: null : null +params.dbsnp = params.genome && ('mapping' in step || 'controlfreec' in tools || 'haplotypecaller' in tools || 'mutect2' in tools || params.sentieon) ? params.genomes[params.genome].dbsnp ?: null : null params.dbsnp_index = params.genome && params.dbsnp ? params.genomes[params.genome].dbsnp_index ?: null : null params.dict = params.genome && params.fasta ? params.genomes[params.genome].dict ?: null : null params.fasta_fai = params.genome && params.fasta ? params.genomes[params.genome].fasta_fai ?: null : null @@ -456,7 +456,7 @@ ch_ac_loci = params.ac_loci && 'ascat' in tools ? Channel.value(file(params.ac_l ch_ac_loci_gc = params.ac_loci_gc && 'ascat' in tools ? Channel.value(file(params.ac_loci_gc)) : "null" ch_chr_dir = params.chr_dir && 'controlfreec' in tools ? Channel.value(file(params.chr_dir)) : "null" ch_chr_length = params.chr_length && 'controlfreec' in tools ? Channel.value(file(params.chr_length)) : "null" -ch_dbsnp = params.dbsnp && ('mapping' in step || 'controlfreec' in tools || 'haplotypecaller' in tools || 'mutect2' in tools) ? Channel.value(file(params.dbsnp)) : "null" +ch_dbsnp = params.dbsnp && ('mapping' in step || 'controlfreec' in tools || 'haplotypecaller' in tools || 'mutect2' in tools || params.sentieon) ? Channel.value(file(params.dbsnp)) : "null" ch_fasta = params.fasta && !('annotate' in step) ? Channel.value(file(params.fasta)) : "null" ch_fai = params.fasta_fai && !('annotate' in step) ? Channel.value(file(params.fasta_fai)) : "null" ch_germline_resource = params.germline_resource && 'mutect2' in tools ? Channel.value(file(params.germline_resource)) : "null" From 7e9ce5aafbcfc4b9cf59f25517c5b00d7513bb5c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 13 Mar 2020 15:18:33 +0100 Subject: [PATCH 22/22] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b16d6abfb2..d678bf7abb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - [#143](https://github.com/nf-core/sarek/pull/143) - Fix annotation CI testing with `snpEff` and `VEP` - [#144](https://github.com/nf-core/sarek/pull/144) - Fix CircleCI for building `VEP` containers - [#146](https://github.com/nf-core/sarek/pull/146) - Fix `--no_intervals` for `Mutect2` cf [#135](https://github.com/nf-core/sarek/issues/135) +- [#156](https://github.com/nf-core/sarek/pull/156) - Fix typos +- [#156](https://github.com/nf-core/sarek/pull/156) - Fix issues with `dbsnp` files while using only `Sention` tools ### `Deprecated`