diff --git a/CHANGELOG.md b/CHANGELOG.md index 26a1a835..9252d553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed +- Switch to generalized resource handling - Update NFTest paths - Fix single tool run logic - Update PipeVal 4.0.0-rc.2 -> 5.1.0 diff --git a/README.md b/README.md index a583acff..6e2186a8 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,8 @@ input: *Providing `intersect_regions` is required and will limit the final output to just those regions. All regions of the reference genome could be provided as a `bed` file with all contigs, however it is HIGHLY recommended to remove `decoy` contigs from the human reference genome. Including these thousands of small contigs will require the user to increase available memory for `Mutect2` and will cause a very long runtime for `Strelka2`. See [Discussion here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/216). For `uclahs-cds` users, a GRCh38 `bed.gz` file can be found here: `/hot/resource/tool-specific-input/pipeline-call-sSNV-6.0.0/GRCh38-BI-20160721/Homo_sapiens_assembly38_no-decoy.bed.gz`. ### Base resource allocation updaters -To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in the [node-specific config files](./config/) +To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in `config/resources.json`. If available resources have matched cpus and memory within `90% - 1GB` of one of the pre-specified configurations, that configuration will be used. Otherwise the default configuration will be used. A spreadsheet view of the resource configuration as of Dec 2024 is [here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/328). For very large or challanging input samples, we suggest using the `m64` configuration or similar. + ```Nextflow base_resource_update { diff --git a/config/F16.config b/config/F16.config deleted file mode 100644 index 51387251..00000000 --- a/config/F16.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 6 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - withName: call_sSNV_Strelka2 { - cpus = 6 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 8.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - withName: call_sSNV_MuSE { - cpus = 6 - memory = 24.GB - retry_strategy { - memory { - strategy = 'add' - operand = 8.GB - } - } - } - } - withName: run_sump_MuSE { - cpus = 8 - memory = 16.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F2.config b/config/F2.config deleted file mode 100644 index 51686737..00000000 --- a/config/F2.config +++ /dev/null @@ -1,37 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. -// The configuration below forces processes to run one at a time by needing -// the total memory available on a lowmem node. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1500.MB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1500.MB - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1500.MB - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1500.MB - } - withName: call_sIndel_Manta { - cpus = 2 - } - withName: call_sSNV_Strelka2 { - cpus = 2 - } - } - withName: plot_VennDiagram_R { - cpus = 1 - } - withName: concat_VCFs_BCFtools { - cpus = 1 - } -} diff --git a/config/F32.config b/config/F32.config deleted file mode 100644 index 49318d96..00000000 --- a/config/F32.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 8 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 12 - memory = 48.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - withName: run_sump_MuSE { - cpus = 12 - memory = 18.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F72.config b/config/F72.config deleted file mode 100644 index 0ee916ca..00000000 --- a/config/F72.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 8 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 12 - memory = 60.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: run_sump_MuSE { - cpus = 12 - memory = 18.GB - retry_strategy { - memory { - strategy = 'add' - operand = 24.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F8.config b/config/F8.config deleted file mode 100644 index 0ea30b4c..00000000 --- a/config/F8.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 4 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 4 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 8.GB - retry_strategy { - memory { - strategy = 'exponential' - operand = 2 - } - } - } - withName: call_sSNV_MuSE { - cpus = 4 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 6.GB - } - } - } - withName: run_sump_MuSE { - cpus = 4 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 4.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/M64.config b/config/M64.config deleted file mode 100644 index 923e4736..00000000 --- a/config/M64.config +++ /dev/null @@ -1,131 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 20.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 100.GB - retry_strategy { - memory { - strategy = 'add' - operand = 100.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: generate_ReadCount_bam_readcount { - cpus = 1 - memory = 20.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 28 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 16 - memory = 120.GB - retry_strategy { - memory { - strategy = 'add' - operand = 60.GB - } - } - } - withName: run_sump_MuSE { - cpus = 6 - memory = 40.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 2 - memory = 5.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/methods.config b/config/methods.config index d6881dc0..c3cdad6e 100644 --- a/config/methods.config +++ b/config/methods.config @@ -1,29 +1,16 @@ import nextflow.util.SysHelper includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/schema/schema.config" -includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/retry/retry.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/methods/common_methods.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/bam/bam_parser.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/store_object_as_json/store_object_as_json.config" +includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/resource_handler/resource_handler.config" methods { set_process = { process.cache = params.cache_intermediate_pipeline_steps } - modify_base_allocations = { - if (!(params.containsKey('base_resource_update') && params.base_resource_update)) { - return - } - - params.base_resource_update.each { resource, updates -> - updates.each { processes, multiplier -> - def processes_to_update = (custom_schema_types.is_string(processes)) ? [processes] : processes - methods.update_base_resource_allocation(resource, multiplier, processes_to_update) - } - } - } - sanitize_string = { raw -> if (![String, GString].any{ raw in it }) { throw new Exception("Input to sanitize is either empty or not a string! Provide a non-empty string.") @@ -147,10 +134,8 @@ methods { } else { methods.get_vcfs_to_process() } + resource_handler.handle_resources("${projectDir}/config/resources.json") methods.set_process() - methods.set_resources_allocation() - methods.modify_base_allocations() - retry.setup_retry() methods.set_env() methods.set_output_directory() methods.set_intersect_regions_params() diff --git a/config/resources.json b/config/resources.json new file mode 100644 index 00000000..59ce78a1 --- /dev/null +++ b/config/resources.json @@ -0,0 +1,1472 @@ +{ + "f2": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.25, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.39, + "max": "1 GB" + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.39, + "max": "1 GB" + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.39, + "max": "1 GB" + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.25, + "max": "1 GB" + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 2, + "fraction": 1, + "max": 2 + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 2, + "fraction": 1, + "max": 2 + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + } + } + }, + "f8": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "3 GB", + "fraction": 0.2, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "3 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "2 GB", + "fraction": 0.13, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.2, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "8 GB", + "fraction": 0.53, + "max": "8 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "8 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "10 GB", + "fraction": 0.66, + "max": "10 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "12 GB", + "fraction": 0.79, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "4 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + }, + "f16": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "3 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "6 GB", + "fraction": 0.2, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "2 GB", + "fraction": 0.07, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "12 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.07, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "8 GB", + "fraction": 0.26, + "max": "8 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "8 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "24 GB", + "fraction": 0.79, + "max": "24 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "8 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "16 GB", + "fraction": 0.53, + "max": "16 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + }, + "f32": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.25, + "max": 8 + }, + "memory": { + "min": "6 GB", + "fraction": 0.1, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 8, + "fraction": 0.25, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.03, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "12 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.03, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "2 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "12 GB", + "fraction": 0.2, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.38, + "max": 12 + }, + "memory": { + "min": "48 GB", + "fraction": 0.79, + "max": "48 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "18 GB", + "fraction": 0.3, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + }, + "f72": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "6 GB", + "fraction": 0.04, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "12 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "3 GB", + "fraction": 0.02, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "3 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "12 GB", + "fraction": 0.09, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.17, + "max": 12 + }, + "memory": { + "min": "60 GB", + "fraction": 0.44, + "max": "60 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.17, + "max": 6 + }, + "memory": { + "min": "18 GB", + "fraction": 0.13, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "24 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + }, + "m64": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.001, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "20 GB", + "fraction": 0.02, + "max": "25 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "100 GB", + "fraction": 0.1, + "max": "100 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "19 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "39 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.13, + "max": 8 + }, + "memory": { + "min": "10 GB", + "fraction": 0.01, + "max": "10 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 28, + "fraction": 0.44, + "max": 28 + }, + "memory": { + "min": "30 GB", + "fraction": 0.03, + "max": "30 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "30 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "6 GB", + "fraction": 0.03, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "18 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "30 GB", + "fraction": 0.03, + "max": "30 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "30 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 16, + "fraction": 0.25, + "max": 16 + }, + "memory": { + "min": "120 GB", + "fraction": 0.12, + "max": "120 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "60 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.09, + "max": 6 + }, + "memory": { + "min": "40 GB", + "fraction": 0.04, + "max": "40 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "14 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + }, + "default": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "16 GB" + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "5 GB" + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 2, + "fraction": 0.2, + "max": 8 + }, + "memory": { + "min": "3 GB", + "fraction": 0.04, + "max": "8 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "4 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 2, + "fraction": 0.25, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.02, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "9 GB" + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "8 GB", + "fraction": 0.2, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.17, + "max": 40 + }, + "memory": { + "min": "10 GB", + "fraction": 0.7, + "max": "60 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "20 GB" + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.17, + "max": 6 + }, + "memory": { + "min": "12 GB", + "fraction": 0.13, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "24 GB" + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "10 GB" + } + } + } + } +} \ No newline at end of file diff --git a/external/pipeline-Nextflow-config b/external/pipeline-Nextflow-config index 23b7b4ea..07c99b8d 160000 --- a/external/pipeline-Nextflow-config +++ b/external/pipeline-Nextflow-config @@ -1 +1 @@ -Subproject commit 23b7b4ea3bdea24aeb180cecb9cf5aeb270892d3 +Subproject commit 07c99b8d93f4232147546302bcae2bdb2a346705 diff --git a/test/configtest-F16.json b/test/configtest-F16.json index ecd72719..3da5ff1e 100644 --- a/test/configtest-F16.json +++ b/test/configtest-F16.json @@ -64,6 +64,60 @@ "muse" ], "bam_readcount_version": "1.0.1", + "base_allocations": { + "call_sIndel_Manta": { + "cpus": "6", + "memory": "6 GB" + }, + "call_sSNV_MuSE": { + "cpus": "6", + "memory": "24 GB" + }, + "call_sSNV_Mutect2": { + "cpus": "1", + "memory": "2 GB" + }, + "call_sSNV_SomaticSniper": { + "cpus": "1", + "memory": "1 GB" + }, + "call_sSNV_Strelka2": { + "cpus": "6", + "memory": "2 GB" + }, + "concat_VCFs_BCFtools": { + "cpus": "1", + "memory": "1 GB" + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "create_IndelCandidate_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": "1 GB" + }, + "plot_VennDiagram_R": { + "cpus": "1", + "memory": "1 GB" + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": "1", + "memory": "8 GB" + }, + "run_sump_MuSE": { + "cpus": "4", + "memory": "16 GB" + }, + "run_validate_PipeVal": { + "cpus": "1", + "memory": "1 GB" + } + }, "bgzip_extra_args": "", "bpg_version": "7.1.0", "cache_intermediate_pipeline_steps": false, @@ -111,8 +165,10 @@ "manta_version": "1.6.0", "max_cpus": "16", "max_memory": "31 GB", + "max_time": "24d 20h 31m 24s", "min_cpus": "1", "min_memory": "1 MB", + "min_time": "1s", "mutect2_extra_args": "", "ncbi_build": "GRCh38", "normal_id": "0192847", @@ -121,128 +177,81 @@ "panel_of_normals_vcf": "", "patient_id": "TWGSAMIN000001", "pipeval_version": "5.1.0", - "proc_resource_params": { + "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", + "retry_information": { "call_sIndel_Manta": { - "cpus": "6", - "memory": "6 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - }, - "withName:call_sSNV_Strelka2": { - "cpus": "6", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - } + "memory": { + "operand": "0", + "strategy": "add" + } + }, + "call_sSNV_MuSE": { + "memory": { + "operand": "8 GB", + "strategy": "add" } }, "call_sSNV_Mutect2": { - "cpus": "1", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "2 GB", - "strategy": "add" - } + "memory": { + "operand": "2 GB", + "strategy": "add" } }, "call_sSNV_SomaticSniper": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "3 GB", - "strategy": "add" - } + "memory": { + "operand": "3 GB", + "strategy": "add" } }, "call_sSNV_Strelka2": { - "ext": { - "retry_codes": [] + "memory": { + "operand": "12 GB", + "strategy": "add" } }, "concat_VCFs_BCFtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "convert_BAM2Pileup_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "create_IndelCandidate_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" + } + }, + "generate_ReadCount_bam_readcount": { + "memory": { + "operand": "6 GB", + "strategy": "add" } }, "plot_VennDiagram_R": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "run_LearnReadOrientationModel_GATK": { - "cpus": "1", - "memory": "8 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - }, - "withName:call_sSNV_MuSE": { - "cpus": "6", - "memory": "24 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - } + "memory": { + "operand": "8 GB", + "strategy": "add" } }, "run_sump_MuSE": { - "cpus": "8", - "memory": "16 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } - }, - "run_validate_PipeVal": { - "cpus": "1", - "memory": "1 GB" } }, - "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", "sample_id": "0192847", "samples_to_process": [ { @@ -364,22 +373,16 @@ } }, "withName:call_sIndel_Manta": { + "cpus": "6", + "memory": "6 GB" + }, + "withName:call_sSNV_MuSE": { "cpus": "6", "memory": { - "1": "6 GB", - "2": "18 GB", - "3": "30 GB", - "closure": "retry_updater(6 GB, add, 12 GB, $task.attempt, memory)" - }, - "withName:call_sSNV_Strelka2": { - "cpus": "6", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - } + "1": "24 GB", + "2": "31 GB", + "3": "31 GB", + "closure": "retry_updater(24 GB, add, 8 GB, $task.attempt, memory)" } }, "withName:call_sSNV_Mutect2": { @@ -401,8 +404,15 @@ } }, "withName:call_sSNV_Strelka2": { + "cpus": "6", "ext": { "retry_codes": [] + }, + "memory": { + "1": "2 GB", + "2": "14 GB", + "3": "26 GB", + "closure": "retry_updater(2 GB, add, 12 GB, $task.attempt, memory)" } }, "withName:concat_VCFs_BCFtools": { @@ -432,6 +442,15 @@ "closure": "retry_updater(1 GB, add, 5 GB, $task.attempt, memory)" } }, + "withName:generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": { + "1": "1 GB", + "2": "7 GB", + "3": "13 GB", + "closure": "retry_updater(1 GB, add, 6 GB, $task.attempt, memory)" + } + }, "withName:plot_VennDiagram_R": { "cpus": "1", "memory": { @@ -448,20 +467,10 @@ "2": "16 GB", "3": "24 GB", "closure": "retry_updater(8 GB, add, 8 GB, $task.attempt, memory)" - }, - "withName:call_sSNV_MuSE": { - "cpus": "6", - "memory": "24 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - } } }, "withName:run_sump_MuSE": { - "cpus": "8", + "cpus": "4", "memory": { "1": "16 GB", "2": "31 GB", @@ -478,6 +487,12 @@ "enabled": true, "file": "/tmp/outputs/call-sSNV-VER.SI.ON/0192847/log-call-sSNV-VER.SI.ON-19970704T165655Z/nextflow-log/report.html" }, + "resource_handler": { + "handle_resources": "closure()", + "set_resource_limit_params": "closure()", + "set_retry": "closure()", + "setup_retry": "closure()" + }, "sm_tag": "0192847", "timeline": { "enabled": true, diff --git a/test/configtest-F32.json b/test/configtest-F32.json index c9319c57..fbaaa5e8 100644 --- a/test/configtest-F32.json +++ b/test/configtest-F32.json @@ -64,6 +64,60 @@ "muse" ], "bam_readcount_version": "1.0.1", + "base_allocations": { + "call_sIndel_Manta": { + "cpus": "8", + "memory": "6 GB" + }, + "call_sSNV_MuSE": { + "cpus": "12", + "memory": "48 GB" + }, + "call_sSNV_Mutect2": { + "cpus": "1", + "memory": "2 GB" + }, + "call_sSNV_SomaticSniper": { + "cpus": "1", + "memory": "1 GB" + }, + "call_sSNV_Strelka2": { + "cpus": "8", + "memory": "2 GB" + }, + "concat_VCFs_BCFtools": { + "cpus": "1", + "memory": "1 GB" + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "create_IndelCandidate_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": "1 GB" + }, + "plot_VennDiagram_R": { + "cpus": "1", + "memory": "1 GB" + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": "1", + "memory": "12 GB" + }, + "run_sump_MuSE": { + "cpus": "6", + "memory": "18 GB" + }, + "run_validate_PipeVal": { + "cpus": "1", + "memory": "1 GB" + } + }, "bgzip_extra_args": "", "bpg_version": "7.1.0", "cache_intermediate_pipeline_steps": false, @@ -111,8 +165,10 @@ "manta_version": "1.6.0", "max_cpus": "32", "max_memory": "64 GB", + "max_time": "24d 20h 31m 24s", "min_cpus": "1", "min_memory": "1 MB", + "min_time": "1s", "mutect2_extra_args": "", "ncbi_build": "GRCh38", "normal_id": "0192847", @@ -121,126 +177,81 @@ "panel_of_normals_vcf": "", "patient_id": "TWGSAMIN000001", "pipeval_version": "5.1.0", - "proc_resource_params": { + "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", + "retry_information": { "call_sIndel_Manta": { - "cpus": "8", - "memory": "6 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "0", + "strategy": "add" } }, "call_sSNV_MuSE": { - "cpus": "12", - "memory": "48 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } }, "call_sSNV_Mutect2": { - "cpus": "1", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "2 GB", - "strategy": "add" - } + "memory": { + "operand": "2 GB", + "strategy": "add" } }, "call_sSNV_SomaticSniper": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "call_sSNV_Strelka2": { - "cpus": "8", - "ext": { - "retry_codes": [] - }, - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } + "memory": { + "operand": "12 GB", + "strategy": "add" } }, "concat_VCFs_BCFtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "convert_BAM2Pileup_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "create_IndelCandidate_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" + } + }, + "generate_ReadCount_bam_readcount": { + "memory": { + "operand": "6 GB", + "strategy": "add" } }, "plot_VennDiagram_R": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "run_LearnReadOrientationModel_GATK": { - "cpus": "1", - "memory": "12 GB", - "retry_strategy": { - "memory": { - "operand": "20 GB", - "strategy": "add" - } + "memory": { + "operand": "20 GB", + "strategy": "add" } }, "run_sump_MuSE": { - "cpus": "12", - "memory": "18 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } - }, - "run_validate_PipeVal": { - "cpus": "1", - "memory": "1 GB" } }, - "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", "sample_id": "0192847", "samples_to_process": [ { @@ -363,12 +374,7 @@ }, "withName:call_sIndel_Manta": { "cpus": "8", - "memory": { - "1": "6 GB", - "2": "11 GB", - "3": "16 GB", - "closure": "retry_updater(6 GB, add, 5 GB, $task.attempt, memory)" - } + "memory": "6 GB" }, "withName:call_sSNV_MuSE": { "cpus": "12", @@ -436,6 +442,15 @@ "closure": "retry_updater(1 GB, add, 5 GB, $task.attempt, memory)" } }, + "withName:generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": { + "1": "1 GB", + "2": "7 GB", + "3": "13 GB", + "closure": "retry_updater(1 GB, add, 6 GB, $task.attempt, memory)" + } + }, "withName:plot_VennDiagram_R": { "cpus": "1", "memory": { @@ -455,7 +470,7 @@ } }, "withName:run_sump_MuSE": { - "cpus": "12", + "cpus": "6", "memory": { "1": "18 GB", "2": "34 GB", @@ -472,6 +487,12 @@ "enabled": true, "file": "/tmp/outputs/call-sSNV-VER.SI.ON/0192847/log-call-sSNV-VER.SI.ON-19970704T165655Z/nextflow-log/report.html" }, + "resource_handler": { + "handle_resources": "closure()", + "set_resource_limit_params": "closure()", + "set_retry": "closure()", + "setup_retry": "closure()" + }, "sm_tag": "0192847", "timeline": { "enabled": true,