From fc56c5d67c4090e7dad94705ca0b5d867b3a58ef Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Tue, 26 Nov 2024 10:54:55 -0800 Subject: [PATCH 01/11] change F configs to resources.json --- config/F16.config | 121 --- config/F2.config | 37 - config/F32.config | 121 --- config/F72.config | 121 --- config/F8.config | 121 --- config/M64.config | 131 ---- config/methods.config | 19 +- config/resources.json | 1654 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1656 insertions(+), 669 deletions(-) delete mode 100644 config/F16.config delete mode 100644 config/F2.config delete mode 100644 config/F32.config delete mode 100644 config/F72.config delete mode 100644 config/F8.config delete mode 100644 config/M64.config create mode 100644 config/resources.json diff --git a/config/F16.config b/config/F16.config deleted file mode 100644 index 51387251..00000000 --- a/config/F16.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 6 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - withName: call_sSNV_Strelka2 { - cpus = 6 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 8.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - withName: call_sSNV_MuSE { - cpus = 6 - memory = 24.GB - retry_strategy { - memory { - strategy = 'add' - operand = 8.GB - } - } - } - } - withName: run_sump_MuSE { - cpus = 8 - memory = 16.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F2.config b/config/F2.config deleted file mode 100644 index 51686737..00000000 --- a/config/F2.config +++ /dev/null @@ -1,37 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. -// The configuration below forces processes to run one at a time by needing -// the total memory available on a lowmem node. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1500.MB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1500.MB - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1500.MB - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1500.MB - } - withName: call_sIndel_Manta { - cpus = 2 - } - withName: call_sSNV_Strelka2 { - cpus = 2 - } - } - withName: plot_VennDiagram_R { - cpus = 1 - } - withName: concat_VCFs_BCFtools { - cpus = 1 - } -} diff --git a/config/F32.config b/config/F32.config deleted file mode 100644 index 49318d96..00000000 --- a/config/F32.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 8 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 12 - memory = 48.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - withName: run_sump_MuSE { - cpus = 12 - memory = 18.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F72.config b/config/F72.config deleted file mode 100644 index 0ee916ca..00000000 --- a/config/F72.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 16.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 6.GB - retry_strategy { - memory { - strategy = 'add' - operand = 5.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 8 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 12.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 12 - memory = 60.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: run_sump_MuSE { - cpus = 12 - memory = 18.GB - retry_strategy { - memory { - strategy = 'add' - operand = 24.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/F8.config b/config/F8.config deleted file mode 100644 index 0ea30b4c..00000000 --- a/config/F8.config +++ /dev/null @@ -1,121 +0,0 @@ -// Other processes will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 4 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 3.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 4 - memory = 2.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 3.GB - retry_strategy { - memory { - strategy = 'add' - operand = 2.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 8.GB - retry_strategy { - memory { - strategy = 'exponential' - operand = 2 - } - } - } - withName: call_sSNV_MuSE { - cpus = 4 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 6.GB - } - } - } - withName: run_sump_MuSE { - cpus = 4 - memory = 12.GB - retry_strategy { - memory { - strategy = 'add' - operand = 4.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/M64.config b/config/M64.config deleted file mode 100644 index 923e4736..00000000 --- a/config/M64.config +++ /dev/null @@ -1,131 +0,0 @@ -// Other processes after create_IndelCandidate_SAMtools will only run one at a time, so -// we don't need to control their resources. - -process { - withName: run_validate_PipeVal { - cpus = 1 - memory = 1.GB - } - if (params.input_type == 'bam') { - withName: call_sSNV_SomaticSniper { - cpus = 1 - memory = 20.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: convert_BAM2Pileup_SAMtools { - cpus = 1 - memory = 100.GB - retry_strategy { - memory { - strategy = 'add' - operand = 100.GB - } - } - } - withName: create_IndelCandidate_SAMtools { - cpus = 1 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: generate_ReadCount_bam_readcount { - cpus = 1 - memory = 20.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - withName: call_sIndel_Manta { - cpus = 8 - memory = 10.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: call_sSNV_Strelka2 { - cpus = 28 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: call_sSNV_Mutect2 { - cpus = 1 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: run_LearnReadOrientationModel_GATK { - cpus = 1 - memory = 30.GB - retry_strategy { - memory { - strategy = 'add' - operand = 30.GB - } - } - } - withName: call_sSNV_MuSE { - cpus = 16 - memory = 120.GB - retry_strategy { - memory { - strategy = 'add' - operand = 60.GB - } - } - } - withName: run_sump_MuSE { - cpus = 6 - memory = 40.GB - retry_strategy { - memory { - strategy = 'add' - operand = 20.GB - } - } - } - } - withName: plot_VennDiagram_R { - cpus = 2 - memory = 5.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } - withName: concat_VCFs_BCFtools { - cpus = 1 - memory = 1.GB - retry_strategy { - memory { - strategy = 'add' - operand = 10.GB - } - } - } -} diff --git a/config/methods.config b/config/methods.config index d6881dc0..9218a77f 100644 --- a/config/methods.config +++ b/config/methods.config @@ -1,29 +1,16 @@ import nextflow.util.SysHelper includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/schema/schema.config" -includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/retry/retry.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/methods/common_methods.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/bam/bam_parser.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/store_object_as_json/store_object_as_json.config" +includeConfig "/hot/code/yashpatel/GitHub/uclahs-cds/pipeline-Nextflow-config/yashpatel-generalized-resource-handler/config/resource_handler/resource_handler.config" methods { set_process = { process.cache = params.cache_intermediate_pipeline_steps } - modify_base_allocations = { - if (!(params.containsKey('base_resource_update') && params.base_resource_update)) { - return - } - - params.base_resource_update.each { resource, updates -> - updates.each { processes, multiplier -> - def processes_to_update = (custom_schema_types.is_string(processes)) ? [processes] : processes - methods.update_base_resource_allocation(resource, multiplier, processes_to_update) - } - } - } - sanitize_string = { raw -> if (![String, GString].any{ raw in it }) { throw new Exception("Input to sanitize is either empty or not a string! Provide a non-empty string.") @@ -147,10 +134,8 @@ methods { } else { methods.get_vcfs_to_process() } + resource_handler.handle_resources("${projectDir}/config/resources.json") methods.set_process() - methods.set_resources_allocation() - methods.modify_base_allocations() - retry.setup_retry() methods.set_env() methods.set_output_directory() methods.set_intersect_regions_params() diff --git a/config/resources.json b/config/resources.json new file mode 100644 index 00000000..c7ce62ec --- /dev/null +++ b/config/resources.json @@ -0,0 +1,1654 @@ +{ + "f2": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.25, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1.5 GB", + "fraction": 0.39, + "max": "1.5 GB" + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1.5 GB", + "fraction": 0.39, + "max": "1.5 GB" + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1.5 GB", + "fraction": 0.39, + "max": "1.5 GB" + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 2, + "fraction": 1.0, + "max": 2 + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 2, + "fraction": 1.0, + "max": 2 + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + } + } + }, + "f8": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "3 GB", + "fraction": 0.20, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 3072, + "kilo": 3145728, + "giga": 3, + "bytes": 3221225472 + } + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "2 GB", + "fraction": 0.13, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "3 GB", + "fraction": 0.20, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "8 GB", + "fraction": 0.53, + "max": "8 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "exponential", + "operand": 2 + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "10 GB", + "fraction": 0.66, + "max": "10 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 6144, + "kilo": 6291456, + "giga": 6, + "bytes": 6442450944 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 4, + "fraction": 0.5, + "max": 4 + }, + "memory": { + "min": "12 GB", + "fraction": 0.79, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 4096, + "kilo": 4194304, + "giga": 4, + "bytes": 4294967296 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + }, + "f16": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 3072, + "kilo": 3145728, + "giga": 3, + "bytes": 3221225472 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "6 GB", + "fraction": 0.20, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 12288, + "kilo": 12582912, + "giga": 12, + "bytes": 12884901888 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.07, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "8 GB", + "fraction": 0.26, + "max": "8 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 8192, + "kilo": 8388608, + "giga": 8, + "bytes": 8589934592 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 8, + "fraction": 0.5, + "max": 8 + }, + "memory": { + "min": "16 GB", + "fraction": 0.53, + "max": "16 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 16384, + "kilo": 16777216, + "giga": 16, + "bytes": 17179869184 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + }, + "f32": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.25, + "max": 8 + }, + "memory": { + "min": "6 GB", + "fraction": 0.10, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 8, + "fraction": 0.25, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.03, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 12288, + "kilo": 12582912, + "giga": 12, + "bytes": 12884901888 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.03, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 2048, + "kilo": 2097152, + "giga": 2, + "bytes": 2147483648 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "12 GB", + "fraction": 0.20, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.38, + "max": 12 + }, + "memory": { + "min": "48 GB", + "fraction": 0.79, + "max": "48 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 16384, + "kilo": 16777216, + "giga": 16, + "bytes": 17179869184 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.38, + "max": 12 + }, + "memory": { + "min": "18 GB", + "fraction": 0.30, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 16384, + "kilo": 16777216, + "giga": 16, + "bytes": 17179869184 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + }, + "f72": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 16384, + "kilo": 16777216, + "giga": 16, + "bytes": 17179869184 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "6 GB", + "fraction": 0.04, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 12288, + "kilo": 12582912, + "giga": 12, + "bytes": 12884901888 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "3 GB", + "fraction": 0.02, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 3072, + "kilo": 3145728, + "giga": 3, + "bytes": 3221225472 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "12 GB", + "fraction": 0.09, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.17, + "max": 12 + }, + "memory": { + "min": "60 GB", + "fraction": 0.44, + "max": "60 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.17, + "max": 12 + }, + "memory": { + "min": "18 GB", + "fraction": 0.13, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 24576, + "kilo": 25165824, + "giga": 24, + "bytes": 25769803776 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + }, + "m64": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.001, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "20 GB", + "fraction": 0.02, + "max": "20 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "100 GB", + "fraction": 0.10, + "max": "100 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 102400, + "kilo": 104857600, + "giga": 100, + "bytes": 107374182400 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "10 GB", + "fraction": 0.01, + "max": "10 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "20 GB", + "fraction": 0.02, + "max": "20 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.13, + "max": 8 + }, + "memory": { + "min": "10 GB", + "fraction": 0.01, + "max": "10 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 28, + "fraction": 0.44, + "max": 28 + }, + "memory": { + "min": "30 GB", + "fraction": 0.03, + "max": "30 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 30720, + "kilo": 31457280, + "giga": 30, + "bytes": 32212254720 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "30 GB", + "fraction": 0.03, + "max": "30 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 30720, + "kilo": 31457280, + "giga": 30, + "bytes": 32212254720 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "30 GB", + "fraction": 0.03, + "max": "30 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 30720, + "kilo": 31457280, + "giga": 30, + "bytes": 32212254720 + } + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 16, + "fraction": 0.25, + "max": 16 + }, + "memory": { + "min": "120 GB", + "fraction": 0.12, + "max": "120 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 61440, + "kilo": 62914560, + "giga": 60, + "bytes": 64424509440 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.09, + "max": 6 + }, + "memory": { + "min": "40 GB", + "fraction": 0.04, + "max": "40 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 2, + "fraction": 0.03, + "max": 2 + }, + "memory": { + "min": "5 GB", + "fraction": 0.01, + "max": "5 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.02, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.00, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + }, + "default": { + "run_validate_PipeVal": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + } + }, + "call_sSNV_SomaticSniper": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 16384, + "kilo": 16777216, + "giga": 16, + "bytes": 17179869184 + } + } + } + }, + "create_IndelCandidate_SAMtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sIndel_Manta": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "6 GB", + "fraction": 0.04, + "max": "6 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 5120, + "kilo": 5242880, + "giga": 5, + "bytes": 5368709120 + } + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 8, + "fraction": 0.11, + "max": 8 + }, + "memory": { + "min": "2 GB", + "fraction": 0.01, + "max": "2 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 12288, + "kilo": 12582912, + "giga": 12, + "bytes": 12884901888 + } + } + } + }, + "call_sSNV_Mutect2": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "3 GB", + "fraction": 0.02, + "max": "3 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 3072, + "kilo": 3145728, + "giga": 3, + "bytes": 3221225472 + } + } + } + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "12 GB", + "fraction": 0.09, + "max": "12 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "call_sSNV_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.17, + "max": 12 + }, + "memory": { + "min": "60 GB", + "fraction": 0.44, + "max": "60 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 20480, + "kilo": 20971520, + "giga": 20, + "bytes": 21474836480 + } + } + } + }, + "run_sump_MuSE": { + "cpus": { + "min": 12, + "fraction": 0.17, + "max": 12 + }, + "memory": { + "min": "18 GB", + "fraction": 0.13, + "max": "18 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 24576, + "kilo": 25165824, + "giga": 24, + "bytes": 25769803776 + } + } + } + }, + "plot_VennDiagram_R": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + }, + "concat_VCFs_BCFtools": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": { + "mega": 10240, + "kilo": 10485760, + "giga": 10, + "bytes": 10737418240 + } + } + } + } + } +} \ No newline at end of file From fe7ebd8753834034d550fa3e595f51c0f3c170a0 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 5 Dec 2024 12:22:22 -0800 Subject: [PATCH 02/11] Update retry strategy operands --- config/resources.json | 448 ++++++------------------------------------ 1 file changed, 64 insertions(+), 384 deletions(-) diff --git a/config/resources.json b/config/resources.json index c7ce62ec..739300f5 100644 --- a/config/resources.json +++ b/config/resources.json @@ -104,12 +104,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -127,12 +122,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -150,12 +140,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -173,12 +158,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 3072, - "kilo": 3145728, - "giga": 3, - "bytes": 3221225472 - } + "operand": "3 GB" } } }, @@ -196,12 +176,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -219,12 +194,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -260,12 +230,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 6144, - "kilo": 6291456, - "giga": 6, - "bytes": 6442450944 - } + "operand": "6 GB" } } }, @@ -283,12 +248,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 4096, - "kilo": 4194304, - "giga": 4, - "bytes": 4294967296 - } + "operand": "4 GB" } } }, @@ -306,12 +266,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -329,12 +284,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } @@ -366,12 +316,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 3072, - "kilo": 3145728, - "giga": 3, - "bytes": 3221225472 - } + "operand": "3 GB" } } }, @@ -389,12 +334,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -412,12 +352,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -435,12 +370,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 12288, - "kilo": 12582912, - "giga": 12, - "bytes": 12884901888 - } + "operand": "12 GB" } } }, @@ -458,12 +388,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -481,12 +406,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 8192, - "kilo": 8388608, - "giga": 8, - "bytes": 8589934592 - } + "operand": "8 GB" } } }, @@ -504,12 +424,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 16384, - "kilo": 16777216, - "giga": 16, - "bytes": 17179869184 - } + "operand": "16 GB" } } }, @@ -527,12 +442,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -550,12 +460,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } @@ -587,12 +492,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -610,12 +510,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -633,12 +528,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -656,12 +546,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -679,12 +564,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 12288, - "kilo": 12582912, - "giga": 12, - "bytes": 12884901888 - } + "operand": "12 GB" } } }, @@ -702,12 +582,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 2048, - "kilo": 2097152, - "giga": 2, - "bytes": 2147483648 - } + "operand": "2 GB" } } }, @@ -725,12 +600,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -748,12 +618,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 16384, - "kilo": 16777216, - "giga": 16, - "bytes": 17179869184 - } + "operand": "16 GB" } } }, @@ -771,12 +636,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 16384, - "kilo": 16777216, - "giga": 16, - "bytes": 17179869184 - } + "operand": "16 GB" } } }, @@ -794,12 +654,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -817,12 +672,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } @@ -854,12 +704,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -877,12 +722,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 16384, - "kilo": 16777216, - "giga": 16, - "bytes": 17179869184 - } + "operand": "16 GB" } } }, @@ -900,12 +740,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -923,12 +758,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -946,12 +776,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 12288, - "kilo": 12582912, - "giga": 12, - "bytes": 12884901888 - } + "operand": "12 GB" } } }, @@ -969,12 +794,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 3072, - "kilo": 3145728, - "giga": 3, - "bytes": 3221225472 - } + "operand": "3 GB" } } }, @@ -992,12 +812,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1015,12 +830,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1038,12 +848,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 24576, - "kilo": 25165824, - "giga": 24, - "bytes": 25769803776 - } + "operand": "24 GB" } } }, @@ -1061,12 +866,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -1084,12 +884,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } @@ -1121,12 +916,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1144,12 +934,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 102400, - "kilo": 104857600, - "giga": 100, - "bytes": 107374182400 - } + "operand": "100 GB" } } }, @@ -1167,12 +952,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -1190,12 +970,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1213,12 +988,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -1236,12 +1006,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 30720, - "kilo": 31457280, - "giga": 30, - "bytes": 32212254720 - } + "operand": "30 GB" } } }, @@ -1259,12 +1024,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 30720, - "kilo": 31457280, - "giga": 30, - "bytes": 32212254720 - } + "operand": "30 GB" } } }, @@ -1282,12 +1042,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 30720, - "kilo": 31457280, - "giga": 30, - "bytes": 32212254720 - } + "operand": "30 GB" } } }, @@ -1305,12 +1060,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 61440, - "kilo": 62914560, - "giga": 60, - "bytes": 64424509440 - } + "operand": "60 GB" } } }, @@ -1328,12 +1078,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1351,12 +1096,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -1374,12 +1114,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } @@ -1411,12 +1146,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -1434,12 +1164,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 16384, - "kilo": 16777216, - "giga": 16, - "bytes": 17179869184 - } + "operand": "16 GB" } } }, @@ -1457,12 +1182,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -1480,12 +1200,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 5120, - "kilo": 5242880, - "giga": 5, - "bytes": 5368709120 - } + "operand": "5 GB" } } }, @@ -1503,12 +1218,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 12288, - "kilo": 12582912, - "giga": 12, - "bytes": 12884901888 - } + "operand": "12 GB" } } }, @@ -1526,12 +1236,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 3072, - "kilo": 3145728, - "giga": 3, - "bytes": 3221225472 - } + "operand": "3 GB" } } }, @@ -1549,12 +1254,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1572,12 +1272,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 20480, - "kilo": 20971520, - "giga": 20, - "bytes": 21474836480 - } + "operand": "20 GB" } } }, @@ -1595,12 +1290,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 24576, - "kilo": 25165824, - "giga": 24, - "bytes": 25769803776 - } + "operand": "24 GB" } } }, @@ -1618,12 +1308,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } }, @@ -1641,12 +1326,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": { - "mega": 10240, - "kilo": 10485760, - "giga": 10, - "bytes": 10737418240 - } + "operand": "10 GB" } } } From a6139298ee0da58ae1a5874a17a28c75ec3cd458 Mon Sep 17 00:00:00 2001 From: Yash Patel Date: Thu, 5 Dec 2024 12:22:37 -0800 Subject: [PATCH 03/11] Update config submodule --- external/pipeline-Nextflow-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/pipeline-Nextflow-config b/external/pipeline-Nextflow-config index 23b7b4ea..8b500b7d 160000 --- a/external/pipeline-Nextflow-config +++ b/external/pipeline-Nextflow-config @@ -1 +1 @@ -Subproject commit 23b7b4ea3bdea24aeb180cecb9cf5aeb270892d3 +Subproject commit 8b500b7d2f733689842d9f5c74d3616935ab2be1 From 9310459d1e09e32d291fa13c03631a2d8bfd3cb4 Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Fri, 13 Dec 2024 15:02:01 -0800 Subject: [PATCH 04/11] add missing f16 process --- config/resources.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/config/resources.json b/config/resources.json index 739300f5..9cfef607 100644 --- a/config/resources.json +++ b/config/resources.json @@ -410,6 +410,24 @@ } } }, + "call_sSNV_MuSE": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "24 GB", + "fraction": 0.79, + "max": "24 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "8 GB" + } + } + }, "run_sump_MuSE": { "cpus": { "min": 8, From 0e96472bf5edf4782f3e3b4970b9c2973d5ae18c Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Fri, 13 Dec 2024 16:58:35 -0800 Subject: [PATCH 05/11] Update submodule generalized resource handler --- external/pipeline-Nextflow-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/pipeline-Nextflow-config b/external/pipeline-Nextflow-config index 8b500b7d..07c99b8d 160000 --- a/external/pipeline-Nextflow-config +++ b/external/pipeline-Nextflow-config @@ -1 +1 @@ -Subproject commit 8b500b7d2f733689842d9f5c74d3616935ab2be1 +Subproject commit 07c99b8d93f4232147546302bcae2bdb2a346705 From 65e5d0d2b192bfcb8a8be4d6aff61139f70377f5 Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Fri, 13 Dec 2024 17:00:31 -0800 Subject: [PATCH 06/11] update resource_handler.config path --- config/methods.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/methods.config b/config/methods.config index 9218a77f..c3cdad6e 100644 --- a/config/methods.config +++ b/config/methods.config @@ -4,7 +4,7 @@ includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/schema/sch includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/methods/common_methods.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/bam/bam_parser.config" includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/store_object_as_json/store_object_as_json.config" -includeConfig "/hot/code/yashpatel/GitHub/uclahs-cds/pipeline-Nextflow-config/yashpatel-generalized-resource-handler/config/resource_handler/resource_handler.config" +includeConfig "${projectDir}/external/pipeline-Nextflow-config/config/resource_handler/resource_handler.config" methods { set_process = { From 3a24958f4758586dc895f5ce3bb4f6b83374ca19 Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Fri, 13 Dec 2024 17:29:15 -0800 Subject: [PATCH 07/11] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26a1a835..9252d553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed +- Switch to generalized resource handling - Update NFTest paths - Fix single tool run logic - Update PipeVal 4.0.0-rc.2 -> 5.1.0 From ed501b7526e9ab438d4140981438d399b4ea15af Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Tue, 17 Dec 2024 14:55:54 -0800 Subject: [PATCH 08/11] update resource specifications --- config/resources.json | 252 +++++++++++++++++++++++++++++++----------- 1 file changed, 186 insertions(+), 66 deletions(-) diff --git a/config/resources.json b/config/resources.json index 9cfef607..59ce78a1 100644 --- a/config/resources.json +++ b/config/resources.json @@ -19,9 +19,9 @@ "max": 1 }, "memory": { - "min": "1.5 GB", + "min": "1 GB", "fraction": 0.39, - "max": "1.5 GB" + "max": "1 GB" } }, "convert_BAM2Pileup_SAMtools": { @@ -31,9 +31,9 @@ "max": 1 }, "memory": { - "min": "1.5 GB", + "min": "1 GB", "fraction": 0.39, - "max": "1.5 GB" + "max": "1 GB" } }, "create_IndelCandidate_SAMtools": { @@ -43,22 +43,34 @@ "max": 1 }, "memory": { - "min": "1.5 GB", + "min": "1 GB", "fraction": 0.39, - "max": "1.5 GB" + "max": "1 GB" + } + }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.5, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.25, + "max": "1 GB" } }, "call_sIndel_Manta": { "cpus": { "min": 2, - "fraction": 1.0, + "fraction": 1, "max": 2 } }, "call_sSNV_Strelka2": { "cpus": { "min": 2, - "fraction": 1.0, + "fraction": 1, "max": 2 } }, @@ -144,6 +156,24 @@ } } }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.13, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.07, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, "call_sIndel_Manta": { "cpus": { "min": 4, @@ -152,7 +182,7 @@ }, "memory": { "min": "3 GB", - "fraction": 0.20, + "fraction": 0.2, "max": "3 GB" }, "retry_strategy": { @@ -187,8 +217,8 @@ "max": 1 }, "memory": { - "min": "3 GB", - "fraction": 0.20, + "min": "2 GB", + "fraction": 0.2, "max": "3 GB" }, "retry_strategy": { @@ -211,8 +241,8 @@ }, "retry_strategy": { "memory": { - "strategy": "exponential", - "operand": 2 + "strategy": "add", + "operand": "8 GB" } } }, @@ -356,6 +386,24 @@ } } }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.06, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.03, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, "call_sIndel_Manta": { "cpus": { "min": 6, @@ -364,9 +412,27 @@ }, "memory": { "min": "6 GB", - "fraction": 0.20, + "fraction": 0.2, "max": "6 GB" }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "0 GB" + } + } + }, + "call_sSNV_Strelka2": { + "cpus": { + "min": 6, + "fraction": 0.38, + "max": 6 + }, + "memory": { + "min": "2 GB", + "fraction": 0.07, + "max": "2 GB" + }, "retry_strategy": { "memory": { "strategy": "add", @@ -430,9 +496,9 @@ }, "run_sump_MuSE": { "cpus": { - "min": 8, + "min": 4, "fraction": 0.5, - "max": 8 + "max": 4 }, "memory": { "min": "16 GB", @@ -550,6 +616,24 @@ } } }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.03, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.02, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, "call_sIndel_Manta": { "cpus": { "min": 8, @@ -558,13 +642,13 @@ }, "memory": { "min": "6 GB", - "fraction": 0.10, + "fraction": 0.1, "max": "6 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "5 GB" + "operand": "0 GB" } } }, @@ -612,7 +696,7 @@ }, "memory": { "min": "12 GB", - "fraction": 0.20, + "fraction": 0.2, "max": "12 GB" }, "retry_strategy": { @@ -642,13 +726,13 @@ }, "run_sump_MuSE": { "cpus": { - "min": 12, + "min": 6, "fraction": 0.38, - "max": 12 + "max": 6 }, "memory": { "min": "18 GB", - "fraction": 0.30, + "fraction": 0.3, "max": "18 GB" }, "retry_strategy": { @@ -762,6 +846,24 @@ } } }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, "call_sIndel_Manta": { "cpus": { "min": 8, @@ -776,7 +878,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": "5 GB" + "operand": "0 GB" } } }, @@ -854,9 +956,9 @@ }, "run_sump_MuSE": { "cpus": { - "min": 12, + "min": 6, "fraction": 0.17, - "max": 12 + "max": 6 }, "memory": { "min": "18 GB", @@ -929,12 +1031,12 @@ "memory": { "min": "20 GB", "fraction": 0.02, - "max": "20 GB" + "max": "25 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "20 GB" + "operand": "0 GB" } } }, @@ -946,13 +1048,13 @@ }, "memory": { "min": "100 GB", - "fraction": 0.10, + "fraction": 0.1, "max": "100 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "100 GB" + "operand": "0 GB" } } }, @@ -963,14 +1065,14 @@ "max": 1 }, "memory": { - "min": "10 GB", + "min": "1 GB", "fraction": 0.01, - "max": "10 GB" + "max": "1 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "10 GB" + "operand": "19 GB" } } }, @@ -981,14 +1083,14 @@ "max": 1 }, "memory": { - "min": "20 GB", + "min": "1 GB", "fraction": 0.02, - "max": "20 GB" + "max": "1 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "20 GB" + "operand": "39 GB" } } }, @@ -1035,14 +1137,14 @@ "max": 1 }, "memory": { - "min": "30 GB", + "min": "6 GB", "fraction": 0.03, - "max": "30 GB" + "max": "6 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "30 GB" + "operand": "18 GB" } } }, @@ -1102,19 +1204,19 @@ }, "plot_VennDiagram_R": { "cpus": { - "min": 2, + "min": 1, "fraction": 0.03, - "max": 2 + "max": 1 }, "memory": { - "min": "5 GB", + "min": "1 GB", "fraction": 0.01, - "max": "5 GB" + "max": "1 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "10 GB" + "operand": "14 GB" } } }, @@ -1126,7 +1228,7 @@ }, "memory": { "min": "1 GB", - "fraction": 0.00, + "fraction": 0, "max": "1 GB" }, "retry_strategy": { @@ -1157,14 +1259,14 @@ "max": 1 }, "memory": { - "min": "2 GB", + "min": "1 GB", "fraction": 0.01, "max": "2 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "5 GB" + "operand": "16 GB" } } }, @@ -1175,7 +1277,7 @@ "max": 1 }, "memory": { - "min": "2 GB", + "min": "1 GB", "fraction": 0.01, "max": "2 GB" }, @@ -1204,28 +1306,46 @@ } } }, + "generate_ReadCount_bam_readcount": { + "cpus": { + "min": 1, + "fraction": 0.01, + "max": 1 + }, + "memory": { + "min": "1 GB", + "fraction": 0.01, + "max": "1 GB" + }, + "retry_strategy": { + "memory": { + "strategy": "add", + "operand": "6 GB" + } + } + }, "call_sIndel_Manta": { "cpus": { - "min": 8, - "fraction": 0.11, + "min": 2, + "fraction": 0.2, "max": 8 }, "memory": { - "min": "6 GB", + "min": "3 GB", "fraction": 0.04, - "max": "6 GB" + "max": "8 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "5 GB" + "operand": "4 GB" } } }, "call_sSNV_Strelka2": { "cpus": { - "min": 8, - "fraction": 0.11, + "min": 2, + "fraction": 0.25, "max": 8 }, "memory": { @@ -1236,7 +1356,7 @@ "retry_strategy": { "memory": { "strategy": "add", - "operand": "12 GB" + "operand": "10 GB" } } }, @@ -1247,14 +1367,14 @@ "max": 1 }, "memory": { - "min": "3 GB", + "min": "2 GB", "fraction": 0.02, "max": "3 GB" }, "retry_strategy": { "memory": { "strategy": "add", - "operand": "3 GB" + "operand": "9 GB" } } }, @@ -1265,8 +1385,8 @@ "max": 1 }, "memory": { - "min": "12 GB", - "fraction": 0.09, + "min": "8 GB", + "fraction": 0.2, "max": "12 GB" }, "retry_strategy": { @@ -1278,13 +1398,13 @@ }, "call_sSNV_MuSE": { "cpus": { - "min": 12, + "min": 4, "fraction": 0.17, - "max": 12 + "max": 40 }, "memory": { - "min": "60 GB", - "fraction": 0.44, + "min": "10 GB", + "fraction": 0.7, "max": "60 GB" }, "retry_strategy": { @@ -1296,12 +1416,12 @@ }, "run_sump_MuSE": { "cpus": { - "min": 12, + "min": 4, "fraction": 0.17, - "max": 12 + "max": 6 }, "memory": { - "min": "18 GB", + "min": "12 GB", "fraction": 0.13, "max": "18 GB" }, From 31e9db6c77c7999b13ba7d780a7452ce93ff7fe7 Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Tue, 17 Dec 2024 15:19:54 -0800 Subject: [PATCH 09/11] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a583acff..9c98cf97 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,8 @@ input: *Providing `intersect_regions` is required and will limit the final output to just those regions. All regions of the reference genome could be provided as a `bed` file with all contigs, however it is HIGHLY recommended to remove `decoy` contigs from the human reference genome. Including these thousands of small contigs will require the user to increase available memory for `Mutect2` and will cause a very long runtime for `Strelka2`. See [Discussion here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/216). For `uclahs-cds` users, a GRCh38 `bed.gz` file can be found here: `/hot/resource/tool-specific-input/pipeline-call-sSNV-6.0.0/GRCh38-BI-20160721/Homo_sapiens_assembly38_no-decoy.bed.gz`. ### Base resource allocation updaters -To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in the [node-specific config files](./config/) +To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in `config/resources.json`. If available resources have matched cpus and memory within `90% - 1GB` of one of the pre-specified configurations, that configuration will be used. Otherwise the default configuration will be used. A spreadsheet view of the resource configuration as of Dec 2024 is [here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/328) + ```Nextflow base_resource_update { From a7afc9bbc5459645f254aa27ee127d1d8aec452b Mon Sep 17 00:00:00 2001 From: sorelfitzgibbon Date: Tue, 17 Dec 2024 23:24:31 +0000 Subject: [PATCH 10/11] Autofix Nextflow configuration regression tests --- test/configtest-F16.json | 247 +++++++++++++++++++++------------------ test/configtest-F32.json | 207 +++++++++++++++++--------------- 2 files changed, 245 insertions(+), 209 deletions(-) diff --git a/test/configtest-F16.json b/test/configtest-F16.json index ecd72719..3da5ff1e 100644 --- a/test/configtest-F16.json +++ b/test/configtest-F16.json @@ -64,6 +64,60 @@ "muse" ], "bam_readcount_version": "1.0.1", + "base_allocations": { + "call_sIndel_Manta": { + "cpus": "6", + "memory": "6 GB" + }, + "call_sSNV_MuSE": { + "cpus": "6", + "memory": "24 GB" + }, + "call_sSNV_Mutect2": { + "cpus": "1", + "memory": "2 GB" + }, + "call_sSNV_SomaticSniper": { + "cpus": "1", + "memory": "1 GB" + }, + "call_sSNV_Strelka2": { + "cpus": "6", + "memory": "2 GB" + }, + "concat_VCFs_BCFtools": { + "cpus": "1", + "memory": "1 GB" + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "create_IndelCandidate_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": "1 GB" + }, + "plot_VennDiagram_R": { + "cpus": "1", + "memory": "1 GB" + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": "1", + "memory": "8 GB" + }, + "run_sump_MuSE": { + "cpus": "4", + "memory": "16 GB" + }, + "run_validate_PipeVal": { + "cpus": "1", + "memory": "1 GB" + } + }, "bgzip_extra_args": "", "bpg_version": "7.1.0", "cache_intermediate_pipeline_steps": false, @@ -111,8 +165,10 @@ "manta_version": "1.6.0", "max_cpus": "16", "max_memory": "31 GB", + "max_time": "24d 20h 31m 24s", "min_cpus": "1", "min_memory": "1 MB", + "min_time": "1s", "mutect2_extra_args": "", "ncbi_build": "GRCh38", "normal_id": "0192847", @@ -121,128 +177,81 @@ "panel_of_normals_vcf": "", "patient_id": "TWGSAMIN000001", "pipeval_version": "5.1.0", - "proc_resource_params": { + "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", + "retry_information": { "call_sIndel_Manta": { - "cpus": "6", - "memory": "6 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - }, - "withName:call_sSNV_Strelka2": { - "cpus": "6", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - } + "memory": { + "operand": "0", + "strategy": "add" + } + }, + "call_sSNV_MuSE": { + "memory": { + "operand": "8 GB", + "strategy": "add" } }, "call_sSNV_Mutect2": { - "cpus": "1", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "2 GB", - "strategy": "add" - } + "memory": { + "operand": "2 GB", + "strategy": "add" } }, "call_sSNV_SomaticSniper": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "3 GB", - "strategy": "add" - } + "memory": { + "operand": "3 GB", + "strategy": "add" } }, "call_sSNV_Strelka2": { - "ext": { - "retry_codes": [] + "memory": { + "operand": "12 GB", + "strategy": "add" } }, "concat_VCFs_BCFtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "convert_BAM2Pileup_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "create_IndelCandidate_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" + } + }, + "generate_ReadCount_bam_readcount": { + "memory": { + "operand": "6 GB", + "strategy": "add" } }, "plot_VennDiagram_R": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "run_LearnReadOrientationModel_GATK": { - "cpus": "1", - "memory": "8 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - }, - "withName:call_sSNV_MuSE": { - "cpus": "6", - "memory": "24 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - } + "memory": { + "operand": "8 GB", + "strategy": "add" } }, "run_sump_MuSE": { - "cpus": "8", - "memory": "16 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } - }, - "run_validate_PipeVal": { - "cpus": "1", - "memory": "1 GB" } }, - "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", "sample_id": "0192847", "samples_to_process": [ { @@ -364,22 +373,16 @@ } }, "withName:call_sIndel_Manta": { + "cpus": "6", + "memory": "6 GB" + }, + "withName:call_sSNV_MuSE": { "cpus": "6", "memory": { - "1": "6 GB", - "2": "18 GB", - "3": "30 GB", - "closure": "retry_updater(6 GB, add, 12 GB, $task.attempt, memory)" - }, - "withName:call_sSNV_Strelka2": { - "cpus": "6", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } - } + "1": "24 GB", + "2": "31 GB", + "3": "31 GB", + "closure": "retry_updater(24 GB, add, 8 GB, $task.attempt, memory)" } }, "withName:call_sSNV_Mutect2": { @@ -401,8 +404,15 @@ } }, "withName:call_sSNV_Strelka2": { + "cpus": "6", "ext": { "retry_codes": [] + }, + "memory": { + "1": "2 GB", + "2": "14 GB", + "3": "26 GB", + "closure": "retry_updater(2 GB, add, 12 GB, $task.attempt, memory)" } }, "withName:concat_VCFs_BCFtools": { @@ -432,6 +442,15 @@ "closure": "retry_updater(1 GB, add, 5 GB, $task.attempt, memory)" } }, + "withName:generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": { + "1": "1 GB", + "2": "7 GB", + "3": "13 GB", + "closure": "retry_updater(1 GB, add, 6 GB, $task.attempt, memory)" + } + }, "withName:plot_VennDiagram_R": { "cpus": "1", "memory": { @@ -448,20 +467,10 @@ "2": "16 GB", "3": "24 GB", "closure": "retry_updater(8 GB, add, 8 GB, $task.attempt, memory)" - }, - "withName:call_sSNV_MuSE": { - "cpus": "6", - "memory": "24 GB", - "retry_strategy": { - "memory": { - "operand": "8 GB", - "strategy": "add" - } - } } }, "withName:run_sump_MuSE": { - "cpus": "8", + "cpus": "4", "memory": { "1": "16 GB", "2": "31 GB", @@ -478,6 +487,12 @@ "enabled": true, "file": "/tmp/outputs/call-sSNV-VER.SI.ON/0192847/log-call-sSNV-VER.SI.ON-19970704T165655Z/nextflow-log/report.html" }, + "resource_handler": { + "handle_resources": "closure()", + "set_resource_limit_params": "closure()", + "set_retry": "closure()", + "setup_retry": "closure()" + }, "sm_tag": "0192847", "timeline": { "enabled": true, diff --git a/test/configtest-F32.json b/test/configtest-F32.json index c9319c57..fbaaa5e8 100644 --- a/test/configtest-F32.json +++ b/test/configtest-F32.json @@ -64,6 +64,60 @@ "muse" ], "bam_readcount_version": "1.0.1", + "base_allocations": { + "call_sIndel_Manta": { + "cpus": "8", + "memory": "6 GB" + }, + "call_sSNV_MuSE": { + "cpus": "12", + "memory": "48 GB" + }, + "call_sSNV_Mutect2": { + "cpus": "1", + "memory": "2 GB" + }, + "call_sSNV_SomaticSniper": { + "cpus": "1", + "memory": "1 GB" + }, + "call_sSNV_Strelka2": { + "cpus": "8", + "memory": "2 GB" + }, + "concat_VCFs_BCFtools": { + "cpus": "1", + "memory": "1 GB" + }, + "convert_BAM2Pileup_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "create_IndelCandidate_SAMtools": { + "cpus": "1", + "memory": "1 GB" + }, + "generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": "1 GB" + }, + "plot_VennDiagram_R": { + "cpus": "1", + "memory": "1 GB" + }, + "run_LearnReadOrientationModel_GATK": { + "cpus": "1", + "memory": "12 GB" + }, + "run_sump_MuSE": { + "cpus": "6", + "memory": "18 GB" + }, + "run_validate_PipeVal": { + "cpus": "1", + "memory": "1 GB" + } + }, "bgzip_extra_args": "", "bpg_version": "7.1.0", "cache_intermediate_pipeline_steps": false, @@ -111,8 +165,10 @@ "manta_version": "1.6.0", "max_cpus": "32", "max_memory": "64 GB", + "max_time": "24d 20h 31m 24s", "min_cpus": "1", "min_memory": "1 MB", + "min_time": "1s", "mutect2_extra_args": "", "ncbi_build": "GRCh38", "normal_id": "0192847", @@ -121,126 +177,81 @@ "panel_of_normals_vcf": "", "patient_id": "TWGSAMIN000001", "pipeval_version": "5.1.0", - "proc_resource_params": { + "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", + "retry_information": { "call_sIndel_Manta": { - "cpus": "8", - "memory": "6 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "0", + "strategy": "add" } }, "call_sSNV_MuSE": { - "cpus": "12", - "memory": "48 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } }, "call_sSNV_Mutect2": { - "cpus": "1", - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "2 GB", - "strategy": "add" - } + "memory": { + "operand": "2 GB", + "strategy": "add" } }, "call_sSNV_SomaticSniper": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "call_sSNV_Strelka2": { - "cpus": "8", - "ext": { - "retry_codes": [] - }, - "memory": "2 GB", - "retry_strategy": { - "memory": { - "operand": "12 GB", - "strategy": "add" - } + "memory": { + "operand": "12 GB", + "strategy": "add" } }, "concat_VCFs_BCFtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "convert_BAM2Pileup_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" } }, "create_IndelCandidate_SAMtools": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "5 GB", - "strategy": "add" - } + "memory": { + "operand": "5 GB", + "strategy": "add" + } + }, + "generate_ReadCount_bam_readcount": { + "memory": { + "operand": "6 GB", + "strategy": "add" } }, "plot_VennDiagram_R": { - "cpus": "1", - "memory": "1 GB", - "retry_strategy": { - "memory": { - "operand": "10 GB", - "strategy": "add" - } + "memory": { + "operand": "10 GB", + "strategy": "add" } }, "run_LearnReadOrientationModel_GATK": { - "cpus": "1", - "memory": "12 GB", - "retry_strategy": { - "memory": { - "operand": "20 GB", - "strategy": "add" - } + "memory": { + "operand": "20 GB", + "strategy": "add" } }, "run_sump_MuSE": { - "cpus": "12", - "memory": "18 GB", - "retry_strategy": { - "memory": { - "operand": "16 GB", - "strategy": "add" - } + "memory": { + "operand": "16 GB", + "strategy": "add" } - }, - "run_validate_PipeVal": { - "cpus": "1", - "memory": "1 GB" } }, - "reference": "/hot/resource/reference-genome/GRCh38-BI-20160721/Homo_sapiens_assembly38.fasta", "sample_id": "0192847", "samples_to_process": [ { @@ -363,12 +374,7 @@ }, "withName:call_sIndel_Manta": { "cpus": "8", - "memory": { - "1": "6 GB", - "2": "11 GB", - "3": "16 GB", - "closure": "retry_updater(6 GB, add, 5 GB, $task.attempt, memory)" - } + "memory": "6 GB" }, "withName:call_sSNV_MuSE": { "cpus": "12", @@ -436,6 +442,15 @@ "closure": "retry_updater(1 GB, add, 5 GB, $task.attempt, memory)" } }, + "withName:generate_ReadCount_bam_readcount": { + "cpus": "1", + "memory": { + "1": "1 GB", + "2": "7 GB", + "3": "13 GB", + "closure": "retry_updater(1 GB, add, 6 GB, $task.attempt, memory)" + } + }, "withName:plot_VennDiagram_R": { "cpus": "1", "memory": { @@ -455,7 +470,7 @@ } }, "withName:run_sump_MuSE": { - "cpus": "12", + "cpus": "6", "memory": { "1": "18 GB", "2": "34 GB", @@ -472,6 +487,12 @@ "enabled": true, "file": "/tmp/outputs/call-sSNV-VER.SI.ON/0192847/log-call-sSNV-VER.SI.ON-19970704T165655Z/nextflow-log/report.html" }, + "resource_handler": { + "handle_resources": "closure()", + "set_resource_limit_params": "closure()", + "set_retry": "closure()", + "setup_retry": "closure()" + }, "sm_tag": "0192847", "timeline": { "enabled": true, From a0e69a574210dbc9a33e30c3b0940cb8ff83d4e5 Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Tue, 17 Dec 2024 16:29:19 -0800 Subject: [PATCH 11/11] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c98cf97..6e2186a8 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ input: *Providing `intersect_regions` is required and will limit the final output to just those regions. All regions of the reference genome could be provided as a `bed` file with all contigs, however it is HIGHLY recommended to remove `decoy` contigs from the human reference genome. Including these thousands of small contigs will require the user to increase available memory for `Mutect2` and will cause a very long runtime for `Strelka2`. See [Discussion here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/216). For `uclahs-cds` users, a GRCh38 `bed.gz` file can be found here: `/hot/resource/tool-specific-input/pipeline-call-sSNV-6.0.0/GRCh38-BI-20160721/Homo_sapiens_assembly38_no-decoy.bed.gz`. ### Base resource allocation updaters -To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in `config/resources.json`. If available resources have matched cpus and memory within `90% - 1GB` of one of the pre-specified configurations, that configuration will be used. Otherwise the default configuration will be used. A spreadsheet view of the resource configuration as of Dec 2024 is [here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/328) +To optionally update the base resource (cpus or memory) allocations for processes, use the following structure and add the necessary parts to the [input.config](config/template.config) file. The default allocations can be found in `config/resources.json`. If available resources have matched cpus and memory within `90% - 1GB` of one of the pre-specified configurations, that configuration will be used. Otherwise the default configuration will be used. A spreadsheet view of the resource configuration as of Dec 2024 is [here](https://github.com/uclahs-cds/pipeline-call-sSNV/discussions/328). For very large or challanging input samples, we suggest using the `m64` configuration or similar. ```Nextflow