Refactor Snakefiles to include SCALE_RESOURCES variable

alsmith151 · Jul 23, 2024 · 3810996 · 3810996
1 parent c914323
commit 3810996
Show file tree

Hide file tree

Showing 17 changed files with 75 additions and 75 deletions.
diff --git a/seqnado/workflow/rules/align.smk b/seqnado/workflow/rules/align.smk
@@ -13,8 +13,8 @@ rule align_paired:
         bam=temp("seqnado_output/aligned/raw/{sample}.bam"),
     threads: config["bowtie2"]["threads"]
     resources:
-        runtime=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}h",
-        mem=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}GB",
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/align/{sample}.log",
     shell:
@@ -34,8 +34,8 @@ rule align_single:
     output:
         bam=temp("seqnado_output/aligned/raw/{sample}.bam"),
     resources:
-        runtime=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}h",
-        mem=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}GB",
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
     threads: config["bowtie2"]["threads"]
     log:
         "seqnado_output/logs/align/{sample}.log",

diff --git a/seqnado/workflow/rules/align_rna.smk b/seqnado/workflow/rules/align_rna.smk
@@ -16,7 +16,7 @@ rule align_paired:
     threads: config["star"]["threads"]
     resources:
         mem=lambda wildcards, attempt: define_memory_requested(initial_value=35, attempts=attempt, scale=SCALE_RESOURCES),
-        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES)
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/align/{sample}.log",
     shell:

diff --git a/seqnado/workflow/rules/alignment_counts.smk b/seqnado/workflow/rules/alignment_counts.smk
@@ -11,8 +11,8 @@ rule feature_counts:
         options=check_options(config["featurecounts"]["options"]),
     threads: config["featurecounts"]["threads"]
     resources:
-        mem=lambda wildcards, attempt: f"{3 * 2 ** (attempt)}GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=3, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/readcounts/featurecounts/featurecounts.log",
     shell:
@@ -41,8 +41,8 @@ rule salmon_counts_paired:
         options=check_options(config["salmon"]["options"]),
     threads: config["salmon"]["threads"]
     resources:
-        mem=lambda wildcards, attempt: f"{3 * 2 ** (attempt)}GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=3, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/readcounts/salmon/salmon_{sample}.log",
     shell:
@@ -61,8 +61,8 @@ rule salmon_counts_single:
         options=check_options(config["salmon"]["options"]),
     threads: config["salmon"]["threads"]
     resources:
-        mem=lambda wildcards, attempt: f"{3 * 2 ** (attempt)}GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=3, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/readcounts/salmon/salmon_{sample}.log",
     shell:

diff --git a/seqnado/workflow/rules/alignment_post_processing.smk b/seqnado/workflow/rules/alignment_post_processing.smk
@@ -7,7 +7,7 @@ rule sort_bam:
     output:
         bam=temp("seqnado_output/aligned/sorted/{sample}.bam"),
     resources:
-        mem=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
     threads: 8
     log:
         "seqnado_output/logs/sorted/{sample}.log",
@@ -26,7 +26,7 @@ rule index_bam:
         bai=temp("seqnado_output/aligned/sorted/{sample}.bam.bai"),
     threads: 1
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2 ** (attempt - 1)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         "samtools index -@ {threads} -b {input.bam}"
 
@@ -46,8 +46,8 @@ if config["remove_blacklist"] and os.path.exists(config.get("blacklist", "")):
         params:
             blacklist=check_options(config["blacklist"]),
         resources:
-            mem="5GB",
-            runtime="4h",
+            mem=lambda wildcards, attempt: define_memory_requested(initial_value=5, attempts=attempt, scale=SCALE_RESOURCES),
+            runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
         log:
             "seqnado_output/logs/blacklist/{sample}.log",
         shell:
@@ -72,7 +72,7 @@ else:
             ),
         threads: 1
         resources:
-            mem="1GB",
+            mem=lambda wildcards, attempt: define_memory_requested(initial_value=1, attempts=attempt, scale=SCALE_RESOURCES),
         log:
             "seqnado_output/logs/blacklist/{sample}.log",
         shell:
@@ -99,8 +99,8 @@ if config["remove_pcr_duplicates_method"] == "picard":
         params:
             options=check_options(config["picard"]["options"]),
         resources:
-            mem="5GB",
-            runtime="4h",
+            mem=lambda wildcards, attempt: define_memory_requested(initial_value=5, attempts=attempt, scale=SCALE_RESOURCES),
+            runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
         log:
             "seqnado_output/logs/duplicates/{sample}.log",
         shell:

diff --git a/seqnado/workflow/rules/exogenous_norm.smk b/seqnado/workflow/rules/exogenous_norm.smk
@@ -5,14 +5,14 @@ use rule align_paired as align_paired_spikein with:
     output:
         bam=temp("seqnado_output/aligned/spikein/raw/{sample}.bam"),
     resources:
-        mem=lambda wildcards, attempt: f"{8 * 2 ** (attempt - 1)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=8, attempts=attempt, scale=SCALE_RESOURCES),
 
 
 use rule align_single as align_single_spikein with:
     output:
         bam=temp("seqnado_output/aligned/spikein/raw/{sample}.bam"),
     resources:
-        mem=lambda wildcards, attempt: f"{8 * 2 ** (attempt - 1)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=8, attempts=attempt, scale=SCALE_RESOURCES),
 
 
 use rule sort_bam as sort_bam_spikein with:
@@ -21,7 +21,7 @@ use rule sort_bam as sort_bam_spikein with:
     output:
         bam=temp("seqnado_output/aligned/spikein/sorted/{sample}.bam"),
     resources:
-        mem=lambda wildcards, attempt: f"{8 * 2 ** (attempt - 1)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=8, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/aligned_spikein/{sample}_sort.log",
 

diff --git a/seqnado/workflow/rules/fastq_screen.smk b/seqnado/workflow/rules/fastq_screen.smk
@@ -54,7 +54,7 @@ rule multiqc_fastqscreen:
     log:
         "seqnado_output/logs/multiqc_fastqscreen.log",
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2**attempt}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         "multiqc -o seqnado_output/qc -n full_fastqscreen_report.html --force seqnado_output/qc/fastq_screen > {log} 2>&1"
 

diff --git a/seqnado/workflow/rules/fastq_trim.smk b/seqnado/workflow/rules/fastq_trim.smk
@@ -11,8 +11,8 @@ rule trimgalore_paired:
         trimmed2=temp("seqnado_output/trimmed/{sample}_2.fastq.gz"),
     threads: 4
     resources:
-        mem="2GB",
-        runtime="4h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
     params:
         options=check_options(config["trim_galore"]["options"]),
         trim_dir="seqnado_output/trimmed",
@@ -34,8 +34,8 @@ rule trimgalore_single:
         trimmed=temp("seqnado_output/trimmed/{sample}.fastq.gz"),
     threads: 4
     resources:
-        mem="2GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     params:
         options=check_options(config["trim_galore"]["options"]),
         trim_dir="seqnado_output/trimmed",

diff --git a/seqnado/workflow/rules/heatmap.smk b/seqnado/workflow/rules/heatmap.smk
@@ -23,7 +23,7 @@ rule heatmap_matrix:
     threads: config["deeptools"]["threads"]
     resources:
         runtime=lambda wildcards, attempt: f"{1 * 2**attempt}h",
-        mem=lambda wildcards, attempt: f"{4 * 2**attempt}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/heatmap/matrix.log",
     shell:
@@ -38,7 +38,7 @@ rule heatmap_plot:
     params:
         colormap=check_options(config["heatmap"]["colormap"]),
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2**attempt}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/heatmap/heatmap.log",
     shell:
@@ -51,7 +51,7 @@ rule heatmap_metaplot:
     output:
         metaplot="seqnado_output/heatmap/metaplot.pdf",
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2**attempt}GB"
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/heatmap/metaplot.log",
     shell:

diff --git a/seqnado/workflow/rules/hub.smk b/seqnado/workflow/rules/hub.smk
@@ -72,7 +72,7 @@ rule bed_to_bigbed:
     params:
         chrom_sizes=config["genome"]["chromosome_sizes"],
     resources:
-        mem="1GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=1, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/bed_to_bigbed/{directory}/{sample}.log",
     shell:

diff --git a/seqnado/workflow/rules/motif.smk b/seqnado/workflow/rules/motif.smk
@@ -8,7 +8,7 @@ rule get_fasta:
     params:
         genome=config["genome"]["fasta"],
     resources:
-        mem=lambda wildcards, attempt: f"{1 * 2 ** (attempt)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=1, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/motifs/fasta/{sample}.log",
     shell:
@@ -28,7 +28,7 @@ rule motif_meme_chip:
         meme_chip_params=config["meme"]["meme_chip_params"],
         meme_chip_db=config["meme"]["meme_chip_db"],
     resources:
-        mem=lambda wildcards, attempt: f"{1 * 2 ** (attempt)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=1, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/motifs/meme/{sample}.log",
     shell:
@@ -47,7 +47,7 @@ rule motif_homer:
         homer_params=config["homer"]["homer_params"],
         homer_bg=config["homer"]["homer_bg"],
     resources:
-        mem=lambda wildcards, attempt: f"{1 * 2 ** (attempt)}GB",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=1, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/motifs/homer/{sample}.log",
     shell:

diff --git a/seqnado/workflow/rules/peak_call_chip.smk b/seqnado/workflow/rules/peak_call_chip.smk
@@ -54,8 +54,8 @@ rule macs2_with_input:
         basename=lambda wc, output: output.peaks.replace(".bed", ""),
     threads: 1
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2 ** (attempt)}GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/macs/{sample}_{treatment}.log",
     shell:
@@ -77,8 +77,8 @@ rule macs2_no_input:
         basename=lambda wc, output: output.peaks.replace(".bed", ""),
     threads: 1
     resources:
-        mem=lambda wildcards, attempt: f"{2 * 2 ** (attempt)}GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/macs/{sample}_{treatment}.log",
     shell:
@@ -100,8 +100,8 @@ rule homer_with_input:
         options=check_options(config["homer"]["findpeaks"]),
     threads: 1
     resources:
-        mem="4GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         findPeaks {input.treatment} {params.options} -o {output.peaks}.tmp  -i {input.control} > {log} 2>&1 &&
@@ -122,8 +122,8 @@ rule homer_no_input:
         options=check_options(config["homer"]["findpeaks"]),
     threads: 1
     resources:
-        mem="4GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         findPeaks {input.treatment} {params.options} -o {output.peaks}.tmp > {log} 2>&1 &&
@@ -149,7 +149,7 @@ rule lanceotron_with_input:
     threads: 1
     resources:
         mem="10GB",
-        runtime="6h",
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         lanceotron callPeaksInput {input.treatment} -i {input.control} -f {params.outdir} --skipheader > {log} 2>&1 &&
@@ -174,7 +174,7 @@ rule lanceotron_no_input:
         "library://asmith151/seqnado/seqnado_extra:latest"
     resources:
         mem=lambda wildcards, attempt: f"{10 * 2 ** (attempt)}GB",
-        runtime="6h",
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         lanceotron callPeaks {input.treatment} -f {params.outdir} --skipheader  {params.options} > {log} 2>&1 &&
@@ -195,8 +195,8 @@ rule seacr:
         prefix=lambda wc, output: pathlib.Path(output.peaks).parent / pathlib.Path(output.peaks).name,
     threads: 1
     resources:
-        mem="5GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=5, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         SEACR_1.3.sh {input.treatment} {params.threshold} {params.norm} {params.stringency} {output.peaks} > {log} 2>&1 || touch {params.prefix}.{params.stringency}.bed

diff --git a/seqnado/workflow/rules/peak_call_grouped.smk b/seqnado/workflow/rules/peak_call_grouped.smk
@@ -7,7 +7,7 @@ rule lanceotron_no_input_consensus:
         peaks="seqnado_output/peaks/merged/lanceotron/{group}.bed",
     threads: 8
     resources:
-        runtime=lambda wildcards, attempt: f"{4 * 2 ** (attempt - 1)}h",
+         runtime=lambda wildcards, attempt: define_time_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
         mem=lambda wildcards, attempt: f"{10 * 2 ** (attempt - 1)}GB",
     params:
         outdir="seqnado_output/peaks/merged/lanceotron",

diff --git a/seqnado/workflow/rules/peak_call_other.smk b/seqnado/workflow/rules/peak_call_other.smk
@@ -21,8 +21,8 @@ rule macs2_no_input:
         basename=lambda wc, output: output.peaks.replace(".bed", ""),
     threads: 1
     resources:
-        mem="2GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     log:
         "seqnado_output/logs/macs/{sample}.bed",
     shell:
@@ -43,8 +43,8 @@ rule homer_no_input:
         options=check_options(config["homer"]["findpeaks"]),
     threads: 1
     resources:
-        mem="4GB",
-        runtime="2h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=4, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=2, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         findPeaks {input.treatment} {params.options} -o {output.peaks}.tmp > {log} 2>&1 &&
@@ -68,7 +68,7 @@ rule lanceotron_no_input:
         "library://asmith151/seqnado/seqnado_extra:latest"
     resources:
         mem=lambda wildcards, attempt: f"{10 * 2 ** (attempt)}GB",
-        runtime="6h",
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES),
     shell:
         """
         lanceotron callPeaks {input.treatment} -f {params.outdir} --skipheader  {params.options} > {log} 2>&1 &&