feat: Add option to scale memory and time resources for pipeline

seqnado/cli.py

@@ -113,6 +113,13 @@ def cli_design(method, files, output="design.csv"):
     is_flag=True,
     help="Remove symlinks created by previous runs. Useful for re-running pipeline after misconfiguration.",
 )
+@click.option(
+    '-s',
+    '--scale-resources',
+    help="Scale factor the memory and time resources for the pipeline",
+    default=1.0,
+    type=float
+)
 @click.option(
     "-v",
     "--verbose",
@@ -128,6 +135,7 @@ def cli_pipeline(
     version=False,
     verbose=False,
     clean_symlinks=False,
+    scale_resources=1.0,
 ):
     """Runs the data processing pipeline"""
 
@@ -151,6 +159,9 @@ def cli_pipeline(
 
     pipeline_options, cores = extract_cores_from_options(pipeline_options)
 
+    # Scale the memory and time resources
+    os.environ["SCALE_RESOURCES"] = str(scale_resources)
+
     # Removes old symlinks if requested
     if clean_symlinks:
         logger.info("Cleaning symlinks")

seqnado/helpers.py

@@ -1,4 +1,4 @@
-from typing import Dict, Union, Optional, List, Tuple
+from typing import Dict, Union, Optional, List, Tuple, Any
 import pathlib
 import numpy as np
 import shlex
@@ -58,6 +58,25 @@ def extract_apptainer_args(options: List[str]) -> Tuple[List[str], str]:
     return options, apptainer_args
 
 
+def define_memory_requested(wildcards: Any, attempts: int = 1, initial_value: int  = 1, scale: float = 1) -> str:
+    """
+    Define the memory requested for the job.
+    """
+    memory = initial_value * 2 ** attempts
+    memory = memory * scale
+    return f"{memory}G"
+
+def define_time_requested(wildcards: Any, attempts: int = 1, initial_value: int = 1, scale: float = 1) -> str:
+    """
+    Define the time requested for the job.
+
+    Base time is 1 hour.
+    """
+    time = initial_value * 2 ** attempts
+    time = time * scale
+    return f"{time}h"
+
+
 def symlink_file(
     output_dir: pathlib.Path, source_path: pathlib.Path, new_file_name: str
 ):

seqnado/workflow/rules/align_rna.smk

@@ -1,4 +1,4 @@
-from seqnado.helpers import check_options
+from seqnado.helpers import check_options, define_memory_requested, define_time_requested
 
 rule align_paired:
     input:
@@ -15,8 +15,8 @@ rule align_paired:
         ),
     threads: config["star"]["threads"]
     resources:
-        mem="35GB",
-        runtime="6h",
+        mem=lambda wildcards, attempt: define_memory_requested(initial_value=35, attempts=attempt, scale=SCALE_RESOURCES),
+        runtime=lambda wildcards, attempt: define_time_requested(initial_value=6, attempts=attempt, scale=SCALE_RESOURCES)
     log:
         "seqnado_output/logs/align/{sample}.log",
     shell:

seqnado/workflow/snakefile_rna

@@ -7,14 +7,14 @@ import sys
 import pandas as pd
 from snakemake.utils import min_version
 
-
 from seqnado.design import Design, RNAOutput, NormGroups
 from seqnado.helpers import format_config_dict, symlink_fastq_files, remove_unwanted_run_files
 
 ####################
 # Hardcoded config #
 ####################
 ASSAY = "RNA"
+SCALE_RESOURCES = os.environ.get("SCALE_RESOURCES", "1")
 
 
 configfile: "config_rna.yml"