fix: Allow concurrent and large refgenie downloads

.github/workflows/conventional-prs.yml

@@ -1,4 +1,4 @@
-name: PR
+name: Lint PR Title
 on:
   pull_request_target:
     types:
@@ -9,8 +9,10 @@ on:
 
 jobs:
   title-format:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, v1.0.2]
     steps:
-      - uses: amannn/action-semantic-pull-request@v3.4.0
+      - uses: amannn/action-semantic-pull-request@v3.6.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          validateSingleCommit: true

.github/workflows/main.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   docs:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, python-3.10]
     steps:
       - uses: actions/checkout@v1
 
@@ -27,21 +27,26 @@ jobs:
           make html
 
   testing:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, python-3.10]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: recursive
           fetch-depth: 0
-
-      - name: Setup mamba
+      # This is a hybrid of things, might need some tweaking
+      - name: create environment with mamba
         uses: conda-incubator/setup-miniconda@v2
+        env:
+          CONDA: "/usr/bin/conda"
         with:
+          mamba-version: "*"
+          # ?miniforge-variant: Mambaforge
+          # ?miniforge-version: latest
+          channels: conda-forge, bioconda
+          auto-activate-base: true
           activate-environment: snakemake
-          channels: "conda-forge, bioconda"
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
-
+          use-only-tar-bz2: true  # IMPORTANT: This needs to be set for caching to work properly!
+          auto-update-conda: true
       - name: Setup Snakemake environment
         shell: bash -el {0}
         run: |

.github/workflows/qc.yml

@@ -2,11 +2,10 @@ name: Code quality
 
 on:
   pull_request:
-    branches_ignore: []
 
 jobs:
   formatting:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, python-3.10]
     steps:
       - uses: actions/checkout@v3
 
@@ -37,7 +36,7 @@ jobs:
       #     snakefmt --check $(git diff origin/master --name-only | grep Snakefile)
 
   linting:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, python-3.10]
     steps:
       - uses: actions/checkout@v1
 

bio/refgenie/test/Snakefile

@@ -1,11 +1,25 @@
 rule obtain_asset:
     output:
         # the name refers to the refgenie seek key (see attributes on http://refgenomes.databio.org)
-        fai="refs/genome.fasta"
+        fai="refs/genome.fasta",
         # Multiple outputs/seek keys are possible here.
     params:
         genome="human_alu",
         asset="fasta",
-        tag="default"
+        tag="default",
+    log:
+        "logs/refgenie/obtain_large_asset.log",
+    wrapper:
+        "master/bio/refgenie"
+
+rule obtain_large_asset:
+    output:
+        star_index=directory("refs/star_index/hg38/star_index"),
+    params:
+        genome="hg38",
+        asset="star_index",
+        tag="default",
+    log:
+        "logs/refgenie/obtain_large_asset.log",
     wrapper:
         "master/bio/refgenie"

bio/refgenie/wrapper.py

@@ -5,17 +5,27 @@
 
 import os
 import refgenconf
+from refgenconf.exceptions import RefgenconfError
 
 genome = snakemake.params.genome
 asset = snakemake.params.asset
 tag = snakemake.params.tag
 
 conf_path = os.environ["REFGENIE"]
 
-rgc = refgenconf.RefGenConf(conf_path, writable=True)
-
+# BUG If there are multiple concurrent refgenie commands, this will fail due to
+# unable to acquire lock of the config file.
+try:
+    rgc = refgenconf.RefGenConf(conf_path, writable=True)
+except RefgenconfError:
+    # If read lock timeout, attempt to skip the read lock
+    rgc = refgenconf.RefGenConf(
+        conf_path, writable=True, skip_read_lock=True, genome_exact=False
+    )
 # pull asset if necessary
-gat, archive_data, server_url = rgc.pull(genome, asset, tag, force=False)
+gat, archive_data, server_url = rgc.pull(
+    genome, asset, tag, force=False, force_large=True
+)
 
 for seek_key, out in snakemake.output.items():
     path = rgc.seek(genome, asset, tag_name=tag, seek_key=seek_key, strict_exists=True)

bio/rsem/calculate-expression/wrapper.py

@@ -5,7 +5,7 @@
 
 
 import os
-
+from pathlib import Path
 from snakemake.shell import shell
 
 bam = snakemake.input.get("bam", "")
@@ -60,15 +60,22 @@
         "output.isoforms_results file name malformed "
         "(rsem will append .isoforms.results suffix)"
     )
-
-reference_prefix = os.path.splitext(snakemake.input.reference[0])[0]
+
+# BUG input_string is 'r' given the input but is should be the reference base path?
+# subprocess.CalledProcessError: Command 'set -euo pipefail;  rsem-calculate-expression --num-threads 24 --estimate-rspd --calc-ci --strandedness reverse --time --paired-end --alignments results/star/D-1/Aligned.toTranscriptome.out.bam r results/rsem/D-1/D-1  > logs/rsem/calculate_expression/D-1.log 2>&1' returned non-zero exit status 255.
+reference_path = Path(snakemake.input.reference[0])
+reference_prefix = str(reference_path.parents[0]/reference_path.stem)
 
 extra = snakemake.params.get("extra", "")
 threads = snakemake.threads
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 shell(
-    "rsem-calculate-expression --num-threads {snakemake.threads} {extra} "
-    "{paired_end_string} {input_bam} {input_string} "
-    "{reference_prefix} {output_prefix} "
+    "rsem-calculate-expression --num-threads {snakemake.threads} "
+    "{extra} "
+    "{paired_end_string} "
+    "{input_bam} "
+    "{input_string} "
+    "{reference_prefix} "
+    "{output_prefix} "
     "{log}"
 )