Badgeo

rules/configuration.smk

@@ -1,17 +1,20 @@
-import os.path
-import glob
-import re
-import time
 import math
-import norns
+import os.path
 import psutil
 import pickle
+import re
 import subprocess
+import time
+
+import norns
 import numpy as np
 import pandas as pd
+import urllib.request
+from bs4 import BeautifulSoup
 from multiprocessing.pool import ThreadPool
-from snakemake.utils import validate
+
 from snakemake.logging import logger
+from snakemake.utils import validate
 
 
 # check config file for correct directory names
@@ -187,8 +190,12 @@ for key, value in config.items():
 logger.info("Checking if samples are single-end or paired-end...")
 layout_cachefile = './.snakemake/layouts.p'
 
-def get_layout(sample):
-    """ sends a request to ncbi checking whether a sample is single-end or paired-end """
+def get_layout_eutils(sample):
+    """
+    Sends a request to ncbi checking whether a sample is single-end or paired-end.
+    Robust method (always returns result), however manually filled in by uploader, so can be wrong.
+    Complementary method of get_layout_trace
+    """
     api_key = config.get('ncbi_key', "")
     if api_key is not "":
         api_key = f'-api_key {api_key}'
@@ -208,38 +215,80 @@ def get_layout(sample):
                          f"manually, and continue the pipeline from there on.")
 
 
+def get_layout_trace(sample):
+    """
+    Parse the ncbi trace website to check if a read has 1, 2, or 3 spots.
+    Will fail if sample is not on ncbi database, however does not have the problem that uploader
+    filled out the form wrongly.
+    Complementary method of get_layout_eutils
+    """
+    try:
+        url = f"https://www.ncbi.nlm.nih.gov/sra/?term={sample}"
+
+        conn = urllib.request.urlopen(url)
+        html = conn.read()
+
+        soup = BeautifulSoup(html, features="html5lib")
+        links = soup.find_all('a')
+
+        for tag in links:
+            link = tag.get('href', None)
+            if link is not None and 'SRR' in link:
+                trace_conn = urllib.request.urlopen("https:" + link)
+                trace_html = trace_conn.read()
+                x = re.search("This run has (\d) read", str(trace_html))
+
+                # if there are spots without info, then just ignore this sample
+                if len(re.findall(", average length: 0", str(trace_html))) > 0:
+                    break
+                elif x.group(1) == '1':
+                    return 'SINGLE'
+                elif x.group(1) == '2':
+                    return 'PAIRED'
+    except:
+        pass
+    return None
+
+
 # try to load the layout cache, otherwise defaults to empty dictionary
 try:
     layout_cache = pickle.load(open(layout_cachefile, "rb"))
 except FileNotFoundError:
     layout_cache = {}
 
 
-tp = ThreadPool(config.get('ncbi_requests', 3) // 2)
+trace_tp = ThreadPool(20)
+eutils_tp = ThreadPool(config.get('ncbi_requests', 3) // 2)
+
+trace_layout = {}
 config['layout'] = {}
 
 # now do a request for each sample that was not in the cache
 for sample in [sample for sample in samples.index if sample not in layout_cache]:
+    config['layout'][sample] = eutils_tp.apply_async(get_layout_eutils, (sample,))
     if os.path.exists(expand(f'{{fastq_dir}}/{sample}.{{fqsuffix}}.gz', **config)[0]):
         config['layout'][sample] ='SINGLE'
     elif all(os.path.exists(path) for path in expand(f'{{fastq_dir}}/{sample}_{{fqext}}.{{fqsuffix}}.gz', **config)):
         config['layout'][sample] ='PAIRED'
     elif sample.startswith(('GSM', 'SRR', 'ERR', 'DRR')):
-        config['layout'][sample] = tp.apply_async(get_layout, (sample,))
-        # sleep 1.25 times the minimum required sleep time
+        # config['layout'][sample] = eutils_tp.apply_async(get_layout_eutils, (sample,))
+        trace_layout[sample] = trace_tp.apply_async(get_layout_trace, (sample,))
+
+        # sleep 1.25 times the minimum required sleep time so eutils don't complain
         time.sleep(1.25 / (config.get('ncbi_requests', 3) // 2))
     else:
         raise ValueError(f"\nsample {sample} was not found..\n"
                          f"We checked for SE file:\n"
-                         f"\t/{config['fastq_dir']}/{sample}.{config['fqsuffix']}.gz \n"
+                         f"\t{config['fastq_dir']}/{sample}.{config['fqsuffix']}.gz \n"
                          f"and for PE files:\n"
                          f"\t{config['fastq_dir']}/{sample}_{config['fqext1']}.{config['fqsuffix']}.gz \n"
                          f"\t{config['fastq_dir']}/{sample}_{config['fqext2']}.{config['fqsuffix']}.gz \n"
                          f"and since the sample did not start with either GSM, SRR, ERR, and DRR we couldn't find it online..\n")
 
 # now parse the output and store the cache, the local files' layout, and the ones that were fetched online
 config['layout'] = {**layout_cache,
-                    **{k: (v if isinstance(v, str) else v.get()) for k, v in config['layout'].items()}}
+                    **{k: (v if isinstance(v, str) else v.get()) for k, v in config['layout'].items()},
+                    **{k: v.get() for k, v in trace_layout.items() if v.get() is not None}}
 
 assert all(layout in ['SINGLE', 'PAIRED'] for sample, layout in config['layout'].items())
 

rules/get_fastq.smk

@@ -1,3 +1,8 @@
+import glob
+import os
+import re
+
+
 rule id2sra:
     """
     Download the SRA of a sample by its unique identifier.
@@ -140,6 +145,7 @@ def get_wrong_fqext(wildcards):
         fastqs.append("impossible input to prevent Snakemake from running the rule without input")
     return sorted(fastqs)
 
+
 rule renamefastq_PE:
     """
     Create symlinks to fastqs with incorrect fqexts (default R1/R2).