fix: issue #366 and #2649

bio/reference/ensembl-annotation/meta.yaml

@@ -4,3 +4,5 @@ authors:
   - Johannes Köster
 output:
   - Ensemble GTF or GFF3 anotation file
+params:
+  - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``)

bio/reference/ensembl-annotation/test/Snakefile

@@ -25,6 +25,8 @@ rule get_annotation_gz:
         # branch="plants",  # optional: specify branch
     log:
         "logs/get_annotation.log",
+    params:
+        url="http://ftp.ensembl.org/pub",
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-annotation"

bio/reference/ensembl-annotation/wrapper.py

@@ -48,17 +48,8 @@
     )
 
 
-url = "ftp://ftp.ensembl.org/pub/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}".format(
-    release=release,
-    gtf_release=gtf_release,
-    build=build,
-    species=species,
-    out_fmt=out_fmt,
-    species_cap=species.capitalize(),
-    suffix=suffix,
-    flavor=flavor,
-    branch=branch,
-)
+url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub")
+url = f"{url}/{branch}release-{release}/{out_fmt}/{species}/{species.capitalize()}.{build}.{gtf_release}.{flavor}{suffix}"
 
 
 try:

bio/reference/ensembl-sequence/meta.yaml

@@ -2,3 +2,7 @@ name: ensembl-sequence
 description: Download sequences (e.g. genome) from ENSEMBL FTP servers, and store them in a single .fasta file.
 authors:
   - Johannes Köster
+output:
+  - fasta file
+params:
+  - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``)

bio/reference/ensembl-sequence/test/Snakefile

@@ -25,6 +25,8 @@ rule get_single_chromosome:
         # branch="plants",  # optional: specify branch
     log:
         "logs/get_genome.log",
+    params:
+        url="http://ftp.ensembl.org/pub",
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-sequence"

bio/reference/ensembl-sequence/wrapper.py

@@ -50,8 +50,9 @@
             "invalid datatype, to select a single chromosome the datatype must be dna"
         )
 
+url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub")
 spec = spec.format(build=build, release=release)
-url_prefix = f"ftp://ftp.ensembl.org/pub/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}"
+url_prefix = f"{url}/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}"
 
 success = False
 for suffix in suffixes:

bio/reference/ensembl-variation/meta.yaml

@@ -2,3 +2,7 @@ name: ensembl-variation
 description: Download known genomic variants from ENSEMBL FTP servers, and store them in a single .vcf.gz file.
 authors:
   - Johannes Köster
+output:
+  - VCF file
+params:
+  - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``)

bio/reference/ensembl-variation/test/Snakefile

@@ -12,6 +12,8 @@ rule get_variation:
         type="all",  # one of "all", "somatic", "structural_variation"
         # chromosome="21", # optionally constrain to chromosome, only supported for homo_sapiens
         # branch="plants",  # optional: specify branch
+    params:
+        url="http://ftp.ensembl.org/pub",
     log:
         "logs/get_variation.log",
     cache: "omit-software"  # save space and time with between workflow caching (see docs)

bio/reference/ensembl-variation/wrapper.py

@@ -62,16 +62,12 @@
 
 species_filename = species if release >= 91 else species.capitalize()
 
+url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub")
 urls = [
-    "ftp://ftp.ensembl.org/pub/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz".format(
-        release=release,
-        species=species,
-        suffix=suffix,
-        species_filename=species_filename,
-        branch=branch,
-    )
+    f"{url}/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz"
     for suffix in suffixes
 ]
+
 names = [os.path.basename(url) for url in urls]
 
 try:

bio/vep/cache/meta.yaml

@@ -3,3 +3,10 @@ description: Download VEP cache for given species, build and release.
 url: http://www.ensembl.org/info/docs/tools/vep/index.html
 authors:
   - Johannes Köster
+output:
+  - directory to store the VEP cache
+params:
+  - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``)
+  - species: species to download cache data
+  - build: build to download cache data
+  - release: release to download cache data

bio/vep/cache/test/Snakefile

@@ -10,3 +10,18 @@ rule get_vep_cache:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/vep/cache"
+
+
+rule get_vep_cache_ebi:
+    output:
+        directory("resources/vep/cache_ebi"),
+    params:
+        url="ftp://ftp.ebi.ac.uk/ensemblgenomes/pub/plants",
+        species="cyanidioschyzon_merolae",
+        build="ASM9120v1",
+        release="58",
+    log:
+        "logs/vep/cache_ebi.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/vep/cache"

bio/vep/cache/wrapper.py

@@ -9,24 +9,25 @@
 
 
 extra = snakemake.params.get("extra", "")
+log = snakemake.log_fmt_shell(stdout=True, stderr=True)
+
 
 try:
     release = int(snakemake.params.release)
 except ValueError:
     raise ValueError("The parameter release is supposed to be an integer.")
 
+
 with tempfile.TemporaryDirectory() as tmpdir:
     # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl).
     # See https://github.com/bcbio/bcbio-nextgen/issues/1080
-    vep_dir = "vep" if release >= 97 else "VEP"
+    cache_url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub")
     cache_tarball = (
         f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz"
     )
-    log = snakemake.log_fmt_shell(stdout=True, stderr=True)
+    vep_dir = "vep" if snakemake.params.get("url") or release >= 97 else "VEP"
     shell(
-        "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/"
-        "variation/{vep_dir}/{cache_tarball} "
-        "-o {tmpdir}/{cache_tarball} {log}"
+        "curl -L {cache_url}/release-{release}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}"
     )
 
     log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True)

test.py

@@ -5915,6 +5915,11 @@ def test_vep_cache():
         ["snakemake", "--cores", "1", "resources/vep/cache", "--use-conda", "-F"],
     )
 
+    run(
+        "bio/vep/cache",
+        ["snakemake", "--cores", "1", "resources/vep/cache_ebi", "--use-conda", "-F"],
+    )
+
 
 @skip_if_not_modified
 def test_vep_plugins():