Merge pull request #79 from databio/staging

v0.4.4
refgenie · Mar 17, 2020 · 52ed286 · 52ed286
2 parents c7d5460 + 6ac9ed6
commit 52ed286
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 23 deletions.
diff --git a/changelog.md b/changelog.md
@@ -2,6 +2,10 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.4.4] -- 2020-03-17
+### Changed
+- `refgenieserver archive` requires all assets to be complete prior to archiving
+
 ## [0.4.3] -- 2020-01-16
 ### Added
 - a possibility to decouple genome archive directory and genome archive config file. `refgenieserver archive` uses new key (`genome_archive_config`) from `refgenconf`

diff --git a/refgenieserver/_version.py b/refgenieserver/_version.py
@@ -1 +1 @@
-__version__ = "0.4.3"
+__version__ = "0.4.4"
diff --git a/refgenieserver/main.py b/refgenieserver/main.py
@@ -10,7 +10,12 @@
 import uvicorn
 from ubiquerg import parse_registry_path
 
-app = FastAPI()
+app = FastAPI(
+    title=PKG_NAME,
+    description="a web interface and RESTful API for reference genome assets",
+    version=server_v
+)
+
 app.mount("/" + STATIC_DIRNAME, StaticFiles(directory=STATIC_PATH), name=STATIC_DIRNAME)
 templates = Jinja2Templates(directory=TEMPLATES_PATH)
 

diff --git a/refgenieserver/routers/version2.py b/refgenieserver/routers/version2.py
@@ -54,7 +54,7 @@ async def list_available_assets():
     """
     Returns a list of all assets that can be downloaded. No inputs required.
     """
-    ret_dict = rgc.assets_dict(include_tags=True)
+    ret_dict = rgc.list(include_tags=True)
     _LOGGER.info("serving assets dict: {}".format(ret_dict))
     return ret_dict
 

diff --git a/refgenieserver/server_builder.py b/refgenieserver/server_builder.py
@@ -4,7 +4,8 @@
 from glob import glob
 from subprocess import run
 from refgenconf import RefGenConf
-from refgenconf.exceptions import GenomeConfigFormatError, ConfigNotCompliantError, RefgenconfError
+from refgenconf.exceptions import RefgenconfError, ConfigNotCompliantError, \
+    GenomeConfigFormatError, MissingConfigDataError
 from ubiquerg import checksum, size, is_command_callable, parse_registry_path
 
 from .const import *
@@ -98,11 +99,11 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
             os.makedirs(target_dir)
         genome_desc = rgc[CFG_GENOMES_KEY][genome].setdefault(CFG_GENOME_DESC_KEY, DESC_PLACEHOLDER) \
             if genomes_desc is None or genome not in descs else descs[genome]
-        genome_checksum = rgc[CFG_GENOMES_KEY][genome].setdefault(CFG_CHECKSUM_KEY, CHECKSUM_PLACEHOLDER)
+        genome_checksum = rgc[CFG_GENOMES_KEY][genome].\
+            setdefault(CFG_CHECKSUM_KEY, CHECKSUM_PLACEHOLDER)
         genome_attrs = {CFG_GENOME_DESC_KEY: genome_desc,
                         CFG_CHECKSUM_KEY: genome_checksum}
-        with rgc_server as r:
-            r.update_genomes(genome, genome_attrs)
+        rgc_server.update_genomes(genome, genome_attrs)
         _LOGGER.debug("Updating '{}' genome attributes...".format(genome))
         asset = asset_list[counter] if asset_list is not None else None
         assets = asset or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY].keys()
@@ -112,23 +113,26 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
         else:
             _LOGGER.debug("Assets to be processed: {}".format(str(assets)))
         for asset_name in assets if isinstance(assets, list) else [assets]:
-            asset_desc = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name].setdefault(CFG_ASSET_DESC_KEY,
-                                                                                             DESC_PLACEHOLDER)
-            default_tag = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name].setdefault(CFG_ASSET_DEFAULT_TAG_KEY,
-                                                                                              DEFAULT_TAG)
+            asset_desc = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name]\
+                .setdefault(CFG_ASSET_DESC_KEY, DESC_PLACEHOLDER)
+            default_tag = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name]\
+                .setdefault(CFG_ASSET_DEFAULT_TAG_KEY, DEFAULT_TAG)
             asset_attrs = {CFG_ASSET_DESC_KEY: asset_desc,
                            CFG_ASSET_DEFAULT_TAG_KEY: default_tag}
             _LOGGER.debug("Updating '{}/{}' asset attributes...".format(genome, asset_name))
             with rgc_server as r:
                 r.update_assets(genome, asset_name, asset_attrs)
+
             tag = tag_list[counter] if tag_list is not None else None
             tags = tag or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY].keys()
             for tag_name in tags if isinstance(tags, list) else [tags]:
                 if not rgc.is_asset_complete(genome, asset_name, tag_name):
-                    _LOGGER.info("'{}/{}:{}' is incomplete, skipping".format(genome, asset_name, tag_name))
-                    with rgc_server as r:
-                        r.remove_assets(genome, asset_name, tag_name)
-                    continue
+                    raise MissingConfigDataError(
+                        "Asset '{}/{}:{}' is incomplete. This probably means an"
+                        " attempt to archive a partially pulled parent. "
+                        "refgenieserver archive requires all assets to be built"
+                        " prior to archiving.".format(genome, asset_name, tag_name)
+                    )
                 file_name = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name][
                     CFG_ASSET_PATH_KEY]
                 target_file = os.path.join(target_dir, "{}__{}".format(asset_name, tag_name) + ".tgz")
@@ -144,7 +148,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                 asset_digest = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
                     setdefault(CFG_ASSET_CHECKSUM_KEY, None)
                 if not os.path.exists(target_file) or force:
-                    _LOGGER.info("Creating asset '{}' from '{}'".format(target_file, input_file))
+                    _LOGGER.info("Creating archive '{}' from '{}' asset".format(target_file, input_file))
                     try:
                         _check_tgz(input_file, target_file, asset_name)
                         _copy_recipe(input_file, target_dir, asset_name, tag_name)
@@ -166,7 +170,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                         with rgc_server as r:
                             for parent in parents:
                                 # here we update any pre-existing parents' children attr with the newly added asset
-                                _LOGGER.debug("updating {} children list with {}".
+                                _LOGGER.debug("Updating {} children list with {}".
                                               format(parent, "{}/{}:{}".format(genome, asset_name, tag_name)))
                                 rp = parse_registry_path(parent)
                                 parent_genome = rp["namespace"]
@@ -267,9 +271,10 @@ def _check_servable(rgc, genome, asset, tag):
             try:
                 for tag_name, tag in asset[CFG_ASSET_TAGS_KEY].items():
                     if not _check_servable(rgc, genome_name, asset_name, tag_name):
-                        rgc.remove_assets(genome_name, asset_name, tag_name)
+                        _LOGGER.debug("Removing '{}/{}:{}', it's not servable".format(genome_name, asset_name, tag_name))
+                        rgc.cfg_remove_assets(genome_name, asset_name, tag_name)
             except KeyError:
-                rgc.remove_assets(genome_name, asset_name)
+                rgc.cfg_remove_assets(genome_name, asset_name)
     return rgc
 
 
@@ -287,9 +292,9 @@ def _remove_archive(rgc, registry_paths, cfg_archive_folder_key=CFG_ARCHIVE_KEY)
         genome, asset, tag = registry_path["namespace"], registry_path["item"], registry_path["tag"]
         try:
             if asset is None:
-                [rgc.remove_assets(genome, x, None) for x in rgc.list_assets_by_genome(genome)]
+                [rgc.cfg_remove_assets(genome, x, None) for x in rgc.list_assets_by_genome(genome)]
             else:
-                rgc.remove_assets(genome, asset, tag)
+                rgc.cfg_remove_assets(genome, asset, tag)
             _LOGGER.info("{}/{}{} removed".format(genome, asset, ":" + tag if tag else ""))
         except KeyError:
             _LOGGER.warning("{}/{}{} not found and not removed.".format(genome, asset, ":" + tag if tag else ""))

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
@@ -3,5 +3,5 @@ fastapi
 jinja2
 logmuse>=0.2
 uvicorn>=0.7.1
-refgenconf>=0.6.2
+refgenconf>=0.7.0
 ubiquerg>=0.5.0
diff --git a/test_refgenie.sh b/test_refgenie.sh
@@ -57,7 +57,8 @@ FROM tiangolo/uvicorn-gunicorn:python3.7-alpine3.8
 LABEL authors="Nathan Sheffield, Michal Stolarczyk"
 
 COPY . /app
-
+# RUN pip install https://github.com/databio/refgenconf/archive/dev.zip
+# RUN pip install https://github.com/databio/refgenieserver/archive/dev.zip
 RUN pip install https://github.com/databio/refgenieserver/archive/master.zip
 
 EOF
@@ -112,6 +113,21 @@ refgenie pull -c $REFGENIE test/fasta || ErrorExit "$LINENO: Failed to pull remo
 echo -e "\n-- List local assets --\n"
 refgenie list -c $REFGENIE || ErrorExit "$LINENO: Failed to list local assets."
 
+echo -e "\n-- Tag asset --\n"
+refgenie tag -c $REFGENIE test/fasta:default --tag test || ErrorExit "$LINENO: Failed to tag asset."
+
+echo -e "\n-- Remove asset --\n"
+refgenie remove -c $REFGENIE test/fasta --force || ErrorExit "$LINENO: Failed to remove asset."
+
+echo -e "\n-- Get asset digest --\n"
+refgenie id -c $REFGENIE rCRS/fasta || ErrorExit "$LINENO: Failed to get asset digest."
+
+echo -e "\n-- Subscribe --\n"
+refgenie subscribe -c $REFGENIE -s http://faulty.com  || ErrorExit "$LINENO: Failed to subscribe."
+
+echo -e "\n-- Unsubscribe --\n"
+refgenie subscribe -c $REFGENIE -s http://faulty.com  || ErrorExit "$LINENO: Failed to unsubscribe."
+
 echo -e "\n-- Shut down local servers --\n"
 docker stop refgenieservercon || ErrorExit "$LINENO: Failed to stop remote rCRS server."
 docker stop refgenieservercon2 || ErrorExit "$LINENO: Failed to stop remote test server."