Merge pull request #72 from databio/staging

v0.4.1
refgenie · Dec 13, 2019 · 893db40 · 893db40
2 parents b935748 + ba83247
commit 893db40
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 19 deletions.
diff --git a/changelog.md b/changelog.md
@@ -2,6 +2,13 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.4.1] -- 2019-12-13
+### Fixed
+- relationship info not being updated during specific asset archivization; [#70](https://github.com/databio/refgenieserver/issues/70) 
+
+### Changed
+- order of the assets adn tags in the table in the index page: sorted alphabetically instead of oldest to newest 
+
 ## [0.4.0] -- 2019-12-06
 ### Added
 - asset splash pages presenting all the asset attributes and related API endpoint links. Available at: `/asset/{genome}/{asset}/splash`

diff --git a/refgenieserver/_version.py b/refgenieserver/_version.py
@@ -1 +1 @@
-__version__ = "0.4.0"
+__version__ = "0.4.1"
diff --git a/refgenieserver/server_builder.py b/refgenieserver/server_builder.py
@@ -4,8 +4,8 @@
 from glob import glob
 from subprocess import run
 from refgenconf import RefGenConf
-from refgenconf.exceptions import GenomeConfigFormatError, ConfigNotCompliantError
-from ubiquerg import checksum, size, is_command_callable
+from refgenconf.exceptions import GenomeConfigFormatError, ConfigNotCompliantError, RefgenconfError
+from ubiquerg import checksum, size, is_command_callable, parse_registry_path
 
 from .const import *
 
@@ -37,10 +37,10 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
         raise GenomeConfigFormatError("The config '{}' is missing a '{}' entry. Can't determine the desired archive.".
                                       format(cfg_path, CFG_ARCHIVE_KEY))
     if force:
-        _LOGGER.info("build forced; file existence will be ignored")
+        _LOGGER.info("Build forced; file existence will be ignored")
         if os.path.exists(server_rgc_path):
             _LOGGER.debug("'{}' file was found and will be updated".format(server_rgc_path))
-    _LOGGER.debug("registry_paths: {}".format(registry_paths))
+    _LOGGER.debug("Registry_paths: {}".format(registry_paths))
 
     # original RefGenConf has been created in read-only mode,
     # make it RW compatible and point to new target path for server use or initialize a new object
@@ -95,7 +95,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                         CFG_CHECKSUM_KEY: genome_checksum}
         with rgc_server as r:
             r.update_genomes(genome, genome_attrs)
-        _LOGGER.debug("updating '{}' genome attributes...".format(genome))
+        _LOGGER.debug("Updating '{}' genome attributes...".format(genome))
         asset = asset_list[counter] if asset_list is not None else None
         assets = asset or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY].keys()
         if not assets:
@@ -110,7 +110,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                                                                                               DEFAULT_TAG)
             asset_attrs = {CFG_ASSET_DESC_KEY: asset_desc,
                            CFG_ASSET_DEFAULT_TAG_KEY: default_tag}
-            _LOGGER.debug("updating '{}/{}' asset attributes...".format(genome, asset_name))
+            _LOGGER.debug("Updating '{}/{}' asset attributes...".format(genome, asset_name))
             with rgc_server as r:
                 r.update_assets(genome, asset_name, asset_attrs)
             tag = tag_list[counter] if tag_list is not None else None
@@ -119,7 +119,6 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                 if not rgc.is_asset_complete(genome, asset_name, tag_name):
                     _LOGGER.info("'{}/{}:{}' is incomplete, skipping".format(genome, asset_name, tag_name))
                     with rgc_server as r:
-                        print(r)
                         r.remove_assets(genome, asset_name, tag_name)
                     continue
                 file_name = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name][
@@ -137,7 +136,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                 asset_digest = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name]. \
                     setdefault(CFG_ASSET_CHECKSUM_KEY, None)
                 if not os.path.exists(target_file) or force:
-                    _LOGGER.info("creating asset '{}' from '{}'".format(target_file, input_file))
+                    _LOGGER.info("Creating asset '{}' from '{}'".format(target_file, input_file))
                     try:
                         _check_tgz(input_file, target_file, asset_name)
                         _copy_recipe(input_file, target_dir, asset_name, tag_name)
@@ -146,7 +145,7 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                         _LOGGER.warning(e)
                         continue
                     else:
-                        _LOGGER.info("updating '{}/{}:{}' tag attributes...".format(genome, asset_name, tag_name))
+                        _LOGGER.info("Updating '{}/{}:{}' tag attributes...".format(genome, asset_name, tag_name))
                         tag_attrs = {CFG_ASSET_PATH_KEY: file_name,
                                      CFG_SEEK_KEYS_KEY: seek_keys,
                                      CFG_ARCHIVE_CHECKSUM_KEY: checksum(target_file),
@@ -157,13 +156,31 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                                      CFG_ASSET_CHECKSUM_KEY: asset_digest}
                         _LOGGER.debug("attr dict: {}".format(tag_attrs))
                         with rgc_server as r:
+                            for parent in parents:
+                                try:
+                                    r.get_asset(parent_genome, parent_asset, parent_tag)
+                                except RefgenconfError:
+                                    _LOGGER.warning("'{}/{}:{}'s parent '{}' does not exist, "
+                                                    "skipping relationship updates".
+                                                    format(genome, asset_name, tag_name, parent))
+                                    continue
+                                # here we update any pre-existing parents' children attr with the newly added asset
+                                _LOGGER.debug("updating {} parents list with {}".
+                                              format(parent, "{}/{}:{}".format(genome, asset_name, tag_name)))
+                                rp = parse_registry_path(parent)
+                                parent_genome = rp["namespace"]
+                                parent_asset = rp["item"]
+                                parent_tag = rp["tag"]
+                                r.update_relatives_assets(parent_genome, parent_asset, parent_tag,
+                                                          ["{}/{}:{}".format(genome, asset_name, tag_name)],
+                                                          children=True)
                             r.update_tags(genome, asset_name, tag_name, tag_attrs)
                 else:
                     _LOGGER.debug("'{}' exists".format(target_file))
         counter += 1
     with rgc_server as r:
         _purge_nonservable(r)
-        _LOGGER.info("builder finished; server config file saved to: '{}'".format(r.write(server_rgc_path)))
+        _LOGGER.info("Builder finished; server config file saved to: '{}'".format(r.write(server_rgc_path)))
 
 
 def _check_tgz(path, output, asset_name):
@@ -190,7 +207,7 @@ def _check_tgz(path, output, asset_name):
         _LOGGER.debug("command: {}".format(cmd.format(p=path, o=output, an=asset_name)))
         run(cmd.format(p=path, o=output, an=asset_name), shell=True)
     else:
-        raise OSError("entity '{}' does not exist".format(path))
+        raise OSError("Entity '{}' does not exist".format(path))
 
 
 def _copy_log(input_dir, target_dir, asset_name, tag_name):
@@ -264,9 +281,9 @@ def _remove_archive(rgc, registry_paths):
                 [rgc.remove_assets(genome, x, None) for x in rgc.list_assets_by_genome(genome)]
             else:
                 rgc.remove_assets(genome, asset, tag)
-            _LOGGER.info("{}/{}:{} removed".format(genome, asset, tag))
+            _LOGGER.info("{}/{}:{} removed".format(genome, asset, ":" + tag if tag else ""))
         except KeyError:
-            _LOGGER.warning("{}/{}:{} not found and not removed.".format(genome, asset, tag))
+            _LOGGER.warning("{}/{}{} not found and not removed.".format(genome, asset, ":" + tag if tag else ""))
             continue
         ret.append(os.path.join(rgc[CFG_ARCHIVE_KEY], genome, "{}__{}".format(asset or "*", tag or "*") + ".tgz"))
         for p in ret:

diff --git a/refgenieserver/templates/index.html b/refgenieserver/templates/index.html
@@ -67,8 +67,8 @@ <h4 id="{{ genome_name }}">Reference genome: <b>{{ genome_name }}</b></h4>
                         </tr>
                     </thead>
                     <tbody>
-                        {% for asset_name, asset_data in values.assets.items() %}
-                            {% for tag_name, tag_data in asset_data.tags.items() %}
+                        {% for asset_name, asset_data in values.assets.items() | sort %}
+                            {% for tag_name, tag_data in asset_data.tags.items() | sort %}
                                 {% if tag_name != "default_tag" %}
                                     <tr>
                                         <td>

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
@@ -3,4 +3,4 @@ fastapi
 jinja2
 logmuse>=0.2
 uvicorn>=0.7.1
-refgenconf>=0.6.0
+refgenconf>=0.6.1
diff --git a/staging.Dockerfile b/staging.Dockerfile
@@ -2,5 +2,4 @@ FROM tiangolo/uvicorn-gunicorn:python3.7-alpine3.8
 LABEL authors="Nathan Sheffield, Michal Stolarczyk"
 
 COPY . /app
-
-RUN pip install .
+RUN pip install .