Merge pull request #65 from databio/dev

v0.4.0

changelog.md

@@ -2,10 +2,21 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.4.0] -- 2019-12-06
+### Added
+- asset splash pages presenting all the asset attributes and related API endpoint links. Available at: `/asset/{genome}/{asset}/splash`
+- archive digest API endpoint. Available at `/asset/{genome}/{asset}/{tag}/archive_digest`
+
+### Changed
+- recipes are served as JSON objects, not files
+- `refgenieserver archive` enhancements; related to config file locking
+
 ## [0.3.4] -- 2019-11-06
 ### Added
 - distribute the license file with the package
 - test script
+- API endpoints for serving asset build logs and asset build recipes. Available at `/asset/{genome}/{asset}/log` and `/asset/{genome}/{asset}/recipe`, respectively
+
 
 ### Fixed
 - `fastapi` - `starlette` dependency issue

refgenieserver/_version.py

@@ -1 +1 @@
-__version__ = "0.3.3"
+__version__ = "0.4.0"

refgenieserver/helpers.py

@@ -95,3 +95,16 @@ def preprocess_attrs(attrs):
             attrs_cpy[CHANGED_KEYS[new_key]] = attrs_cpy[new_key]
             del attrs_cpy[new_key]
     return attrs_cpy
+
+
+def get_openapi_version(app):
+    """
+    Get the OpenAPI version from the OpenAPI description JSON
+
+    :param fastapi.FastAPI app: app object
+    :return str: openAPI version in use
+    """
+    try:
+        return app.openapi()["openapi"]
+    except Exception as e:
+        return "3.0.2"

refgenieserver/routers/version1.py

@@ -1,9 +1,11 @@
 from starlette.responses import FileResponse
 from starlette.requests import Request
 from fastapi import HTTPException, APIRouter
+
 from ..const import *
 from ..helpers import preprocess_attrs
-from ..main import rgc, templates, _LOGGER
+from ..main import rgc, templates, _LOGGER, app
+from ..helpers import get_openapi_version
 
 router = APIRouter()
 
@@ -15,9 +17,10 @@ async def index(request: Request):
     Returns a landing page HTML with the server resources ready do download. No inputs required.
     """
     _LOGGER.debug("RefGenConf object:\n{}".format(rgc))
-    vars = {"request": request, "genomes": rgc[CFG_GENOMES_KEY], "rgc": rgc[CFG_GENOMES_KEY]}
-    _LOGGER.debug("merged vars: {}".format(dict(vars, **ALL_VERSIONS)))
-    return templates.TemplateResponse("index.html", dict(vars, **ALL_VERSIONS))
+    templ_vars = {"request": request, "genomes": rgc[CFG_GENOMES_KEY], "rgc": rgc[CFG_GENOMES_KEY],
+                  "openapi_version": get_openapi_version(app)}
+    _LOGGER.debug("merged vars: {}".format(dict(templ_vars, **ALL_VERSIONS)))
+    return templates.TemplateResponse("index.html", dict(templ_vars, **ALL_VERSIONS))
 
 
 @router.get("/genomes")

refgenieserver/routers/version2.py

@@ -1,8 +1,14 @@
-from starlette.responses import FileResponse
+from starlette.responses import FileResponse, JSONResponse
 from starlette.requests import Request
 from fastapi import HTTPException, APIRouter
+
+from ubiquerg import parse_registry_path
+from refgenconf.refgenconf import map_paths_by_id
+
 from ..const import *
-from ..main import rgc, templates, _LOGGER
+from ..main import rgc, templates, _LOGGER, app
+from ..helpers import get_openapi_version
+
 router = APIRouter()
 
 
@@ -13,13 +19,29 @@ async def index(request: Request):
     Returns a landing page HTML with the server resources ready do download. No inputs required.
     """
     _LOGGER.debug("RefGenConf object:\n{}".format(rgc))
-    vars = {"request": request, "genomes": rgc[CFG_GENOMES_KEY], "rgc": rgc[CFG_GENOMES_KEY]}
-    _LOGGER.debug("merged vars: {}".format(dict(vars, **ALL_VERSIONS)))
-    return templates.TemplateResponse("index.html", dict(vars, **ALL_VERSIONS))
+    templ_vars = {"request": request, "genomes": rgc[CFG_GENOMES_KEY], "rgc": rgc[CFG_GENOMES_KEY],
+                  "openapi_version": get_openapi_version(app)}
+    _LOGGER.debug("merged vars: {}".format(dict(templ_vars, **ALL_VERSIONS)))
+    return templates.TemplateResponse("index.html", dict(templ_vars, **ALL_VERSIONS))
+
+
+@router.get("/asset/{genome}/{asset}/splash")
+async def asset_splash_page(request: Request, genome: str, asset: str, tag: str = None):
+    """
+    Returns an asset splash page
+    """
+    tag = tag or rgc.get_default_tag(genome, asset)  # returns 'default' for nonexistent genome/asset; no need to catch
+    links_dict = {OPERATION_IDS["asset"][oid]: path.format(genome=genome, asset=asset, tag=tag)
+                  for oid, path in map_paths_by_id(app.openapi()).items() if oid in OPERATION_IDS["asset"].keys()}
+    templ_vars = {"request": request, "genome": genome, "asset": asset,
+                  "tag": tag, "rgc": rgc, "prp": parse_registry_path, "links_dict": links_dict,
+                  "openapi_version": get_openapi_version(app)}
+    _LOGGER.debug("merged vars: {}".format(dict(templ_vars, **ALL_VERSIONS)))
+    return templates.TemplateResponse("asset.html", dict(templ_vars, **ALL_VERSIONS))
 
 
 @router.get("/genomes")
-def list_available_genomes():
+async def list_available_genomes():
     """
     Returns a list of genomes this server holds at least one asset for. No inputs required.
     """
@@ -28,7 +50,7 @@ def list_available_genomes():
 
 
 @router.get("/assets", operation_id=API_ID_ASSETS)
-def list_available_assets():
+async def list_available_assets():
     """
     Returns a list of all assets that can be downloaded. No inputs required.
     """
@@ -77,6 +99,19 @@ async def get_asset_digest(genome: str, asset: str, tag: str):
         raise HTTPException(status_code=404, detail=msg)
 
 
+@router.get("/asset/{genome}/{asset}/{tag}/archive_digest", operation_id=API_ID_ARCHIVE_DIGEST)
+async def get_asset_digest(genome: str, asset: str, tag: str):
+    """
+    Returns the archive digest. Requires genome name asset name and tag name as an input.
+    """
+    try:
+        return rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset][CFG_ASSET_TAGS_KEY][tag][CFG_ARCHIVE_CHECKSUM_KEY]
+    except KeyError:
+        msg = MSG_404.format("genome/asset:tag combination ({}/{}:{})".format(genome, asset, tag))
+        _LOGGER.warning(msg)
+        raise HTTPException(status_code=404, detail=msg)
+
+
 @router.get("/asset/{genome}/{asset}/log", operation_id=API_ID_LOG)
 async def download_asset_build_log(genome: str, asset: str, tag: str = None):
     """
@@ -103,20 +138,23 @@ async def download_asset_build_recipe(genome: str, asset: str, tag: str = None):
 
     Optionally, 'tag' query parameter can be specified to get a tagged asset archive. Default tag is returned otherwise.
     """
+    import json
     tag = tag or rgc.get_default_tag(genome, asset)  # returns 'default' for nonexistent genome/asset; no need to catch
     file_name = TEMPLATE_RECIPE_JSON.format(asset, tag)
     recipe_file = "{base}/{genome}/{file_name}".format(base=BASE_DIR, genome=genome, file_name=file_name)
     _LOGGER.info("serving build recipe file: '{}'".format(recipe_file))
     if os.path.isfile(recipe_file):
-        return FileResponse(recipe_file, filename=file_name, media_type="application/octet-stream")
+        with open(recipe_file, 'r') as f:
+            recipe = json.load(f)
+        return JSONResponse(recipe)
     else:
         msg = MSG_404.format("asset ({})".format(asset))
         _LOGGER.warning(msg)
         raise HTTPException(status_code=404, detail=msg)
 
 
 @router.get("/asset/{genome}/{asset}", operation_id=API_ID_ASSET_ATTRS)
-def download_asset_attributes(genome: str, asset: str, tag: str = None):
+async def download_asset_attributes(genome: str, asset: str, tag: str = None):
     """
     Returns a dictionary of asset attributes, like archive size, archive digest etc.
     Requires the genome name and the asset name as an input.
@@ -165,7 +203,7 @@ async def download_genome_attributes(genome: str):
 
 
 @router.get("/genomes/{asset}")
-def list_genomes_by_asset(asset: str):
+async def list_genomes_by_asset(asset: str):
     """
     Returns a list of genomes that have the requested asset defined. Requires the asset name as an input.
     """

refgenieserver/server_builder.py

@@ -45,24 +45,25 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
     # original RefGenConf has been created in read-only mode,
     # make it RW compatible and point to new target path for server use or initialize a new object
     if os.path.exists(server_rgc_path):
-        rgc_server = RefGenConf(filepath=server_rgc_path, writable=True)
+        rgc_server = RefGenConf(filepath=server_rgc_path)
         if remove:
             if not registry_paths:
                 _LOGGER.error("To remove archives you have to specify them. Use 'asset_registry_path' argument.")
                 exit(1)
-            _remove_archive(rgc_server, registry_paths)
-            rgc_server.write()
+            with rgc_server as r:
+                _remove_archive(r, registry_paths)
             exit(0)
     else:
         if remove:
             _LOGGER.error("You can't remove archives since the genome_archive path does not exist yet.")
             exit(1)
         rgc_server = RefGenConf(filepath=rgc.file_path)
         rgc_server.make_writable(filepath=server_rgc_path)
+        rgc_server.make_readonly()
     if registry_paths:
-        genomes, asset_list, tag_list = _get_paths_element(registry_paths, "namespace"), \
-                                        _get_paths_element(registry_paths, "item"), _get_paths_element(registry_paths,
-                                                                                                       "tag")
+        genomes = _get_paths_element(registry_paths, "namespace")
+        asset_list = _get_paths_element(registry_paths, "item")
+        tag_list = _get_paths_element(registry_paths, "tag")
     else:
         genomes = rgc.genomes_list()
         asset_list, tag_list = None, None
@@ -92,7 +93,8 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
         genome_checksum = rgc[CFG_GENOMES_KEY][genome].setdefault(CFG_CHECKSUM_KEY, CHECKSUM_PLACEHOLDER)
         genome_attrs = {CFG_GENOME_DESC_KEY: genome_desc,
                         CFG_CHECKSUM_KEY: genome_checksum}
-        rgc_server.update_genomes(genome, genome_attrs)
+        with rgc_server as r:
+            r.update_genomes(genome, genome_attrs)
         _LOGGER.debug("updating '{}' genome attributes...".format(genome))
         asset = asset_list[counter] if asset_list is not None else None
         assets = asset or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY].keys()
@@ -109,16 +111,17 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
             asset_attrs = {CFG_ASSET_DESC_KEY: asset_desc,
                            CFG_ASSET_DEFAULT_TAG_KEY: default_tag}
             _LOGGER.debug("updating '{}/{}' asset attributes...".format(genome, asset_name))
-            rgc_server.update_assets(genome, asset_name, asset_attrs)
+            with rgc_server as r:
+                r.update_assets(genome, asset_name, asset_attrs)
             tag = tag_list[counter] if tag_list is not None else None
             tags = tag or rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY].keys()
             for tag_name in tags if isinstance(tags, list) else [tags]:
                 if not rgc.is_asset_complete(genome, asset_name, tag_name):
                     _LOGGER.info("'{}/{}:{}' is incomplete, skipping".format(genome, asset_name, tag_name))
-                    rgc_server.remove_assets(genome, asset_name, tag_name)
+                    with rgc_server as r:
+                        print(r)
+                        r.remove_assets(genome, asset_name, tag_name)
                     continue
-                rgc_server.write()
-                del rgc_server
                 file_name = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset_name][CFG_ASSET_TAGS_KEY][tag_name][
                     CFG_ASSET_PATH_KEY]
                 target_file = os.path.join(target_dir, "{}__{}".format(asset_name, tag_name) + ".tgz")
@@ -141,10 +144,8 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                         _copy_log(input_file, target_dir, asset_name, tag_name)
                     except OSError as e:
                         _LOGGER.warning(e)
-                        rgc_server = RefGenConf(filepath=server_rgc_path, writable=True)
                         continue
                     else:
-                        rgc_server = RefGenConf(filepath=server_rgc_path, writable=True)
                         _LOGGER.info("updating '{}/{}:{}' tag attributes...".format(genome, asset_name, tag_name))
                         tag_attrs = {CFG_ASSET_PATH_KEY: file_name,
                                      CFG_SEEK_KEYS_KEY: seek_keys,
@@ -155,14 +156,14 @@ def archive(rgc, registry_paths, force, remove, cfg_path, genomes_desc):
                                      CFG_ASSET_CHILDREN_KEY: children,
                                      CFG_ASSET_CHECKSUM_KEY: asset_digest}
                         _LOGGER.debug("attr dict: {}".format(tag_attrs))
-                        rgc_server.update_tags(genome, asset_name, tag_name, tag_attrs)
-                        rgc_server.write()
+                        with rgc_server as r:
+                            r.update_tags(genome, asset_name, tag_name, tag_attrs)
                 else:
                     _LOGGER.debug("'{}' exists".format(target_file))
-                    rgc_server = RefGenConf(filepath=server_rgc_path, writable=True)
         counter += 1
-    rgc_server = _purge_nonservable(rgc_server)
-    _LOGGER.info("builder finished; server config file saved to: '{}'".format(rgc_server.write(server_rgc_path)))
+    with rgc_server as r:
+        _purge_nonservable(r)
+        _LOGGER.info("builder finished; server config file saved to: '{}'".format(r.write(server_rgc_path)))
 
 
 def _check_tgz(path, output, asset_name):
@@ -199,13 +200,13 @@ def _copy_log(input_dir, target_dir, asset_name, tag_name):
     :param str input_dir: path to the directory to copy the recipe from
     :param str target_dir: path to the directory to copy the recipe to
     """
-    log_path = "{}/_refgenie_build/refgenie_log.md".format(input_dir)
+    log_path = "{}/{}/{}".format(input_dir, BUILD_STATS_DIR, ORI_LOG_NAME)
     if log_path and os.path.exists(log_path):
         run("cp " + log_path + " " +
-            os.path.join(target_dir, "build_log_{}__{}.md".format(asset_name, tag_name)), shell=True)
+            os.path.join(target_dir, TEMPLATE_LOG.format(asset_name, tag_name)), shell=True)
         _LOGGER.debug("Log copied to: {}".format(target_dir))
     else:
-        _LOGGER.debug("Log not found: ".format(log_path))
+        _LOGGER.warning("Log not found: {}".format(log_path))
 
 
 def _copy_recipe(input_dir, target_dir, asset_name, tag_name):
@@ -217,13 +218,12 @@ def _copy_recipe(input_dir, target_dir, asset_name, tag_name):
     :param str asset_name: asset name
     :param str tag_name: tag name
     """
-    recipe_path = "{}/_refgenie_build/build_recipe_{}__{}.json". \
-        format(input_dir, asset_name, tag_name)
+    recipe_path = "{}/{}/{}".format(input_dir, BUILD_STATS_DIR, TEMPLATE_RECIPE_JSON.format(asset_name, tag_name))
     if recipe_path and os.path.exists(recipe_path):
         run("cp " + recipe_path + " " + target_dir, shell=True)
         _LOGGER.debug("Recipe copied to: {}".format(target_dir))
     else:
-        _LOGGER.debug("Recipe not found: {}".format(recipe_path))
+        _LOGGER.warning("Recipe not found: {}".format(recipe_path))
 
 
 def _purge_nonservable(rgc):
@@ -233,7 +233,6 @@ def _purge_nonservable(rgc):
     :param refgenconf.RefGenConf rgc: object to check
     :return refgenconf.RefGenConf: object with just the servable entries
     """
-
     def _check_servable(rgc, genome, asset, tag):
         tag_data = rgc[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset][CFG_ASSET_TAGS_KEY][tag]
         return all([r in tag_data for r in [CFG_ARCHIVE_CHECKSUM_KEY, CFG_ARCHIVE_SIZE_KEY]])
@@ -293,7 +292,6 @@ def _correct_registry_paths(registry_paths):
     :param list[dict] registry_paths: output of parse_registry_path
     :return list[dict]: corrected registry paths
     """
-
     def _swap(rp):
         """
         Swaps dict values of 'namespace' with 'item' keys

refgenieserver/static/style.css

@@ -1,8 +1,3 @@
-* {
-  margin: 0;
-  padding: 0;
-}
-
 html,
 body {
   height: 100%;
@@ -44,10 +39,6 @@ body {
   /*margin-bottom:15px;*/
 }
 
-code.footer {
-  color:#464c56
-}
-
 span {
   margin-left: 5px;
 }