apps: Publish Apps with layers metadata

1. Download layers metadata of images built by each build run,
   and merge them into a single dict.
2. Pass the overall Apps' layers metadata to the compose-publish tool.
   The publish tool extracts metadata of specific App's layers and
   put them as a second layer/blob into the App manifest.

Signed-off-by: Mike <[email protected]>

apps/apps_publisher.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2020 Foundries.io
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import logging
 from tempfile import NamedTemporaryFile
 
@@ -15,11 +16,13 @@
 
 
 class AppsPublisher:
-    def __init__(self, factory, publish_tool: str, archs: str, registry_host=DockerRegistryClient.DefaultRegistryHost):
+    def __init__(self, factory, publish_tool: str, archs: str,
+                 registry_host=DockerRegistryClient.DefaultRegistryHost, layers_meta: dict = None):
         self._factory = factory
         self._publish_tool = publish_tool
         self._archs = archs
         self._registry_host = registry_host
+        self._layers_meta = layers_meta
 
         self._image_base_url = '{}/{}'.format(registry_host, self._factory)
         self._allowed_tags = ['${TAG}', 'latest']
@@ -83,6 +86,10 @@ def __publish(self, app: ComposeApps.App, tag: str):
         app_base_url = self._image_base_url + '/' + app.name
         self._app_tagged_url = app_base_url + ':app-' + tag
         # TODO: Consider implementation of the "publish tool" in DockerRegistryClient
-        with NamedTemporaryFile(mode="w+") as f:
-            cmd_exe(self._publish_tool, '-d', f.name, self._app_tagged_url, self._archs, cwd=app.dir)
-            return app_base_url + '@' + f.read().strip()
+        with NamedTemporaryFile(mode="w+") as layers_meta_file:
+            json.dump(self._layers_meta, layers_meta_file)
+            layers_meta_file.flush()
+            with NamedTemporaryFile(mode="w+") as f:
+                cmd_exe(self._publish_tool, '-d', f.name, '-l',  layers_meta_file.name,
+                        self._app_tagged_url, self._archs, cwd=app.dir)
+                return app_base_url + '@' + f.read().strip()

apps/publish.py

@@ -3,11 +3,12 @@
 # Copyright (c) 2020 Foundries.io
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import logging
 import argparse
 
 
-from helpers import fio_dnsbase, status
+from helpers import fio_dnsbase, status, jobserv_get
 from apps.target_manager import create_target
 from apps.compose_apps import ComposeApps
 from apps.apps_publisher import AppsPublisher
@@ -18,6 +19,16 @@
 logger = logging.getLogger(__name__)
 
 
+def get_layers_metadata(factory: str, ver: str, archs: []) -> dict:
+    layers_meta = {}
+    project = f"{factory}/lmp" if factory != "lmp" else "lmp"
+    for a in archs:
+        run_name = {"amd64": "build-amd64", "arm64": "build-aarch64", "arm": "build-armhf"}[a]
+        status(f"Downloading layers metadata built by `{run_name}` run", prefix="=== ")
+        layers_meta[a] = jobserv_get(f"/projects/{project}/builds/{ver}/runs/{run_name}/layers_meta.json")
+    return layers_meta
+
+
 def main(factory: str, sha: str, targets_json: str, machines: [], platforms: [], app_root_dir: str,
          publish_tool: str, apps_version: str, target_tag: str, target_version: str, new_targets_file: str):
     publish_manifest_lists()
@@ -28,9 +39,13 @@ def main(factory: str, sha: str, targets_json: str, machines: [], platforms: [],
 
     status('Compose Apps has been validated: {}'.format(apps.str))
 
+    status('Downloading Apps\' layers metadata...')
+    layers_meta = get_layers_metadata(factory, target_version, platforms)
+    status('Apps\' layers metadata have been downloaded')
+
     reg_host = "hub." + fio_dnsbase()
     archs = ','.join(platforms) if platforms else ''
-    apps_to_add_to_target = AppsPublisher(factory, publish_tool, archs, reg_host).publish(apps, apps_version)
+    apps_to_add_to_target = AppsPublisher(factory, publish_tool, archs, reg_host, layers_meta).publish(apps, apps_version)
 
     status('Creating Targets that refer to the published Apps; tag: {}, version: {}, machines: {}, platforms: {} '
            .format(target_tag, target_version, ','.join(machines) if machines else '[]',