pull meta data with squashfs

lib/oras.py

@@ -1,3 +1,4 @@
+import json
 import os
 import pathlib
 import shutil
@@ -67,37 +68,69 @@ def run_command(args):
         terminal.error(f"oras command failed with unknown exception: {e}")
 
 
-def pull_uenv(source_address, image_path, target):
+def pull_uenv(source_address, image_path, target, pull_meta, pull_sqfs):
     # download the image using oras
     try:
-        # run the oras command in a separate process so that this process can
-        # draw a progress bar.
-        proc = run_command_internal(["pull", "-o", image_path, source_address])
-
-        # remove the old path if it exists
-        if os.path.exists(image_path):
-            shutil.rmtree(image_path)
-            time.sleep(0.2)
-
-        sqfs_path = image_path + "/store.squashfs"
-        total_mb = target.size/(1024*1024)
-        while proc.poll() is None:
-            time.sleep(1.0)
-            if os.path.exists(sqfs_path) and terminal.is_tty():
-                current_size = os.path.getsize(sqfs_path)
-                current_mb = current_size / (1024*1024)
-                p = current_mb/total_mb
-                msg = f"{int(current_mb)}/{int(total_mb)} MB"
-                progress.progress_bar(p, width=50, msg=msg)
-        stdout, stderr = proc.communicate()
-        if proc.returncode == 0:
-            # draw a final complete progress bar
-            progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB")
-            terminal.stdout("")
-            terminal.info(f"oras command successful: {stdout}")
-        else:
-            msg = error_message_from_stderr(stderr)
-            terminal.error(f"image pull failed: {stderr}\n{msg}")
+        terms = source_address.rsplit(":", 1)
+        source_address = terms[0]
+        tag = terms[1]
+
+        if pull_meta:
+            if os.path.exists(image_path+"/meta"):
+                shutil.rmtree(image_path+"/meta")
+                time.sleep(0.05)
+
+            # step 1: download the meta data if there is any
+            #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1
+            terminal.info(f"discovering meta data")
+            proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"])
+            stdout, stderr = proc.communicate()
+            if proc.returncode == 0:
+                terminal.info(f"successfully downloaded meta data info: {stdout}")
+            else:
+                msg = error_message_from_stderr(stderr)
+                terminal.error(f"failed to find meta data: {stderr}\n{msg}")
+
+            manifests = json.loads(stdout)["manifests"]
+            if len(manifests)==0:
+                terminal.error(f"no meta data is available")
+
+            digest = manifests[0]["digest"]
+            terminal.info(f"meta data digest: {digest}")
+
+            proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"])
+            stdout, stderr = proc.communicate()
+
+        if pull_sqfs:
+            sqfs_path = image_path + "/store.squashfs"
+            if os.path.exists(sqfs_path):
+                os.remove(sqfs_path)
+                time.sleep(0.05)
+
+            # step 2: download the image itself
+            # run the oras command in a separate process so that this process can
+            # draw a progress bar.
+            proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"])
+
+            total_mb = target.size/(1024*1024)
+            while proc.poll() is None:
+                time.sleep(0.25)
+                if os.path.exists(sqfs_path) and terminal.is_tty():
+                    current_size = os.path.getsize(sqfs_path)
+                    current_mb = current_size / (1024*1024)
+                    p = current_mb/total_mb
+                    msg = f"{int(current_mb)}/{int(total_mb)} MB"
+                    progress.progress_bar(p, width=50, msg=msg)
+            stdout, stderr = proc.communicate()
+            if proc.returncode == 0:
+                # draw a final complete progress bar
+                progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB")
+                terminal.stdout("")
+                terminal.info(f"oras command successful: {stdout}")
+            else:
+                msg = error_message_from_stderr(stderr)
+                terminal.error(f"image pull failed: {stderr}\n{msg}")
+
     except KeyboardInterrupt:
         proc.terminate()
         terminal.stdout("")

todo-meta.md

@@ -0,0 +1,9 @@
+TODO list for meta data operations
+
++ pull meta data alongside squashfs images
++ optionally download meta data
++ image inspect
+    + add {meta} format option that will print the meta data path
++ uenv image pull: pull missing meta data of already downloaded images
+- image ls: annotate meta-data only pulls
+- image start / run : handle meta-data only pulls

todo.md

@@ -49,12 +49,11 @@ uenv start --view=develop icon
         setenv("PATH", ...)
         setenv("LD_LIBRARY_PATH", ...)
         |
+        execve(squashfs-mount)
+        |
         execve(bash)
         |
-        |_fork__
-                |
-                |
-                bash
+        bash
 ```
 
 ## Features

uenv-image

@@ -109,7 +109,8 @@ sha256:  {{sha256}}
 system:  {{system}}
 date:    {{date}}
 path:    {{path}}
-sqfs:    {{sqfs}}\
+sqfs:    {{sqfs}}
+meta:    {{meta}}\
 """ ,
                              help="optional format string")
     path_parser.add_argument("uenv", type=str)
@@ -153,6 +154,8 @@ downloaded uenv with the list command. For more information:
     find_parser.add_argument("-a", "--uarch", required=False, type=str)
     find_parser.add_argument("--build", action="store_true",
                              help="Search undeployed builds.", required=False)
+    find_parser.add_argument("--no-header", action="store_true",
+                             help="Do not print header in output.", required=False)
     find_parser.add_argument("uenv", nargs="?", default=None, type=str)
 
     pull_parser = subparsers.add_parser("pull",
@@ -188,6 +191,10 @@ with appropriate JFrog access and with the JFrog token in their oras keychain.
     pull_parser.add_argument("-a", "--uarch", required=False, type=str)
     pull_parser.add_argument("--build", action="store_true", required=False,
                              help="enable undeployed builds")
+    pull_parser.add_argument("--only-meta", action="store_true", required=False,
+                             help="only download meta data, if it is available")
+    pull_parser.add_argument("--force", action="store_true", required=False,
+                             help="force download if the image has already been downloaded")
     pull_parser.add_argument("uenv", nargs="?", default=None, type=str)
 
     list_parser = subparsers.add_parser("ls",
@@ -219,6 +226,8 @@ List uenv that are available.
 """)
     list_parser.add_argument("-s", "--system", required=False, type=str)
     list_parser.add_argument("-a", "--uarch", required=False, type=str)
+    list_parser.add_argument("--no-header", action="store_true",
+                             help="Do not print header in output.", required=False)
     list_parser.add_argument("uenv", nargs="?", default=None, type=str)
 
     deploy_parser = subparsers.add_parser("deploy",
@@ -292,6 +301,8 @@ def get_filter(args):
 
 def inspect_string(record: record.Record, image_path, format_string: str) -> str:
     try:
+        meta = image_path+"/meta" if os.path.exists(image_path+"/meta") else "none"
+        sqfs = image_path+"/store.squashfs" if os.path.exists(image_path+"/store.squashfs") else "none"
         return format_string.format(
             system = record.system,
             uarch = record.uarch,
@@ -302,7 +313,8 @@ def inspect_string(record: record.Record, image_path, format_string: str) -> str
             id = record.id,
             sha256 = record.sha256,
             path = image_path,
-            sqfs = image_path + "/store.squashfs",
+            sqfs = sqfs,
+            meta = meta,
         )
     except Exception as err:
         terminal.error(f"unable to format {str(err)}")
@@ -317,11 +329,12 @@ def image_size_string(size):
     return f"{(size/(1024*1024*1024)):<.1f}GB"
 
 # pretty print a list of Record
-def print_records(recordset):
+def print_records(recordset, no_header=False):
     records = recordset.records
 
     if not recordset.is_empty:
-        terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow"))
+        if not args.no_header:
+            terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow"))
         for r in recordset.records:
             namestr = f"{r.name}/{r.version}"
             tagstr = f"{r.tag}"
@@ -406,7 +419,7 @@ if __name__ == "__main__":
             terminal.error(f"no uenv matches the spec: {colorize(results.request, 'white')}")
 
         if args.command == "find":
-            print_records(results)
+            print_records(results, no_header=args.no_header)
 
         elif args.command == "pull":
             if not results.is_unique_sha:
@@ -423,32 +436,67 @@ if __name__ == "__main__":
             t = records.records[0]
             source_address = jfrog.address(t, 'build' if args.build else 'deploy')
 
-            terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB")
+            terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB with only-meta={args.only_meta}")
 
             terminal.info(f"repo path: {repo_path}")
 
             cache = safe_repo_open(repo_path)
             image_path = cache.image_path(t)
+            terminal.info(f"image path: {image_path}")
 
-            # if the record isn't already in the filesystem repo download it
+            # at this point the request is for an sha that is in the remote repository
+            do_download=False
+
+            meta_path=image_path+"/meta"
+            sqfs_path=image_path+"/store.squashfs"
+            meta_exists=os.path.exists(meta_path)
+            sqfs_exists=os.path.exists(sqfs_path)
+
+            only_meta=meta_exists and not sqfs_exists
+
+            # if there is no entry in the local database do a full clean download
             if cache.database.get_record(t.sha256).is_empty:
-                terminal.stdout(f"downloading image {t.sha256} {image_size_string(t.size)}")
-                # clean up the path if it already exists: sometimes this causes an oras error.
-                if os.path.exists(image_path):
-                    terminal.info(f"removing existing path {image_path}")
-                    shutil.rmtree(image_path)
-                oras.pull_uenv(source_address, image_path, t)
+                terminal.info("===== is_empty")
+                do_download=True
+                pull_meta=True
+                pull_sqfs=not args.only_meta
+            elif args.force:
+                terminal.info("===== force")
+                do_download=True
+                pull_meta=True
+                pull_sqfs=not args.only_meta
+            # a record exists, so check whether any components are missing
             else:
-                terminal.stdout(f"image {t.sha256} is already available locally")
+                terminal.info("===== else")
+                pull_meta=not meta_exists
+                pull_sqfs=not sqfs_exists and (not args.only_meta)
+                do_download=pull_meta or pull_sqfs
+
+            terminal.info(f"pull {t.sha256} exists:  meta={meta_exists} sqfs={sqfs_exists}")
+            terminal.info(f"pull {t.sha256} pulling: meta={pull_meta} sqfs={pull_sqfs}")
+            if do_download:
+                terminal.info(f"downloading")
+            else:
+                terminal.info(f"nothing to pull: use --force to force the download")
+
+            # determine whether to perform download
+            # check whether the image is in the database, or when only meta-data has been downloaded
+            if do_download:
+                terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} matches remote image {t.sha256}")
+                if pull_meta:
+                    terminal.stdout(f"{t.id} pulling meta data")
+                if pull_sqfs:
+                    terminal.stdout(f"{t.id} pulling squashfs")
+                oras.pull_uenv(source_address, image_path, t, pull_meta, pull_sqfs)
+            else:
+                terminal.stdout(f"{t.name}/{t.version}:{t.tag} meta data and image with id {t.id} have already been pulled")
 
             # update all the tags associated with the image.
             terminal.info(f"updating the local repository database")
             for r in records.records:
                 terminal.stdout(f"updating local reference {r.name}/{r.version}:{r.tag}")
                 cache.add_record(r)
 
-            terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} downloaded")
-
         sys.exit(0)
 
     elif args.command == "ls":
@@ -459,7 +507,7 @@ if __name__ == "__main__":
         fscache = safe_repo_open(repo_path)
 
         records = fscache.database.find_records(**img_filter)
-        print_records(records)
+        print_records(records, no_header=args.no_header)
 
         sys.exit(0)