wip

img

@@ -95,22 +95,21 @@ def relative_path_from_record(record):
 def print_records(records):
     if len(records)>0:
         print(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'sha256':16} {'size':<10}", "yellow"))
-        for rset in records:
-            for r in rset:
-                namestr = f"{r.name}/{r.version}"
-                tagstr = f"{r.tag}"
-                label = namestr + ":" + tagstr
-                datestr = r.date.strftime("%Y-%m-%d")
-                S = r.size
-                if S<1024:
-                    size_str = f"{S:<} bytes"
-                elif S<1024*1024:
-                    size_str = f"{(S/1024):<.0f} kB"
-                elif S<1024*1024*1024:
-                    size_str = f"{(S/(1024*1024)):<.0f} MB"
-                else:
-                    size_str = f"{(S/(1024*1024*1024)):<.1f} GB"
-                print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256[:16]:16} {size_str:<10}")
+        for r in records:
+            namestr = f"{r.name}/{r.version}"
+            tagstr = f"{r.tag}"
+            label = namestr + ":" + tagstr
+            datestr = r.date.strftime("%Y-%m-%d")
+            S = r.size
+            if S<1024:
+                size_str = f"{S:<} bytes"
+            elif S<1024*1024:
+                size_str = f"{(S/1024):<.0f} kB"
+            elif S<1024*1024*1024:
+                size_str = f"{(S/(1024*1024)):<.0f} MB"
+            else:
+                size_str = f"{(S/(1024*1024*1024)):<.1f} GB"
+            print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256[:16]:16} {size_str:<10}")
 
 # return dictionary {"name", "version", "tag"} from a uenv description string
 #       "prgenv_gnu"              -> ("prgenv_gnu", None, None)
@@ -169,6 +168,7 @@ if __name__ == "__main__":
 
     parser = make_argparser()
     args = parser.parse_args()
+    terminal.use_colored_output(args.no_color)
     if args.command is None:
         parser.print_help()
         sys.exit(0)
@@ -185,12 +185,14 @@ if __name__ == "__main__":
         except RuntimeError as err:
             terminal.error(f"{str(err)}")
 
-        terminal.info(f"downloaded jfrog meta data: build->{len(build.images)} images, deploy->{len(deploy.images)}")
+        terminal.info(f"downloaded jfrog meta data: build->{len(build.images)}, deploy->{len(deploy.images)}")
 
         remote_database = build if args.build else deploy
 
         records = remote_database.find_records(**img_filter)
 
+        terminal.info(f"The following records matched the query: {records}")
+
         if args.command == "find":
             if len(records)>0:
                 print_records(records)
@@ -230,7 +232,7 @@ if __name__ == "__main__":
                 image_path = cache.image_path(t)
 
                 # if the record isn't already in the filesystem repo download it
-                if cache.get_record(t.sha256) is None:
+                if not cache.get_record(t.sha256):
                     terminal.info(f"downloading {t.sha256}")
                     # download the image using oras
                     oras.run_command(["pull", "-o", image_path, source_address])
@@ -283,11 +285,11 @@ if __name__ == "__main__":
         terminal.info(f"downloaded jfrog build meta data: {len(build_database.images)} images")
 
         # after this block, record is the record of the requested source
-        if is_valid_sha256(source) or is_short_sha256(source):
+        if DataStore.is_valid_sha(source):
             # lookup using sha256
             source_record = build_database.get_record(source)
             terminal.info(f"image to deploy: {source_record}")
-            if source_record is None:
+            if not source_record:
                 terminal.error(f"no record in the build repository matches the hash {source}")
             source_record = source_record[0]
         else:

lib/datastore.py

@@ -7,15 +7,24 @@
 
 UENV_CLI_API_VERSION=1
 
-def is_sha256(s: str):
+def is_full_sha256(s: str):
     pattern = re.compile(r'^[a-fA-F-0-9]{64}$')
     return True if pattern.match(s) else False
 
 def is_short_sha256(s: str):
     pattern = re.compile(r'^[a-fA-F-0-9]{16}$')
     return True if pattern.match(s) else False
 
+
 class DataStore:
+    @staticmethod
+    def is_valid_sha(sha:str) -> bool:
+        if is_full_sha256(sha):
+            return True
+        if is_short_sha256(sha):
+            return True
+        return False
+
     def __init__(self):
         # all images store with (key,value) = (sha256,Record)
         self._images = {}
@@ -58,28 +67,36 @@ def find_records(self, **constraints):
         else:
             unique = set()
 
-        results = [self._images[sha] for sha in unique]
+        results = []
+        for sha in unique:
+            results += (self._images[sha])
+        #results = [self._images[sha] for sha in unique]
         results.sort(reverse=True)
         return results
 
     @property
     def images(self):
         return self._images
 
+    # return a list of records that match a sha
     def get_record(self, sha: str) -> Record:
-        if is_sha256(sha):
-            return self._images.get(sha, None)
+        if is_full_sha256(sha):
+            return self._images.get(sha, [])
         elif is_short_sha256(sha):
-            return self._images.get(self._short_sha[sha], None)
+            return self._images.get(self._short_sha[sha], [])
         raise ValueError(f"{sha} is not a valid sha256 or short (16 character) sha")
 
     # Convert to a dictionary that can be written to file as JSON
     # The serialisation and deserialisation are central: able to represent
     # uenv that are available in both JFrog and filesystem directory tree.
     def serialise(self, version: int=UENV_CLI_API_VERSION):
+        image_list = []
+        for x in self._images.values():
+            image_list += x
+        terminal.info(f"serialized image list in datastore: {image_list}")
         return {
                 "API_VERSION": version,
-                "images": [img.dictionary for img in self._images.values()]
+                "images": [img.dictionary for img in image_list]
         }
 
     # Convert to a dictionary that can be written to file as JSON
@@ -137,6 +154,8 @@ def image_path(self, r: Record) -> str:
     # Return the full record for a given hash
     # Returns None if no image with that hash is stored in the repo.
     def get_record(self, sha256: str):
+        if not DataStore.is_valid_sha(sha256):
+            raise ValueError(f"{sha256} is not a valid image sha256 (neither full 64 or short 16 character form)")
         return self._database.get_record(sha256)
 
     def publish(self):

lib/jfrog.py

@@ -1,17 +1,10 @@
-from datetime import datetime, timezone
 import requests
 
 from datastore import DataStore
 from record import Record
 import terminal
 
 
-def to_datetime(date: str):
-    # In Python 3.6, datetime.fromisoformat is not available.
-    # Manually parsing the string.
-    dt_format = '%Y-%m-%dT%H:%M:%S.%fZ'
-    return datetime.strptime(date, dt_format).replace(tzinfo=timezone.utc)
-
 # The https://cicd-ext-mw.cscs.ch/uenv/list API endpoint returns
 # a list of images in the jfrog uenv.
 #
@@ -57,7 +50,7 @@ def query() -> tuple:
         for record in raw_records["results"]:
             path = record["path"]
 
-            date = to_datetime(record["created"])
+            date = Record.to_datetime(record["created"])
             sha256 = record["sha256"]
             size = record["size"]
             if path.startswith("build/"):

lib/record.py

@@ -1,4 +1,13 @@
+from datetime import datetime, timezone
+
 class Record:
+    @staticmethod
+    def to_datetime(date: str):
+        # In Python 3.6, datetime.fromisoformat is not available.
+        # Manually parsing the string.
+        dt_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        return datetime.strptime(date, dt_format).replace(tzinfo=timezone.utc)
+
 
     def __init__(self, system: str, uarch: str, name: str, version: str, tag: str, date: str, size_bytes: int, sha256: str):
         self._system  = system
@@ -27,7 +36,7 @@ def fromjson(cls, raw: dict):
         name = raw["name"]
         version = raw["version"]
         tag = raw["tag"]
-        date = to_datetime(raw["date"])
+        date = Record.to_datetime(raw["date"])
         size_bytes = raw["size"]
         sha256 = raw["sha256"]
 
@@ -47,6 +56,8 @@ def __lt__(self, other):
         if other.name   < self.name: return False
         if self.version < other.version: return True
         if other.version< self.version: return False
+        if self.tag=="latest"  and other.tag!="latest": return False
+        if other.tag=="latest" and self.tag!="latest":  return True
         if self.tag     < other.tag: return True
         #if other.tag    < self.tag: return False
         return False

lib/terminal.py

@@ -1,16 +1,19 @@
+import os
 import sys
 
 colored_output = True
+debug_level = 1
 
 # Choose whether to use colored output.
 # - by default colored output is ON
 # - if the flag --no-color is passed it is OFF
 # - if the environment variable NO_COLOR is set it is OFF
 def use_colored_output(cli_arg):
-    colored_output = True
+    global colored_output
 
     # The --no-color argument overrides all environment variables if passed.
     if cli_arg:
+        print(colorize("nope", "cyan"))
         colored_output = False
         return
 
@@ -38,6 +41,10 @@ def colorize(string, color):
     else:
         return string
 
+def deubg_level(level: int):
+    global debug_level
+    debug_level = level
+
 def error(message):
     print(f"{colorize('[error]', 'red')} {message}", file=sys.stderr)
     exit(1)
@@ -46,6 +53,7 @@ def exit_with_success():
     exit(0)
 
 def info(message):
-    print(f"{colorize('[log]', 'yellow')} {message}", file=sys.stderr)
+    if debug_level>1:
+        print(f"{colorize('[log]', 'blue')} {message}", file=sys.stderr)