From ceaef610958b4fa7cba7bc0c8e9760729f66d796 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 25 Jun 2024 08:45:48 +0200 Subject: [PATCH 1/7] purge modules command --- uenv-impl | 189 ++---------------------------------------------------- 1 file changed, 6 insertions(+), 183 deletions(-) diff --git a/uenv-impl b/uenv-impl index 08dac93..1f7bc38 100755 --- a/uenv-impl +++ b/uenv-impl @@ -69,10 +69,6 @@ Note how the command to execute comes after the two dashes '--'. {colorize("Example", "blue")} - run the script job.sh in an evironmnent with a view loaded: {colorize("uenv run prgenv-gnu/24.2:v1 --view=default -- ./job.sh", "white")} -{colorize("Example", "blue")} - run the script job.sh in an evironmnent with modules enabled: - {colorize("uenv run prgenv-gnu/24.2:v1 --modules -- ./job.sh", "white")} -If the uenv provides modules, they will be visible through "module avail" and "module load". - {colorize("Note", "cyan")} - the spec must uniquely identify the uenv. To ensure this, always use a fully qualified spec in the form of name/version:tag, the unique 16 digit id, or sha256 of a uenv. If more than one uenv match the spec, an error message @@ -89,7 +85,7 @@ Here the mount point for each image is specified using a ":". {colorize("Note", "cyan")} - uenv must be mounted at the mount point for which they were built. If mounted at the wrong location, a warning message will be printed, and -features like views and modules will be disabled. +views will be disabled. {colorize("Example", "blue")} - the run command can be used to execute workflow steps with separate environments: @@ -103,9 +99,6 @@ separate environments: run_parser.add_argument("-v", "--view", help="The name of the environment view to activate when the uenv is started.", required=False, type=str) - run_parser.add_argument("-m", "--modules", - help="Enable any modules provided by the uenv.", - action="store_true") run_parser.add_argument("runline", nargs=argparse.REMAINDER, type=str) #### start @@ -122,10 +115,6 @@ This will mount prgenv-gnu at /user-environment, and start a new shell. {colorize("uenv start prgenv-gnu/24.2:v1 --view=default", "white")} Mount prgenv-gnu at /user-environment, and start a new shell with the view named "default" activated. -{colorize("Example", "blue")} - start a uenv with modules enabled: - {colorize("uenv start prgenv-gnu/24.2:v1 --modules", "white")} -Mount prgenv-gnu at /user-environment, and start a new shell with any modules provided by the uenv visible via "module avail". - {colorize("Note", "cyan")} - the spec must uniquely identify the uenv. To ensure this, always use a fully qualified spec in the form of name/version:tag, the unique 16 digit id, or sha256 of a uenv. If more than one uenv match the spec, an error message @@ -145,7 +134,7 @@ Here the mount point for each image is specified using a ":". {colorize("Note", "cyan")} - uenv must be mounted at the mount point for which they were built. If mounted at the wrong location, a warning message will be printed, and -features like views and modules will be disabled. +features like views will be disabled. """ )) start_parser.add_argument("-a", "--uarch", @@ -154,9 +143,6 @@ features like views and modules will be disabled. start_parser.add_argument("-v", "--view", help="The name of the environment view to activate when the uenv is started.", required=False, type=str) - start_parser.add_argument("-m", "--modules", - help="Enable any modules provided by the uenv.", - action="store_true") start_parser.add_argument("image", nargs='+', type=str, help="the uenv to start") @@ -168,44 +154,6 @@ features like views and modules will be disabled. status_parser = subparsers.add_parser("status", help="print information about any loaded uenv") - #### modules - modules_parser = subparsers.add_parser("modules", - help="use modules if they are available", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent( -f"""\ -{colorize("Example", "blue")} - get information about uenv modules: - {colorize("uenv modules", "white")} - -The sample output below is generated when there is one started uenv that -provides modules. In this case the modules have already been made available -using the {colorize("uenv modules use", "white")} command (see below). - -{colorize("the following loaded uenv provide modules:", "gray")} -{colorize(" prgenv-gnu:/user-environment (loaded)", "gray")} - -{colorize("Example", "blue")} - enable the modules provided by the current uenv: - {colorize("uenv modules use", "white")} - -Once enabled, the modules will be available through the system module command, -e.g. {colorize("module avail", "white")} and {colorize("module load ...", "white")}. - -{colorize("Note", "cyan")} - the {colorize("uenv modules use", "white")} command above is eqivalent to the module command -{colorize("module use /user-environment/modules", "white")}. - -{colorize("Example", "blue")} - enable modules of a uenv by name: - {colorize("uenv modules use prgenv-gnu", "white")} -This command is useful if more than one uenv is mounted. Modules provided by -more than one mounted uenv can be used by providing a space separated list of -uenv names. -""" - )) - modules_subparsers = modules_parser.add_subparsers(dest="modules_command") - modules_use_parser = modules_subparsers.add_parser("use", - help="use modules if they are available") - modules_use_parser.add_argument("image", nargs='*', type=str, - help="the uenv(s) with the modules to load") - #### view views_parser = subparsers.add_parser("view", help="activate a view", @@ -271,9 +219,6 @@ def get_uenv_version(): except: return None -def parse_uenv_modules(desc): - return [pathlib.Path(p) for p in desc.split(',')] - def parse_uenv_view(desc): mount, uenv, name = desc.split(':') return {"path": pathlib.Path(mount), "uenv": uenv, "name": name} @@ -319,14 +264,9 @@ class environment: # /user-environment /dev/loop12 squashfs ro,nosuid,nodev,relatime - # Read environment variables set by previous calls to uenv, that record which - # views and modules have been loaded. + # Read environment variables set by previous calls to uenv, that record which views have been loaded. # Additionally read the vcluster name, if set. def load_status(self): - modules = None - if os.environ.get('UENV_MODULE_PATH') is not None: - modules = parse_uenv_modules(os.environ.get('UENV_MODULE_PATH')) - view = None if os.environ.get('UENV_VIEW') is not None: view = parse_uenv_view(os.environ.get('UENV_VIEW')) @@ -335,7 +275,7 @@ class environment: if os.environ.get('CLUSTER_NAME') is not None: system = os.environ.get('CLUSTER_NAME') - return {"modules": modules, "view": view, "vcluster": vcluster} + return {"view": view, "vcluster": vcluster} @property def uenvs(self): @@ -346,11 +286,6 @@ class environment: """true if one or more uenv has been mounted""" return len(self._uenvs)>0 - @property - def modules_loaded(self): - """true if the modules have been activated""" - return self._status["modules"] is not None - @property def loaded_view(self): """the loaded view, if any""" @@ -381,21 +316,9 @@ class uenv: self._image = self.load_image(path) self.native_mount = self.get_native_mount() - # check whether this environment provides modules that have - # been loaded - self.modules_loaded = False - if self.modules is not None: - env_module_paths = [] - if os.environ.get('UENV_MODULE_PATH') is not None: - env_module_paths = parse_uenv_modules(os.environ.get('UENV_MODULE_PATH')) - if self.modules in [str(e) for e in env_module_paths]: - self.modules_loaded = True - if self.modules_loaded: - terminal.info(f"uenv {self.name} at {str(self.mount)} has modules loaded") - if not self.is_native_mounted: terminal.warning(f"The uenv mounted at {self.mount} should be mounted at {self.native_mount}.") - terminal.warning(f"Features like modules and views will be be disabled.") + terminal.warning(f"Features like views will be be disabled.") def load_image(self, path): """return information (if any) about the image mounted at path""" @@ -440,11 +363,6 @@ class uenv: terminal.info(f"image at {self.mount} has native mount {pathlib.Path(e['root']).parents[1]} inferred from views") return pathlib.Path(e["root"]).parents[1] - # check whether it can be inferred from a module path - if self.modules is not None: - terminal.info(f"image at {self.mount} has native mount {pathlib.Path(e['root']).parents[1]} inferred from modules") - return pathlib.Path(self.modules).parents[0] - # no mount information found, so assume that the actual mount point is valid. terminal.info(f"image at {self.mount} has native mount {self.mount} assumed") return self.mount @@ -463,16 +381,6 @@ class uenv: """true if the mounted image provided meta data that describes a uenv""" return self._image["uenv"] is not None - @property - def modules(self): - """the module path if modules are provided by the uenv""" - if not self.is_uenv: - return None - m = self._image["uenv"].get("modules", None) - if m is None: - return None - return m.get("root", None) - @property def description(self): """description of the uenv""" @@ -546,8 +454,6 @@ def generate_command(args, env_vars): return generate_status_command(args, env) elif args.command == "view": return generate_view_command(args, env, env_vars) - elif args.command == "modules": - return generate_modules_command(args, env) terminal.error(f"unknown command '{args.command}'", abort=False) return shell_error @@ -765,10 +671,6 @@ def generate_run_command(args, env, env_vars): terminal.info(f"start with view {args.view}") env_vars.set_scalar("UENV_FORWARD_VIEW", args.view) - if args.modules: - terminal.info(f"enabling modules") - env_vars.set_scalar("UENV_FORWARD_MODULES", "1") - # TODO: remove UENV_MOUNT_* variables when the slurm plugin on Balfrin/Tasna is updated main_mount=mount_pairs[0].split(":") env_vars.set_scalar("UENV_MOUNT_FILE", f"{main_mount[0]}") @@ -794,10 +696,6 @@ def generate_start_command(args, env, env_vars): terminal.info(f"enabling the view '{args.view}'") env_vars.set_scalar("UENV_FORWARD_VIEW", args.view) - if args.modules: - terminal.info(f"enabling modules") - env_vars.set_scalar("UENV_FORWARD_MODULES", "1") - # TODO: remove UENV_MOUNT_* variables when the slurm plugin on Balfrin/Tasna is updated main_mount=mount_pairs[0].split(":") env_vars.set_scalar("UENV_MOUNT_FILE", f"{main_mount[0]}") @@ -806,74 +704,6 @@ def generate_start_command(args, env, env_vars): return [f"squashfs-mount {mount_string} -- $UENV_WRAPPER_CMD bash", "local _exitcode=$?",] -def generate_modules_command(args, env): - - terminal.info(f"parsing modules command: {args}") - - if not env.active: - terminal.error(f'there is no uenv loaded', abort=False) - return shell_error - - # generate a list of all the mounted environments that provide modules - module_envs = [ - {"name": e.name, "mount": e.mount, "modules": e.modules, "loaded": e.modules_loaded} - for e in env.uenvs - if (e.modules is not None) and e.is_native_mounted] - - terminal.info(f"modules are provided by {module_envs}") - - # No use command, i.e. the folloing CLI command was made: - # uenv modules - # print the status of the modules - if not args.modules_command=="use": - if len(module_envs)==0: - return "echo 'no loaded environments provide modules'" - output = ["echo 'the following loaded uenv provide modules:'"] - loaded = colorize("(loaded)", "yellow") - for e in module_envs: - name = e["name"] - mount = e["mount"] - if e["loaded"]: - output.append(f"echo ' {name}:{mount} {loaded}'") - else: - output.append(f"echo ' {name}:{mount}'") - output.append(shell_noop) - return output - # uenv modules use images[] - else: - images = args.image - mounts=[] - # no images were specified, i.e. the user simpley requested: - # uenv modules use - # in which case the list of mounts is the mount points for all images - # that provide modules. - if len(images)==0: - mounts = [e["mount"] for e in module_envs] - else: - for i in images: - matches = [e.mount for e in env.uenvs - if (e.matches_name(i)) - and (e.modules is not None) - and (e.is_native_mounted)] - if len(matches)==0: - terminal.error(f"no uenv matching {i} provides modules", abort=False) - return shell_error - terminal.info(f" uenv {i} mounted at {matches[0]}") - mounts.append(matches[0]) - - modulepaths = [str(p / 'modules') for p in mounts] - for p in modulepaths: - terminal.info(f" using modules in {p}") - modulecmds = [f"module use {p}" for p in modulepaths] - env_vars.set_scalar("UENV_MODULE_PATH", f"{','.join(modulepaths)}") - - env_vars.set_post(False) - - return modulecmds - - - return shell_noop - def generate_view_command(args, env, env_vars): if not env.active: @@ -1010,15 +840,8 @@ def generate_status_command(args, env): else: lines.append(f" {description}") if not uenv.is_native_mounted: - lines.append(f" {colorize('warning', 'yellow')}: mount the image at {uenv.native_mount} to use views and modules") + lines.append(f" {colorize('warning', 'yellow')}: mount the image at {uenv.native_mount} to use views") else: - if uenv.modules_loaded: - lines.append(f" modules: {colorize('(loaded)', 'yellow')}") - elif uenv.modules is not None: - lines.append(f" modules: available") - else: - lines.append(f" modules: no modules available") - views = uenv.views view = None if (loaded_view is not None) and (loaded_view["path"]==uenv.mount): From d6bb2987b3c0d8be21c65de5c2bb0ffd8e061034 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 25 Jun 2024 09:05:04 +0200 Subject: [PATCH 2/7] further purging of modules --- activate | 1 - docs/implementation.md | 8 -------- docs/index.md | 42 ------------------------------------------ lib/envvars.py | 2 +- test/commands.sh | 6 ------ todo.md | 33 +++++++++++++++++++++++++++++---- uenv-wrapper | 5 ----- 7 files changed, 30 insertions(+), 67 deletions(-) diff --git a/activate b/activate index 3a8d336..caa246b 100644 --- a/activate +++ b/activate @@ -20,7 +20,6 @@ function uenv { echo " start start a new shell with an environment loaded" echo " stop stop a shell with an environment loaded" echo " status print information about each running environment" - echo " modules view module status and activate with 'use'" echo " view activate a view" echo " image query and pull uenv images" echo "" diff --git a/docs/implementation.md b/docs/implementation.md index 0111eea..85895ad 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -4,14 +4,6 @@ ### Variables set and read by UENV -* `UENV_MODULE_PATH`: a comma separated list of module files that have been loaded. - -For example: -``` -UENV_MODULE_PATH=/user-environment/modules,/user-tools/modules -UENV_MODULE_PATH=/user-tools/modules -``` - * `UENV_VIEW`: the view that has been loaded (only one view can be loaded at a time ``` diff --git a/docs/index.md b/docs/index.md index 3374241..683d215 100644 --- a/docs/index.md +++ b/docs/index.md @@ -69,45 +69,3 @@ To get the status of all loaded environments: uenv status ``` -### Modules - -Uenv can be configured to provide modules, however not all uenv provide modules. -If a uenv provides modules, they are not directly available for querying and accessing using `module avail` or `module load` etc. - -To find information about which running uenv provide modules, and whether modules have been activated: - -```bash -uenv modules -``` - -If a loaded uenv provides modules, these can be enabled using the `modules use` command: - -```bash -uenv modules use [environments] -``` - -where `[environments]'` is a list of uenv whose modules are to be made available. - -!!! info - If `[environments]` is not provided, the module files from all loaded uenv that provide modules will be made available. - -**Example 1**: use the modules provided by `/user-environment`: -```bash -uenv modules use /user-environment -``` - -**Example 2**: use the modules provided by the loaded uenv with name `gromacs/2023`: -```bash -uenv modules use gromacs/2023 -``` - -**Example 3**: the modules provided by all loaded uenv that provide modules: -```bash -uenv modules use -``` - -**Example 4**: use the modules provided by the uenv mounted at `/user-tools` and the uenv with name `gromacs/2023` -```bash -uenv modules use gromacs/2023 /user-tools -``` - diff --git a/lib/envvars.py b/lib/envvars.py index 177c38f..68f0074 100644 --- a/lib/envvars.py +++ b/lib/envvars.py @@ -234,7 +234,7 @@ def update(self, other: 'EnvVarSet'): # "post": the list of commands to be executed to revert the environment # # The "post" list is optional, and should not be used for commands that - # update the environment like "uenv view" and "uenv modules use", instead + # update the environment like "uenv view", instead # it should be used for commands that should not alter the calling environment, # like "uenv run" and "uenv start". # diff --git a/test/commands.sh b/test/commands.sh index 0029f01..52b03ef 100644 --- a/test/commands.sh +++ b/test/commands.sh @@ -22,8 +22,6 @@ echo ==================== stop uenv stop --help echo ==================== status uenv status --help -echo ==================== modules -uenv modules --help echo echo ==================== uenv status @@ -37,10 +35,6 @@ echo echo ==================== uenv run --uarch=gh200 prgenv-gnu -- ls /user-environment time uenv run --uarch=gh200 prgenv-gnu -- ls /user-environment -echo -echo ==================== uenv run --modules prgenv-gnu -- bash -c "module avail; ls /user-environment; module load gcc; gcc --version; which gcc;" -time uenv run --modules prgenv-gnu -- bash -c "module avail; ls /user-environment; module load gcc; gcc --version; which gcc;" - echo echo ==================== uenv run --view=default prgenv-gnu -- bash -c "which gcc; mpic++ --version; nvcc --version;" time uenv run --view=default prgenv-gnu -- bash -c "which gcc; mpic++ --version; nvcc --version;" diff --git a/todo.md b/todo.md index e2dbcec..27df4fc 100644 --- a/todo.md +++ b/todo.md @@ -10,10 +10,6 @@ This document outlines a single design for how this should be achieved for all u Activating a view sets environment variables like `PATH`, `PKG_CONFIG_PATH`, etc. -### Enabling modules - -To automatically enable modules, the MODULEPATH variable needs to be prenended. - ### Configuration files We want to support configruation files that specify a uenv, and how the environment should be configured, fo users. @@ -32,6 +28,35 @@ Environment modification: - create a new namespace and mount squashfs images. - set environment variables using `spank_setenv`, `os.setenv`, etc. +## Design + +The current design, a bash function wrapper that `exec`s commands returned by the `uenv-impl` app, supports modifying the environment of the calling shell. +This enables `uenv view x` to "load a view" in the running shell, by modifying environment variables. +Unfortunately, this approach is fragile and complicated. + +A more robust design would drop the ability to modify the calling shell's environment. + +There would no longer be separate `uenv-impl` ane `uenv-image` commands - these would be merged into a single `uenv` executable that would be called directly on the CLI (instead of indirectly calling them via a bash function). + +Modifications to the environment would only be performed via calls to `uenv start` and `uenv run`, each of which starts a new shell. + +``` +uenv start --view=develop icon +| +|_fork__ + | + | + setenv("PATH", ...) + setenv("LD_LIBRARY_PATH", ...) + | + execve(bash) + | + |_fork__ + | + | + bash +``` + ## Features 1. support for scalar and list variables. diff --git a/uenv-wrapper b/uenv-wrapper index 30871fe..0c0a592 100644 --- a/uenv-wrapper +++ b/uenv-wrapper @@ -6,11 +6,6 @@ if [ -n "$UENV_FORWARD_VIEW" ]; then uenv view $UENV_FORWARD_VIEW fi -if [ -n "$UENV_FORWARD_MODULES" ]; then - uenv modules use -fi - unset UENV_FORWARD_VIEW -unset UENV_FORWARD_MODULES exec "$@" From b5a8254c6fffb4f96cf02f8d8a83aaeb264ad51a Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 26 Jun 2024 16:55:28 +0200 Subject: [PATCH 3/7] uenv view now loads using json data if available --- lib/envvars.py | 383 ++++++++++++++++++++++++++++++++++++++++--------- uenv-impl | 38 +++-- 2 files changed, 345 insertions(+), 76 deletions(-) diff --git a/lib/envvars.py b/lib/envvars.py index 68f0074..f73ff89 100644 --- a/lib/envvars.py +++ b/lib/envvars.py @@ -1,30 +1,39 @@ -from enum import Enum +#!/usr/bin/python3 + +import argparse import json import os -from typing import Optional, List +from enum import Enum +from typing import List, Optional + +import yaml -class EnvVarOp (Enum): - PREPEND=1 - APPEND=2 - SET=3 + +class EnvVarOp(Enum): + PREPEND = 1 + APPEND = 2 + SET = 3 def __str__(self): return self.name.lower() -class EnvVarKind (Enum): - SCALAR=2 - LIST=2 + +class EnvVarKind(Enum): + SCALAR = 2 + LIST = 2 + list_variables = { - "ACLOCAL_PATH", - "CMAKE_PREFIX_PATH", - "CPATH", - "LD_LIBRARY_PATH", - "LIBRARY_PATH", - "MANPATH", - "PATH", - "PKG_CONFIG_PATH", - } + "ACLOCAL_PATH", + "CMAKE_PREFIX_PATH", + "CPATH", + "LD_LIBRARY_PATH", + "LIBRARY_PATH", + "MANPATH", + "PATH", + "PKG_CONFIG_PATH", +} + class EnvVarError(Exception): """Exception raised when there is an error with environment variable manipulation.""" @@ -36,13 +45,15 @@ def __init__(self, message): def __str__(self): return self.message + def is_env_value_list(v): return isinstance(v, list) and all(isinstance(item, str) for item in v) -class ListEnvVarUpdate(): + +class ListEnvVarUpdate: def __init__(self, value: List[str], op: EnvVarOp): - # strip white space from each entry - self._value = [v.strip() for v in value] + # clean up paths as they are inserted + self._value = [os.path.normpath(p) for p in value] self._op = op @property @@ -53,12 +64,21 @@ def op(self): def value(self): return self._value + def set_op(self, op: EnvVarOp): + self._op = op + + # remove all paths that have root as common root + def remove_root(self, root: str): + root = os.path.normpath(root) + self._value = [p for p in self._value if root != os.path.commonprefix([root, p])] + def __repr__(self): return f"envvar.ListEnvVarUpdate({self.value}, {self.op})" def __str__(self): return f"({self.value}, {self.op})" + class EnvVar: def __init__(self, name: str): self._name = name @@ -67,60 +87,76 @@ def __init__(self, name: str): def name(self): return self._name + class ListEnvVar(EnvVar): def __init__(self, name: str, value: List[str], op: EnvVarOp): super().__init__(name) self._updates = [ListEnvVarUpdate(value, op)] - def update(self, value: List[str], op:EnvVarOp): + def update(self, value: List[str], op: EnvVarOp): self._updates.append(ListEnvVarUpdate(value, op)) + def remove_root(self, root: str): + for i in range(len(self._updates)): + self._updates[i].remove_root(root) + @property def updates(self): return self._updates - def concat(self, other: 'ListEnvVar'): + def concat(self, other: "ListEnvVar"): self._updates += other.updates + def make_dirty(self): + if len(self._updates) > 0: + self._updates[0].set_op(EnvVarOp.PREPEND) + + @property + def paths(self): + paths = [] + for u in self._updates: + paths += u.value + return paths + # Given the current value, return the value that should be set # current is None implies that the variable is not set # # dirty allows for not overriding the current value of the variable. - def get_value(self, current: Optional[str], dirty: bool=False): + def get_value(self, current: Optional[str], dirty: bool = False): v = current # if the variable is currently not set, first initialise it as empty. if v is None: - if len(self._updates)==0: + if len(self._updates) == 0: return None v = "" first = True for update in self._updates: joined = ":".join(update.value) - if first and dirty and update.op==EnvVarOp.SET: + if first and dirty and update.op == EnvVarOp.SET: op = EnvVarOp.PREPEND else: op = update.op - if v == "" or op==EnvVarOp.SET: + if v == "" or op == EnvVarOp.SET: v = joined - elif op==EnvVarOp.APPEND: + elif op == EnvVarOp.APPEND: v = ":".join([v, joined]) - elif op==EnvVarOp.PREPEND: + elif op == EnvVarOp.PREPEND: v = ":".join([joined, v]) else: - raise EnvVarError(f"Internal error: implement the operation {update.op}"); + raise EnvVarError(f"Internal error: implement the operation {update.op}") first = False # strip any leading/trailing ":" - v = v.strip(':') + v = v.strip(":") return v def __repr__(self): - return f"envvars.ListEnvVar(\"{self.name}\", {self._updates})" + return f'envvars.ListEnvVar("{self.name}", {self._updates})' def __str__(self): return f"(\"{self.name}\": [{','.join([str(u) for u in self._updates])}])" @@ -148,10 +184,11 @@ def get_value(self, value: Optional[str]): return self._value def __repr__(self): - return f"envvars.ScalarEnvVar(\"{self.name}\", \"{self.value}\")" + return f'envvars.ScalarEnvVar("{self.name}", "{self.value}")' def __str__(self): - return f"(\"{self.name}\": \"{self.value}\")" + return f'("{self.name}": "{self.value}")' + class Env: def __init__(self): @@ -160,11 +197,13 @@ def __init__(self): def apply(self, var: EnvVar): self._vars[var.name] = var + # returns true if the environment variable with name is a list variable, # e.g. PATH, LD_LIBRARY_PATH, PKG_CONFIG_PATH, etc. def is_list_var(name: str) -> bool: return name in list_variables + class EnvVarSet: """ A set of environment variable updates. @@ -190,19 +229,26 @@ def clear(self): def scalars(self): return self._scalars + def make_dirty(self): + for name in self._lists: + self._lists[name].make_dirty() + + def remove_root(self, root: str): + for name in self._lists: + self._lists[name].remove_root(root) + def set_scalar(self, name: str, value: str): self._scalars[name] = ScalarEnvVar(name, value) def set_list(self, name: str, value: List[str], op: EnvVarOp): var = ListEnvVar(name, value, op) if var.name in self._lists.keys(): - old = self._lists[var.name] self._lists[var.name].concat(var) else: self._lists[var.name] = var def __repr__(self): - return f"envvars.EnvVarSet(\"{self.lists}\", \"{self.scalars}\")" + return f'envvars.EnvVarSet("{self.lists}", "{self.scalars}")' def __str__(self): s = "EnvVarSet:\n" @@ -219,7 +265,7 @@ def __str__(self): # Update the environment variables using the values in another EnvVarSet. # This operation is used when environment variables are sourced from more # than one location, e.g. multiple activation scripts. - def update(self, other: 'EnvVarSet'): + def update(self, other: "EnvVarSet"): for name, var in other.scalars.items(): self.set_scalar(name, var.value) for name, var in other.lists.items(): @@ -234,7 +280,7 @@ def update(self, other: 'EnvVarSet'): # "post": the list of commands to be executed to revert the environment # # The "post" list is optional, and should not be used for commands that - # update the environment like "uenv view", instead + # update the environment like "uenv view" and "uenv modules use", instead # it should be used for commands that should not alter the calling environment, # like "uenv run" and "uenv start". # @@ -277,9 +323,26 @@ def export(self, dirty=False): return {"pre": pre, "post": post} + def as_dict(self) -> dict: + # create a dictionary with the information formatted for JSON + d = {"list": {}, "scalar": {}} + + for name, var in self.lists.items(): + ops = [] + for u in var.updates: + op = "set" if u.op == EnvVarOp.SET else ("prepend" if u.op == EnvVarOp.PREPEND else "append") + ops.append({"op": op, "value": u.value}) + + d["list"][name] = ops + + for name, var in self.scalars.items(): + d["scalar"][name] = var.value + + return d + # returns a string that represents the environment variable modifications # in json format - #{ + # { # "list": { # "PATH": [ # {"op": "set", "value": "/user-environment/bin"}, @@ -294,62 +357,46 @@ def export(self, dirty=False): # "CUDA_HOME": "/user-environment/env/default", # "MPIF90": "/user-environment/env/default/bin/mpif90" # } - #} - def json(self) -> str: - # create a dictionary with the information formatted for JSON - d = {"list": {}, "scalar": {}} - - for name, var in self.lists.items(): - ops = [] - for u in var.updates: - op = "set" if u.op == EnvVarOp.SET else ("prepend" if u.op==EnvVarOp.PREPEND else "append") - ops.append({"op": op, "value": u.value}) - - d["list"][name] = ops - - for name, var in self.scalars.items(): - d["scalar"][name] = var.value - - return json.dumps(d, separators=(',', ':')) + # } + def as_json(self) -> str: + return json.dumps(self.as_dict(), separators=(",", ":")) def set_post(self, value: bool): self._generate_post = value -def read_activation_script(filename: str, env: Optional[EnvVarSet]=None) -> EnvVarSet: - scalars = {} - lists = {} + +def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> EnvVarSet: if env is None: env = EnvVarSet() with open(filename) as fid: for line in fid: - l = line.strip().rstrip(";") + ls = line.strip().rstrip(";") # skip empty lines and comments - if (len(l)==0) or (l[0]=='#'): + if (len(ls) == 0) or (ls[0] == "#"): continue # split on the first whitespace # this splits lines of the form # export Y # where Y is an arbitray string into ['export', 'Y'] - fields = l.split(maxsplit=1) + fields = ls.split(maxsplit=1) # handle lines of the form 'export Y' - if len(fields)>1 and fields[0]=='export': - fields = fields[1].split('=', maxsplit=1) + if len(fields) > 1 and fields[0] == "export": + fields = fields[1].split("=", maxsplit=1) # get the name of the environment variable name = fields[0] # if there was only one field, there was no = sign, so pass - if len(fields)<2: + if len(fields) < 2: continue # rhs the value that is assigned to the environment variable rhs = fields[1] if name in list_variables: - fields = [f for f in rhs.split(":") if len(f.strip())>0] - lists[name] = fields + fields = [f for f in rhs.split(":") if len(f.strip()) > 0] # look for $name as one of the fields (only works for append or prepend) - if len(fields)==0: + if len(fields) == 0: env.set_list(name, fields, EnvVarOp.SET) elif fields[0] == f"${name}": env.set_list(name, fields[1:], EnvVarOp.APPEND) @@ -362,4 +409,204 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet]=None) -> EnvV return env +def read_dictionary(d: dict, env: Optional[EnvVarSet] = None) -> EnvVarSet: + if env is None: + env = EnvVarSet() + + if "scalar" in d: + for name, value in d["scalar"].items(): + env.set_scalar(name, value) + + if "list" in d: + for name, updates in d["list"].items(): + for u in updates: + if u["op"] == "set": + env.set_list(name, u["value"], EnvVarOp.SET) + elif u["op"] == "prepend": + env.set_list(name, u["value"], EnvVarOp.PREPEND) + elif u["op"] == "append": + env.set_list(name, u["value"], EnvVarOp.APPEND) + # just ignore updtes with invalid "op" + + return env + + +def view_impl(args): + print( + f"parsing view {args.root}\n compilers {args.compilers}\n prefix_paths '{args.prefix_paths}'\n \ + build_path '{args.build_path}'" + ) + + if not os.path.isdir(args.root): + print(f"error - environment root path {args.root} does not exist") + exit(1) + + root_path = args.root + activate_path = root_path + "/activate.sh" + if not os.path.isfile(activate_path): + print(f"error - activation script {activate_path} does not exist") + exit(1) + + envvars = read_activation_script(activate_path) + + # force all prefix path style variables (list vars) to use PREPEND the first operation. + envvars.make_dirty() + envvars.remove_root(args.build_path) + + if args.compilers is not None: + if not os.path.isfile(args.compilers): + print(f"error - compiler yaml file {args.compilers} does not exist") + exit(1) + + with open(args.compilers, "r") as file: + data = yaml.safe_load(file) + compilers = [c["compiler"] for c in data["compilers"]] + + compiler_paths = [] + for c in compilers: + local_paths = set([os.path.dirname(v) for _, v in c["paths"].items() if v is not None]) + compiler_paths += local_paths + print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}') + + envvars.set_list("PATH", compiler_paths, EnvVarOp.PREPEND) + + if args.prefix_paths: + # get the root path of the env + print(f"prefix_paths: searching in {root_path}") + + for p in args.prefix_paths.split(","): + name, value = p.split("=") + paths = [] + for path in [os.path.normpath(p) for p in value.split(":")]: + test_path = f"{root_path}/{path}" + if os.path.isdir(test_path): + paths.append(test_path) + + print(f"{name}:") + for p in paths: + print(f" {p}") + + if len(paths) > 0: + if name in envvars.lists: + ld_paths = envvars.lists[name].paths + final_paths = [p for p in paths if p not in ld_paths] + envvars.set_list(name, final_paths, EnvVarOp.PREPEND) + else: + envvars.set_list(name, paths, EnvVarOp.PREPEND) + + json_path = os.path.join(root_path, "env.json") + print(f"writing JSON data to {json_path}") + envvar_dict = {"version": 1, "values": envvars.as_dict()} + with open(json_path, "w") as fid: + json.dump(envvar_dict, fid) + fid.write("\n") + + +def meta_impl(args): + # verify that the paths exist + if not os.path.exists(args.mount): + print(f"error - uenv mount '{args.mount}' does not exist.") + exit(1) + + # parse the uenv meta data from file + meta_in_path = os.path.normpath(f"{args.mount}/meta/env.json.in") + meta_path = os.path.normpath(f"{args.mount}/meta/env.json") + print(f"loading meta data to update: {meta_in_path}") + with open(meta_in_path) as fid: + meta = json.load(fid) + + for name, data in meta["views"].items(): + env_root = data["root"] + + # read the json view data from file + json_path = os.path.join(env_root, "env.json") + print(f"reading view {name} data rom {json_path}") + + if not os.path.exists(json_path): + print(f"error - meta data file '{json_path}' does not exist.") + exit(1) + + with open(json_path, "r") as fid: + envvar_dict = json.load(fid) + + # update the global meta data to include the environment variable state + meta["views"][name]["env"] = envvar_dict + meta["views"][name]["type"] = "spack-view" + + # process spack and modules + if args.modules: + module_path = f"{args.mount}/modules" + meta["views"]["modules"] = { + "activate": "/dev/null", + "description": "activate modules", + "root": module_path, + "env": { + "version": 1, + "type": "augment", + "values": {"list": {"MODULEPATH": [{"op": "prepend", "value": [module_path]}]}, "scalar": {}}, + }, + } + + if args.spack is not None: + spack_url, spack_version = args.spack.split(",") + spack_path = f"{args.mount}/config".replace("//", "/") + meta["views"]["spack"] = { + "activate": "/dev/null", + "description": "configure spack upstream", + "root": spack_path, + "env": { + "version": 1, + "type": "augment", + "values": { + "list": {}, + "scalar": { + "UENV_SPACK_CONFIG_PATH": spack_path, + "UENV_SPACK_COMMIT": spack_version, + "UENV_SPACK_URL": spack_url, + }, + }, + }, + } + # update the uenv meta data file with the new env. variable description + with open(meta_path, "w") as fid: + # write updated meta data + json.dump(meta, fid) + fid.write("\n") + print(f"wrote the uenv meta data {meta_path}") + + +if __name__ == "__main__": + # parse CLI arguments + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + view_parser = subparsers.add_parser( + "view", formatter_class=argparse.RawDescriptionHelpFormatter, help="generate env.json for a view" + ) + view_parser.add_argument("root", help="root path of the view", type=str) + view_parser.add_argument("build_path", help="build_path", type=str) + view_parser.add_argument( + "--prefix_paths", help="a list of relative prefix path searchs of the form X=y:z,Y=p:q", default="", type=str + ) + # only add compilers if this argument is passed + view_parser.add_argument("--compilers", help="path of the compilers.yaml file", type=str, default=None) + + uenv_parser = subparsers.add_parser( + "uenv", + formatter_class=argparse.RawDescriptionHelpFormatter, + help="generate meta.json meta data file for a uenv.", + ) + uenv_parser.add_argument("mount", help="mount point of the image", type=str) + uenv_parser.add_argument("--modules", help="configure a module view", action="store_true") + uenv_parser.add_argument( + "--spack", help='configure a spack view. Format is "spack_url,git_commit"', type=str, default=None + ) + + args = parser.parse_args() + + if args.command == "uenv": + print("!!! running meta") + meta_impl(args) + elif args.command == "view": + print("!!! running view") + view_impl(args) diff --git a/uenv-impl b/uenv-impl index 1f7bc38..cbd02d2 100755 --- a/uenv-impl +++ b/uenv-impl @@ -408,11 +408,22 @@ class uenv: views = [] for name, info in vlist.items(): - views.append({ - "name": name, - "activate": info["activate"], - "description": info["description"], - "root": info["root"]}) + if "env" in info: + if info["env"]["version"] != 1: + terminal.warning(f"this uenv image is too recent for uenv - please upgrade.") + return [] + views.append({ + "name": name, + "activate": info["activate"], + "description": info["description"], + "root": info["root"], + "env": info["env"]["values"]}) + else: + views.append({ + "name": name, + "activate": info["activate"], + "description": info["description"], + "root": info["root"]}) return views @@ -796,10 +807,21 @@ def generate_view_command(args, env, env_vars): terminal.error(f'the view "{requested_view}" is not available', abort=False) return help_message(shell_error) - path = next((v['activate'] for v in uenv.views if v['name']==vname)) + # load the raw view data + view_meta = next((v for v in uenv.views if v["name"]==vname)) + + # check whether the view provides environment variable information + terminal.info(f"view meta: {view_meta}") + if "env" in view_meta: + terminal.info(f"the image provides environment variable meta data") + env_vars.update(ev.read_dictionary(view_meta["env"])) + terminal.info(f"{env_vars}") + # else use the raw activate.sh + else: + path = view_meta['activate'] + terminal.info(f"full path for view activation script: '{path}'") + env_vars.update(ev.read_activation_script(path)) - terminal.info(f"full path for view activation script: '{path}'") - env_vars.update(ev.read_activation_script(path)) env_vars.set_scalar("UENV_VIEW", f"{uenv.mount}:{uenv.name}:{vname}") env_vars.set_post(False) From 6a30e400361f4ef6e4cd18141761bcfc4c13c1d9 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 27 Jun 2024 16:25:43 +0200 Subject: [PATCH 4/7] support loading multiple views --- lib/terminal.py | 2 - uenv-impl | 170 +++++++++++++++++++++++------------------------- 2 files changed, 83 insertions(+), 89 deletions(-) diff --git a/lib/terminal.py b/lib/terminal.py index 9810190..5f39d02 100644 --- a/lib/terminal.py +++ b/lib/terminal.py @@ -33,8 +33,6 @@ def use_colored_output(cli_arg: bool): colored_output = False return - colored_output = is_tty() - def colorize(string, color): colors = { "red": "31", diff --git a/uenv-impl b/uenv-impl index cbd02d2..7c099da 100755 --- a/uenv-impl +++ b/uenv-impl @@ -220,8 +220,11 @@ def get_uenv_version(): return None def parse_uenv_view(desc): - mount, uenv, name = desc.split(':') - return {"path": pathlib.Path(mount), "uenv": uenv, "name": name} + views = [] + for v in desc.split(','): + mount, uenv, name = v.split(':') + views.append({"path": pathlib.Path(mount), "uenv": uenv, "name": name}) + return views ############################################################################### # Types that read and represent uenv status @@ -721,108 +724,101 @@ def generate_view_command(args, env, env_vars): terminal.error(f'there is no uenv loaded', abort=False) return shell_error - requested_view = args.view_name + # the requested view(s) + request = args.view_name - # A dictionary that provides a list of views for each mounted uenv - # {key=uenv-name: value=[views]} - # only uenv with views are added to the dictionary - available_views = {} - # A dictionary with view name as a key, and a list of uenv that provide - # view with that name as the values - view2uenv = {} + # build a list of all views that are provided by the mounted uenv + available_views = [] for uenv in env.uenvs: - name = uenv.name - views = [v["name"] for v in uenv.views] - if len(views)>0: - available_views[name] = views - for v in views: - view2uenv.setdefault(v, []).append(uenv) + for name in [v["name"] for v in uenv.views]: + available_views.append({"uenv": uenv.name, "name": name}) + + terminal.info(f"available views: {available_views}") # A helper function that generates a help message to echo to the screen. def help_message(final_op=shell_noop): output = [] - if not available_views: + if len(available_views)==0: output.append("echo 'no views are provided by the loaded uenv'") - output.append("echo 'the following views are available:'") - output.append("echo ''") - # if only one uenv provides views, there is no need to disambiguate view names - disambiguate = len(available_views)>1 - for name, views in available_views.items(): - for view in views: - output.append(f"echo '{colorize(name+':'+view, 'cyan')}'") - if disambiguate: - command = f"uenv view {name}:{view}" - else: - command = f"uenv view {view}" + else: + output.append("echo 'the following views are available:'") + output.append("echo ''") + # if only one uenv provides views, there is no need to disambiguate view names + disambiguate = len(available_views)>1 + for v in available_views: + uenv = v["uenv"] + name = v["name"] + output.append(f"echo '{colorize(uenv+':'+name, 'cyan')}'") + command = f"uenv view {uenv}:{name}" output.append(f"echo ' {colorize(command, 'white')}'") + output.append(final_op) + return output # handle the case where `uenv view` is called with no further arguments # print status information and suggested commands before quitting without error - if requested_view is None: + if request is None: # handle the case that a view is already loaded - if env.loaded_view is not None: - loaded_view = f"{env.loaded_view['uenv']}:{env.loaded_view['name']}" - return [f"echo 'the view {colorize(loaded_view, 'cyan')} is loaded'", - shell_noop] + loaded_views = env.loaded_view + if loaded_views is not None: + view_strings = [f"{v['uenv']}:{v['name']}" for v in loaded_views] + if len(loaded_views)==1: + return [f"echo 'the view {colorize(view_strings[0], 'cyan')} is loaded'", + shell_noop] + else: + return [f"echo 'the views {', '.join([colorize(s, 'cyan') for s in view_strings])} are loaded'", + shell_noop] + else: return help_message(shell_noop) if env.loaded_view is not None: - loaded_view = f"{env.loaded_view['uenv']}:{env.loaded_view['name']}" - terminal.error(f"the view {colorize(loaded_view, 'cyan')} is already loaded", abort=False) + terminal.error(f"views are already loaded", abort=False) return shell_error - view_components = requested_view.split(':') + requested_views = [] + for r in request.split(','): + components = r.split(':') - # handle the case where no uenv name was provided - # e.g. "develop" - if len(view_components)==1: - vname = requested_view - uenvs = view2uenv.get(vname, [None]) - # more than one uenv provides a view with the requested name - if len(uenvs)>1: - terminal.error(f'the view"{name}" is provided by {[e.name for e in uenvs]}. Use `uenv view` to see the available options.', abort=False) + if len(components)==1: + request = {"uenv": None, "name": components[0]} + else: + request = {"uenv": components[0], "name": components[1]} + + matches = [v for v in available_views if (request == v) or (request["uenv"] is None and request["name"]==v["name"])] + + if len(matches)>1: + terminal.error(f'the view "{r}" is provided by {[e["name"] for e in matches]}. Use "uenv view" to see the available options.', abort=False) + return help_message(shell_error) + elif len(matches)==0: + terminal.error(f'the view "{r}" is not available', abort=False) return help_message(shell_error) - # this will be either - # the single uenv that provides the view - # None -> no matching image - uenv = uenvs[0] - # handle the case where both uenv name and view are provided - # e.g. "prgenv-gnu:default" - else: - ename = view_components[0] - vname = view_components[1] - # find all uenvs that provide this view - uenvs = view2uenv.get(vname, []) - # this will be either - # the uenv that provides the view - # None -> no matching image - uenv = next((e for e in uenvs if e.name==ename), None) - - # no view with the requested name exists - if uenv is None: - terminal.error(f'the view "{requested_view}" is not available', abort=False) - return help_message(shell_error) - - # load the raw view data - view_meta = next((v for v in uenv.views if v["name"]==vname)) - - # check whether the view provides environment variable information - terminal.info(f"view meta: {view_meta}") - if "env" in view_meta: - terminal.info(f"the image provides environment variable meta data") - env_vars.update(ev.read_dictionary(view_meta["env"])) - terminal.info(f"{env_vars}") - # else use the raw activate.sh - else: - path = view_meta['activate'] - terminal.info(f"full path for view activation script: '{path}'") - env_vars.update(ev.read_activation_script(path)) + terminal.info(f"found compatible view: {request} -> {matches[0]}") + requested_views.append(matches[0]) + + uenv_view_strings_long = [] + uenv_view_strings_short = [] + for request in requested_views: + uenv = next((u for u in env.uenvs if u.name==request["uenv"])) + view_meta = next((v for v in uenv.views if v["name"]==request["name"])) + + # check whether the view provides environment variable information + terminal.info(f"loading the view: {view_meta}") + if "env" in view_meta: + terminal.info(f"the image provides environment variable meta data") + env_vars.update(ev.read_dictionary(view_meta["env"])) + terminal.info(f"{env_vars}") + # else use the raw activate.sh + else: + path = view_meta['activate'] + terminal.info(f"full path for view activation script: '{path}'") + env_vars.update(ev.read_activation_script(path)) + uenv_view_strings_long.append(f"{uenv.mount}:{uenv.name}:{request['name']}") + uenv_view_strings_short.append(f"{uenv.name}:{request['name']}") - env_vars.set_scalar("UENV_VIEW", f"{uenv.mount}:{uenv.name}:{vname}") + env_vars.set_scalar("UENV_VIEW", ",".join(uenv_view_strings_long)) env_vars.set_post(False) @@ -832,8 +828,8 @@ def generate_view_command(args, env, env_vars): env_vars.clear() return [f"echo '{json_string}'"] - qualified_view_name = f"{uenv.name}:{vname}" - return [f"echo 'loading the view {colorize(qualified_view_name, 'cyan')}'"] + qualified_view_name = " ".join(uenv_view_strings_short) + return [f"echo 'loading the view{'s' if len(requested_views)>1 else ''} {colorize(qualified_view_name, 'cyan')}'"] def generate_status_command(args, env): num_env = len(env.uenvs) @@ -843,8 +839,10 @@ def generate_status_command(args, env): lines = [] first = True - loaded_view = env.loaded_view - terminal.info(f"loaded view: {loaded_view}") + loaded_views = env.loaded_view + if loaded_views is None: + loaded_views = [] + terminal.info(f"loaded views: {loaded_views}") for uenv in env.uenvs: if not first: @@ -866,15 +864,13 @@ def generate_status_command(args, env): else: views = uenv.views view = None - if (loaded_view is not None) and (loaded_view["path"]==uenv.mount): - view = loaded_view["name"] if len(views)==0: lines.append(" views: no views available") else: lines.append(" views:") for v in views: name = v["name"] - if name == view: + if {"path": uenv.mount, "uenv": uenv.name, "name": name} in loaded_views: name += f" {colorize('(loaded)', 'yellow')}" description = v["description"] From bbaaddd9a7c05c197362050edce0fb3a88bc3c68 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 27 Jun 2024 16:43:59 +0200 Subject: [PATCH 5/7] update uenv registry query to new (long term) end point --- lib/jfrog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jfrog.py b/lib/jfrog.py index a755a0a..d498706 100644 --- a/lib/jfrog.py +++ b/lib/jfrog.py @@ -37,7 +37,7 @@ def query() -> tuple: try: # GET request to the middleware - url = "https://cicd-ext-mw.cscs.ch/uenv/list" + url = "https://uenv-list.svc.cscs.ch/list" terminal.info(f"querying jfrog at {url}") response = requests.get(url) response.raise_for_status() From c316eaeb90b8b2f81c596a48bc3dc34b026b47cd Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 27 Jun 2024 18:56:01 +0200 Subject: [PATCH 6/7] download meta data alongside squashfs images --- lib/oras.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/lib/oras.py b/lib/oras.py index b2dcbfd..146dbd9 100644 --- a/lib/oras.py +++ b/lib/oras.py @@ -1,3 +1,4 @@ +import json import os import pathlib import shutil @@ -70,14 +71,40 @@ def run_command(args): def pull_uenv(source_address, image_path, target): # download the image using oras try: - # run the oras command in a separate process so that this process can - # draw a progress bar. - proc = run_command_internal(["pull", "-o", image_path, source_address]) - # remove the old path if it exists if os.path.exists(image_path): shutil.rmtree(image_path) - time.sleep(0.2) + time.sleep(0.1) + + terms = source_address.rsplit(":", 1) + source_address = terms[0] + tag = terms[1] + + # step 1: download the meta data if there is any + #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1 + terminal.info(f"discovering meta data") + proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"]) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + terminal.info(f"successfully downloaded meta data info: {stdout}") + else: + msg = error_message_from_stderr(stderr) + terminal.error(f"failed to find meta data: {stderr}\n{msg}") + + manifests = json.loads(stdout)["manifests"] + if len(manifests)==0: + terminal.error(f"no meta data is available") + + digest = manifests[0]["digest"] + terminal.info(f"meta data digest: {digest}") + + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"]) + stdout, stderr = proc.communicate() + + # step 2: download the image itself + # run the oras command in a separate process so that this process can + # draw a progress bar. + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"]) sqfs_path = image_path + "/store.squashfs" total_mb = target.size/(1024*1024) From b865552fe67d87d360b02efa992f414fbcc67744 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 19 Jul 2024 17:35:38 +0200 Subject: [PATCH 7/7] reasonably robust --only-meta support, --no-header flag, --force pull flag --- lib/oras.py | 106 +++++++++++++++++++++++++++------------------------ todo-meta.md | 9 +++++ todo.md | 7 ++-- uenv-image | 82 ++++++++++++++++++++++++++++++--------- 4 files changed, 133 insertions(+), 71 deletions(-) create mode 100644 todo-meta.md diff --git a/lib/oras.py b/lib/oras.py index 146dbd9..f87ee28 100644 --- a/lib/oras.py +++ b/lib/oras.py @@ -68,63 +68,69 @@ def run_command(args): terminal.error(f"oras command failed with unknown exception: {e}") -def pull_uenv(source_address, image_path, target): +def pull_uenv(source_address, image_path, target, pull_meta, pull_sqfs): # download the image using oras try: - # remove the old path if it exists - if os.path.exists(image_path): - shutil.rmtree(image_path) - time.sleep(0.1) - terms = source_address.rsplit(":", 1) source_address = terms[0] tag = terms[1] - # step 1: download the meta data if there is any - #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1 - terminal.info(f"discovering meta data") - proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"]) - stdout, stderr = proc.communicate() - if proc.returncode == 0: - terminal.info(f"successfully downloaded meta data info: {stdout}") - else: - msg = error_message_from_stderr(stderr) - terminal.error(f"failed to find meta data: {stderr}\n{msg}") - - manifests = json.loads(stdout)["manifests"] - if len(manifests)==0: - terminal.error(f"no meta data is available") - - digest = manifests[0]["digest"] - terminal.info(f"meta data digest: {digest}") + if pull_meta: + if os.path.exists(image_path+"/meta"): + shutil.rmtree(image_path+"/meta") + time.sleep(0.05) + + # step 1: download the meta data if there is any + #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1 + terminal.info(f"discovering meta data") + proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"]) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + terminal.info(f"successfully downloaded meta data info: {stdout}") + else: + msg = error_message_from_stderr(stderr) + terminal.error(f"failed to find meta data: {stderr}\n{msg}") + + manifests = json.loads(stdout)["manifests"] + if len(manifests)==0: + terminal.error(f"no meta data is available") + + digest = manifests[0]["digest"] + terminal.info(f"meta data digest: {digest}") + + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"]) + stdout, stderr = proc.communicate() + + if pull_sqfs: + sqfs_path = image_path + "/store.squashfs" + if os.path.exists(sqfs_path): + os.remove(sqfs_path) + time.sleep(0.05) + + # step 2: download the image itself + # run the oras command in a separate process so that this process can + # draw a progress bar. + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"]) + + total_mb = target.size/(1024*1024) + while proc.poll() is None: + time.sleep(0.25) + if os.path.exists(sqfs_path) and terminal.is_tty(): + current_size = os.path.getsize(sqfs_path) + current_mb = current_size / (1024*1024) + p = current_mb/total_mb + msg = f"{int(current_mb)}/{int(total_mb)} MB" + progress.progress_bar(p, width=50, msg=msg) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + # draw a final complete progress bar + progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB") + terminal.stdout("") + terminal.info(f"oras command successful: {stdout}") + else: + msg = error_message_from_stderr(stderr) + terminal.error(f"image pull failed: {stderr}\n{msg}") - proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"]) - stdout, stderr = proc.communicate() - - # step 2: download the image itself - # run the oras command in a separate process so that this process can - # draw a progress bar. - proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"]) - - sqfs_path = image_path + "/store.squashfs" - total_mb = target.size/(1024*1024) - while proc.poll() is None: - time.sleep(1.0) - if os.path.exists(sqfs_path) and terminal.is_tty(): - current_size = os.path.getsize(sqfs_path) - current_mb = current_size / (1024*1024) - p = current_mb/total_mb - msg = f"{int(current_mb)}/{int(total_mb)} MB" - progress.progress_bar(p, width=50, msg=msg) - stdout, stderr = proc.communicate() - if proc.returncode == 0: - # draw a final complete progress bar - progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB") - terminal.stdout("") - terminal.info(f"oras command successful: {stdout}") - else: - msg = error_message_from_stderr(stderr) - terminal.error(f"image pull failed: {stderr}\n{msg}") except KeyboardInterrupt: proc.terminate() terminal.stdout("") diff --git a/todo-meta.md b/todo-meta.md new file mode 100644 index 0000000..81fc0e7 --- /dev/null +++ b/todo-meta.md @@ -0,0 +1,9 @@ +TODO list for meta data operations + ++ pull meta data alongside squashfs images ++ optionally download meta data ++ image inspect + + add {meta} format option that will print the meta data path ++ uenv image pull: pull missing meta data of already downloaded images +- image ls: annotate meta-data only pulls +- image start / run : handle meta-data only pulls diff --git a/todo.md b/todo.md index 27df4fc..1309d6e 100644 --- a/todo.md +++ b/todo.md @@ -49,12 +49,11 @@ uenv start --view=develop icon setenv("PATH", ...) setenv("LD_LIBRARY_PATH", ...) | + execve(squashfs-mount) + | execve(bash) | - |_fork__ - | - | - bash + bash ``` ## Features diff --git a/uenv-image b/uenv-image index fe165fa..cbfe13b 100755 --- a/uenv-image +++ b/uenv-image @@ -109,7 +109,8 @@ sha256: {{sha256}} system: {{system}} date: {{date}} path: {{path}} -sqfs: {{sqfs}}\ +sqfs: {{sqfs}} +meta: {{meta}}\ """ , help="optional format string") path_parser.add_argument("uenv", type=str) @@ -153,6 +154,8 @@ downloaded uenv with the list command. For more information: find_parser.add_argument("-a", "--uarch", required=False, type=str) find_parser.add_argument("--build", action="store_true", help="Search undeployed builds.", required=False) + find_parser.add_argument("--no-header", action="store_true", + help="Do not print header in output.", required=False) find_parser.add_argument("uenv", nargs="?", default=None, type=str) pull_parser = subparsers.add_parser("pull", @@ -188,6 +191,10 @@ with appropriate JFrog access and with the JFrog token in their oras keychain. pull_parser.add_argument("-a", "--uarch", required=False, type=str) pull_parser.add_argument("--build", action="store_true", required=False, help="enable undeployed builds") + pull_parser.add_argument("--only-meta", action="store_true", required=False, + help="only download meta data, if it is available") + pull_parser.add_argument("--force", action="store_true", required=False, + help="force download if the image has already been downloaded") pull_parser.add_argument("uenv", nargs="?", default=None, type=str) list_parser = subparsers.add_parser("ls", @@ -219,6 +226,8 @@ List uenv that are available. """) list_parser.add_argument("-s", "--system", required=False, type=str) list_parser.add_argument("-a", "--uarch", required=False, type=str) + list_parser.add_argument("--no-header", action="store_true", + help="Do not print header in output.", required=False) list_parser.add_argument("uenv", nargs="?", default=None, type=str) deploy_parser = subparsers.add_parser("deploy", @@ -292,6 +301,8 @@ def get_filter(args): def inspect_string(record: record.Record, image_path, format_string: str) -> str: try: + meta = image_path+"/meta" if os.path.exists(image_path+"/meta") else "none" + sqfs = image_path+"/store.squashfs" if os.path.exists(image_path+"/store.squashfs") else "none" return format_string.format( system = record.system, uarch = record.uarch, @@ -302,7 +313,8 @@ def inspect_string(record: record.Record, image_path, format_string: str) -> str id = record.id, sha256 = record.sha256, path = image_path, - sqfs = image_path + "/store.squashfs", + sqfs = sqfs, + meta = meta, ) except Exception as err: terminal.error(f"unable to format {str(err)}") @@ -317,11 +329,12 @@ def image_size_string(size): return f"{(size/(1024*1024*1024)):<.1f}GB" # pretty print a list of Record -def print_records(recordset): +def print_records(recordset, no_header=False): records = recordset.records if not recordset.is_empty: - terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow")) + if not args.no_header: + terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow")) for r in recordset.records: namestr = f"{r.name}/{r.version}" tagstr = f"{r.tag}" @@ -406,7 +419,7 @@ if __name__ == "__main__": terminal.error(f"no uenv matches the spec: {colorize(results.request, 'white')}") if args.command == "find": - print_records(results) + print_records(results, no_header=args.no_header) elif args.command == "pull": if not results.is_unique_sha: @@ -423,23 +436,60 @@ if __name__ == "__main__": t = records.records[0] source_address = jfrog.address(t, 'build' if args.build else 'deploy') - terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB") + terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB with only-meta={args.only_meta}") terminal.info(f"repo path: {repo_path}") cache = safe_repo_open(repo_path) image_path = cache.image_path(t) + terminal.info(f"image path: {image_path}") - # if the record isn't already in the filesystem repo download it + # at this point the request is for an sha that is in the remote repository + do_download=False + + meta_path=image_path+"/meta" + sqfs_path=image_path+"/store.squashfs" + meta_exists=os.path.exists(meta_path) + sqfs_exists=os.path.exists(sqfs_path) + + only_meta=meta_exists and not sqfs_exists + + # if there is no entry in the local database do a full clean download if cache.database.get_record(t.sha256).is_empty: - terminal.stdout(f"downloading image {t.sha256} {image_size_string(t.size)}") - # clean up the path if it already exists: sometimes this causes an oras error. - if os.path.exists(image_path): - terminal.info(f"removing existing path {image_path}") - shutil.rmtree(image_path) - oras.pull_uenv(source_address, image_path, t) + terminal.info("===== is_empty") + do_download=True + pull_meta=True + pull_sqfs=not args.only_meta + elif args.force: + terminal.info("===== force") + do_download=True + pull_meta=True + pull_sqfs=not args.only_meta + # a record exists, so check whether any components are missing else: - terminal.stdout(f"image {t.sha256} is already available locally") + terminal.info("===== else") + pull_meta=not meta_exists + pull_sqfs=not sqfs_exists and (not args.only_meta) + do_download=pull_meta or pull_sqfs + + terminal.info(f"pull {t.sha256} exists: meta={meta_exists} sqfs={sqfs_exists}") + terminal.info(f"pull {t.sha256} pulling: meta={pull_meta} sqfs={pull_sqfs}") + if do_download: + terminal.info(f"downloading") + else: + terminal.info(f"nothing to pull: use --force to force the download") + + # determine whether to perform download + # check whether the image is in the database, or when only meta-data has been downloaded + if do_download: + terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} matches remote image {t.sha256}") + if pull_meta: + terminal.stdout(f"{t.id} pulling meta data") + if pull_sqfs: + terminal.stdout(f"{t.id} pulling squashfs") + oras.pull_uenv(source_address, image_path, t, pull_meta, pull_sqfs) + else: + terminal.stdout(f"{t.name}/{t.version}:{t.tag} meta data and image with id {t.id} have already been pulled") # update all the tags associated with the image. terminal.info(f"updating the local repository database") @@ -447,8 +497,6 @@ if __name__ == "__main__": terminal.stdout(f"updating local reference {r.name}/{r.version}:{r.tag}") cache.add_record(r) - terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} downloaded") - sys.exit(0) elif args.command == "ls": @@ -459,7 +507,7 @@ if __name__ == "__main__": fscache = safe_repo_open(repo_path) records = fscache.database.find_records(**img_filter) - print_records(records) + print_records(records, no_header=args.no_header) sys.exit(0)