diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 835acf593..e15c690ce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,9 +34,27 @@ jobs: run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black + pyflakes shpc/utils/fileio.py + pyflakes shpc/utils/terminal.py pyflakes shpc/main/*.py pyflakes shpc/main/modules pyflakes shpc/main/container/base.py pyflakes shpc/main/container/podman.py pyflakes shpc/main/container/docker.py pyflakes shpc/main/container/singularity.py + pyflakes shpc/tests + pyflakes shpc/*.py + pyflakes shpc/client/add.py + pyflakes shpc/client/check.py + pyflakes shpc/client/config.py + pyflakes shpc/client/docgen.py + pyflakes shpc/client/get.py + pyflakes shpc/client/inspect.py + pyflakes shpc/client/install.py + pyflakes shpc/client/listing.py + pyflakes shpc/client/namespace.py + pyflakes shpc/client/pull.py + pyflakes shpc/client/show.py + pyflakes shpc/client/test.py + pyflakes shpc/client/uninstall.py + pyflakes shpc/main/wrappers diff --git a/CHANGELOG.md b/CHANGELOG.md index 64878ccad..636041647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x) + - refactor to "add" to generate a container.yaml first (0.0.49) - Properly cleanup empty module directories, and asking to remove a container that doesn't exist now logs a _warning_ (0.0.48) - wrapper script generation permissions error (0.0.47) - fixing but with stream command repeating output (0.0.46) diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 8086122b1..0bf4bee77 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -1116,17 +1116,61 @@ Add It might be the case that you have a container locally, and you want to make it available as a module (without pulling it from a registry). Although this is discouraged because it means you will need to manually maintain -versions, shpc does support the "add" command to do this. You can simply provide -the container path and the unique resource identifier: +versions, shpc does support the "add" command to do this. The steps for adding a +container are: + +1. Running ``shpc add`` to create a container.yaml in the registry namespace +2. Customizing the container.yaml to your liking +3. Running ``shpc install`` to formally install your new container. + +As an example, let's start with the container ``salad_latest.sif``. We have it +on our local machine and cannot pull it from a registry. First, let's run ``shpc add`` +and tell shpc that we want it under the ``dinosaur/salad`` namespace. + +.. code-block:: console + + $ shpc add salad_latest.sif dinosaur/salad:latest + Registry entry dinosaur/salad:latest was added! Before shpc install, edit: + /home/vanessa/Desktop/Code/shpc/registry/dinosaur/salad/container.yaml + +At this point, you should open up the container.yaml generated and edit to your liking. +This usually means updating the description, maintainer, aliases, and possibly providing a url +to find more information or support. Also notice we've provided the tag to be latest. If you update this registry +entry in the future with a new version, you'll want to provide a new tag. If you provide +an existing tag, you'll be asked to confirm before continuing. When you are happy, +it's time to install it, just as you would a regular container! + +.. code-block:: console + + $ shpc install dinosaur/salad:latest + + +And this will generate the expected module and container in your respective directory bases: + .. code-block:: console - $ shpc add salad_latest.sif vanessa/salad:latest + $ tree modules/dinosaur/salad/ + modules/dinosaur/salad/ + └── latest + ├── 99-shpc.sh + └── module.lua + + 1 directory, 2 files + + $ tree containers/dinosaur/salad/ + containers/dinosaur/salad/ + └── latest + └── sha256:77c7326e74d0e8b46d4e50d99e848fc950ed047babd60203e17449f5df8f39d4.sif + + 1 directory, 1 file + +And that's it! Note that ``add`` previously would add the container directly to the module +directory, and as of version 0.0.49 it's been updated to generate the container.yaml first. +Also note that ``add`` is only supported for Singularity, as Docker and Podman containers are +typically provided via registries. If you are looking for support for add for another +container technology, please `open a new issue `_. -If the unique resource identifier corresponds with a registry entry, you -will not be allowed to create it, as this would create a namespace conflict. -Since we don't have a configuration file to define custom aliases, the container -will just be exposed as it's command to run it. Get --- diff --git a/shpc/__init__.py b/shpc/__init__.py index 0851293a1..9feb67825 100644 --- a/shpc/__init__.py +++ b/shpc/__init__.py @@ -1 +1,3 @@ from shpc.version import __version__ + +assert __version__ diff --git a/shpc/main/container/__init__.py b/shpc/main/container/__init__.py index 87f33cfb9..b65e855ae 100644 --- a/shpc/main/container/__init__.py +++ b/shpc/main/container/__init__.py @@ -45,6 +45,9 @@ def get(self, key, default=None): if digest: return Tag(key, digest) + def set(self, key, value): + self._tags[key] = value + class Tag: """ @@ -67,10 +70,11 @@ class ContainerConfig: A ContainerConfig wraps a container.yaml file, intended for install. """ - def __init__(self, package_file): + def __init__(self, package_file, validate=True): """Load a package file for a container.""" self.load(package_file) - self.validate() + if validate: + self.validate() def __str__(self): return "[container:%s]" % self.name @@ -92,7 +96,7 @@ def flatname(self): """ Flatten the docker uri into a filesystem appropriate name """ - name = self.docker or self.oras or self.gh + name = self.docker or self.oras or self.gh or self.path return name.replace("/", "-") @property @@ -102,6 +106,8 @@ def name(self): """ from .base import ContainerName + if hasattr(self, "path") and self.path is not None: + return ContainerName("/".join(self.package_dir.split("/")[-2:])) name = self.docker or self.oras or self.gh return ContainerName(name) @@ -112,6 +118,17 @@ def latest(self): """ return self.tags.latest + def set(self, key, value): + """ + Update loaded config with keys and values + """ + if key not in self._config: + logger.warning("%s is not already defined for this container config!" % key) + self._config[key] = value + + def add_tag(self, key, value): + self._config["tags"][key] = value + def set_tag(self, tag): """ Set a tag to be the config default (defaults to latest otherwise) @@ -135,25 +152,54 @@ def get_url(self): Given a loaded container recipe, get the registry url. """ # Not in json schema, but currently required - if "docker" not in self._config and "gh" not in self._config: - logger.exit("A docker or gh field is currently required in the config.") - return self._config.get("docker") or self._config.get("gh") + if ( + "docker" not in self._config + and "gh" not in self._config + and "path" not in self._config + ): + logger.exit( + "A docker, gh, or path field is currently required in the config." + ) + return ( + self._config.get("docker") + or self._config.get("gh") + or self._config.get("path") + ) def get(self, key, default=None): return self._config.get(key, default) + @property + def package_dir(self): + """ + Get the directory of the container.yaml, for finding local containers. + """ + if self.package_file: + return os.path.dirname(self.package_file) + def get_pull_type(self): if self.oras: return "oras" if self.gh: return "gh" - return "docker" + if self.docker: + return "docker" + if self.path: + return self.path + logger.exit( + "Cannot identify pull type: one of oras, docker, gh, or path is required." + ) def get_uri(self): """ Return the unique resource identifier """ - return getattr(self, "docker") or getattr(self, "oras") or getattr(self, "gh") + return ( + getattr(self, "docker") + or getattr(self, "oras") + or getattr(self, "gh") + or getattr(self, "path") + ) def __getattr__(self, key): """ @@ -203,9 +249,18 @@ def get_aliases(self): seen.add(key) return aliases - def load(self, package_file): - """Load the settings file into the settings object""" + def save(self, package_file): + """ + Save the container.yaml to file. This is usually for shpc add. + """ + yaml = YAML() + with open(package_file, "w") as fd: + yaml.dump(self._config, fd) + def load(self, package_file): + """ + Load the settings file into the settings object + """ # Exit quickly if the package does not exist if not os.path.exists(package_file): logger.exit("%s does not exist." % package_file) diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index c5d1f97cc..e872a9f6c 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -4,8 +4,9 @@ from shpc.logger import logger -import shpc.utils -from .base import ContainerTechnology, ContainerName +import shpc.utils as utils +import shpc.main.wrappers +from .base import ContainerTechnology from datetime import datetime from glob import glob @@ -72,50 +73,55 @@ def get(self, module_name, env_file=False): logger.exit("Found more than one sif in module folder.") return sif[0] - def add(self, sif, module_name, modulefile, template, **kwargs): + def add(self, module_name, image, config, container_yaml, **kwargs): """ - Manually add a registry container. + Manually add a registry container, e.g., generating a container.yaml + for an existing container file. container_yaml is the destination file. + If it's already exisitng, it's loaded into config. Otherwise we are + using a template config. """ - module_dir = os.path.dirname(modulefile) - - # Ensure the container exists - sif = os.path.abspath(sif) - if not os.path.exists(sif): - logger.exit("%s does not exist." % sif) - - # First ensure that we aren't using a known namespace - for registry_dir, _ in self.iter_registry(): - for subfolder in module_name.split("/"): - registry_dir = os.path.join(registry_dir, subfolder) - if os.path.exists(registry_dir): - logger.exit( - "%s is a known registry namespace, choose another for a custom addition." - % subfolder - ) - - # The user can have a different container directory defined - container_dir = self.container_dir(module_name) - shpc.utils.mkdirp([container_dir, module_dir]) - - # Name the container appropriately - name = module_name.replace("/", "-") - digest = shpc.utils.get_file_hash(sif) - dest = os.path.join(container_dir, "%s-sha256:%s.sif" % (name, digest)) - shutil.copyfile(sif, dest) - - # Parse the module name for the user - parsed_name = ContainerName(module_name) - - self.install( - modulefile, - dest, - module_name, - template, - parsed_name=parsed_name, - features=kwargs.get("features"), + if ":" not in module_name: + tag = "latest" + else: + name, tag = module_name.split(":", 1) + + # Ensure the sif exists + if not os.path.exists(image): + logger.exit(f"{image} does not exist.") + + digest = utils.get_file_hash(image) + + # Cut out early if the tag isn't latest, and we already have it + if tag != "latest" and tag in config.tags: + if not utils.confirm_action( + "Tag %s already is defined, are you sure you want to overwrite it? " + % tag + ): + return + + # Destination for container in registry + dest_dir = os.path.dirname(container_yaml) + + # The destination container in the registry folder + container_digest = "sha256:%s" % digest + container_name = "%s.sif" % container_digest + dest_container = os.path.join(dest_dir, container_name) + + # Update the config path and latest + config.set("path", container_name) + config.set("latest", {tag: container_digest}) + config.add_tag(tag, container_digest) + + # Only copy if it's not there yet (enforces naming by hash) + utils.mkdir_p(dest_dir) + if not os.path.exists(dest_container): + shutil.copyfile(image, dest_container) + + config.save(container_yaml) + logger.info( + "Registry entry %s was added! Before shpc install, edit:" % module_name ) - self.add_environment(module_dir, {}, self.settings.environment_file) - logger.info("Module %s was created." % (module_name)) + print(container_yaml) def install( self, @@ -199,7 +205,7 @@ def install( parsed_name=parsed_name, wrapper_scripts=wrapper_scripts, ) - shpc.utils.write_file(module_path, out) + utils.write_file(module_path, out) def registry_pull(self, module_dir, container_dir, config, tag): """ @@ -357,7 +363,7 @@ def test_script(self, image, test_script): Given a test file, run it and respond accordingly. """ command = [self.command, "exec", image, "/bin/bash", test_script] - result = shpc.utils.run_command(command) + result = utils.run_command(command) # We can't run on incompatible hosts if ( diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 878b7d6ce..cec880ac8 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -7,6 +7,7 @@ import shpc.utils as utils import shpc.defaults as defaults import shpc.main.templates +import shpc.main.container as container from jinja2 import Template from datetime import datetime @@ -19,18 +20,26 @@ here = os.path.abspath(os.path.dirname(__file__)) + class ModuleBase(BaseClient): def __init__(self, **kwargs): super(ModuleBase, self).__init__(**kwargs) self.here = os.path.dirname(inspect.getfile(self.__class__)) - def _load_template(self, template_name): + def _get_template(self, template_name): """ - Load the default module template. + Get a template from templates """ template_file = os.path.join(here, "templates", template_name) if not os.path.exists(template_file): template_file = os.path.abspath(template_name) + return template_file + + def _load_template(self, template_name): + """ + Load the default module template. + """ + template_file = self._get_template(template_name) # Make all substitutions here with open(template_file, "r") as temp: @@ -41,7 +50,9 @@ def substitute(self, template): """ For all known identifiers, substitute user specified format strings. """ - subs = {"{|module_name|}": self.settings.module_name or "{{ parsed_name.tool }}"} + subs = { + "{|module_name|}": self.settings.module_name or "{{ parsed_name.tool }}" + } for key, replacewith in subs.items(): template = template.replace(key, replacewith) return template @@ -138,14 +149,26 @@ def _test(self, module_name, module_dir, tag, template="test.sh"): utils.write_file(test_file, out) return subprocess.call(["/bin/bash", test_file]) - def add(self, sif, module_name, **kwargs): + def add(self, image, module_name, **kwargs): """ - Add a container directly as a module, copying the file. + Add a container to the registry to enable install. """ module_name = self.add_namespace(module_name) - template = self._load_template(self.templatefile) - modulefile = os.path.join(self.settings.module_base, module_name.replace(":", os.sep), self.modulefile) - self.container.add(sif, module_name, modulefile, template, **kwargs) + + # Assume adding to default registry + dest = os.path.join( + self.settings.registry[0], module_name.split(":")[0], "container.yaml" + ) + + # if the container.yaml already exists, use it + template = self._get_template("container.yaml") + if os.path.exists(dest): + logger.warning("%s already exists and will be updated!" % module_name) + template = dest + + # Load config (but don't validate yet!) + config = container.ContainerConfig(template, validate=False) + self.container.add(module_name, image, config, container_yaml=dest, **kwargs) def get(self, module_name, env_file=False): """ @@ -289,9 +312,13 @@ def install(self, name, tag=None, **kwargs): % (name, "\n".join(config.tags.keys())) ) - # We currently support gh, docker, or oras + # We currently support gh, docker, path, or oras uri = config.get_uri() + # If we have a path, the URI comes from the name + if ".sif" in uri: + uri = name.split(":", 1)[0] + # This is a tag object with name and digest tag = config.tag @@ -301,6 +328,24 @@ def install(self, name, tag=None, **kwargs): container_dir = self.container.container_dir(subfolder) shpc.utils.mkdirp([module_dir, container_dir]) + # If we have a sif URI provided by path, the container needs to exist + container_path = None + if config.path: + container_path = os.path.join(config.package_dir, config.path) + if not os.path.exists(container_path): + logger.exit( + "Expected container defined by path %s not found in %s." + % (config.path, config.package_dir) + ) + container_dest = os.path.join(container_dir, config.path) + + # Note that here we are *duplicating* the container, assuming we + # cannot use a link, and the registry won't be deleted but the + # module container might! + if not os.path.exists(container_dest): + shutil.copyfile(container_path, container_dest) + container_path = container_dest + # Add a .version file to indicate the level of versioning (not for tcl) if self.module_extension != "tcl" and self.settings.default_version == True: version_dir = os.path.join(self.settings.module_base, uri) @@ -310,9 +355,10 @@ def install(self, name, tag=None, **kwargs): # For Singularity this is a path, podman is a uri. If None is returned # there was an error and we cleanup - container_path = self.container.registry_pull( - module_dir, container_dir, config, tag - ) + if not container_path: + container_path = self.container.registry_pull( + module_dir, container_dir, config, tag + ) if not container_path: self._cleanup(container_dir) logger.exit("There was an issue pulling %s" % container_path) @@ -324,7 +370,7 @@ def install(self, name, tag=None, **kwargs): # If the module has a version, overrides version version = tag.name if ":" in name: - name, version = name.split(':', 1) + name, version = name.split(":", 1) # Install the container self.container.install( @@ -354,6 +400,6 @@ def install(self, name, tag=None, **kwargs): ) if ":" not in name: - name = "%s:%s" %(name, tag.name) + name = "%s:%s" % (name, tag.name) logger.info("Module %s was created." % name) return container_path diff --git a/shpc/main/modules/templates/container.yaml b/shpc/main/modules/templates/container.yaml new file mode 100644 index 000000000..9e4c70272 --- /dev/null +++ b/shpc/main/modules/templates/container.yaml @@ -0,0 +1,26 @@ +path: null + +# change this to a URL to describe your container, or to get help +url: https://singularity-hpc.readthedocs.io + +# change this to your GitHub alias (or another contact or name) +maintainer: 'Dinosaur' + +# A description to describe your container +description: A custom container to do X. +latest: + latest: null +tags: + latest: null + +# Any custom features? +# features: +# gpu: true + +# Put custom aliases here +# aliases: +# echo: /usr/bin/echo + +# custom environment variables +# env: +# breakfast: pancakes diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index a2beaf393..a9aaac432 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -57,18 +57,10 @@ } -latest = { - "type": "object", - "minProperties": 1, - "maxProperties": 1, - "patternProperties": { - "\\w[\\w-]*": {"type": "string"}, - }, -} - containerConfigProperties = { "latest": keyvals, "docker": {"type": "string"}, + "path": {"type": "string"}, "oras": {"type": "string"}, "gh": {"type": "string"}, "url": {"type": "string"}, diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index d378b2e8e..c29e45377 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -3,7 +3,7 @@ __license__ = "MPL 2.0" from shpc.logger import logger -from jinja2 import Template, Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader import shpc.utils import os diff --git a/shpc/tests/test_client.py b/shpc/tests/test_client.py index f15afb4c8..a091785bc 100644 --- a/shpc/tests/test_client.py +++ b/shpc/tests/test_client.py @@ -162,11 +162,24 @@ def test_check(tmp_path, module_sys, container_tech): @pytest.mark.parametrize("module_sys", [("lmod"), ("tcl")]) def test_add(tmp_path, module_sys): - """Test adding a custom container""" + """ + Test adding a custom container + """ client = init_client(str(tmp_path), module_sys, "singularity") # Create a copy of the latest image to add container = os.path.join(str(tmp_path), "salad_latest.sif") shutil.copyfile(os.path.join(here, "testdata", "salad_latest.sif"), container) - client.add(container, "dinosaur/salad/latest") + client.add(container, "dinosaur/salad:latest") + + # Ensure this creates a container.yaml in the registry + container_yaml = os.path.join( + client.settings.registry[0], "dinosaur", "salad", "container.yaml" + ) + assert os.path.exists(container_yaml) + + # Add does not install! + with pytest.raises(SystemExit): + client.get("dinosaur/salad:latest") + client.install("dinosaur/salad:latest") assert client.get("dinosaur/salad:latest") diff --git a/shpc/tests/test_client.sh b/shpc/tests/test_client.sh index b1f8880b3..9955e43fe 100755 --- a/shpc/tests/test_client.sh +++ b/shpc/tests/test_client.sh @@ -53,6 +53,7 @@ echo echo "#### Testing add " runTest 0 $output shpc --settings-file $settings add --help runTest 0 $output shpc --settings-file $settings add "$container" salad/latest +runTest 0 $output shpc --settings-file $settings install salad/latest echo echo "#### Testing install " diff --git a/shpc/tests/test_container.py b/shpc/tests/test_container.py index d4cc00537..77e976807 100644 --- a/shpc/tests/test_container.py +++ b/shpc/tests/test_container.py @@ -6,8 +6,6 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import pytest -import shutil import os import shpc.main.container as container diff --git a/shpc/tests/test_container_config.py b/shpc/tests/test_container_config.py index 0092b7187..d5ad36ae3 100644 --- a/shpc/tests/test_container_config.py +++ b/shpc/tests/test_container_config.py @@ -6,8 +6,6 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import pytest -import shutil import os import shpc.main.container as container diff --git a/shpc/tests/test_settings.py b/shpc/tests/test_settings.py index dfd632876..fd7d03bb6 100644 --- a/shpc/tests/test_settings.py +++ b/shpc/tests/test_settings.py @@ -7,7 +7,6 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. import pytest -import shutil import os from shpc.main.settings import Settings diff --git a/shpc/version.py b/shpc/version.py index eaa88df08..4f1f4e642 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.48" +__version__ = "0.0.49" AUTHOR = "Vanessa Sochat" NAME = "singularity-hpc" PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc"