diff --git a/src/ec_cli/autocomplete.py b/src/ec_cli/autocomplete.py index 3d6c1bff..01e10ddd 100644 --- a/src/ec_cli/autocomplete.py +++ b/src/ec_cli/autocomplete.py @@ -46,7 +46,7 @@ def read_cached_dict(cache_folder: str, cached_file: str) -> dict: return read_dict -def fetch_ioc_graph(beamline_repo: str) -> dict: +def fetch_service_graph(beamline_repo: str) -> dict: ioc_graph = read_cached_dict(url_encode(beamline_repo), globals.IOC_CACHE) if not ioc_graph: tmp_dir = Path(tempfile.mkdtemp()) @@ -59,12 +59,12 @@ def fetch_ioc_graph(beamline_repo: str) -> dict: def avail_IOCs(ctx: typer.Context) -> List[str]: params = ctx.parent.parent.params # type: ignore - beamline_repo = params["repo"] or globals.EC_SERVICES_REPO + services_repo = params["repo"] or globals.EC_SERVICES_REPO # This block prevents getting a stack trace during autocompletion try: - ioc_graph = fetch_ioc_graph(beamline_repo) - return list(ioc_graph.keys()) + services_graph = fetch_service_graph(services_repo) + return list(services_graph.keys()) except typer.Exit: return [" "] except CalledProcessError: @@ -78,7 +78,7 @@ def avail_versions(ctx: typer.Context) -> List[str]: # This block prevents getting a stack trace during autocompletion try: - ioc_graph = fetch_ioc_graph(beamline_repo) + ioc_graph = fetch_service_graph(beamline_repo) ioc_versions = ioc_graph[ioc_name] return ioc_versions except KeyError: diff --git a/src/ec_cli/cmds/helm.py b/src/ec_cli/cmds/helm.py index dccbd767..698fb809 100644 --- a/src/ec_cli/cmds/helm.py +++ b/src/ec_cli/cmds/helm.py @@ -59,11 +59,6 @@ def deploy_local(self, service_path: Path, yes: bool = False): if not typer.confirm("Are you sure ?"): raise typer.Abort() - # to update the local helm charts we may need to remove the lock files - lock_files = service_path.glob("../../**/Chart.lock") - for lock_file in lock_files: - lock_file.unlink() - self._do_deploy(service_path) def deploy(self): @@ -92,11 +87,11 @@ def _do_deploy(self, service_folder: Path): # package up the charts to get the appVersion set for chart in chart_paths: - shell.run_command(f"helm dependency build {chart}", interactive=False) + shell.run_command(f"helm dependency update {chart}", interactive=False) with chdir(service_folder): shell.run_command( - f"helm dependency build {service_folder}; " + f"helm dependency update {service_folder}; " f"helm package {service_folder} --app-version {self.version}", interactive=False, ) diff --git a/src/ec_cli/cmds/ioc_cli.py b/src/ec_cli/cmds/ioc_cli.py index f345c5df..17a78309 100644 --- a/src/ec_cli/cmds/ioc_cli.py +++ b/src/ec_cli/cmds/ioc_cli.py @@ -16,7 +16,7 @@ from ec_cli.cmds.local_commands import IocLocalCommands from ec_cli.git import create_ioc_graph from ec_cli.logging import log -from ec_cli.utils import cleanup_temp, drop_ioc_path +from ec_cli.utils import cleanup_temp, drop_path cli = typer.Typer(pretty_exceptions_show_locals=False) @@ -143,7 +143,7 @@ def deploy( """ Pull an IOC helm chart version from the domain repo and deploy it to the cluster """ - service_name = drop_ioc_path(service_name) + service_name = drop_path(service_name) if ctx.obj.namespace == globals.LOCAL_NAMESPACE: IocLocalCommands(ctx.obj, service_name).deploy(service_name, version, args) else: diff --git a/src/ec_cli/cmds/kubectl.py b/src/ec_cli/cmds/kubectl.py index 5dad1570..8d35dcdf 100644 --- a/src/ec_cli/cmds/kubectl.py +++ b/src/ec_cli/cmds/kubectl.py @@ -37,4 +37,6 @@ r'{"\n"}{end}' "'" ) -json_service_types = {"name": str, "ready": bool, "restarts": int, "started": str} + +# force all the values to be strings so there are never parsing errors +json_service_types = {"name": str, "ready": str, "restarts": str, "started": str} diff --git a/src/ec_cli/cmds/local_commands.py b/src/ec_cli/cmds/local_commands.py index c8b4ebda..2e76a53a 100644 --- a/src/ec_cli/cmds/local_commands.py +++ b/src/ec_cli/cmds/local_commands.py @@ -52,7 +52,7 @@ def __init__( self.ioc_name: str = ioc_name self.tmp = Path(tempfile.mkdtemp()) - self.ioc_folder = self.tmp / "iocs" / ioc_name + self.ioc_folder = self.tmp / "services" / ioc_name self.docker = Docker(check=with_docker) def __del__(self): diff --git a/src/ec_cli/git.py b/src/ec_cli/git.py index 609fb44d..75977a1c 100644 --- a/src/ec_cli/git.py +++ b/src/ec_cli/git.py @@ -107,20 +107,18 @@ def repo2registry(repo_name: str) -> str: return registry -def create_ioc_graph(beamline_repo: str, folder: Path, branch="main") -> Dict: +def create_ioc_graph(repo: str, folder: Path, branch="main") -> Dict: """ return a dictionary of the available IOCs (by discovering the children - to the iocs/ folder in the beamline repo) as well as a list of the corresponding + to the services/ folder in the beamline repo) as well as a list of the corresponding available versions for each IOC (by discovering the tags in the beamline repo at which changes to the instance were made since the last tag) and the respective list of available versions """ ioc_graph = {} - check_services_repo(beamline_repo) - shell.run_command( - f"git clone {beamline_repo} {folder} -b {branch}", interactive=False - ) + check_services_repo(repo) + shell.run_command(f"git clone {repo} {folder} -b {branch}", interactive=False) path_list = os.listdir(os.path.join(folder, "services")) service_list = [ path diff --git a/src/ec_cli/utils.py b/src/ec_cli/utils.py index d024e46e..41ccaec0 100644 --- a/src/ec_cli/utils.py +++ b/src/ec_cli/utils.py @@ -34,23 +34,23 @@ def get_instance_image_name(ioc_instance: Path, tag: Optional[str] = None) -> st return image -def check_instance_path(ioc_path: Path): +def check_instance_path(service_path: Path): """ - verify that the ioc instance path is valid + verify that the service instance path is valid """ - ioc_path = ioc_path.absolute() - ioc_name = ioc_path.name.lower() + service_path = service_path.absolute() + service_name = service_path.name.lower() - log.info(f"checking IOC instance {ioc_name} at {ioc_path}") - if ioc_path.is_dir(): - if not (ioc_path / "values.yaml").exists(): + log.info(f"checking instance {service_name} at {service_path}") + if service_path.is_dir(): + if not (service_path / "values.yaml").exists(): log.error("IOC instance requires values.yaml") raise typer.Exit(1) else: - log.error(f"IOC instance path {ioc_path} does not exist") + log.error(f"instance path {service_path} does not exist") raise typer.Exit(1) - return ioc_name, ioc_path + return service_name, service_path def generic_ioc_from_image(image_name: str) -> str: @@ -65,12 +65,12 @@ def generic_ioc_from_image(image_name: str) -> str: return match[0] -def drop_ioc_path(raw_input: str): +def drop_path(raw_input: str): """ Extracts the IOC name if is a path through ioc """ match = re.findall( - r"iocs\/(.*?)(?:/|\s|$)", raw_input + r"services\/(.*?)(?:/|\s|$)", raw_input ) # https://regex101.com/r/L3GUvk/1 if not match: return raw_input diff --git a/tests/data/beamline-chart/.helmignore b/tests/data/beamline-chart/.helmignore deleted file mode 100644 index 7c54e53b..00000000 --- a/tests/data/beamline-chart/.helmignore +++ /dev/null @@ -1,28 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ - -# from this project -ghcr-deploy.sh -*.tgz -.github diff --git a/tests/data/beamline-chart/Chart.yaml.jinja b/tests/data/beamline-chart/Chart.yaml.jinja deleted file mode 100644 index 57264d8c..00000000 --- a/tests/data/beamline-chart/Chart.yaml.jinja +++ /dev/null @@ -1,25 +0,0 @@ -# A template for the Chart.yaml file for an IOC instance -# -# We generate a new helm chart for each IOC instance deployment. This is -# primarily because helm cannot package the config folder at install time. -# This approach could be reviewed when this PR is released: -# -# Two other reasons for this approach: -# 1. We prefer for the chart name to be the same as the IOC name as this looks -# better in K8S dashboard. (this can only be set in Chart.yaml) -# 2. We prefer to set the Chart version as this is the primary version that -# is shown in the dashboard. (this can be changed at package time so does -# not strictly need to set be in this template) -# -# To deploy at IOC using beamline chart: -# 1. Render this Jinja template to create a Chart.yaml file -# 2. package the chart using helm package -# 3. install the chart using helm upgrade --install passing the image name e.g. -# --set image=ghcr.io/epics-containers/ioc-adaravis-linux-runtime:23.9.4 -# You may also use --set to override any of the values in the values.yaml file - -apiVersion: v2 -name: {{ ioc_name }} - -version: {{ ioc_version }} -appVersion: {{ ioc_version }} diff --git a/tests/data/beamline-chart/values.yaml b/tests/data/beamline-chart/values.yaml deleted file mode 100644 index 428d5380..00000000 --- a/tests/data/beamline-chart/values.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# default values for all IOCs on bl45p -# see end of file for values that may be overriden per individual IOC - -beamline: bl45p -serviceAccount: bl45p-priv - -# useHostNetwork - use host network for IOC - required for Channel Access -# to work outside of the cluster -hostNetwork: true - -# useAffinity - only run on nodes with label beamline:bl45p -useAffinity: true - -# root folder for ioc source/binaries inside generic IOC container -# not expected to change -iocFolder: /epics/ioc - -# the following are added to the pod's environment -globalenv: - # Where to find RTEMS IOC files - - name: "K8S_IOC_TFTP_ADDR" - value: "172.23.168.220" - - name: "K8S_IOC_TFTP_PORT" - value: "69" - - name: "K8S_IOC_NFS_MOUNT" - value: "172.23.168.220:iocs" - -# reasonable defaults for securityContext -securityContext: - allowPrivilegeEscalation: false - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -######################################################################################## -# The following values are expected to be overridden in individual IOC values.yaml -######################################################################################## - -# this value is the only mandatory override -# note that the configuration for the generic IOC is provided by placing its -# config folder into beamline-chart/config. When the chart is packaged, the -# template -image: - -# provide a PVC for autosave -autosave: true - -# Somewhere to mount at /data if needed -dataVolume: - # A PVC to write data into - pvc: true - # A path on the host machine to write data into, ignored if dataVolume.pvc is true - hostPath: "" - -# default resource limits -resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 100m - memory: 64Mi diff --git a/tests/data/ioc.yaml b/tests/data/ioc.yaml index 83fee88b..c54ea70c 100644 --- a/tests/data/ioc.yaml +++ b/tests/data/ioc.yaml @@ -1,8 +1,8 @@ checks: - cmd: kubectl get namespace bl45p -o name rsp: namespace/bl45p - - cmd: kubectl get -n bl45p statefulset/bl45p-ea-ioc-01 - rsp: bl45p-ea-ioc-01 1/1 1 1 5d5h + - cmd: helm list -n bl45p -qf bl45p-ea-ioc-01 + rsp: bl45p-ea-ioc-01 attach: - cmd: kubectl -it -n bl45p attach statefulset/bl45p-ea-ioc-01 @@ -13,31 +13,28 @@ delete: rsp: True template: - - cmd: helm dependency update /tmp/ec_tests/beamline-chart + - cmd: 'helm dependency update .*\/tests\/data\/services\/bl45p-ea-ioc-01\; helm package .*\/tests\/data\/services\/bl45p-ea-ioc-01 --app-version .*' rsp: "" - - cmd: - 'bash -c "helm template bl45p-ea-ioc-01 /tmp/ec_tests/beamline-chart --version - 20.*--namespace bl45p -f \/.*\/data\/iocs\/bl45p-ea-ioc-01\/values.yaml.*' + - cmd: 'bash -c "helm template bl45p-ea-ioc-01 .*pretend_helm.tgz --namespace bl45p --debug' rsp: | # Source: bl45p-ea-ioc-01/templates/configmap.yaml apiVersion: v1 ... -# TODO blank response is that OK? deploy_local: - - cmd: helm dependency update /tmp/ec_tests/beamline-chart + - cmd: 'helm dependency update .*\/tests\/data\/services\/bl45p-ea-ioc-01\; helm package .*\/tests\/data\/services\/bl45p-ea-ioc-01 --app-version .*' rsp: "" - - cmd: - 'bash -c "helm upgrade --install bl45p-ea-ioc-01 /tmp/ec_tests/beamline-chart - --version 20.*--namespace bl45p -f \/.*\/data\/iocs\/bl45p-ea-ioc-01\/values.yaml.*' + - cmd: 'bash -c "helm upgrade --install bl45p-ea-ioc-01 .*pretend_helm.tgz --namespace bl45p' rsp: "" deploy: - - cmd: git clone https://github.com/epics-containers/bl45p /tmp/ec_tests --depth=1 --single-branch --branch=2.0 + - cmd: kubectl get namespace bl45p -o name + rsp: namespace/bl45p + - cmd: git clone https://github.com/epics-containers/bl45p /tmp/ec_tests --depth=1 --single-branch --branch=main rsp: "" - - cmd: helm dependency update /tmp/ec_tests/beamline-chart + - cmd: helm dependency update /tmp/ec_tests/services/bl45p-ea-ioc-01; helm package /tmp/ec_tests/services/bl45p-ea-ioc-01 --app-version 2.0 rsp: "" - - cmd: 'bash -c "helm upgrade --install bl45p-ea-ioc-01 /tmp/ec_tests/beamline-chart --version 2.0 --namespace bl45p -f /tmp/ec_tests/iocs/bl45p-ea-ioc-01/values.yaml.*' + - cmd: 'bash -c "helm upgrade --install bl45p-ea-ioc-01 .*pretend_helm.tgz --namespace bl45p.*' rsp: "" instances: diff --git a/tests/data/local.yaml b/tests/data/local.yaml index b5c9f587..ba3becbc 100644 --- a/tests/data/local.yaml +++ b/tests/data/local.yaml @@ -30,7 +30,7 @@ deploy_local: rsp: "" - cmd: docker container create --name busybox -v bl45p-ea-ioc-01_config:/copyto busybox rsp: "" - - cmd: docker cp {data}/iocs/bl45p-ea-ioc-01/config/ioc.yaml busybox:copyto + - cmd: docker cp {data}/services/bl45p-ea-ioc-01/config/ioc.yaml busybox:copyto rsp: "" - cmd: docker rm -f busybox rsp: "" @@ -54,7 +54,7 @@ deploy: rsp: "" - cmd: docker container create --name busybox -v bl45p-ea-ioc-01_config:/copyto busybox rsp: "" - - cmd: docker cp /tmp/ec_tests/iocs/bl45p-ea-ioc-01/config/ioc.yaml busybox:copyto + - cmd: docker cp /tmp/ec_tests/services/bl45p-ea-ioc-01/config/ioc.yaml busybox:copyto rsp: "" - cmd: docker rm -f busybox rsp: "" @@ -72,9 +72,6 @@ instances: - cmd: git diff --name-only 2.0 2.0\^ rsp: | 2.0 - - cmd: git diff --name-only 2.0 $(git hash-object -t tree /dev/null) - rsp: | - "" exec: - cmd: docker ps -f name=bl45p-ea-ioc-01 --format .* diff --git a/tests/data/iocs/bl45p-ea-ioc-01/config/ioc.yaml b/tests/data/services/bl45p-ea-ioc-01/config/ioc.yaml similarity index 100% rename from tests/data/iocs/bl45p-ea-ioc-01/config/ioc.yaml rename to tests/data/services/bl45p-ea-ioc-01/config/ioc.yaml diff --git a/tests/data/services/bl45p-ea-ioc-01/pretend_helm.tgz b/tests/data/services/bl45p-ea-ioc-01/pretend_helm.tgz new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/iocs/bl45p-ea-ioc-01/values.yaml b/tests/data/services/bl45p-ea-ioc-01/values.yaml similarity index 100% rename from tests/data/iocs/bl45p-ea-ioc-01/values.yaml rename to tests/data/services/bl45p-ea-ioc-01/values.yaml diff --git a/tests/test_autocomplete.py b/tests/test_autocomplete.py index c4830b94..da78593a 100644 --- a/tests/test_autocomplete.py +++ b/tests/test_autocomplete.py @@ -41,7 +41,7 @@ def test_all_iocs_local(mock_run, mocker, autocomplete, ctx): def test_avail_IOCs(mock_run, data, autocomplete, ctx): mock_run.set_seq(autocomplete.avail_IOCs) - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") ctx.parent.parent.params["repo"] = "" # use env variable result = mock_run.call(avail_IOCs, ctx) @@ -50,7 +50,7 @@ def test_avail_IOCs(mock_run, data, autocomplete, ctx): def test_avail_versions(mock_run, data, autocomplete, ctx): mock_run.set_seq(autocomplete.avail_versions) - # shutil.copytree(data / "iocs", TMPDIR / "iocs") already exists + # shutil.copytree(data / "services", TMPDIR / "services") already exists ctx.parent.parent.params["repo"] = "" # use env variable ctx.parent.parent.params["ioc_name"] = "bl45p-ea-ioc-01" diff --git a/tests/test_ioc.py b/tests/test_ioc.py index 22aee45c..70c8703d 100644 --- a/tests/test_ioc.py +++ b/tests/test_ioc.py @@ -1,6 +1,8 @@ import shutil from pathlib import Path +import pytest + from tests.conftest import TMPDIR @@ -16,37 +18,35 @@ def test_delete(mock_run, ioc): def test_template(mock_run, data, ioc): mock_run.set_seq(ioc.checks[:1] + ioc.template) - mock_run.run_cli(f"template {data / 'iocs/bl45p-ea-ioc-01'}") + mock_run.run_cli(f"template {data / 'services/bl45p-ea-ioc-01'}") def test_deploy_local(mock_run, data, ioc): mock_run.set_seq(ioc.checks[:1] + ioc.deploy_local) - mock_run.run_cli(f"deploy-local {data / 'iocs/bl45p-ea-ioc-01'}") + mock_run.run_cli(f"deploy-local {data / 'services/bl45p-ea-ioc-01'}") def test_deploy(mock_run, data: Path, ioc): - mock_run.set_seq(ioc.checks + ioc.deploy) + mock_run.set_seq(ioc.deploy) # prep what deploy expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "beamline-chart", TMPDIR / "beamline-chart") - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") mock_run.run_cli("deploy bl45p-ea-ioc-01 2.0") def test_deploy_path(mock_run, data: Path, ioc): - mock_run.set_seq(ioc.checks + ioc.deploy) + mock_run.set_seq(ioc.deploy) # prep what deploy expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "beamline-chart", TMPDIR / "beamline-chart") - shutil.copytree(data / "iocs", TMPDIR / "iocs") - mock_run.run_cli("deploy bl45p/iocs/bl45p-ea-ioc-01 2.0") + shutil.copytree(data / "services", TMPDIR / "services") + mock_run.run_cli("deploy bl45p/services/bl45p-ea-ioc-01 2.0") def test_list(mock_run, ioc, data: Path): mock_run.set_seq(ioc.instances) # prep what instances expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") mock_run.run_cli("list") @@ -54,7 +54,7 @@ def test_instances(mock_run, ioc, data: Path): mock_run.set_seq(ioc.instances) # prep what instances expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") mock_run.run_cli("instances bl45p-ea-ioc-01") @@ -88,6 +88,7 @@ def test_stop(mock_run, ioc): mock_run.run_cli("stop bl45p-ea-ioc-01") +@pytest.mark.skip(reason="under review - and in flux") def test_ps(mock_run, ioc): mock_run.set_seq(ioc.ps) mock_run.run_cli("ps") diff --git a/tests/test_local.py b/tests/test_local.py index 7428b0d1..7fae65a7 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -21,14 +21,14 @@ def test_delete(mock_run, local): def test_deploy_local(mock_run, data, local): mock_run.set_seq(local.setup + local.deploy_local) - mock_run.run_cli(f"deploy-local {data / 'iocs/bl45p-ea-ioc-01'}") + mock_run.run_cli(f"deploy-local {data / 'services/bl45p-ea-ioc-01'}") def test_deploy(mock_run, data, local): mock_run.set_seq(local.setup + local.deploy) # prep what deploy expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") mock_run.run_cli("deploy bl45p-ea-ioc-01 2.0") @@ -36,7 +36,7 @@ def test_instances(mock_run, local, data: Path): mock_run.set_seq(local.instances) # prep what instances expects to find after it cloned bl45p repo TMPDIR.mkdir() - shutil.copytree(data / "iocs", TMPDIR / "iocs") + shutil.copytree(data / "services", TMPDIR / "services") mock_run.run_cli("instances bl45p-ea-ioc-01") @@ -73,4 +73,4 @@ def test_ps(mock_run, local): def test_validate(mock_run, local, data): mock_run.set_seq(local.setup + local.validate) os.chdir(Path(__file__).parent) - mock_run.run_cli(f"validate {data / 'iocs/bl45p-ea-ioc-01'}") + mock_run.run_cli(f"validate {data / 'services/bl45p-ea-ioc-01'}") diff --git a/tests/test_system.py b/tests/test_system.py index 1128cdfa..3c47f96d 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -24,7 +24,7 @@ def test_validate(): cli, [ "validate", - f"{THIS_DIR}/data/iocs/bl45p-ea-ioc-01", + f"{THIS_DIR}/data/services/bl45p-ea-ioc-01", ], ) @@ -44,7 +44,7 @@ def test_validate_chdir(): cli, [ "validate", - f"{THIS_DIR}/data/iocs/bl45p-ea-ioc-01", + f"{THIS_DIR}/data/services/bl45p-ea-ioc-01", ], )