From 642dfebdb50ddd08e75f27f386ba0dcb99a0de52 Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 21 Mar 2024 13:24:56 +0530 Subject: [PATCH 01/54] feat(disks): use v2 APIs for Disk operations --- riocli/apply/resolver.py | 18 +- riocli/constants/__init__.py | 3 +- riocli/constants/regions.py | 17 ++ riocli/disk/create.py | 37 +++- riocli/disk/delete.py | 103 ++++++++-- riocli/disk/list.py | 36 +--- riocli/disk/model.py | 67 ++---- riocli/disk/util.py | 126 ++++-------- riocli/v2client/client.py | 389 +++++++++++++++++++++++++++++++++++ 9 files changed, 588 insertions(+), 208 deletions(-) create mode 100644 riocli/constants/regions.py diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index 46e21a0a..5b2674c1 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -123,7 +123,7 @@ def _guid_functor(self, kind): "deployment": lambda x: munchify(x)['deploymentId'], "network": lambda x: munchify(x).guid, # This is only temporarily like this - "disk": lambda x: munchify(x)['internalDeploymentGUID'], + "disk": lambda x: munchify(x)['metadata']['guid'], "device": lambda x: munchify(x)['uuid'], "managedservice": lambda x: munchify(x)['metadata']['name'], "usergroup": lambda x: munchify(x).guid @@ -140,7 +140,7 @@ def _list_functors(self, kind): phases=[DeploymentPhaseConstants.SUCCEEDED, DeploymentPhaseConstants.PROVISIONING]), "network": self._list_networks, - "disk": self._list_disks, + "disk": self.v2client.list_disks, "device": self.client.get_all_devices, "managedservice": self._list_managedservices, "usergroup": self.client.list_usergroups @@ -182,20 +182,6 @@ def _list_networks(self): return networks - def _list_disks(self): - config = Configuration() - catalog_host = config.data.get( - 'catalog_host', 'https://gacatalog.apps.okd4v2.prod.rapyuta.io') - url = '{}/disk'.format(catalog_host) - headers = config.get_auth_header() - response = RestClient(url).method( - HttpMethod.GET).headers(headers).execute() - data = json.loads(response.text) - if not response.ok: - err_msg = data.get('error') - raise Exception(err_msg) - return munchify(data) - def _list_managedservices(self): instances = ManagedService.list_instances() return munchify(instances) diff --git a/riocli/constants/__init__.py b/riocli/constants/__init__.py index 67fb31a4..fbe68368 100644 --- a/riocli/constants/__init__.py +++ b/riocli/constants/__init__.py @@ -14,5 +14,6 @@ from riocli.constants.colors import Colors from riocli.constants.symbols import Symbols +from riocli.constants.regions import Regions -__all__ = [Colors, Symbols] +__all__ = [Colors, Symbols, Regions] diff --git a/riocli/constants/regions.py b/riocli/constants/regions.py new file mode 100644 index 00000000..a416414e --- /dev/null +++ b/riocli/constants/regions.py @@ -0,0 +1,17 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class Regions: + JP = 'jp' + US = 'us' \ No newline at end of file diff --git a/riocli/disk/create.py b/riocli/disk/create.py index 3af8cc1e..d78f2c84 100644 --- a/riocli/disk/create.py +++ b/riocli/disk/create.py @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import click +import time + from click_help_colors import HelpColorsCommand from rapyuta_io.clients.persistent_volumes import DiskCapacity -from riocli.constants import Colors, Symbols -from riocli.disk.util import create_cloud_disk +from riocli.constants import Colors, Symbols, Regions +from riocli.disk.util import is_disk_ready from riocli.utils.spinner import with_spinner +from riocli.config import new_v2_client SUPPORTED_CAPACITIES = [ DiskCapacity.GiB_4.value, @@ -30,6 +33,10 @@ DiskCapacity.GiB_512.value, ] +SUPPORTED_REGIONS = [ + Regions.JP, + Regions.US, +] @click.command( 'create', @@ -40,21 +47,39 @@ @click.argument('disk-name', type=str) @click.option('--capacity', 'capacity', type=click.Choice(SUPPORTED_CAPACITIES), default=DiskCapacity.GiB_4.value, help='Disk size in GiB') +@click.option('--region', 'region', type=click.Choice(SUPPORTED_REGIONS), + default=Regions.JP, help='Region to create the disk in') + @with_spinner(text="Creating a new disk...") def create_disk( disk_name: str, capacity: int = 4, + region: str = 'jp', spinner=None, ) -> None: """ Creates a new disk """ try: - disk = create_cloud_disk(disk_name, capacity) - + client = new_v2_client() + payload = { + "metadata": { + "name": disk_name, + "region": region + }, + "spec": { + "capacity": capacity, + "runtime": "cloud" + } + } + + client.create_disk(payload) + while not is_disk_ready(client, disk_name): + time.sleep(3) + spinner.text = click.style( - 'Disk {} ({}) created successfully.'. - format(disk['name'], disk['guid']), fg=Colors.GREEN) + 'Disk {} created successfully.'. + format(disk_name), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except Exception as e: spinner.text = click.style('Failed to create disk: {}'.format(e), fg=Colors.RED) diff --git a/riocli/disk/delete.py b/riocli/disk/delete.py index e3b76eec..7ea2b6aa 100644 --- a/riocli/disk/delete.py +++ b/riocli/disk/delete.py @@ -14,10 +14,27 @@ import click from click_help_colors import HelpColorsCommand from rapyuta_io.utils.rest_client import HttpMethod +from yaspin.api import Yaspin +from queue import Queue +import functools +from riocli.config import new_v2_client from riocli.constants import Symbols, Colors -from riocli.disk.util import name_to_guid, _api_call +from riocli.disk.model import Disk +from riocli.disk.util import fetch_disks, display_disk_list from riocli.utils.spinner import with_spinner +from riocli.utils.execute import apply_func_with_result +from riocli.utils import tabulate_data + +from rapyuta_io import Client + + +def _apply_delete(client: Client, result: Queue, disk: Disk) -> None: + try: + client.delete_disk(name = disk.metadata.name) + result.put((disk.metadata.name, True)) + except Exception as e: + result.put((disk.metadata.name, False)) @click.command( @@ -26,31 +43,81 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -@click.option('--force', '-f', 'force', is_flag=True, default=False, - help='Skip confirmation') -@click.argument('disk-name', required=True, type=str) -@name_to_guid -@with_spinner(text="Deleting disk...") +@click.option('--force', '-f', is_flag=True, default=False, + help='Skip confirmation', type=bool) +@click.option('--workers', '-w', + help="Number of parallel workers while running deleting disks. Defaults to 10", + type=int, default=10) +@click.argument('disk-name-or-regex', type=str) +@with_spinner(text='Deleting disk...') def delete_disk( - disk_name: str, - disk_guid: str, force: bool, - spinner=None + disk_name_or_regex: str, + delete_all: bool = False, + workers: int = 10, + spinner: Yaspin = None ) -> None: """ - Delete a disk + Deletes a disk """ + client = new_v2_client() + + if not (disk_name_or_regex or delete_all): + spinner.text = "Nothing to delete" + spinner.green.ok(Symbols.SUCCESS) + return + + try: + disks = fetch_disks(client, disk_name_or_regex, delete_all) + except Exception as e: + spinner.text = click.style( + 'Failed to find disk(s): {}'.format(e), Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) from e + + if not disks: + spinner.text = click.style("Disk(s) not found", Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) + with spinner.hidden(): - if not force: - click.confirm( - 'Deleting disk {} ({})'.format(disk_name, disk_guid), abort=True) + display_disk_list(disks) + + spinner.write('') + + if not force: + with spinner.hidden(): + click.confirm('Do you want to delete the above disk(s)?', default=True, abort=True) try: - _api_call(HttpMethod.DELETE, guid=disk_guid, load_response=False) + f = functools.partial(_apply_delete, client) + result = apply_func_with_result( + f=f, items=disks, + workers=workers, key=lambda x: x[0] + ) + data, statuses = [], [] + for name, status in result: + fg = Colors.GREEN if status else Colors.RED + icon = Symbols.SUCCESS if status else Symbols.ERROR - spinner.text = click.style('Disk deleted successfully.', fg=Colors.GREEN) - spinner.green.ok(Symbols.SUCCESS) + statuses.append(status) + data.append([ + click.style(name, fg), + click.style(icon, fg) + ]) + + with spinner.hidden(): + tabulate_data(data, headers=['Name', 'Status']) + + icon = Symbols.SUCCESS if all(statuses) else Symbols.WARNING + fg = Colors.GREEN if all(statuses) else Colors.YELLOW + text = "successfully" if all(statuses) else "partially" + + spinner.text = click.style( + 'Disk(s) deleted {}.'.format(text), fg) + spinner.ok(click.style(icon, fg)) except Exception as e: - spinner.text = click.style('Failed to delete disk: {}'.format(str(e)), fg=Colors.RED) + spinner.text = click.style( + 'Failed to delete disk(s): {}'.format(e), Colors.RED) spinner.red.fail(Symbols.ERROR) - raise SystemExit(1) + raise SystemExit(1) from e diff --git a/riocli/disk/list.py b/riocli/disk/list.py index db37d23b..7b5a8ff2 100644 --- a/riocli/disk/list.py +++ b/riocli/disk/list.py @@ -11,15 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing - import click -from rapyuta_io.utils.rest_client import HttpMethod from riocli.constants import Colors -from riocli.disk.util import _api_call -from riocli.utils import tabulate_data - +from riocli.config import new_v2_client +from riocli.disk.util import display_disk_list @click.command('list') def list_disks() -> None: @@ -27,29 +23,9 @@ def list_disks() -> None: List the disks in the selected project """ try: - disks = _api_call(HttpMethod.GET) - disks = sorted(disks, key=lambda d: d['name'].lower()) - _display_disk_list(disks, show_header=True) + client = new_v2_client(with_project=True) + disks = client.list_disks() + display_disk_list(disks, show_header=True) except Exception as e: click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) - - -def _display_disk_list(disks: typing.Any, show_header: bool = True): - headers = [] - if show_header: - headers = ( - 'Disk ID', 'Name', 'Status', 'Capacity', - 'Used', 'Available', 'Used By', - ) - - data = [[d['guid'], - d['name'], - d['status'], - d['capacity'], - d.get('used', 'NA'), - d.get('available', 'NA'), - d['usedBy']] - for d in disks] - - tabulate_data(data, headers) + raise SystemExit(1) \ No newline at end of file diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 98eae4a0..88a04308 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,21 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from os import stat import typing from time import sleep +from munch import unmunchify import click from munch import munchify from rapyuta_io import Client from rapyuta_io.utils.rest_client import HttpMethod +from riocli.config import new_v2_client from riocli.constants import Colors, Symbols -from riocli.disk.util import _api_call + from riocli.jsonschema.validate import load_schema from riocli.model import Model class Disk(Model): + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + def find_object(self, client: Client) -> typing.Any: _, disk = self.rc.find_depends({ 'kind': 'disk', @@ -38,59 +44,28 @@ def find_object(self, client: Client) -> typing.Any: return disk def create_object(self, client: Client, **kwargs) -> typing.Any: - labels = self.metadata.get('labels', None) - payload = { - "labels": labels, - "name": self.metadata.name, - "diskType": "ssd", - "runtime": self.spec.runtime, - "capacity": self.spec.capacity, - } - - result = _api_call(HttpMethod.POST, payload=payload) - result = munchify(result) - disk_dep_guid, disk = self.rc.find_depends({ - 'kind': self.kind.lower(), - 'nameOrGUID': self.metadata.name - }) - - volume_instance = client.get_volume_instance(disk_dep_guid) + v2_client = new_v2_client() - retry_count = int(kwargs.get('retry_count')) - retry_interval = int(kwargs.get('retry_interval')) - try: - volume_instance.poll_deployment_till_ready( - retry_count=retry_count, - sleep_interval=retry_interval - ) - except Exception as e: - click.secho(">> {}: Error polling for disk ({}:{})".format( - Symbols.WARNING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW) - - return result + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + self.pop("rc", None) + disk = unmunchify(self) + r = v2_client.create_disk(disk) + return unmunchify(r) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass - def delete_object(self, client: Client, obj: typing.Any, sleep_interval=10, retries=12) -> typing.Any: - volume_instance = client.get_volume_instance(obj.internalDeploymentGUID) - for attempt in range(retries): - sleep(sleep_interval) - try: - volume_instance.destroy_volume_instance() - except Exception as e: - if attempt == retries - 1: - raise e - else: - return + @staticmethod + def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: + v2_client = new_v2_client() + v2_client.delete_disk(obj.metadata.name) @classmethod def pre_process(cls, client: Client, d: typing.Dict) -> None: pass @staticmethod - def validate(data): + def validate(d): schema = load_schema('disk') - schema.validate(data) + schema.validate(d) diff --git a/riocli/disk/util.py b/riocli/disk/util.py index 3da90161..c55bc881 100644 --- a/riocli/disk/util.py +++ b/riocli/disk/util.py @@ -11,116 +11,60 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import functools -import json -import time import typing +import re -import click from rapyuta_io import Client from rapyuta_io.clients.persistent_volumes import DiskCapacity, DiskType from rapyuta_io.utils.rest_client import RestClient, HttpMethod from riocli.config import Configuration, new_client from riocli.constants import Colors, Symbols - +from riocli.disk.model import Disk +from riocli.utils import tabulate_data class DiskNotFound(Exception): def __init__(self): super().__init__('Disk not found') -def _api_call( - method: str, - guid: typing.Union[str, None] = None, - payload: typing.Union[typing.Dict, None] = None, - load_response: bool = True, -) -> typing.Any: - config = Configuration() - catalog_host = config.data.get( - 'catalog_host', 'https://gacatalog.apps.okd4v2.prod.rapyuta.io') - - url = '{}/disk'.format(catalog_host) - if guid: - url = '{}/{}'.format(url, guid) - - headers = config.get_auth_header() - response = RestClient(url).method(method).headers( - headers).execute(payload=payload) - - data = None - if load_response: - data = json.loads(response.text) - - if not response.ok: - err_msg = data.get('error') - raise Exception(err_msg) - return data - - -def name_to_guid(f: typing.Callable) -> typing.Callable: - @functools.wraps(f) - def decorated(**kwargs: typing.Any): - client = new_client() - name = kwargs.pop('disk_name') - guid = None - - if name.startswith('disk-'): - guid = name - name = None - - if name is None: - name = get_disk_name(client, guid) - - if guid is None: - try: - guid = find_disk_guid(client, name) - except Exception as e: - click.secho('{} {}'.format(Symbols.ERROR, e), fg=Colors.RED) - raise SystemExit(1) from e - - kwargs['disk_name'] = name - kwargs['disk_guid'] = guid - f(**kwargs) - - return decorated - - -def get_disk_name(client: Client, guid: str) -> str: - disk = _api_call(HttpMethod.GET, guid=guid) - return disk['name'] - +def fetch_disks( + client: Client, + disk_name_or_regex: str, + include_all: bool, +) -> typing.List[Disk]: -def find_disk_guid(client: Client, name: str) -> str: - try: - disks = _api_call(HttpMethod.GET) - for disk in disks: - if disk['name'] == name: - return disk['guid'] - raise DiskNotFound() - except Exception: - raise DiskNotFound() + disks = client.list_disks() + if include_all: + return disks -def create_cloud_disk(disk_name: str, capacity: int) -> typing.Dict: - """ - Creates a new cloud disk and waits until it is provisioned - """ - payload = { - "name": disk_name, - "diskType": DiskType.SSD, - "runtime": "cloud", - "capacity": DiskCapacity(capacity).value, - } + result = [] + for n in disks: + if re.search(disk_name_or_regex, n.metadata.name): + result.append(n) - disk = _api_call(HttpMethod.POST, payload=payload) + return result - while not is_disk_ready(disk.get('guid')): - time.sleep(5) +def is_disk_ready(client: Client , name: str) -> bool: + disk = client.get_disk(name) + return disk.status.get("status", "") == "Available" - return disk +def display_disk_list(disks: typing.Any, show_header: bool = True): + headers = [] + if show_header: + headers = ( + 'Disk ID', 'Name', 'Status', 'Capacity', + 'Used', 'Available', 'Used By', + ) + data = [[d.metadata.guid, + d.metadata.name, + d.status.get("status"), + d.spec.capacity, + d.spec.get("CapacityUsed"), + d.spec.get("CapacityAvailable"), + d.get("diskBound", {}).get("DeploymentName")] + for d in disks] -def is_disk_ready(disk_guid: str) -> bool: - disk = _api_call(HttpMethod.GET, disk_guid, load_response=True) - return disk.get('status') != 'Pending' + tabulate_data(data, headers) \ No newline at end of file diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 16e6b618..32e3782c 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -712,3 +712,392 @@ def _walk_pages(self, c: RestClient, params: dict = {}, limit: Optional[int] = N result.extend(items) return munchify(result) + + + def list_packages( + self, + query: dict = None + ) -> Munch: + """ + List all packages in a project + """ + url = "{}/v2/packages/".format(self._host) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + offset, result = 0, [] + while True: + params.update({ + "continue": offset, + }) + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("packages: {}".format(err_msg)) + packages = data.get('items', []) + if not packages: + break + offset = data['metadata']['continue'] + result.extend(packages) + + return munchify(result) + + def create_package(self, payload: dict) -> Munch: + """ + Create a new package + """ + url = "{}/v2/packages/".format(self._host) + headers = self._config.get_auth_header() + response = RestClient(url).method(HttpMethod.POST).headers( + headers).execute(payload=payload) + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("package: {}".format(err_msg)) + + return munchify(data) + + def get_package( + self, + name: str, + query: dict = None + ) -> Munch: + """ + List all packages in a project + """ + url = "{}/v2/packages/{}/".format(self._host, name) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("package: {}".format(err_msg)) + + return munchify(data) + + def delete_package(self, package_name: str, + query: dict = None) -> Munch: + """ + Delete a secret + """ + url = "{}/v2/packages/{}/".format(self._host, package_name) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + + response = RestClient(url).method(HttpMethod.DELETE).query_param( + params).headers(headers).execute() + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("package: {}".format(err_msg)) + + return munchify(data) + + + def list_networks( + self, + query: dict = None + ) -> Munch: + """ + List all networks in a project + """ + url = "{}/v2/networks/".format(self._host) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + offset, result = 0, [] + while True: + params.update({ + "continue": offset, + }) + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("networks: {}".format(err_msg)) + networks = data.get('items', []) + if not networks: + break + offset = data['metadata']['continue'] + result.extend(networks) + + return munchify(result) + + def create_network(self, payload: dict) -> Munch: + """ + Create a new network + """ + url = "{}/v2/networks/".format(self._host) + headers = self._config.get_auth_header() + response = RestClient(url).method(HttpMethod.POST).headers( + headers).execute(payload=payload) + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("network: {}".format(err_msg)) + + return munchify(data) + + def get_network( + self, + name: str, + query: dict = None + ) -> Munch: + """ + get a network in a project + """ + url = "{}/v2/networks/{}/".format(self._host, name) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + + if response.status_code == http.HTTPStatus.NOT_FOUND: + raise NetworkNotFound() + + if not response.ok: + err_msg = data.get('error') + raise Exception("network: {}".format(err_msg)) + + return munchify(data) + + def delete_network(self, network_name: str, + query: dict = None) -> Munch: + """ + Delete a secret + """ + url = "{}/v2/networks/{}/".format(self._host, network_name) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + + response = RestClient(url).method(HttpMethod.DELETE).query_param( + params).headers(headers).execute() + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("package: {}".format(err_msg)) + + return munchify(data) + + return munchify(data) + + + def list_deployments( + self, + query: dict = None + ) -> Munch: + """ + List all deployments in a project + """ + url = "{}/v2/deployments/".format(self._host) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + offset, result = 0, [] + while True: + params.update({ + "continue": offset, + "limit": 50, + }) + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("deployments: {}".format(err_msg)) + deployments = data.get('items', []) + if not deployments: + break + offset = data['metadata']['continue'] + for deployment in deployments: + result.append(deployment['metadata']) + + return munchify(result) + + def create_deployment(self, deployment: dict) -> Munch: + """ + Create a new deployment + """ + url = "{}/v2/deployments/".format(self._host) + headers = self._config.get_auth_header() + + deployment["metadata"]["projectGUID"] = headers["project"] + response = RestClient(url).method(HttpMethod.POST).headers( + headers).execute(payload=deployment) + handle_server_errors(response) + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("deployment: {}".format(err_msg)) + + return munchify(data) + + def get_deployment( + self, + name: str, + query: dict = None + ): + url = "{}/v2/deployments/{}/".format(self._host, name) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + + if response.status_code == http.HTTPStatus.NOT_FOUND: + raise Exception("deployment: {} not found".format(name)) + + if not response.ok: + err_msg = data.get('error') + raise Exception("deployment: {}".format(err_msg)) + + return munchify(data) + + def update_deployment(self, name: str, dep: dict) -> Munch: + """ + Update a deployment + """ + url = "{}/v2/deployments/{}/".format(self._host, name) + headers = self._config.get_auth_header() + response = RestClient(url).method(HttpMethod.PUT).headers( + headers).execute(payload=dep) + handle_server_errors(response) + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("deployment: {}".format(err_msg)) + + return munchify(data) + + def delete_deployment(self, name: str, query: dict = None) -> Munch: + """ + Delete a deployment + """ + url = "{}/v2/deployments/{}/".format(self._host, name) + headers = self._config.get_auth_header() + params = {} + params.update(query or {}) + response = RestClient(url).method( + HttpMethod.DELETE).headers(headers).execute() + handle_server_errors(response) + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("deployment: {}".format(err_msg)) + + return munchify(data) + + def list_disks( + self, + query: dict = None + ) -> Munch: + """ + List all disks in a project + """ + url = "{}/v2/disks/".format(self._host) + headers = self._config.get_auth_header() + + params = {} + params.update(query or {}) + offset, result = 0, [] + while True: + params.update({ + "continue": offset, + }) + response = RestClient(url).method(HttpMethod.GET).query_param( + params).headers(headers).execute() + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("disks: {}".format(err_msg)) + disks = data.get('items', []) + if not disks: + break + offset = data['metadata']['continue'] + result.extend(disks) + + return munchify(result) + + def get_disk(self, name: str) -> Munch: + """ + Get a Disk by its name + """ + url = "{}/v2/disks/{}/".format(self._host, name) + headers = self._config.get_auth_header() + response = RestClient(url).method( + HttpMethod.GET).headers(headers).execute() + + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("disks: {}".format(err_msg)) + + return munchify(data) + + def create_disk(self, disk: dict) -> Munch: + """ + Create a new disk + """ + url = "{}/v2/disks/".format(self._host) + headers = self._config.get_auth_header() + response = RestClient(url).method(HttpMethod.POST).headers( + headers).execute(payload=disk) + + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("disks: {}".format(err_msg)) + + return munchify(data) + + def delete_disk(self, name: str) -> Munch: + """ + Delete a disk by its name + """ + url = "{}/v2/disks/{}/".format(self._host, name) + headers = self._config.get_auth_header() + response = RestClient(url).method( + HttpMethod.DELETE).headers(headers).execute() + + handle_server_errors(response) + + data = json.loads(response.text) + if not response.ok: + err_msg = data.get('error') + raise Exception("disks: {}".format(err_msg)) + + return munchify(data) From 3a6a7d044eb74e42178db83454ed766ed78d14c3 Mon Sep 17 00:00:00 2001 From: romill Date: Thu, 21 Mar 2024 09:20:36 +0530 Subject: [PATCH 02/54] feat(networks): uses v2 networks APIs --- .../apply/manifests/network-native-cloud.yaml | 2 +- .../apply/manifests/network-routed-cloud.yaml | 1 + riocli/apply/manifests/network.yaml | 2 + riocli/apply/resolver.py | 7 +- riocli/jsonschema/schemas/network-schema.yaml | 27 ++- riocli/network/__init__.py | 2 - riocli/network/create.py | 53 ------ riocli/network/delete.py | 100 +++++++--- riocli/network/inspect.py | 36 ++-- riocli/network/list.py | 37 ++-- riocli/network/model.py | 78 ++------ riocli/network/native_network.py | 75 -------- riocli/network/routed_network.py | 86 --------- riocli/network/util.py | 175 ++++++------------ riocli/v2client/client.py | 10 + 15 files changed, 216 insertions(+), 475 deletions(-) delete mode 100644 riocli/network/create.py delete mode 100644 riocli/network/native_network.py delete mode 100644 riocli/network/routed_network.py diff --git a/riocli/apply/manifests/network-native-cloud.yaml b/riocli/apply/manifests/network-native-cloud.yaml index 6878abf2..2096dccd 100644 --- a/riocli/apply/manifests/network-native-cloud.yaml +++ b/riocli/apply/manifests/network-native-cloud.yaml @@ -2,9 +2,9 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-native-cloud" - project: "project-guid" labels: app: test + #rapyuta.io/region: us spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] diff --git a/riocli/apply/manifests/network-routed-cloud.yaml b/riocli/apply/manifests/network-routed-cloud.yaml index 8b2c543e..d15e6e94 100644 --- a/riocli/apply/manifests/network-routed-cloud.yaml +++ b/riocli/apply/manifests/network-routed-cloud.yaml @@ -5,6 +5,7 @@ metadata: project: "project-guid" labels: app: test + #rapyuta.io/region: us spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] diff --git a/riocli/apply/manifests/network.yaml b/riocli/apply/manifests/network.yaml index d18d301d..a5f2adee 100644 --- a/riocli/apply/manifests/network.yaml +++ b/riocli/apply/manifests/network.yaml @@ -6,6 +6,7 @@ metadata: project: "project-guid" labels: app: test + #rapyuta.io/region: us spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] @@ -36,6 +37,7 @@ metadata: project: "project-guid" labels: app: test + #rapyuta.io/region: us spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index 5b2674c1..17a7e912 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -121,7 +121,7 @@ def _guid_functor(self, kind): "package": lambda x: munchify(x)['id'], "staticroute": lambda x: munchify(x)['metadata']['guid'], "deployment": lambda x: munchify(x)['deploymentId'], - "network": lambda x: munchify(x).guid, + "network": lambda x: munchify(x)['metadata']['guid'], # This is only temporarily like this "disk": lambda x: munchify(x)['metadata']['guid'], "device": lambda x: munchify(x)['uuid'], @@ -139,8 +139,8 @@ def _list_functors(self, kind): "deployment": functools.partial(self.client.get_all_deployments, phases=[DeploymentPhaseConstants.SUCCEEDED, DeploymentPhaseConstants.PROVISIONING]), - "network": self._list_networks, "disk": self.v2client.list_disks, + "network": self.v2client.list_networks, "device": self.client.get_all_devices, "managedservice": self._list_managedservices, "usergroup": self.client.list_usergroups @@ -157,7 +157,8 @@ def _find_functors(self, kind): "staticroute": lambda name, obj_list: filter( lambda x: name == x.metadata.name.rsplit('-', 1)[0], obj_list), "deployment": self._generate_find_guid_functor(), - "network": self._generate_find_guid_functor(), + "network": lambda name, obj_list, network_type: filter( + lambda x: name == x.metadata.name and network_type == x.spec.type, obj_list), "disk": self._generate_find_guid_functor(), "device": self._generate_find_guid_functor(), "managedservice": lambda name, instances: filter(lambda i: i.metadata.name == name, instances), diff --git a/riocli/jsonschema/schemas/network-schema.yaml b/riocli/jsonschema/schemas/network-schema.yaml index 766b2ff3..2c2116e1 100644 --- a/riocli/jsonschema/schemas/network-schema.yaml +++ b/riocli/jsonschema/schemas/network-schema.yaml @@ -28,6 +28,23 @@ definitions: "$ref": "#/definitions/rosDistro" runtime: "$ref": "#/definitions/runtime" + discoveryServer: + allOf: + - "$ref": "#/definitions/discoveryServerData" + - if: + properties: + rosDistro: + const: foxy + then: + properties: + serverID: + type: integer + default: 0 + serverPort: + type: integer + default: 11811 + else: + not: { } required: - type - rosDistro @@ -65,6 +82,7 @@ definitions: - melodic - kinetic - noetic + - foxy restartPolicy: enum: - always @@ -119,4 +137,11 @@ definitions: multipleOf: 128 required: - cpu - - memory \ No newline at end of file + - memory + discoveryServerData: + type: object + properties: + serverID: + type: integer + serverPort: + type: integer \ No newline at end of file diff --git a/riocli/network/__init__.py b/riocli/network/__init__.py index 6099ea8d..0c03b234 100644 --- a/riocli/network/__init__.py +++ b/riocli/network/__init__.py @@ -15,7 +15,6 @@ from click_help_colors import HelpColorsGroup from riocli.constants import Colors -from riocli.network.create import create_network from riocli.network.delete import delete_network from riocli.network.inspect import inspect_network from riocli.network.list import list_networks @@ -35,7 +34,6 @@ def network() -> None: pass -network.add_command(create_network) network.add_command(delete_network) network.add_command(list_networks) network.add_command(inspect_network) diff --git a/riocli/network/create.py b/riocli/network/create.py deleted file mode 100644 index b47bb226..00000000 --- a/riocli/network/create.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2021 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import typing - -import click - -from riocli.device.util import name_to_guid as device_name_to_guid -from riocli.network.native_network import create_native_network -from riocli.network.routed_network import create_routed_network - - -@click.command('create', hidden=True) -@click.argument('name', type=str) -@click.option('--network', help='Type of Network', - type=click.Choice(['routed', 'native']), default='routed') -@click.option('--ros', help='Version of ROS', - type=click.Choice(['kinetic', 'melodic', 'noetic']), - default='melodic') -@click.option('--device', 'device_name', - help='Device ID of the Device where Network will run (device only)') -@click.option('--cpu', help='cpu limit for Network (cloud only) ', type=float) -@click.option('--memory', help='memory limit for Network (cloud only) ', - type=int) -@click.option('--network-interface', '-nic', type=str, - help='Network Interface on which Network will listen (device only)') -@click.option('--restart-policy', - help='Restart policy for the Network (device only)', - type=click.Choice(['always', 'no', 'on-failure']), - default='always') -@device_name_to_guid -def create_network(name: str, network: str, **kwargs: typing.Any) -> None: - """ - Create a new network - """ - try: - if network == 'routed': - create_routed_network(name, **kwargs) - else: - create_native_network(name, **kwargs) - except Exception as e: - click.secho(str(e), fg='red') - raise SystemExit(1) diff --git a/riocli/network/delete.py b/riocli/network/delete.py index ab787327..cd718fc9 100644 --- a/riocli/network/delete.py +++ b/riocli/network/delete.py @@ -14,12 +14,26 @@ import click from click_help_colors import HelpColorsCommand from yaspin.api import Yaspin +import functools +from queue import Queue -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors, Symbols -from riocli.network.util import name_to_guid +from riocli.network.util import fetch_networks, print_networks_for_confirmation from riocli.utils.spinner import with_spinner +from riocli.utils import tabulate_data +from riocli.utils.execute import apply_func_with_result +from rapyuta_io import Client +from riocli.network.model import Network + + +def _apply_delete(client: Client, result: Queue, network: Network) -> None: + try: + client.delete_network(network_name=network.metadata.name) + result.put((network.metadata.name, True)) + except Exception: + result.put((network.metadata.name, False)) @click.command( 'delete', @@ -29,45 +43,79 @@ ) @click.option('--force', '-f', is_flag=True, default=False, help='Skip confirmation', type=bool) -@click.option('--network', 'network_type', help='Type of Network', - default=None, - type=click.Choice(['routed', 'native'])) -@click.argument('network-name', type=str) -@name_to_guid +@click.option('--workers', '-w', + help="Number of parallel workers while running deleting networks. Defaults to 10", + type=int, default=10) +@click.argument('network-name-or-regex', type=str) @with_spinner(text='Deleting network...') def delete_network( force: bool, - network_name: str, - network_guid: str, - network_type: str, + network_name_or_regex: str, + delete_all: bool = False, + workers: int = 10, spinner: Yaspin = None ) -> None: """ Deletes a network """ + client = new_v2_client() + + if not (network_name_or_regex or delete_all): + spinner.text = "Nothing to delete" + spinner.green.ok(Symbols.SUCCESS) + return + + try: + networks = fetch_networks(client, network_name_or_regex, "", delete_all) + except Exception as e: + spinner.text = click.style( + 'Failed to find network(s): {}'.format(e), Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) from e + + if not networks: + spinner.text = click.style("Network(s) not found", Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) + + with spinner.hidden(): + print_networks_for_confirmation(networks) + + spinner.write('') + if not force: with spinner.hidden(): - click.confirm( - 'Deleting {} network {} ({})'. - format(network_type, network_name, network_guid), - abort=True) + click.confirm('Do you want to delete the above network(s)?', default=True, abort=True) try: - client = new_client() + f = functools.partial(_apply_delete, client) + result = apply_func_with_result( + f=f, items=networks, + workers=workers, key=lambda x: x[0] + ) + data, statuses = [], [] + for name, status in result: + fg = Colors.GREEN if status else Colors.RED + icon = Symbols.SUCCESS if status else Symbols.ERROR - if network_type == 'routed': - client.delete_routed_network(network_guid) - elif network_type == 'native': - client.delete_native_network(network_guid) - else: - raise Exception('invalid network type') + statuses.append(status) + data.append([ + click.style(name, fg), + click.style(icon, fg) + ]) + + with spinner.hidden(): + tabulate_data(data, headers=['Name', 'Status']) + + icon = Symbols.SUCCESS if all(statuses) else Symbols.WARNING + fg = Colors.GREEN if all(statuses) else Colors.YELLOW + text = "successfully" if all(statuses) else "partially" spinner.text = click.style( - '{} network deleted successfully!'.format(network_type.capitalize()), - fg=Colors.GREEN) - spinner.green.ok(Symbols.SUCCESS) + 'Networks(s) deleted {}.'.format(text), fg) + spinner.ok(click.style(icon, fg)) except Exception as e: - spinner.text = click.style('Failed to delete network: {}'.format(e), - fg=Colors.RED) + spinner.text = click.style( + 'Failed to delete network(s): {}'.format(e), Colors.RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e diff --git a/riocli/network/inspect.py b/riocli/network/inspect.py index dfeb5b43..3fdde41b 100644 --- a/riocli/network/inspect.py +++ b/riocli/network/inspect.py @@ -15,35 +15,27 @@ from click_help_colors import HelpColorsCommand from riocli.constants import Colors -from riocli.network.native_network import inspect_native_network -from riocli.network.routed_network import inspect_routed_network -from riocli.network.util import name_to_guid from riocli.utils import inspect_with_format +from riocli.config import new_v2_client +from munch import unmunchify - -@click.command( - 'inspect', - cls=HelpColorsCommand, - help_headers_color=Colors.YELLOW, - help_options_color=Colors.GREEN, -) -@click.option('--format', '-f', 'format_type', - type=click.Choice(['json', 'yaml'], case_sensitive=False), - default='yaml') -@click.argument('network-name', type=str) -@name_to_guid -def inspect_network(format_type: str, network_name: str, network_guid: str, - network_type: str) -> None: +@click.command('inspect') +@click.option('--format', '-f', 'format_type', default='yaml', + type=click.Choice(['json', 'yaml'], case_sensitive=False)) +@click.argument('network-name') +def inspect_network(format_type: str, network_name: str) -> None: """ Inspect the network resource """ try: - if network_type == 'routed': - data = inspect_routed_network(network_guid) - else: - data = inspect_native_network(network_guid) + client = new_v2_client() + network_obj = client.get_network(network_name) + + if not network_obj: + click.secho("network not found", fg='red') + raise SystemExit(1) - inspect_with_format(data, format_type) + inspect_with_format(unmunchify(network_obj), format_type) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) from e diff --git a/riocli/network/list.py b/riocli/network/list.py index 5db25b51..c895691c 100644 --- a/riocli/network/list.py +++ b/riocli/network/list.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import typing - import click from click_help_colors import HelpColorsCommand +from munch import Munch + from rapyuta_io import DeploymentPhaseConstants -from rapyuta_io.clients.native_network import NativeNetwork -from rapyuta_io.clients.routed_network import RoutedNetwork -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.utils import tabulate_data @@ -37,17 +36,14 @@ def list_networks(network: str) -> None: List the networks in the selected project """ try: - client = new_client() - - networks = [] - if network in ['routed', 'both']: - networks += client.get_all_routed_networks() + client = new_v2_client(with_project=True) - if network in ['native', 'both']: - networks += client.list_native_networks() - - networks = sorted(networks, key=lambda n: n.name.lower()) + if network in ("both", ""): + networks = client.list_networks() + else: + networks = client.list_networks(query={"network_type": network}) + networks = sorted(networks, key=lambda n: n.metadata.name.lower()) _display_network_list(networks, show_header=True) except Exception as e: click.secho(str(e), fg='red') @@ -55,27 +51,20 @@ def list_networks(network: str) -> None: def _display_network_list( - networks: typing.List[typing.Union[RoutedNetwork, NativeNetwork]], + networks: typing.List[Munch], show_header: bool = True, ) -> None: headers = [] if show_header: headers = ('Network ID', 'Network Name', 'Runtime', 'Type', 'Phase') - data = [] for network in networks: - phase = None - network_type = None - if isinstance(network, RoutedNetwork): - network_type = 'routed' - phase = network.phase - elif isinstance(network, NativeNetwork): - network_type = 'native' - phase = network.internal_deployment_status.phase + phase = network.status.phase if network.status else "" + network_type = network.spec.type if phase and phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: continue data.append( - [network.guid, network.name, network.runtime, network_type, phase]) + [network.metadata.guid, network.metadata.name, network.spec.runtime, network_type, phase]) tabulate_data(data, headers) diff --git a/riocli/network/model.py b/riocli/network/model.py index 3599113d..1fc3694f 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -13,6 +13,7 @@ # limitations under the License. import typing from typing import Union, Any, Dict +from munch import munchify, unmunchify from rapyuta_io import Client from rapyuta_io.clients.common_models import Limits @@ -23,8 +24,8 @@ from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.network.util import find_network_name, NetworkNotFound - +from riocli.v2client.client import NetworkNotFound +from riocli.config import new_v2_client class Network(Model): def __init__(self, *args, **kwargs): @@ -32,82 +33,41 @@ def __init__(self, *args, **kwargs): def find_object(self, client: Client) -> bool: try: - network, _ = find_network_name(client, self.metadata.name, - self.spec.type, - is_resolve_conflict=False) - return network + network, obj = self.rc.find_depends({"kind": self.kind.lower(), + "nameOrGUID": self.metadata.name}, self.spec.type) + if not network: + return False + + return obj except NetworkNotFound: return False - def create_object(self, client: Client, **kwargs) -> Union[NativeNetwork, RoutedNetwork]: - retry_count = int(kwargs.get('retry_count')) - retry_interval = int(kwargs.get('retry_interval')) - if self.spec.type == 'routed': - network = self._create_routed_network(client) - network.poll_routed_network_till_ready(retry_count=retry_count, sleep_interval=retry_interval) - return network + def create_object(self, client: Client, **kwargs) -> typing.Any: + client = new_v2_client() - network = client.create_native_network(self.to_v1(client)) - network.poll_native_network_till_ready(retry_count=retry_count, sleep_interval=retry_interval) - return network + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + network = unmunchify(self) + network.pop("rc", None) + r = client.create_network(network) + return unmunchify(r) def update_object(self, client: Client, obj: Union[RoutedNetwork, NativeNetwork]) -> Any: pass def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - obj.delete() + client = new_v2_client() + client.delete_network(obj.metadata.name) @classmethod def pre_process(cls, client: Client, d: Dict) -> None: pass - def to_v1(self, client: Client) -> NativeNetwork: - if self.spec.runtime == 'cloud': - limits = self._get_limits() - parameters = NativeNetworkParameters(limits=limits) - else: - device = client.get_device(self.spec.deviceGUID) - parameters = NativeNetworkParameters( - device=device, - network_interface=self.spec.networkInterface) - - return NativeNetwork( - self.metadata.name, - self.spec.runtime.lower(), - self.spec.rosDistro, - parameters=parameters - ) - def _get_limits(self): return Limits(self.spec.resourceLimits['cpu'], self.spec.resourceLimits['memory']) - def _create_routed_network(self, client: Client) -> RoutedNetwork: - if self.spec.runtime == 'cloud': - network = self._create_cloud_routed_network(client) - else: - network = self._create_device_routed_network(client) - - return network - - def _create_cloud_routed_network(self, client: Client) -> RoutedNetwork: - limits = self._get_limits() - parameters = RoutedNetworkParameters(limits) - return client.create_cloud_routed_network(self.metadata.name, - self.spec.rosDistro, True, - parameters=parameters) - - def _create_device_routed_network(self, client: Client) -> RoutedNetwork: - device = client.get_device(self.spec.deviceGUID) - return client.create_device_routed_network( - name=self.metadata.name, - ros_distro=self.spec.rosDistro, - shared=True, - device=device, - network_interface=self.spec.networkInterface, - ) - @staticmethod def validate(data): """ diff --git a/riocli/network/native_network.py b/riocli/network/native_network.py deleted file mode 100644 index 86aac499..00000000 --- a/riocli/network/native_network.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2021 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import typing - -import click -from rapyuta_io.clients.common_models import Limits -from rapyuta_io.clients.native_network import NativeNetwork, Parameters -from rapyuta_io.clients.package import Runtime, ROSDistro - -from riocli.config import new_client - - -def create_native_network(name: str, ros: str, device_guid: str = None, - network_interface: str = None, - cpu: float = 0, memory: int = 0, - restart_policy: str = None, - **kwargs: typing.Any) -> None: - client = new_client() - - ros_distro = ROSDistro(ros) - runtime = Runtime.CLOUD - - limit = None - if cpu or memory: - if device_guid: - raise Exception( - 'Native network for device does not support cpu or memory') - limit = Limits(cpu, memory) - - device = None - if device_guid: - runtime = Runtime.DEVICE - device = client.get_device(device_id=device_guid) - - parameters = Parameters(limits=limit, device=device, - network_interface=network_interface, - restart_policy=restart_policy) - - client.create_native_network(NativeNetwork(name, runtime=runtime, - ros_distro=ros_distro, - parameters=parameters)) - - click.secho('Native Network created successfully!', fg='green') - - -def inspect_native_network(network_guid: str) -> dict: - client = new_client() - network = client.get_native_network(network_guid) - - return { - 'created_at': network.created_at, - 'updated_at': network.updated_at, - 'creator': network.creator, - 'guid': network.guid, - 'internal_deployment_guid': network.internal_deployment_guid, - 'internal_deployment_status': network.internal_deployment_status.__dict__, - 'name': network.name, - 'owner_project': network.owner_project, - 'parameters': { - 'limits': network.parameters.limits.__dict__, - }, - 'ros_distro': network.ros_distro.value, - 'runtime': network.runtime.value, - } diff --git a/riocli/network/routed_network.py b/riocli/network/routed_network.py deleted file mode 100644 index 29d1664d..00000000 --- a/riocli/network/routed_network.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2021 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import typing - -import click -from rapyuta_io import ROSDistro -from rapyuta_io.clients.common_models import Limits -from rapyuta_io.clients.package import RestartPolicy -from rapyuta_io.clients.routed_network import Parameters - -from riocli.config import new_client - - -def create_routed_network(name: str, ros: str, device_guid: str = None, - network_interface: str = None, - cpu: float = 0, memory: int = 0, - restart_policy: str = None, - **kwargs: typing.Any) -> None: - client = new_client() - ros_distro = ROSDistro(ros) - - limit = None - if cpu or memory: - if device_guid: - raise Exception( - 'Routed network for device does not support cpu or memory') - limit = Limits(cpu, memory) - - if restart_policy: - restart_policy = RestartPolicy(restart_policy) - - if device_guid: - device = client.get_device(device_id=device_guid) - client.create_device_routed_network(name=name, ros_distro=ros_distro, - shared=False, - device=device, - network_interface=network_interface, - restart_policy=restart_policy) - else: - parameters = None if not limit else Parameters(limit) - client.create_cloud_routed_network(name, ros_distro=ros_distro, - shared=False, - parameters=parameters) - - click.secho('Routed Network created successfully!', fg='green') - - -def inspect_routed_network(network_guid: str) -> dict: - client = new_client() - network = client.get_routed_network(network_guid) - - # TODO: Validate if all fields are present (Keep it consistent with Native Network's Inspect) - - return { - 'created_at': network.CreatedAt, - 'updated_at': network.UpdatedAt, - 'creator': network.creator, - 'guid': network.guid, - 'internal_deployment_guid': network.internalDeploymentGUID, - 'internal_deployment_status': { - 'phase': network.internalDeploymentStatus.phase, - 'status': network.internalDeploymentStatus.status, - }, - 'name': network.name, - 'owner_project': network.ownerProject, - 'parameters': { - 'limits': { - 'memory': network.parameters.limits.memory, - 'cpu': network.parameters.limits.cpu, - }, - }, - 'ros_distro': network.rosDistro, - 'runtime': network.runtime, - 'status': network.status, - } diff --git a/riocli/network/util.py b/riocli/network/util.py index 4b013dc1..9d1e1065 100644 --- a/riocli/network/util.py +++ b/riocli/network/util.py @@ -12,41 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. import functools -from typing import Optional, Union, Tuple, Callable, Any - +from typing import Optional, Union, Tuple, Callable, Any, List +from munch import Munch import click +import re + from rapyuta_io import DeploymentPhaseConstants, Client from rapyuta_io.clients.native_network import NativeNetwork from rapyuta_io.clients.routed_network import RoutedNetwork from riocli.config import new_client from riocli.constants import Colors +from riocli.utils import tabulate_data from riocli.utils.selector import show_selection - +from riocli.config import new_v2_client +from riocli.network.model import Network def name_to_guid(f: Callable) -> Callable: @functools.wraps(f) def decorated(**kwargs: Any): - client = new_client() + client = new_v2_client() name = kwargs.pop('network_name') network_type = kwargs.pop('network', None) guid = None - if name.startswith('net-'): guid = name name = None try: if guid: - network, network_type = find_network_guid( + network = find_network_guid( client, guid, network_type) - name = network.name + name = network.metadata.name - if name: - network, network_type = find_network_name( + else: + network = find_network_name( client, name, network_type) - guid = network.guid except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) from e @@ -63,134 +65,61 @@ def find_network_guid( client: Client, guid: str, network_type: str, -) -> Tuple[Union[RoutedNetwork, NativeNetwork], str]: - if network_type is None or network_type == 'routed': - routed_networks = client.get_all_routed_networks() - for network in routed_networks: - if network.phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: - continue - if network.guid == guid: - return network, 'routed' - - if network_type is None or network_type == 'native': - native_network = client.list_native_networks() - for network in native_network: - phase = network.internal_deployment_status.phase - if phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: - continue - if network.guid == guid: - return network, 'native' - - raise NetworkNotFound() - - -def find_network_name( - client: Client, - name: str, - network_type: Optional[str], - is_resolve_conflict: bool = True -) -> Tuple[Optional[Union[RoutedNetwork, NativeNetwork]], str]: - routed, native = None, None - if network_type in [None, 'routed']: - routed = find_routed_network_name(client, name) - - if network_type in [None, 'native']: - native = find_native_network_name(client, name) - - return resolve_conflict(routed, native, network_type, is_resolve_conflict) - - -def find_native_network_name(client: Client, name: str) -> Optional[ - NativeNetwork]: - native_networks = client.list_native_networks() - for network in native_networks: - phase = network.internal_deployment_status.phase - if phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: - continue - if network.name == name: - return network +) -> Munch: + if network_type: + networks = client.list_networks(query={'network_type': network_type}) + else: + networks = client.list_networks() -def find_routed_network_name(client: Client, name: str) -> Optional[ - RoutedNetwork]: - routed_networks = client.get_all_routed_networks() - for network in routed_networks: - if network.phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: + for network in networks: + if network.status and network.status.phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: continue - if network.name == name: + if guid == network.metadata.guid: return network + raise NetworkNotFound() -def resolve_conflict( - routed: Optional[RoutedNetwork], - native: Optional[NativeNetwork], - network_type: Optional[str], - is_resolve_conflict: bool = True -) -> Tuple[Optional[Union[RoutedNetwork, NativeNetwork]], str]: - if not routed and not native: - raise NetworkNotFound() +def find_network_guid(client: Client, name: str, version: str = None) -> str: + packages = client.get_all_packages(name=name, version=version) + if len(packages) == 0: + click.secho("package not found", fg='red') + raise SystemExit(1) - # If only routed, or only native network was found, there is no conflict to - # resolve. - if routed and not native: - return routed, 'routed' - elif native and not routed: - return native, 'native' + if len(packages) == 1: + return packages[0].packageId - if not is_resolve_conflict: - raise NetworkConflict() + options = {} + for pkg in packages: + options[pkg.packageId] = '{} ({})'.format(pkg.packageName, pkg.packageVersion) - # Check if user already offered a choice in case of conflict - if network_type: - choice = network_type - else: - # Ask user to help us resolve conflict by selecting one network - options = { - 'routed': '{} ({})'.format(routed.name, routed.guid), - 'native': '{} ({})'.format(native.name, native.guid), - } - choice = show_selection(options, - header='Both Routed and Native networks were found with ' - 'the same name') - - if choice == 'routed': - return routed, choice - elif choice == 'native': - return native, choice - else: - click.secho('Invalid choice. Try again', fg=Colors.RED) - raise SystemExit(1) + choice = show_selection(options, header='Following packages were found with the same name') + return choice -def get_network( +def fetch_networks( client: Client, - network_guid: str, + network_name_or_regex: str, network_type: str, -) -> Optional[Union[RoutedNetwork, NativeNetwork]]: - if network_type == 'routed': - return client.get_routed_network(network_guid) - elif network_type == 'native': - return client.get_native_network(network_guid) - + include_all: bool, +) -> List[Network]: -def get_network_internal_deployment( - network: Union[RoutedNetwork, NativeNetwork], - network_type: str, -) -> Optional[str]: - if network_type == 'routed': - return network.internalDeploymentGUID - elif network_type == 'native': - return network.internal_deployment_guid + if network_type: + networks = client.list_networks(query={'network_type': network_type}) + else: + networks = client.list_networks() + if include_all: + return networks -class NetworkNotFound(Exception): - def __init__(self, message='network not found!'): - self.message = message - super().__init__(self.message) + result = [] + for n in networks: + if re.search(network_name_or_regex, n.metadata.name): + result.append(n) + return result -class NetworkConflict(Exception): - def __init__(self, - message='both routed and native networks exist with the same name!'): - self.message = message - super().__init__(self.message) +def print_networks_for_confirmation(networks: List[Munch]) -> None: + headers = ['Name', 'Type'] + data = [[n.metadata.name, n.spec.type] for n in networks] + tabulate_data(data, headers) \ No newline at end of file diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 32e3782c..91e1b6b6 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -22,8 +22,13 @@ import magic import requests from munch import munchify, Munch + from rapyuta_io.utils.rest_client import HttpMethod, RestClient +class DeploymentNotFound(Exception): + def __init__(self, message='deployment not found!'): + self.message = message + super().__init__(self.message) def handle_server_errors(response: requests.Response): status_code = response.status_code @@ -46,6 +51,10 @@ def handle_server_errors(response: requests.Response): if status_code > 504: raise Exception('unknown server error') +class NetworkNotFound(Exception): + def __init__(self, message='network not found!'): + self.message = message + super().__init__(self.message) class Client(object): """ @@ -714,6 +723,7 @@ def _walk_pages(self, c: RestClient, params: dict = {}, limit: Optional[int] = N return munchify(result) + def list_packages( self, query: dict = None From b16edb08bfb942d82a7c914f1555311d9a84c7da Mon Sep 17 00:00:00 2001 From: romill Date: Fri, 23 Feb 2024 18:03:43 +0530 Subject: [PATCH 03/54] feat(packages): uses v2 packages APIs --- examples/kiba-robots/03_packages.yaml | 3 +- .../apply/manifests/package-nonros-cloud.yaml | 4 +- .../manifests/package-nonros-device.yaml | 14 +- riocli/apply/manifests/package-ros-cloud.yaml | 4 +- .../package-ros-device-no-rosbag.yaml | 1 - .../manifests/package-ros-device-rosbag.yaml | 6 +- riocli/apply/manifests/package.yaml | 61 ++++++-- riocli/apply/resolver.py | 6 +- riocli/deployment/model.py | 10 -- riocli/jsonschema/schemas/package-schema.yaml | 44 +++--- riocli/package/delete.py | 22 +-- riocli/package/inspect.py | 66 +++++---- riocli/package/list.py | 30 ++-- riocli/package/model.py | 137 ++---------------- riocli/package/util.py | 13 +- riocli/v2client/client.py | 7 +- 16 files changed, 172 insertions(+), 256 deletions(-) diff --git a/examples/kiba-robots/03_packages.yaml b/examples/kiba-robots/03_packages.yaml index 206be437..f471f6e6 100644 --- a/examples/kiba-robots/03_packages.yaml +++ b/examples/kiba-robots/03_packages.yaml @@ -147,7 +147,8 @@ spec: executables: - type: docker name: django - command: /code/docker/entrypoint.sh + command: + - /code/docker/entrypoint.sh docker: image: rrdockerhub/sootballs_wcs:1.14.0-rc4 pullSecret: diff --git a/riocli/apply/manifests/package-nonros-cloud.yaml b/riocli/apply/manifests/package-nonros-cloud.yaml index 7b29f174..8d01abe0 100644 --- a/riocli/apply/manifests/package-nonros-cloud.yaml +++ b/riocli/apply/manifests/package-nonros-cloud.yaml @@ -4,7 +4,6 @@ metadata: name: "package-nonros-cloud" # Required version: "v1.0.0" # Required description: "A RIO non ROS cloud package" - project: "project-guid" labels: app: test spec: @@ -14,7 +13,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "sleep infinity" + command: + - "sleep infinity" runAsBash: True simulation: False limits: diff --git a/riocli/apply/manifests/package-nonros-device.yaml b/riocli/apply/manifests/package-nonros-device.yaml index ab63e04b..484b898e 100644 --- a/riocli/apply/manifests/package-nonros-device.yaml +++ b/riocli/apply/manifests/package-nonros-device.yaml @@ -4,7 +4,7 @@ metadata: name: "package-nonros-device" # Required version: "v1.0.0" # Required description: "A RIO non ROS device package" - project: "project-guid" + #project: "project-guid" labels: app: test spec: @@ -14,8 +14,9 @@ spec: restart: always # Options: [always, never, onfailure] executables: # Required - name: "exec-docker" - type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + type: preInstalled # Options: [docker (default), preInstalled] + command: + - "roslaunch talker talker.launch" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -24,7 +25,7 @@ spec: httpGet: path: "/" port: 90 - initialDelaySeconds: 5 # Number of seconds after the container has started before liveness probes are initiated. + initialDelaySeconds: 50 # Number of seconds after the container has started before liveness probes are initiated. periodSeconds: 10 # How often (in seconds) to perform the probe. failureThreshold: 1 # Minimum consecutive failures for the probe to be considered failed after having succeeded. successThreshold: 3 # Minimum consecutive successes for the probe to be considered successful after having failed. @@ -39,8 +40,9 @@ spec: nameOrGUID: "secret-docker" - name: "exec-preInstalled" type: preInstalled # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" - environmentVars: + command: + - "roslaunch talker talker.launch" + environmentArgs: - name: "key1" # Required default: "key1-value" description: "A environment variable" diff --git a/riocli/apply/manifests/package-ros-cloud.yaml b/riocli/apply/manifests/package-ros-cloud.yaml index b852ee40..48f18e1a 100644 --- a/riocli/apply/manifests/package-ros-cloud.yaml +++ b/riocli/apply/manifests/package-ros-cloud.yaml @@ -4,7 +4,6 @@ metadata: name: "package-ros-cloud" # Required version: "v1.0.0" # Required description: "A RIO cloud ROS package" - project: "project-guid" labels: app: test spec: @@ -14,7 +13,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "roslaunch talker talker.launch" runAsBash: True simulation: False limits: diff --git a/riocli/apply/manifests/package-ros-device-no-rosbag.yaml b/riocli/apply/manifests/package-ros-device-no-rosbag.yaml index f0e894e2..9487ed32 100644 --- a/riocli/apply/manifests/package-ros-device-no-rosbag.yaml +++ b/riocli/apply/manifests/package-ros-device-no-rosbag.yaml @@ -5,7 +5,6 @@ metadata: name: "package-ros-device-no-rosbag" # Required version: "v1.0.0" # Required description: "A RIO device ROS package without rosbag" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/package-ros-device-rosbag.yaml b/riocli/apply/manifests/package-ros-device-rosbag.yaml index 8c1eec66..445654c6 100644 --- a/riocli/apply/manifests/package-ros-device-rosbag.yaml +++ b/riocli/apply/manifests/package-ros-device-rosbag.yaml @@ -5,7 +5,6 @@ metadata: name: "package-ros-device-rosbag" # Required version: "v1.0.0" # Required description: "A RIO device ROS package with rosbag" - project: "project-guid" labels: app: test spec: @@ -16,7 +15,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "roslaunch talker talker.launch" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -114,7 +114,7 @@ spec: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" compression: False - scoped: False + scoped: True targeted: False qos: low # Options: [low, med, hi, max] - type: service # Required, Options: [ topic, service, action ] diff --git a/riocli/apply/manifests/package.yaml b/riocli/apply/manifests/package.yaml index aecaf737..55346c5a 100644 --- a/riocli/apply/manifests/package.yaml +++ b/riocli/apply/manifests/package.yaml @@ -4,7 +4,6 @@ metadata: name: "package-ros-cloud" # Required version: "v1.0.0" # Required description: "A RIO cloud ROS package" - project: "project-guid" labels: app: test spec: @@ -14,7 +13,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "roslaunch talker talker.launch" runAsBash: True simulation: False limits: @@ -138,7 +138,6 @@ metadata: name: "package-ros-device-rosbag" # Required version: "v1.0.0" # Required description: "A RIO device ROS package with rosbag" - project: "project-guid" labels: app: test spec: @@ -149,7 +148,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "shell-cmd" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -268,7 +268,6 @@ metadata: name: "package-nonros-cloud" # Required version: "v1.0.0" # Required description: "A RIO non ROS cloud package" - project: "project-guid" labels: app: test spec: @@ -278,7 +277,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "sleep infinity" + command: + - "sleep infinity" runAsBash: True simulation: False limits: @@ -334,7 +334,6 @@ metadata: name: "package-nonros-device" # Required version: "v1.0.0" # Required description: "A RIO non ROS device package" - project: "project-guid" labels: app: test spec: @@ -345,7 +344,8 @@ spec: executables: # Required - name: "exec-docker-01" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "roslaunch talker talker.launch" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -385,8 +385,9 @@ spec: imagePullPolicy: "Always" # Always, Never, IfNotPresent(default) - name: "exec-preInstalled" type: preInstalled # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" - environmentVars: + command: + - "roslaunch talker talker.launch" + environmentArgs: - name: "key1" # Required default: "key1-value" description: "A environment variable" @@ -403,7 +404,6 @@ metadata: name: "package-ros-device-no-rosbag" # Required version: "v1.0.0" # Required description: "A RIO device ROS package without rosbag" - project: "project-guid" labels: app: test spec: @@ -414,7 +414,8 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: + - "roslaunch talker talker.launch" runAsBash: True limits: cpu: 0.025 # Unit: Core (Optional) @@ -437,8 +438,9 @@ spec: nameOrGUID: "secret-docker" - name: "exec-preInstalled" type: preInstalled # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" - environmentVars: + command: + - "roslaunch talker talker.launch" + environmentArgs: - name: "key1" # Required default: "key1-value" description: "A environment variable" @@ -471,4 +473,33 @@ spec: compression: False scoped: False targeted: False - targeted: False \ No newline at end of file + targeted: False + +--- +apiVersion: "apiextensions.rapyuta.io/v1" +kind: "Package" +metadata: + name: "tc128-talker" # Required + version: "v1.0.0" # Required + description: "A RIO cloud ROS package" + labels: + app: test +spec: + runtime: cloud # Options: [device, cloud (default)] + cloud: + replicas: 1 # Required + executables: # Required + - name: "exec-build" + type: docker # Options: [docker (default), build, preInstalled] + command: + - "roslaunch talker talker.launch" + simulation: False + runAsBash: False + limits: + cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] + memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] + docker: + image: "quay.io/rapyuta/io_tutorials:latest" + ros: + enabled: True + version: melodic # Required, Options: [ kinetic, melodic, noetic ] diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index 17a7e912..91f0b546 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -118,7 +118,7 @@ def _guid_functor(self, kind): mapping = { 'secret': lambda x: munchify(x).metadata.guid, "project": lambda x: munchify(x).metadata.guid, - "package": lambda x: munchify(x)['id'], + "package": lambda x: munchify(x)['metadata']['guid'], "staticroute": lambda x: munchify(x)['metadata']['guid'], "deployment": lambda x: munchify(x)['deploymentId'], "network": lambda x: munchify(x)['metadata']['guid'], @@ -134,7 +134,7 @@ def _list_functors(self, kind): mapping = { 'secret': self.v2client.list_secrets, "project": self.v2client.list_projects, - "package": self.client.get_all_packages, + "package": self.v2client.list_packages, "staticroute": self.v2client.list_static_routes, "deployment": functools.partial(self.client.get_all_deployments, phases=[DeploymentPhaseConstants.SUCCEEDED, @@ -153,7 +153,7 @@ def _find_functors(self, kind): 'secret': lambda name, secrets: filter(lambda i: i.metadata.name == name, secrets), "project": lambda name, projects: filter(lambda i: i.metadata.name == name, projects), "package": lambda name, obj_list, version: filter( - lambda x: name == x.name and version == x['packageVersion'], obj_list), + lambda x: name == x.metadata.name and version == x.metadata.version, obj_list), "staticroute": lambda name, obj_list: filter( lambda x: name == x.metadata.name.rsplit('-', 1)[0], obj_list), "deployment": self._generate_find_guid_functor(), diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index 000495e3..df90a442 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -29,7 +29,6 @@ from riocli.deployment.util import add_mount_volume_provision_config, process_deployment_errors from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.package.util import find_package_guid from riocli.parameter.utils import list_trees from riocli.static_route.util import find_static_route_guid from riocli.utils.cache import get_cache @@ -271,15 +270,6 @@ def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: def pre_process(cls, client: Client, d: typing.Dict) -> None: pass - def _get_package(self, client: Client) -> Package: - name = self.metadata.depends.nameOrGUID - if name.startswith('pkg-') or name.startswith('io-'): - guid = name - else: - guid = find_package_guid(client, name, self.metadata.depends.version) - - return client.get_package(package_id=guid) - def _get_provision_config(self, client: Client, pkg: Package): comp_name = pkg['plans']['components'][0]['name'] prov_config = pkg.get_provision_configuration() diff --git a/riocli/jsonschema/schemas/package-schema.yaml b/riocli/jsonschema/schemas/package-schema.yaml index 58e4f230..be4334f8 100644 --- a/riocli/jsonschema/schemas/package-schema.yaml +++ b/riocli/jsonschema/schemas/package-schema.yaml @@ -67,6 +67,7 @@ definitions: "$ref": "#/definitions/deviceComponentInfoSpec" executables: type: array + minItems: 1 items: "$ref": "#/definitions/deviceExecutableSpec" environmentArgs: @@ -88,6 +89,7 @@ definitions: executables: type: array + minItems: 1 items: "$ref": "#/definitions/cloudExecutableSpec" @@ -121,11 +123,11 @@ definitions: enum: - docker - preInstalled - command: - type: array - items: - type: string - description: Command to execute for the Exec probe. + command: + type: array + items: + type: string + description: Command to execute for the Exec probe. runAsBash: type: boolean default: True @@ -178,13 +180,13 @@ definitions: enum: - docker command: - type: string + type: array + items: + type: string + description: Command to execute for the Exec probe. runAsBash: type: boolean default: True - simulation: - type: boolean - default: False limits: type: object properties: @@ -413,6 +415,7 @@ definitions: - kinetic - melodic - noetic + - foxy default: melodic inboundScopedTargeted: type: boolean @@ -475,6 +478,15 @@ definitions: type: enum: - action + allOf: + - anyOf: + - not: + properties: + scoped: + const: true + targeted: + const: true + cloudROSBagJobSpec: type: object @@ -679,13 +691,8 @@ definitions: type: string secretDepends: properties: - kind: - const: secret - default: secret nameOrGUID: type: string - guid: - type: string diskDepends: properties: kind: @@ -766,11 +773,4 @@ definitions: type: integer default: 3 minimum: 1 - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. - oneOf: - - required: - - httpGet - - required: - - tcpSocket - - required: - - exec \ No newline at end of file + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. \ No newline at end of file diff --git a/riocli/package/delete.py b/riocli/package/delete.py index 7cfe3390..7709fabd 100644 --- a/riocli/package/delete.py +++ b/riocli/package/delete.py @@ -14,15 +14,17 @@ from queue import Queue import click -from rapyuta_io.clients.package import Package +import functools from yaspin.api import Yaspin -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Symbols, Colors from riocli.package.util import fetch_packages, print_packages_for_confirmation from riocli.utils import tabulate_data from riocli.utils.execute import apply_func_with_result from riocli.utils.spinner import with_spinner +from rapyuta_io import Client +from rapyuta_io.clients.package import Package @click.command('delete') @@ -46,7 +48,7 @@ def delete_package( """ Delete the package from the Platform """ - client = new_client() + client = new_v2_client() if not (package_name_or_regex or delete_all): spinner.text = "Nothing to delete" @@ -54,7 +56,7 @@ def delete_package( return try: - packages = fetch_packages(client, package_name_or_regex, delete_all, package_version) + packages = fetch_packages(client, package_name_or_regex, delete_all) except Exception as e: spinner.text = click.style( 'Failed to find package(s): {}'.format(e), Colors.RED) @@ -76,8 +78,9 @@ def delete_package( click.confirm('Do you want to delete the above package(s)?', default=True, abort=True) try: + f = functools.partial(_apply_delete, client) result = apply_func_with_result( - f=_apply_delete, items=packages, + f=f, items=packages, workers=workers, key=lambda x: x[0] ) @@ -107,11 +110,10 @@ def delete_package( spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e - -def _apply_delete(result: Queue, package: Package) -> None: - name_version = "{}@{}".format(package.packageName, package.packageVersion) +def _apply_delete(client: Client ,result: Queue, package: Package) -> None: + name_version = "{}@{}".format(package.metadata.name, package.metadata.version) try: - package.delete() + client.delete_package(package_name=package.metadata.name, query={"version": package.metadata.version}) result.put((name_version, True)) except Exception: - result.put((name_version, False)) + result.put((name_version, False)) \ No newline at end of file diff --git a/riocli/package/inspect.py b/riocli/package/inspect.py index 8d24efe7..63898ba6 100644 --- a/riocli/package/inspect.py +++ b/riocli/package/inspect.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import click -from rapyuta_io.clients.package import Package +from munch import unmunchify -from riocli.config import new_client -from riocli.package.util import name_to_guid +from riocli.config import new_v2_client from riocli.utils import inspect_with_format - +from riocli.utils.selector import show_selection @click.command('inspect') @click.option('--version', 'package_version', type=str, @@ -25,38 +24,41 @@ @click.option('--format', '-f', 'format_type', default='yaml', type=click.Choice(['json', 'yaml'], case_sensitive=False)) @click.argument('package-name') -@name_to_guid -def inspect_package(format_type: str, package_name: str, package_guid: str) -> None: +def inspect_package(format_type: str, package_name: str, package_version: str) -> None: """ Inspect the package resource """ try: - client = new_client() - package = client.get_package(package_guid) - data = make_package_inspectable(package) - inspect_with_format(data, format_type) + client = new_v2_client() + package_obj = None + if package_name.startswith("pkg-"): + packages = client.list_packages(query = {"guid": package_name}) + if packages: + package_obj = packages[0] + + elif package_name and package_version: + package_obj = client.get_package(package_name, query = {"version": package_version}) + elif package_name: + packages = client.list_packages(query={"name": package_name}) + + if len(packages) == 0: + click.secho("package not found", fg='red') + raise SystemExit(1) + + options = {} + package_objs = {} + for pkg in packages: + options[pkg.metadata.guid] = '{} ({})'.format(pkg.metadata.name, pkg.metadata.version) + package_objs[pkg.metadata.guid] = pkg + choice = show_selection(options, header='Following packages were found with the same name') + package_obj = package_objs[choice] + + if not package_obj: + click.secho("package not found", fg='red') + raise SystemExit(1) + + inspect_with_format(unmunchify(package_obj), format_type) + except Exception as e: click.secho(str(e), fg='red') raise SystemExit(1) - - -def make_package_inspectable(package: Package) -> dict: - return { - 'created_at': package.CreatedAt, - 'updated_at': package.UpdatedAt, - 'deleted_at': package.DeletedAt, - 'guid': package.guid, - 'package_version': package.packageVersion, - 'description': package.description, - 'package_name': package.packageName, - 'creator': package.creator, - 'owner_project': package.ownerProject, - 'tags': package.tags, - 'plans': package.plans, - 'status': package.status, - 'is_public': package.isPublic, - 'category': package.category, - 'bindable': package.bindable, - 'api_version': package.apiVersion, - 'package_id': package.packageId, - } diff --git a/riocli/package/list.py b/riocli/package/list.py index d0875418..3ccea526 100644 --- a/riocli/package/list.py +++ b/riocli/package/list.py @@ -16,10 +16,9 @@ import click from rapyuta_io.clients.package import Package -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.utils import tabulate_data - @click.command('list') @click.option('--filter', 'filter_word', type=str, default=None, help='A sub-string can be provided to filter down the package list') @@ -28,16 +27,16 @@ def list_packages(filter_word: str) -> None: List the packages in the selected project """ try: - client = new_client() - packages = client.get_all_packages(name=filter_word) - _display_package_list(packages, show_header=True) + client = new_v2_client(with_project=True) + packages = client.list_packages() + _display_package_list(packages, filter_word,show_header=True) except Exception as e: click.secho(str(e), fg='red') raise SystemExit(1) - def _display_package_list( packages: typing.List[Package], + filter_word: str, show_header: bool = True, truncate_limit: int = 48, ) -> None: @@ -46,26 +45,31 @@ def _display_package_list( headers = ('Name', 'Version', 'Package ID', 'Description') # Show IO Packages first - iter_pkg = list(map(lambda x: x.packageName, packages)) + iter_pkg = list(map(lambda x: x.metadata.name, packages)) iter_pkg.sort() package_dict = {} for pkgName in iter_pkg: - filtered_pkg = list(filter(lambda x: x.packageName == pkgName, packages)) - filtered_pkg.sort(key=lambda x: x.packageVersion) + filtered_pkg = list(filter(lambda x: x.metadata.name == pkgName, packages)) + filtered_pkg.sort(key=lambda x: x.metadata.version) package_dict[pkgName] = filtered_pkg data = [] for pkgName, pkgVersionList in package_dict.items(): for package in pkgVersionList: - description = package.description - name = package.packageName + description = package.metadata.get("description", "") + name = package.metadata.name + + # check if filter word was passed. + # if filter word was passed and it is not present in package name then continue + if filter_word and not package.metadata.name.find(filter_word): + continue + if truncate_limit: if len(description) > truncate_limit: description = description[:truncate_limit] + '..' if len(name) > truncate_limit: name = name[:truncate_limit] + '..' - data.append([name, package.packageVersion, package.packageId, description]) - + data.append([name, package.metadata.version, package.metadata.guid, description]) tabulate_data(data, headers=headers) diff --git a/riocli/package/model.py b/riocli/package/model.py index c802ed68..5c8cb3c1 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -14,13 +14,14 @@ import os import typing -from munch import munchify +from munch import munchify, unmunchify + from rapyuta_io import Client from rapyuta_io.clients.package import RestartPolicy from riocli.jsonschema.validate import load_schema from riocli.model import Model - +from riocli.config import new_v2_client class Package(Model): RESTART_POLICY = { @@ -40,132 +41,23 @@ def find_object(self, client: Client): return obj - def create_object(self, client: Client, **kwargs): - pkg_object = munchify({ - 'name': 'default', - 'packageVersion': 'v1.0.0', - 'apiVersion': "2.1.0", - 'description': '', - 'bindable': True, - 'plans': [ - { - "inboundROSInterfaces": { - "anyIncomingScopedOrTargetedRosConfig": False - }, - 'singleton': False, - 'metadata': {}, - 'name': 'default', - 'dependentDeployments': [], - 'exposedParameters': [], - 'includePackages': [], - 'components': [ - ] - } - ], - }) - component_obj = munchify({ - 'requiredRuntime': 'cloud', - 'architecture': 'amd64', - 'executables': [], - 'parameters': [], - 'ros': {'services': [], 'topics': [], 'isROS': False, 'actions': []}, - 'exposedParameters': [], - 'includePackages': [], - 'rosBagJobDefs': [] - }) + def create_object(self, client: Client, **kwargs) -> typing.Any: + client = new_v2_client() - # metadata - # ✓ name, ✓ description, ✓ version - - pkg_object.name = self.metadata.name - pkg_object.packageVersion = self.metadata.version - - if 'description' in self.metadata: - pkg_object.description = self.metadata.description - - # spec - # executables - component_obj.name = 'default' # self.metadata.name #package == component in the single component model - - # TODO validate transform. specially nested secret. - component_obj.executables = list(map(self._map_executable, self.spec.executables)) - for exec in component_obj.executables: - if hasattr(exec, 'cmd') is False: - setattr(exec, 'cmd', []) - component_obj.requiredRuntime = self.spec.runtime - - # ✓ parameters - # TODO validate transform. - if 'environmentVars' in self.spec: - fixed_default = [] - for envVar in self.spec.environmentVars: - obj = envVar.copy() - if 'defaultValue' in obj: - obj['default'] = obj['defaultValue'] - del obj['default'] - - fixed_default.append(obj) - component_obj.parameters = fixed_default - # handle exposed params - exposed_parameters = [] - for entry in filter(lambda x: 'exposed' in x and x.exposed, self.spec.environmentVars): - if os.environ.get('DEBUG'): - print(entry.name) - exposed_parameters.append( - {'component': component_obj.name, 'param': entry.name, 'targetParam': entry.exposedName}) - pkg_object.plans[0].exposedParameters = exposed_parameters - - # device - # ✓ arch, ✓ restart - if self.spec.runtime == 'device': - component_obj.required_runtime = 'device' - component_obj.architecture = self.spec.device.arch - if 'restart' in self.spec.device: - component_obj.restart_policy = self.RESTART_POLICY[self.spec.device.restart.lower()] - - # cloud - # ✓ replicas - # ✓ endpoints - if 'cloud' in self.spec: - component_obj.cloudInfra = munchify(dict()) - if 'replicas' in self.spec.cloud: - component_obj.cloudInfra.replicas = self.spec.cloud.replicas - else: - component_obj.cloudInfra.replicas = 1 - - if 'endpoints' in self.spec: - endpoints = list(map(self._map_endpoints, self.spec.endpoints)) - component_obj.cloudInfra.endpoints = endpoints - - # ros: - # ✓ isros - # ✓ topic - # ✓ service - # ✓ action - # rosbagjob - if 'ros' in self.spec and self.spec.ros.enabled: - component_obj.ros.isROS = True - component_obj.ros.ros_distro = self.spec.ros.version - pkg_object.plans[0].inboundROSInterfaces = munchify({}) - - pkg_object.plans[ - 0].inboundROSInterfaces.anyIncomingScopedOrTargetedRosConfig = self.spec.ros.inboundScopedTargeted if 'inboundScopedTargeted' in self.spec.ros else False - if 'rosEndpoints' in self.spec.ros: - component_obj.ros.topics = list(self._get_rosendpoint_struct(self.spec.ros.rosEndpoints, 'topic')) - component_obj.ros.services = list(self._get_rosendpoint_struct(self.spec.ros.rosEndpoints, 'service')) - component_obj.ros.actions = list(self._get_rosendpoint_struct(self.spec.ros.rosEndpoints, 'action')) - - if 'rosBagJobs' in self.spec: - component_obj.rosBagJobDefs = self.spec.rosBagJobs - - pkg_object.plans[0].components = [component_obj] - return client.create_package(pkg_object) + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + package = unmunchify(self) + package.pop("rc", None) + r = client.create_package(package) + return unmunchify(r) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: + pass def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - client.delete_package(obj.packageId) + client = new_v2_client() + client.delete_package(obj.metadata.name, query={"version": obj.metadata.version}) def to_v1(self): # return v1Project(self.metadata.name) @@ -183,7 +75,6 @@ def _get_rosendpoint_struct(self, rosEndpoints, filter_type): return return_list def _map_executable(self, exec): - exec_object = munchify({ "name": exec.name, "simulationOptions": { diff --git a/riocli/package/util.py b/riocli/package/util.py index f218943a..fc939353 100644 --- a/riocli/package/util.py +++ b/riocli/package/util.py @@ -81,20 +81,16 @@ def fetch_packages( client: Client, package_name_or_regex: str, include_all: bool, - version: str = None ) -> List[Package]: - packages = client.get_all_packages(version=version) + packages = client.list_packages() result = [] for pkg in packages: # We cannot delete public packages. Skip them instead. - if 'io-public' in pkg.packageId: + if 'io-public' in pkg.metadata.guid: continue - if (include_all or package_name_or_regex == pkg.packageName or - pkg.packageId == package_name_or_regex or - (package_name_or_regex not in pkg.packageName and - re.search(r'^{}$'.format(package_name_or_regex), pkg.packageName))): + if include_all or re.search(package_name_or_regex, pkg.metadata.name): result.append(pkg) return result @@ -102,6 +98,5 @@ def fetch_packages( def print_packages_for_confirmation(packages: List[Package]) -> None: headers = ['Name', 'Version'] - data = [[p.packageName, p.packageVersion] for p in packages] - + data = [[p.metadata.name, p.metadata.version] for p in packages] tabulate_data(data, headers) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 91e1b6b6..aee7efc1 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -915,8 +915,6 @@ def delete_network(self, network_name: str, return munchify(data) - return munchify(data) - def list_deployments( self, @@ -950,7 +948,7 @@ def list_deployments( result.append(deployment['metadata']) return munchify(result) - + def create_deployment(self, deployment: dict) -> Munch: """ Create a new deployment @@ -968,7 +966,7 @@ def create_deployment(self, deployment: dict) -> Munch: raise Exception("deployment: {}".format(err_msg)) return munchify(data) - + def get_deployment( self, name: str, @@ -1019,6 +1017,7 @@ def delete_deployment(self, name: str, query: dict = None) -> Munch: params.update(query or {}) response = RestClient(url).method( HttpMethod.DELETE).headers(headers).execute() + handle_server_errors(response) data = json.loads(response.text) if not response.ok: From b78f41daa5aeb0f69c1f9a125a8119033e39d58e Mon Sep 17 00:00:00 2001 From: romill Date: Tue, 23 Apr 2024 11:31:12 +0530 Subject: [PATCH 04/54] feat(deployment): uses v2 deployments APIs --- riocli/apply/parse.py | 9 +- riocli/apply/resolver.py | 14 +- riocli/deployment/__init__.py | 1 - riocli/deployment/delete.py | 20 +- riocli/deployment/execute.py | 1 + riocli/deployment/inspect.py | 50 +--- riocli/deployment/list.py | 26 +- riocli/deployment/logs.py | 1 + riocli/deployment/model.py | 238 ++---------------- riocli/deployment/status.py | 9 +- riocli/deployment/update.py | 24 +- riocli/deployment/util.py | 63 +---- riocli/deployment/wait.py | 9 +- riocli/disk/model.py | 1 - .../jsonschema/schemas/deployment-schema.yaml | 2 +- riocli/jsonschema/schemas/disk-schema.yaml | 2 +- riocli/jsonschema/schemas/package-schema.yaml | 14 +- riocli/jsonschema/schemas/primitives.yaml | 2 +- riocli/package/inspect.py | 7 +- riocli/v2client/client.py | 12 +- 20 files changed, 106 insertions(+), 399 deletions(-) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 03742bb2..5a746880 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -319,8 +319,13 @@ def _resolve_dependency(self, dependent_key, dependency): self._add_remote_object_to_resolve_tree( dependent_key, obj_guid, dependency, obj) +<<<<<<< HEAD if (name_or_guid == obj_name) and ( 'version' in dependency and obj['packageVersion'] == dependency.get('version')): +======= + if (name_or_guid == obj_name) and ('version' in dependency and + obj.metadata.version == dependency.get('version')): +>>>>>>> 52515c9 (feat(deployment): uses v2 deployments APIs) self._add_remote_object_to_resolve_tree( dependent_key, obj_guid, dependency, obj) @@ -357,10 +362,10 @@ def _add_remote_object_to_resolve_tree(self, dependent_key, guid, dependency['guid'] = guid if kind.lower() == "disk": - dependency['depGuid'] = obj['internalDeploymentGUID'] + dependency['depGuid'] = obj.metadata.guid if kind.lower() == "deployment": - dependency['guid'] = obj['deploymentId'] + dependency['guid'] = obj.metadata.guid def _initialize_kind_dependency(self, kind): if not self.dependencies.get(kind): diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index 91f0b546..f20ede66 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -62,7 +62,7 @@ class ResolverCache(object, metaclass=_Singleton): "organization": "^org-[a-z]{24}$", "project": "^project-[a-z]{24}$", "secret": "^secret-[a-z]{24}$", - "package": "^pkg-[a-z]{24}$", + "package": "^pkg-[a-z0-9]{20}$", "staticroute": "^staticroute-[a-z]{24}$", "disk": "^disk-[a-z]{24}$", "deployment": "^dep-[a-z]{24}$", @@ -120,7 +120,7 @@ def _guid_functor(self, kind): "project": lambda x: munchify(x).metadata.guid, "package": lambda x: munchify(x)['metadata']['guid'], "staticroute": lambda x: munchify(x)['metadata']['guid'], - "deployment": lambda x: munchify(x)['deploymentId'], + "deployment": lambda x: munchify(x)['metadata']['guid'], "network": lambda x: munchify(x)['metadata']['guid'], # This is only temporarily like this "disk": lambda x: munchify(x)['metadata']['guid'], @@ -136,11 +136,9 @@ def _list_functors(self, kind): "project": self.v2client.list_projects, "package": self.v2client.list_packages, "staticroute": self.v2client.list_static_routes, - "deployment": functools.partial(self.client.get_all_deployments, - phases=[DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.PROVISIONING]), "disk": self.v2client.list_disks, "network": self.v2client.list_networks, + "deployment": functools.partial(self.v2client.list_deployments), "device": self.client.get_all_devices, "managedservice": self._list_managedservices, "usergroup": self.client.list_usergroups @@ -156,10 +154,12 @@ def _find_functors(self, kind): lambda x: name == x.metadata.name and version == x.metadata.version, obj_list), "staticroute": lambda name, obj_list: filter( lambda x: name == x.metadata.name.rsplit('-', 1)[0], obj_list), - "deployment": self._generate_find_guid_functor(), "network": lambda name, obj_list, network_type: filter( lambda x: name == x.metadata.name and network_type == x.spec.type, obj_list), - "disk": self._generate_find_guid_functor(), + "deployment": lambda name, obj_list: filter( + lambda x: name == x.metadata.name, obj_list), + "disk": lambda name, obj_list: filter( + lambda x: name == x.metadata.name, obj_list), "device": self._generate_find_guid_functor(), "managedservice": lambda name, instances: filter(lambda i: i.metadata.name == name, instances), "usergroup": lambda name, groups: filter(lambda i: i.name == name, groups), diff --git a/riocli/deployment/__init__.py b/riocli/deployment/__init__.py index 1ada4f60..ef23b60e 100644 --- a/riocli/deployment/__init__.py +++ b/riocli/deployment/__init__.py @@ -17,7 +17,6 @@ from riocli.constants import Colors from riocli.deployment.delete import delete_deployment from riocli.deployment.execute import execute_command -from riocli.deployment.execute import execute_command from riocli.deployment.inspect import inspect_deployment from riocli.deployment.list import list_deployments from riocli.deployment.logs import deployment_logs diff --git a/riocli/deployment/delete.py b/riocli/deployment/delete.py index 980a178c..88324e6a 100644 --- a/riocli/deployment/delete.py +++ b/riocli/deployment/delete.py @@ -16,8 +16,11 @@ import click from click_help_colors import HelpColorsCommand from rapyuta_io.clients.deployment import Deployment +import functools +from queue import Queue + -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors, Symbols from riocli.deployment.util import fetch_deployments from riocli.deployment.util import print_deployments_for_confirmation @@ -25,6 +28,7 @@ from riocli.utils.execute import apply_func_with_result from riocli.utils.spinner import with_spinner +from rapyuta_io import Client @click.command( 'delete', @@ -51,8 +55,7 @@ def delete_deployment( """ Deletes one or more deployments given a name or a pattern """ - client = new_client() - + client = new_v2_client() if not (deployment_name_or_regex or delete_all): spinner.text = "Nothing to delete" spinner.green.ok(Symbols.SUCCESS) @@ -83,8 +86,9 @@ def delete_deployment( spinner.write('') try: + f = functools.partial(_apply_delete, client) result = apply_func_with_result( - f=_apply_delete, items=deployments, + f=f, items=deployments, workers=workers, key=lambda x: x[0] ) @@ -116,9 +120,9 @@ def delete_deployment( raise SystemExit(1) from e -def _apply_delete(result: Queue, deployment: Deployment) -> None: +def _apply_delete(client: Client, result: Queue, deployment: Deployment) -> None: try: - deployment.deprovision() - result.put((deployment.name, True)) + client.delete_deployment(name=deployment.metadata.name) + result.put((deployment.metadata.name, True)) except Exception: - result.put((deployment.name, False)) + result.put((deployment.metadata.name, False)) diff --git a/riocli/deployment/execute.py b/riocli/deployment/execute.py index fee280a5..32b9b55d 100644 --- a/riocli/deployment/execute.py +++ b/riocli/deployment/execute.py @@ -50,6 +50,7 @@ def execute_command( """ Execute commands on cloud deployment """ + # TODO(Romil): Move to V2 client try: comp_id, exec_id, pod_name = select_details(deployment_guid, component_name, exec_name) diff --git a/riocli/deployment/inspect.py b/riocli/deployment/inspect.py index 8dd8d0c7..e9e0f391 100644 --- a/riocli/deployment/inspect.py +++ b/riocli/deployment/inspect.py @@ -14,8 +14,9 @@ import click from click_help_colors import HelpColorsCommand from rapyuta_io.clients.deployment import Deployment +from munch import unmunchify -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.deployment.util import name_to_guid from riocli.utils import inspect_with_format @@ -30,53 +31,22 @@ @click.option('--format', '-f', 'format_type', default='yaml', type=click.Choice(['json', 'yaml'], case_sensitive=False)) @click.argument('deployment-name') -@name_to_guid def inspect_deployment( format_type: str, deployment_name: str, - deployment_guid: str, ) -> None: """ Inspect the deployment resource """ try: - client = new_client() - deployment = client.get_deployment(deployment_guid) - data = make_deployment_inspectable(deployment) - inspect_with_format(data, format_type) + client = new_v2_client() + deployment_obj = client.get_deployment(deployment_name) + + if not deployment_obj: + click.secho("deployment not found", fg='red') + raise SystemExit(1) + + inspect_with_format(unmunchify(deployment_obj), format_type) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) - - -def make_deployment_inspectable(deployment: Deployment) -> dict: - return { - 'created_at': deployment.CreatedAt, - 'updated_at': deployment.UpdatedAt, - 'deleted_at': deployment.DeletedAt, - 'package_id': deployment.packageId, - 'package_name': deployment.packageName, - 'package_api_version': deployment.packageAPIVersion, - 'owner_project': deployment.ownerProject, - 'creator': deployment.creator, - 'plan_id': deployment.planId, - 'deployment_id': deployment.deploymentId, - 'current_generation': deployment.currentGeneration, - 'bindable': deployment.bindable, - 'name': deployment.name, - 'parameters': deployment.parameters, - 'provision_context': deployment.provisionContext, - 'component_instance_ids': deployment.componentInstanceIds, - 'dependent_deployments': deployment.dependentDeployments, - 'labels': deployment.labels, - 'in_use': deployment.inUse, - 'used_by': deployment.usedBy, - 'phase': deployment.phase, - 'status': deployment.status, - 'component_info': deployment.componentInfo, - 'dependent_deployment_status': deployment.get( - 'dependentDeploymentStatus'), - 'errors': deployment.get('errors'), - 'package_dependency_status': deployment.get('packageDependencyStatus'), - 'core_networks': deployment.get('coreNetworks'), - } diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index 8d6a711c..3814fef4 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -17,7 +17,7 @@ from click_help_colors import HelpColorsCommand from rapyuta_io.clients.deployment import Deployment, DeploymentPhaseConstants -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.deployment.util import process_deployment_errors from riocli.utils import tabulate_data @@ -62,10 +62,10 @@ def list_deployments( List the deployments in the selected project """ try: - client = new_client() - deployments = client.get_all_deployments(device_id=device, phases=phase) - deployments = sorted(deployments, key=lambda d: d.name.lower()) - display_deployment_list(deployments, show_header=True, wide=wide) + client = new_v2_client(with_project=True) + deployments = client.list_deployments() + deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) + display_deployment_list(deployments, show_header=True) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) @@ -84,17 +84,9 @@ def display_deployment_list( data = [] for deployment in deployments: - package_name_version = "{} ({})".format(deployment.packageName, deployment.packageVersion) - row = [ - deployment.name, - deployment.status, - deployment.phase, - process_deployment_errors(deployment.get('errorCode', []), no_action=True) - ] - - if wide: - row += [package_name_version, deployment.deploymentId] - - data.append(row) + package_name_version = "{} ({})".format(deployment.metadata.depends.nameOrGUID, deployment.metadata.depends.version) + phase = deployment.status.phase if deployment.status else "" + data.append([deployment.metadata.guid, deployment.metadata.name, + phase, package_name_version]) tabulate_data(data, headers=headers) diff --git a/riocli/deployment/logs.py b/riocli/deployment/logs.py index 83113bf6..2133b041 100644 --- a/riocli/deployment/logs.py +++ b/riocli/deployment/logs.py @@ -44,6 +44,7 @@ def deployment_logs( """ Stream live logs from cloud deployments (not supported for device deployments) """ + # TODO(Romil): Move to V2 client try: comp_id, exec_id, pod_name = select_details(deployment_guid, component_name, exec_name) stream_deployment_logs(deployment_guid, comp_id, exec_id, pod_name) diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index df90a442..f3a4ff45 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -11,27 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import typing +from munch import unmunchify from rapyuta_io import Client -from rapyuta_io.clients import ObjDict from rapyuta_io.clients.catalog_client import Package -from rapyuta_io.clients.deployment import DeploymentNotRunningException, DeploymentPhaseConstants -from rapyuta_io.clients.native_network import NativeNetwork -from rapyuta_io.clients.package import ExecutableMount, ProvisionConfiguration, RestartPolicy +from rapyuta_io.clients.package import ProvisionConfiguration, RestartPolicy from rapyuta_io.clients.rosbag import (OverrideOptions, ROSBagCompression, ROSBagJob, ROSBagOnDemandUploadOptions, ROSBagOptions, ROSBagTimeRange, ROSBagUploadTypes, TopicOverrideInfo, UploadOptions) -from rapyuta_io.clients.routed_network import RoutedNetwork -from rapyuta_io.clients.static_route import StaticRoute -from riocli.deployment.util import add_mount_volume_provision_config, process_deployment_errors +from riocli.config import new_v2_client from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.parameter.utils import list_trees from riocli.static_route.util import find_static_route_guid -from riocli.utils.cache import get_cache class Deployment(Model): @@ -41,230 +34,29 @@ class Deployment(Model): 'onfailure': RestartPolicy.OnFailure } + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + def find_object(self, client: Client) -> typing.Any: - guid, obj = self.rc.find_depends({ - "kind": "deployment", - "nameOrGUID": self.metadata.name, - }) + guid, obj = self.rc.find_depends({"kind": "deployment", "nameOrGUID": self.metadata.name}) return obj if guid else False def create_object(self, client: Client, **kwargs) -> typing.Any: - pkg_guid, pkg = self.rc.find_depends( - self.metadata.depends, - self.metadata.depends.version) - - if pkg_guid: - pkg = client.get_package(pkg_guid) - - if not pkg: - raise ValueError('package not found: {}'.format(self.metadata.depends)) - - pkg.update() - - default_plan = pkg['plans'][0] - plan_id = default_plan['planId'] - internal_component = default_plan['internalComponents'][0] - component_name = internal_component.componentName - component = default_plan['components']['components'][0] - executables = component['executables'] - runtime = internal_component['runtime'] - - retry_count = int(kwargs.get('retry_count')) - retry_interval = int(kwargs.get('retry_interval')) - - if 'runtime' in self.spec and runtime != self.spec.runtime: - raise Exception('>> runtime mismatch => deployment:{}.runtime !== package:{}.runtime '.format( - self.metadata.name, pkg['packageName'] - )) - - provision_config = pkg.get_provision_configuration(plan_id) - - # add label - if 'labels' in self.metadata: - for key, value in self.metadata.labels.items(): - provision_config.add_label(key, value) - - # Add envArgs - if 'envArgs' in self.spec: - for items in self.spec.envArgs: - provision_config.add_parameter(component_name, items.name, - items.value) - - # Add Dependent Deployment - if 'depends' in self.spec: - for item in self.spec.depends: - dep_guid, dep = self.rc.find_depends(item) - if dep is None and dep_guid: - dep = client.get_deployment(dep_guid) - provision_config.add_dependent_deployment(dep, ready_phases=[ - DeploymentPhaseConstants.PROVISIONING.value, - DeploymentPhaseConstants.SUCCEEDED.value]) - - # Add Network - if 'rosNetworks' in self.spec: - for network_depends in self.spec.rosNetworks: - network_guid, network_obj = self.rc.find_depends(network_depends.depends) - - if type(network_obj) == RoutedNetwork: - provision_config.add_routed_network( - network_obj, network_interface=network_depends.get('interface', None)) - if type(network_obj) == NativeNetwork: - provision_config.add_native_network( - network_obj, network_interface=network_depends.get('interface', None)) - - if 'rosBagJobs' in self.spec: - for req_job in self.spec.rosBagJobs: - provision_config.add_rosbag_job(component_name, - self._form_rosbag_job(req_job)) - - if self.spec.runtime == 'cloud': - if 'staticRoutes' in self.spec: - for stroute in self.spec.staticRoutes: - route_guid, route = self.rc.find_depends(stroute.depends) - # TODO: Remove this once we transition to v2 - route = StaticRoute(ObjDict({"guid": route_guid})) - provision_config.add_static_route(component_name, - stroute.name, route) - - # Add Disk - if 'volumes' in self.spec: - disk_mounts = {} - for vol in self.spec.volumes: - disk_guid, disk = self.rc.find_depends(vol.depends) - if disk_guid not in disk_mounts: - disk_mounts[disk_guid] = [] - - disk_mounts[disk_guid].append( - ExecutableMount(vol.execName, vol.mountPath, vol.subPath)) - - for disk_guid in disk_mounts.keys(): - disk = client.get_volume_instance(disk_guid) - provision_config.mount_volume(component_name, volume=disk, - executable_mounts= - disk_mounts[disk_guid]) - - # TODO: Managed Services is currently limited to `cloud` deployments - # since we don't expose `elasticsearch` outside Openshift. This may - # change in the future. - if 'managedServices' in self.spec: - managed_services = [] - for managed_service in self.spec.managedServices: - managed_services.append({ - 'instance': managed_service.depends.nameOrGUID, - }) - provision_config.context['managedServices'] = managed_services - - # inject the vpn managedservice instance if the flag is set to - # true. 'rio-internal-headscale' is the default vpn instance - if 'features' in self.spec: - if 'vpn' in self.spec.features and self.spec.features.vpn.enabled: - provision_config.context['managedServices'] = [{ - "instance": "rio-internal-headscale" - }] - - if self.spec.runtime == 'device': - device_guid, device = self.rc.find_depends( - self.spec.device.depends) - if device is None and device_guid: - device = client.get_device(device_guid) - - provision_config.add_device( - component_name, - device=device, - set_component_alias=False - ) + client = new_v2_client() - if 'restart' in self.spec: - provision_config.add_restart_policy( - component_name, - self.RESTART_POLICY[self.spec.restart.lower()]) - - # Add Disk - exec_mounts = [] - if 'volumes' in self.spec: - for vol in self.spec.volumes: - uid = getattr(vol, 'uid', None) - gid = getattr(vol, 'gid', None) - perm = getattr(vol, 'perm', None) - exec_mounts.append( - ExecutableMount(vol.execName, vol.mountPath, - vol.subPath, uid, gid, perm)) - - if len(exec_mounts) > 0: - provision_config = add_mount_volume_provision_config( - provision_config, component_name, device, exec_mounts) - - if 'features' in self.spec: - if 'params' in self.spec.features and self.spec.features.params.enabled: - component_id = internal_component.componentId - disable_sync = self.spec.features.params.get('disableSync', False) - - # Validate trees in the manifest with the ones available - # to avoid misconfigurations. - tree_names = self.spec.features.params.get('trees', []) - - # For multiple deployments in the same project, the list of - # available config trees is going to remain the same. Hence, - # we cache it once and keep fetching it from the cache. - cache_key = '{}-trees'.format(pkg.get('ownerProject')) - with get_cache() as c: - if c.get(cache_key) is None: - c.set(cache_key, set(list_trees())) - - available_trees = c.get(cache_key) - - if not available_trees: - raise ValueError("One or more trees are incorrect. Please run `rio parameter list` to confirm.") - - if not set(tree_names).issubset(available_trees): - raise ValueError("One or more trees are incorrect. Please run `rio parameter list` to confirm.") - - args = [] - for e in executables: - args.append({ - 'executableId': e['id'], - 'paramTreeNames': tree_names, - 'enableParamSync': not disable_sync - }) - - context = provision_config.context - if 'component_context' not in context: - context['component_context'] = {} - - component_context = context['component_context'] - if component_id not in component_context: - component_context[component_id] = {} - - component_context[component_id]['param_sync_exec_args'] = args - - provision_config.set_component_alias(component_name, - self.metadata.name) - - if os.environ.get('DEBUG'): - print(provision_config) - - deployment = pkg.provision(self.metadata.name, provision_config) - - try: - deployment.poll_deployment_till_ready(retry_count=retry_count, sleep_interval=retry_interval, - ready_phases=[DeploymentPhaseConstants.PROVISIONING.value, - DeploymentPhaseConstants.SUCCEEDED.value]) - except DeploymentNotRunningException as e: - errors = process_deployment_errors(e.deployment_status.errors) - raise Exception(errors) from e - except Exception as e: - raise e - - deployment.get_status() - - return deployment + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + deployment = unmunchify(self) + deployment.pop("rc", None) + r = client.create_deployment(deployment) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - obj.deprovision() + client = new_v2_client() + client.delete_deployment(obj.metadata.name) @classmethod def pre_process(cls, client: Client, d: typing.Dict) -> None: diff --git a/riocli/deployment/status.py b/riocli/deployment/status.py index cd4cb93c..9e9779fa 100644 --- a/riocli/deployment/status.py +++ b/riocli/deployment/status.py @@ -14,7 +14,7 @@ import click from click_help_colors import HelpColorsCommand -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.deployment.util import name_to_guid @@ -26,14 +26,13 @@ help_options_color=Colors.GREEN, ) @click.argument('deployment-name', type=str) -@name_to_guid -def status(deployment_name: str, deployment_guid: str) -> None: +def status(deployment_name: str) -> None: """ Current status of the deployment """ try: - client = new_client() - deployment = client.get_deployment(deployment_guid) + client = new_v2_client() + deployment = client.get_deployment(deployment_name) click.secho(deployment.status) except Exception as e: click.secho(str(e), fg=Colors.RED) diff --git a/riocli/deployment/update.py b/riocli/deployment/update.py index 068290fb..5f9a78ae 100644 --- a/riocli/deployment/update.py +++ b/riocli/deployment/update.py @@ -21,7 +21,7 @@ from rapyuta_io.clients.deployment import Deployment from yaspin.api import Yaspin -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Symbols, Colors from riocli.deployment.util import fetch_deployments from riocli.deployment.util import print_deployments_for_confirmation @@ -55,7 +55,7 @@ def update_deployment( """ Updates one more deployments """ - client = new_client() + client = new_v2_client() if not (deployment_name_or_regex or update_all): spinner.text = "Nothing to update" spinner.green.ok(Symbols.SUCCESS) @@ -157,17 +157,11 @@ def _apply_update( deployment: Deployment, ) -> None: try: - dep = client.get_deployment(deployment['deploymentId']) - component_context = get_component_context(dep.get("componentInfo", {})) - payload = { - "service_id": dep["packageId"], - "plan_id": dep["planId"], - "deployment_id": dep["deploymentId"], - "context": { - "component_context": component_context - } - } - client.update_deployment(payload) - result.put((deployment["name"], True)) + dep = client.get_deployment(deployment.metadata.name) + if not dep: + result.put((deployment.metadata.name, False)) + return + client.update_deployment(deployment.metadata.name, deployment) + result.put((deployment.metadata.name, True)) except Exception: - result.put((deployment["name"], False)) + result.put((deployment.metadata.name, False)) diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index 51dea6ef..0781f0b9 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -122,66 +122,18 @@ def __init__(self, message='deployment not found!'): self.message = message super().__init__(self.message) - -def add_mount_volume_provision_config(provision_config, component_name, device, executable_mounts): - if not isinstance(device, Device): - raise InvalidParameterException('device must be of type Device') - - component_id = provision_config.plan.get_component_id(component_name) - if not isinstance(executable_mounts, list) or not all( - isinstance(mount, ExecutableMount) for mount in executable_mounts): - raise InvalidParameterException( - 'executable_mounts must be a list of rapyuta_io.clients.package.ExecutableMount') - if device.get_runtime() != Device.DOCKER_COMPOSE and not device.is_docker_enabled(): - raise OperationNotAllowedError('Device must be a {} device'.format(Device.DOCKER_COMPOSE)) - component_params = provision_config.parameters.get(component_id) - if component_params.get(DEVICE_ID) != device.deviceId: - raise OperationNotAllowedError('Device must be added to the component') - # self._add_disk_mount_info(device.deviceId, component_id, executable_mounts) - - dep_info = dict() - dep_info['diskResourceId'] = device.deviceId - dep_info['applicableComponentId'] = component_id - dep_info['config'] = dict() - - for mount in executable_mounts: - exec_mount = { - 'mountPath': mount.mount_path - } - if mount.uid is not None: - exec_mount['uid'] = mount.uid - if mount.gid is not None: - exec_mount['gid'] = mount.gid - if mount.perm is not None: - exec_mount['perm'] = mount.perm - if mount.sub_path: - exec_mount['subPath'] = mount.sub_path - else: - exec_mount['subPath'] = '/' - - tmp_info = copy.deepcopy(dep_info) - tmp_info['config']['mountPaths'] = { - mount.exec_name: exec_mount, - } - provision_config.context['diskMountInfo'].append(tmp_info) - - return provision_config - - def fetch_deployments( client: Client, deployment_name_or_regex: str, include_all: bool, ) -> List[Deployment]: - deployments = client.get_all_deployments( - phases=[DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.PROVISIONING]) + deployments = client.list_deployments() result = [] for deployment in deployments: - if (include_all or deployment_name_or_regex == deployment.name or - deployment_name_or_regex == deployment.deploymentId or - (deployment_name_or_regex not in deployment.name and - re.search(r'^{}$'.format(deployment_name_or_regex), deployment.name))): + if (include_all or deployment_name_or_regex == deployment.metadata.name or + deployment_name_or_regex == deployment.metadata.guid or + (deployment_name_or_regex not in deployment.metadata.name and + re.search(r'^{}$'.format(deployment_name_or_regex), deployment.metadata.name))): result.append(deployment) return result @@ -189,7 +141,10 @@ def fetch_deployments( def print_deployments_for_confirmation(deployments: List[Deployment]): headers = ['Name', 'GUID', 'Phase', 'Status'] - data = [[d.name, d.deploymentId, d.phase, d.status] for d in deployments] + + data = [] + for deployment in deployments: + data.append([deployment.metadata.name, deployment.metadata.guid, deployment.status.phase, deployment.status.status]) tabulate_data(data, headers) diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index b02de14f..bf34fe5c 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -15,7 +15,7 @@ from click_help_colors import HelpColorsCommand from rapyuta_io.utils import RetriesExhausted, DeploymentNotRunningException -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.constants import Symbols from riocli.deployment.util import name_to_guid @@ -29,19 +29,18 @@ help_options_color=Colors.GREEN, ) @click.argument('deployment-name', type=str) -@name_to_guid @with_spinner(text="Waiting for deployment...", timer=True) def wait_for_deployment( deployment_name: str, - deployment_guid: str, spinner=None, ) -> None: """ Wait until the deployment succeeds/fails """ try: - client = new_client() - deployment = client.get_deployment(deployment_guid) + # TODO(Romil): test once the v2 deployment client is ready with status + client = new_v2_client() + deployment = client.get_deployment(deployment_name) # TODO(ankit): Fix the poll_deployment_till_ready for Runtime Error status = deployment.poll_deployment_till_ready() spinner.text = click.style('Deployment status: {}'.format(status.status), fg=Colors.GREEN) diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 88a04308..5c521cc6 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -56,7 +56,6 @@ def create_object(self, client: Client, **kwargs) -> typing.Any: def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass - @staticmethod def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: v2_client = new_v2_client() v2_client.delete_disk(obj.metadata.name) diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index d81492ca..c38b8d0b 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -405,7 +405,7 @@ definitions: pattern: "^disk-[a-z]{24}$" packageGUID: type: string - pattern: "^pkg-[a-z]{24}$" + pattern: "^pkg-[a-z0-9]{20}$" deploymentGUID: type: string pattern: "^dep-[a-z]{24}$" diff --git a/riocli/jsonschema/schemas/disk-schema.yaml b/riocli/jsonschema/schemas/disk-schema.yaml index c15097f1..8b8fec2a 100644 --- a/riocli/jsonschema/schemas/disk-schema.yaml +++ b/riocli/jsonschema/schemas/disk-schema.yaml @@ -44,7 +44,7 @@ definitions: pattern: "^disk-[a-z]{24}$" packageGUID: type: string - pattern: "^pkg-[a-z]{24}$" + pattern: "^pkg-[a-z0-9]{20}$" stringMap: type: object additionalProperties: diff --git a/riocli/jsonschema/schemas/package-schema.yaml b/riocli/jsonschema/schemas/package-schema.yaml index be4334f8..bd5007ed 100644 --- a/riocli/jsonschema/schemas/package-schema.yaml +++ b/riocli/jsonschema/schemas/package-schema.yaml @@ -306,7 +306,6 @@ definitions: type: enum: - internal-udp - port: "$ref": "#/definitions/portNumber" default: 80 @@ -321,18 +320,14 @@ definitions: - internal-tcp-range portRange: type: string - default: 22,80, 1024-1030 required: - portRange - - properties: type: enum: - internal-udp-range - portRange: type: string - default: 53,1024-1025 required: - portRange @@ -588,9 +583,10 @@ definitions: type: integer to: type: integer - required: - - from - - to + anyOf: + - required: ["from", "to"] + - not: + required: ["from", "to"] required: - timeRange @@ -641,7 +637,7 @@ definitions: pattern: "^disk-[a-z]{24}$" packageGUID: type: string - pattern: "^pkg-[a-z]{24}$" + pattern: "^pkg-[a-z0-9]{20}$" deploymentGUID: type: string pattern: "^dep-[a-z]{24}$" diff --git a/riocli/jsonschema/schemas/primitives.yaml b/riocli/jsonschema/schemas/primitives.yaml index a1c1aac9..d909f400 100644 --- a/riocli/jsonschema/schemas/primitives.yaml +++ b/riocli/jsonschema/schemas/primitives.yaml @@ -18,7 +18,7 @@ definitions: pattern: "^disk-[a-z]{24}$" packageGUID: type: string - pattern: "^pkg-[a-z]{24}$" + pattern: "^pkg-[a-z0-9]{20}$" deploymentGUID: type: string pattern: "^dep-[a-z]{24}$" diff --git a/riocli/package/inspect.py b/riocli/package/inspect.py index 63898ba6..e13e97f1 100644 --- a/riocli/package/inspect.py +++ b/riocli/package/inspect.py @@ -34,7 +34,8 @@ def inspect_package(format_type: str, package_name: str, package_version: str) - if package_name.startswith("pkg-"): packages = client.list_packages(query = {"guid": package_name}) if packages: - package_obj = packages[0] + obj = packages[0] + package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) elif package_name and package_version: package_obj = client.get_package(package_name, query = {"version": package_version}) @@ -51,7 +52,9 @@ def inspect_package(format_type: str, package_name: str, package_version: str) - options[pkg.metadata.guid] = '{} ({})'.format(pkg.metadata.name, pkg.metadata.version) package_objs[pkg.metadata.guid] = pkg choice = show_selection(options, header='Following packages were found with the same name') - package_obj = package_objs[choice] + obj = package_objs[choice] + package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) + if not package_obj: click.secho("package not found", fg='red') diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index aee7efc1..42fe7607 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -914,7 +914,6 @@ def delete_network(self, network_name: str, raise Exception("package: {}".format(err_msg)) return munchify(data) - def list_deployments( self, @@ -933,6 +932,7 @@ def list_deployments( params.update({ "continue": offset, "limit": 50, + "phases": ["Succeeded", "InProgress", "Provisioning", "FailedToUpdate", "FailedToStart"], }) response = RestClient(url).method(HttpMethod.GET).query_param( params).headers(headers).execute() @@ -944,11 +944,10 @@ def list_deployments( if not deployments: break offset = data['metadata']['continue'] - for deployment in deployments: - result.append(deployment['metadata']) + result.extend(deployments) return munchify(result) - + def create_deployment(self, deployment: dict) -> Munch: """ Create a new deployment @@ -966,7 +965,7 @@ def create_deployment(self, deployment: dict) -> Munch: raise Exception("deployment: {}".format(err_msg)) return munchify(data) - + def get_deployment( self, name: str, @@ -997,7 +996,7 @@ def update_deployment(self, name: str, dep: dict) -> Munch: """ url = "{}/v2/deployments/{}/".format(self._host, name) headers = self._config.get_auth_header() - response = RestClient(url).method(HttpMethod.PUT).headers( + response = RestClient(url).method(HttpMethod.PATCH).headers( headers).execute(payload=dep) handle_server_errors(response) data = json.loads(response.text) @@ -1017,7 +1016,6 @@ def delete_deployment(self, name: str, query: dict = None) -> Munch: params.update(query or {}) response = RestClient(url).method( HttpMethod.DELETE).headers(headers).execute() - handle_server_errors(response) data = json.loads(response.text) if not response.ok: From 58cd08b725938b306f43984280f8ae86816b3d6c Mon Sep 17 00:00:00 2001 From: romill Date: Mon, 8 Jul 2024 23:35:25 +0530 Subject: [PATCH 05/54] fix(deployments): implements waiting mechanism for disk and network deployments to reach running state --- riocli/constants/__init__.py | 3 ++- riocli/constants/status.py | 10 ++++++++++ riocli/deployment/__init__.py | 1 - riocli/deployment/wait.py | 26 +++++++++++++++++++------- riocli/disk/model.py | 28 +++++++++++++++++++++++++++- riocli/network/model.py | 30 ++++++++++++++++++++++++++++++ riocli/package/model.py | 6 ++++++ 7 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 riocli/constants/status.py diff --git a/riocli/constants/__init__.py b/riocli/constants/__init__.py index fbe68368..b905f913 100644 --- a/riocli/constants/__init__.py +++ b/riocli/constants/__init__.py @@ -15,5 +15,6 @@ from riocli.constants.colors import Colors from riocli.constants.symbols import Symbols from riocli.constants.regions import Regions +from riocli.constants.status import Status -__all__ = [Colors, Symbols, Regions] +__all__ = [Colors, Symbols, Regions, Status] diff --git a/riocli/constants/status.py b/riocli/constants/status.py new file mode 100644 index 00000000..c3822076 --- /dev/null +++ b/riocli/constants/status.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class Status(str, Enum): + + def __str__(self): + return str(self.value).lower() + + RUNNING = 'Running' + AVAILABLE = 'Available' diff --git a/riocli/deployment/__init__.py b/riocli/deployment/__init__.py index ef23b60e..c53f4964 100644 --- a/riocli/deployment/__init__.py +++ b/riocli/deployment/__init__.py @@ -24,7 +24,6 @@ from riocli.deployment.update import update_deployment from riocli.deployment.wait import wait_for_deployment - @click.group( invoke_without_command=False, cls=HelpColorsGroup, diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index bf34fe5c..56d0f4fd 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -12,15 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import click +import typing +import time + from click_help_colors import HelpColorsCommand from rapyuta_io.utils import RetriesExhausted, DeploymentNotRunningException - from riocli.config import new_v2_client -from riocli.constants import Colors -from riocli.constants import Symbols +from riocli.constants import Colors, Symbols, Status from riocli.deployment.util import name_to_guid from riocli.utils.spinner import with_spinner +from rapyuta_io import Client + +def poll_deployment_till_ready(client: Client, deployment: typing.Any, retry_count=50, sleep_interval=6): + for _ in range(retry_count): + if deployment.status.status == Status.RUNNING: + return deployment + + time.sleep(sleep_interval) + deployment = client.get_deployment(deployment.metadata.name) + return deployment @click.command( 'wait', @@ -38,12 +49,13 @@ def wait_for_deployment( Wait until the deployment succeeds/fails """ try: - # TODO(Romil): test once the v2 deployment client is ready with status client = new_v2_client() deployment = client.get_deployment(deployment_name) - # TODO(ankit): Fix the poll_deployment_till_ready for Runtime Error - status = deployment.poll_deployment_till_ready() - spinner.text = click.style('Deployment status: {}'.format(status.status), fg=Colors.GREEN) + + retry_count = int(kwargs.get('retry_count')) + retry_interval = int(kwargs.get('retry_interval')) + deployment = poll_deployment_till_ready(client, deployment, retry_count, retry_interval) + spinner.text = click.style('Deployment status: {}'.format(deployment.status.status), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except RetriesExhausted as e: spinner.write(click.style(str(e), fg=Colors.RED)) diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 5c521cc6..411be2a2 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -15,6 +15,7 @@ import typing from time import sleep from munch import unmunchify +import time import click from munch import munchify @@ -22,7 +23,7 @@ from rapyuta_io.utils.rest_client import HttpMethod from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols +from riocli.constants import Colors, Symbols, Status from riocli.jsonschema.validate import load_schema from riocli.model import Model @@ -51,8 +52,33 @@ def create_object(self, client: Client, **kwargs) -> typing.Any: self.pop("rc", None) disk = unmunchify(self) r = v2_client.create_disk(disk) + + retry_count = int(kwargs.get('retry_count')) + retry_interval = int(kwargs.get('retry_interval')) + try: + r = self.poll_deployment_till_ready( + client = v2_client, + disk = r, + retry_count=retry_count, + sleep_interval=retry_interval, + ) + except Exception as e: + click.secho(">> {}: Error polling for disk ({}:{})".format( + Symbols.WARNING, + self.kind.lower(), + self.metadata.name), fg=Colors.YELLOW) + return unmunchify(r) + def poll_deployment_till_ready(self, client: Client, disk: typing.Any, retry_count = 50, sleep_interval = 6): + for _ in range(retry_count): + if disk.status.status == Status.AVAILABLE: + return disk + + time.sleep(sleep_interval) + disk = client.get_disk(disk.metadata.name) + return disk + def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass diff --git a/riocli/network/model.py b/riocli/network/model.py index 1fc3694f..3994059b 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -14,6 +14,8 @@ import typing from typing import Union, Any, Dict from munch import munchify, unmunchify +import click +import time from rapyuta_io import Client from rapyuta_io.clients.common_models import Limits @@ -26,6 +28,7 @@ from riocli.model import Model from riocli.v2client.client import NetworkNotFound from riocli.config import new_v2_client +from riocli.constants import Colors, Symbols, Status class Network(Model): def __init__(self, *args, **kwargs): @@ -42,6 +45,15 @@ def find_object(self, client: Client) -> bool: except NetworkNotFound: return False + def poll_deployment_till_ready(self, client: Client, network: typing.Any, retry_count = 50, sleep_interval = 6): + for _ in range(retry_count): + if network.status and network.status.status == Status.RUNNING: + return network + + time.sleep(sleep_interval) + network = client.get_network(network.metadata.name) + return network + def create_object(self, client: Client, **kwargs) -> typing.Any: client = new_v2_client() @@ -50,6 +62,24 @@ def create_object(self, client: Client, **kwargs) -> typing.Any: network = unmunchify(self) network.pop("rc", None) r = client.create_network(network) + + retry_count = int(kwargs.get('retry_count')) + retry_interval = int(kwargs.get('retry_interval')) + + try: + r = self.poll_deployment_till_ready( + client = client, + network = r, + retry_count=retry_count, + sleep_interval=retry_interval, + ) + except Exception as e: + click.secho(">> {}: Error polling for network ({}:{})".format( + Symbols.WARNING, + self.kind.lower(), + self.metadata.name), fg=Colors.YELLOW) + + return unmunchify(r) def update_object(self, client: Client, diff --git a/riocli/package/model.py b/riocli/package/model.py index 5c8cb3c1..f9a61534 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -44,6 +44,12 @@ def find_object(self, client: Client): def create_object(self, client: Client, **kwargs) -> typing.Any: client = new_v2_client() + # incase createdAt and updatedAt are present they are in datetime format + # they need to be converted to isoformat for serializing request + # or just dont pass the values as they are not needed. + self.metadata.createdAt = None + self.metadata.updatedAt = None + # convert to a dict and remove the ResolverCache # field since it's not JSON serializable package = unmunchify(self) From 0cacf383721ae97127c50b1669a9ece15e14a5c3 Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 11 Jul 2024 16:43:36 +0530 Subject: [PATCH 06/54] fix(package): allow executable command to be string, array or null Following the familiar conventions, we should support string, array of strings or null as the value for executable commands in the package manifests. This commit updates the jsonschema as well as the package model to accomodate the change. Wrike Ticket: https://www.wrike.com/open.htm?id=1449756230 --- riocli/jsonschema/schemas/package-schema.yaml | 15 +-- riocli/package/model.py | 123 +++++------------- 2 files changed, 42 insertions(+), 96 deletions(-) diff --git a/riocli/jsonschema/schemas/package-schema.yaml b/riocli/jsonschema/schemas/package-schema.yaml index bd5007ed..1535e03f 100644 --- a/riocli/jsonschema/schemas/package-schema.yaml +++ b/riocli/jsonschema/schemas/package-schema.yaml @@ -124,10 +124,7 @@ definitions: - docker - preInstalled command: - type: array - items: - type: string - description: Command to execute for the Exec probe. + "$ref": "#/definitions/commandSpec" runAsBash: type: boolean default: True @@ -180,10 +177,7 @@ definitions: enum: - docker command: - type: array - items: - type: string - description: Command to execute for the Exec probe. + "$ref": "#/definitions/commandSpec" runAsBash: type: boolean default: True @@ -226,6 +220,11 @@ definitions: required: - image + commandSpec: + type: ["array", "string", "null"] + items: + type: string + portNumber: type: integer min: 1 diff --git a/riocli/package/model.py b/riocli/package/model.py index f9a61534..d5ab9f4f 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -42,107 +42,54 @@ def find_object(self, client: Client): return obj def create_object(self, client: Client, **kwargs) -> typing.Any: - client = new_v2_client() + v2_client = new_v2_client() - # incase createdAt and updatedAt are present they are in datetime format - # they need to be converted to isoformat for serializing request - # or just dont pass the values as they are not needed. - self.metadata.createdAt = None - self.metadata.updatedAt = None - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - package = unmunchify(self) - package.pop("rc", None) - r = client.create_package(package) + r = v2_client.create_package(self._sanitize_package()) return unmunchify(r) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass + @staticmethod def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - client = new_v2_client() - client.delete_package(obj.metadata.name, query={"version": obj.metadata.version}) - - def to_v1(self): - # return v1Project(self.metadata.name) - pass - - def _get_rosendpoint_struct(self, rosEndpoints, filter_type): - topic_list = filter(lambda x: x.type == filter_type, rosEndpoints) - return_list = [] - for topic in topic_list: - if topic.compression is False: - topic.compression = "" - else: - topic.compression = "snappy" - return_list.append(topic) - return return_list - - def _map_executable(self, exec): - exec_object = munchify({ - "name": exec.name, - "simulationOptions": { - "simulation": exec.simulation if 'simulation' in exec else False - } - }) - - if 'limits' in exec: - exec_object.limits = { - 'cpu': exec.limits.get('cpu', 0.0), - 'memory': exec.limits.get('memory', 0) - } - - if 'livenessProbe' in exec: - exec_object.livenessProbe = exec.livenessProbe - - if 'command' in exec: - c = [] - - if exec.get('runAsBash'): - c = ['/bin/bash', '-c'] - - if isinstance(exec.command, list): - c.extend(exec.command) - else: - c.append(exec.command) - - exec_object.cmd = c - - if exec.type == 'docker': - exec_object.docker = exec.docker.image - if 'pullSecret' in exec.docker and exec.docker.pullSecret.depends: - secret_guid, secret = self.rc.find_depends(exec.docker.pullSecret.depends) - exec_object.secret = secret_guid - - if exec.docker.get('imagePullPolicy'): - exec_object.imagePullPolicy = exec.docker.imagePullPolicy - - # TODO handle preinstalled - - return exec_object - - def _map_endpoints(self, endpoint): - exposedExternally = endpoint.type.split("-")[0] == 'external' - proto = "-".join(endpoint.type.split("-")[1:]) - if 'tls-tcp' in proto: - proto = 'tcp' - - if 'range' in endpoint.type: - proto = proto.replace("-range", '') - return { - "name": endpoint.name, "exposeExternally": exposedExternally, - "portRange": endpoint.portRange, "proto": proto.upper()} - else: - return { - "name": endpoint.name, "exposeExternally": exposedExternally, - "port": endpoint.port, "targetPort": endpoint.targetPort, "proto": proto.upper()} + v2_client = new_v2_client() + v2_client.delete_package(obj.metadata.name, query={"version": obj.metadata.version}) @classmethod def pre_process(cls, client: Client, d: typing.Dict) -> None: pass + def _sanitize_package(self) -> typing.Dict: + # Unset createdAt and updatedAt to avoid timestamp parsing issue. + self.metadata.createdAt = None + self.metadata.updatedAt = None + + self._convert_command() + + data = unmunchify(self) + + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + data.pop("rc", None) + + return data + + def _convert_command(self): + for exec in self.spec.executables: + if exec.get('command') is not None: + c = [] + + if exec.get('runAsBash'): + c = ['/bin/bash', '-c'] + + if isinstance(exec.command, list): + c.extend(exec.command) + else: + c.append(exec.command) + + exec.command = c + @staticmethod def validate(data): """ From 04f03d635316818c3819fb7f6a04c5709875ce51 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Tue, 16 Jul 2024 17:24:10 +0530 Subject: [PATCH 07/54] fix(deployments): fix deployment status and wait command --- riocli/deployment/list.py | 28 ++++++++-------- riocli/deployment/status.py | 2 +- riocli/deployment/wait.py | 34 ++++--------------- riocli/v2client/client.py | 65 ++++++++++++++++++++++++++++++------- riocli/v2client/enums.py | 33 +++++++++++++++++++ riocli/v2client/error.py | 9 +++++ 6 files changed, 118 insertions(+), 53 deletions(-) create mode 100644 riocli/v2client/enums.py create mode 100644 riocli/v2client/error.py diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index 3814fef4..f2b590ab 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -15,7 +15,8 @@ import click from click_help_colors import HelpColorsCommand -from rapyuta_io.clients.deployment import Deployment, DeploymentPhaseConstants +from riocli.v2client.enums import DeploymentPhaseConstants +from rapyuta_io.clients.deployment import Deployment from riocli.config import new_v2_client from riocli.constants import Colors @@ -23,19 +24,20 @@ from riocli.utils import tabulate_data ALL_PHASES = [ - DeploymentPhaseConstants.INPROGRESS, - DeploymentPhaseConstants.PROVISIONING, - DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.FAILED_TO_START, - DeploymentPhaseConstants.PARTIALLY_DEPROVISIONED, - DeploymentPhaseConstants.DEPLOYMENT_STOPPED, + DeploymentPhaseConstants.DeploymentPhaseInProgress, + DeploymentPhaseConstants.DeploymentPhaseProvisioning, + DeploymentPhaseConstants.DeploymentPhaseSucceeded, + DeploymentPhaseConstants.DeploymentPhaseStopped, + DeploymentPhaseConstants.DeploymentPhaseFailedToStart, + DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate, ] DEFAULT_PHASES = [ - DeploymentPhaseConstants.INPROGRESS, - DeploymentPhaseConstants.PROVISIONING, - DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.FAILED_TO_START, + DeploymentPhaseConstants.DeploymentPhaseInProgress, + DeploymentPhaseConstants.DeploymentPhaseProvisioning, + DeploymentPhaseConstants.DeploymentPhaseSucceeded, + DeploymentPhaseConstants.DeploymentPhaseFailedToStart, + DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate ] @@ -46,7 +48,7 @@ help_options_color=Colors.GREEN, ) @click.option('--device', prompt_required=False, default='', type=str, - help='Filter the Deployment list by Device ID') + help='Filter the Deployment list by Device name') @click.option('--phase', prompt_required=False, multiple=True, type=click.Choice(ALL_PHASES), default=DEFAULT_PHASES, @@ -63,7 +65,7 @@ def list_deployments( """ try: client = new_v2_client(with_project=True) - deployments = client.list_deployments() + deployments = client.list_deployments(query={"phases": phase, "deviceName": device}) deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) display_deployment_list(deployments, show_header=True) except Exception as e: diff --git a/riocli/deployment/status.py b/riocli/deployment/status.py index 9e9779fa..d5636876 100644 --- a/riocli/deployment/status.py +++ b/riocli/deployment/status.py @@ -33,7 +33,7 @@ def status(deployment_name: str) -> None: try: client = new_v2_client() deployment = client.get_deployment(deployment_name) - click.secho(deployment.status) + click.secho(deployment.status.aggregateStatus) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index 56d0f4fd..1a0c7d8c 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -12,26 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import click -import typing -import time from click_help_colors import HelpColorsCommand -from rapyuta_io.utils import RetriesExhausted, DeploymentNotRunningException from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols, Status -from riocli.deployment.util import name_to_guid +from riocli.constants import Colors, Symbols from riocli.utils.spinner import with_spinner +from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning -from rapyuta_io import Client - -def poll_deployment_till_ready(client: Client, deployment: typing.Any, retry_count=50, sleep_interval=6): - for _ in range(retry_count): - if deployment.status.status == Status.RUNNING: - return deployment - - time.sleep(sleep_interval) - deployment = client.get_deployment(deployment.metadata.name) - return deployment @click.command( 'wait', @@ -50,25 +37,18 @@ def wait_for_deployment( """ try: client = new_v2_client() - deployment = client.get_deployment(deployment_name) - - retry_count = int(kwargs.get('retry_count')) - retry_interval = int(kwargs.get('retry_interval')) - deployment = poll_deployment_till_ready(client, deployment, retry_count, retry_interval) + deployment = client.poll_deployment(deployment_name) spinner.text = click.style('Deployment status: {}'.format(deployment.status.status), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except RetriesExhausted as e: spinner.write(click.style(str(e), fg=Colors.RED)) - spinner.text = click.style('Try again?', Colors.RED) + spinner.text = click.style('Try again', Colors.RED) spinner.red.fail(Symbols.ERROR) - except DeploymentNotRunningException as e: - if 'DEP_E151' in e.deployment_status.errors: - spinner.text = click.style('Device is either offline or not reachable', fg=Colors.RED) - else: - spinner.text = click.style(str(e), fg=Colors.RED) + except DeploymentNotRunning as e: + spinner.text = click.style(str(e), fg=Colors.RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) except Exception as e: spinner.text = click.style(str(e), fg=Colors.RED) spinner.red.fail(Symbols.ERROR) - raise SystemExit(1) + raise SystemExit(1) \ No newline at end of file diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 42fe7607..85b6bb04 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -16,20 +16,26 @@ import http import json import os +import time from hashlib import md5 from typing import List, Optional, Dict, Any +import click import magic import requests from munch import munchify, Munch - from rapyuta_io.utils.rest_client import HttpMethod, RestClient +from riocli.v2client.enums import DeploymentPhaseConstants +from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning + + class DeploymentNotFound(Exception): def __init__(self, message='deployment not found!'): self.message = message super().__init__(self.message) + def handle_server_errors(response: requests.Response): status_code = response.status_code # 500 Internal Server Error @@ -51,11 +57,13 @@ def handle_server_errors(response: requests.Response): if status_code > 504: raise Exception('unknown server error') + class NetworkNotFound(Exception): def __init__(self, message='network not found!'): self.message = message super().__init__(self.message) + class Client(object): """ v2 API Client @@ -722,8 +730,6 @@ def _walk_pages(self, c: RestClient, params: dict = {}, limit: Optional[int] = N return munchify(result) - - def list_packages( self, query: dict = None @@ -816,7 +822,6 @@ def delete_package(self, package_name: str, raise Exception("package: {}".format(err_msg)) return munchify(data) - def list_networks( self, @@ -911,13 +916,13 @@ def delete_network(self, network_name: str, data = json.loads(response.text) if not response.ok: err_msg = data.get('error') - raise Exception("package: {}".format(err_msg)) + raise Exception("network: {}".format(err_msg)) return munchify(data) def list_deployments( - self, - query: dict = None + self, + query: dict = None ) -> Munch: """ List all deployments in a project @@ -947,7 +952,7 @@ def list_deployments( result.extend(deployments) return munchify(result) - + def create_deployment(self, deployment: dict) -> Munch: """ Create a new deployment @@ -965,11 +970,11 @@ def create_deployment(self, deployment: dict) -> Munch: raise Exception("deployment: {}".format(err_msg)) return munchify(data) - + def get_deployment( - self, - name: str, - query: dict = None + self, + name: str, + query: dict = None ): url = "{}/v2/deployments/{}/".format(self._host, name) headers = self._config.get_auth_header() @@ -1024,6 +1029,42 @@ def delete_deployment(self, name: str, query: dict = None) -> Munch: return munchify(data) + def poll_deployment( + self, + name: str, + retry_count: int = 50, + sleep_interval: int = 6, + ready_phases: List[str] = None, + ) -> Munch: + if ready_phases is None: + ready_phases = [] + + deployment = self.get_deployment(name) + + status = deployment.status + + for _ in range(retry_count): + if status.phase in ready_phases: + return deployment + + if status.phase == DeploymentPhaseConstants.DeploymentPhaseProvisioning.value: + errors = status.error_codes or [] + if 'DEP_E153' in errors: # DEP_E153 (image-pull error) will persist across retries + return deployment + elif status.phase == DeploymentPhaseConstants.DeploymentPhaseSucceeded.value: + return deployment + elif status.phase in [DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate.value, + DeploymentPhaseConstants.DeploymentPhaseFailedToStart.value, + DeploymentPhaseConstants.DeploymentPhaseStopped.value]: + raise DeploymentNotRunning('Deployment not running. Deployment status: {}'.format(status.phase)) + + time.sleep(sleep_interval) + deployment = self.get_deployment(name) + status = deployment.status + + raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Deployment status: {}'.format( + retry_count, sleep_interval, status.phase)) + def list_disks( self, query: dict = None diff --git a/riocli/v2client/enums.py b/riocli/v2client/enums.py new file mode 100644 index 00000000..1a7435fa --- /dev/null +++ b/riocli/v2client/enums.py @@ -0,0 +1,33 @@ +import enum + + +class DeploymentPhaseConstants(str, enum.Enum): + """ + Enumeration variables for the deployment phase + """ + + def __str__(self): + return str(self.value) + + DeploymentPhaseInProgress = "InProgress" + DeploymentPhaseProvisioning = "Provisioning" + DeploymentPhaseSucceeded = "Succeeded" + DeploymentPhaseFailedToUpdate = "FailedToUpdate" + DeploymentPhaseFailedToStart = "FailedToStart" + DeploymentPhaseStopped = "Stopped" + + +class DeploymentStatusConstants(str, enum.Enum): + """ + Enumeration variables for the deployment status + + """ + + def __str__(self): + return str(self.value) + + DeploymentStatusRunning = "Running" + DeploymentStatusPending = "Pending" + DeploymentStatusError = "Error" + DeploymentStatusUnknown = "Unknown" + DeploymentStatusStopped = "Stopped" \ No newline at end of file diff --git a/riocli/v2client/error.py b/riocli/v2client/error.py new file mode 100644 index 00000000..98881e67 --- /dev/null +++ b/riocli/v2client/error.py @@ -0,0 +1,9 @@ + +class RetriesExhausted(Exception): + def __init__(self, msg=None): + Exception.__init__(self, msg) + + +class DeploymentNotRunning(Exception): + def __init__(self, msg=None): + Exception.__init__(self, msg) \ No newline at end of file From 8b84b46067cea3032295780bb4cc37c81ff32f4c Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Tue, 16 Jul 2024 17:24:52 +0530 Subject: [PATCH 08/54] fix(networks): fix device network schema --- riocli/jsonschema/schemas/network-schema.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/riocli/jsonschema/schemas/network-schema.yaml b/riocli/jsonschema/schemas/network-schema.yaml index 2c2116e1..366b94d9 100644 --- a/riocli/jsonschema/schemas/network-schema.yaml +++ b/riocli/jsonschema/schemas/network-schema.yaml @@ -65,8 +65,8 @@ definitions: runtime: enum: - device - deviceGUID: - "$ref": "#/definitions/uuid" + depends: + "$ref": "#/definitions/depends" networkInterface: type: string restartPolicy: @@ -74,7 +74,7 @@ definitions: default: always required: - - deviceGUID + - depends - networkInterface rosDistro: @@ -126,6 +126,15 @@ definitions: type: string pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" + depends: + type: object + properties: + kind: + const: Device + default: Device + nameOrGUID: + type: string + resourceLimits: type: object properties: From 6a7ae654e4d737b5545c6a8848c1ca7136740752 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Tue, 16 Jul 2024 17:25:21 +0530 Subject: [PATCH 09/54] fix(networks): fix network type flag --- riocli/network/delete.py | 16 ++++++++-------- riocli/network/list.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/riocli/network/delete.py b/riocli/network/delete.py index cd718fc9..ed721fb2 100644 --- a/riocli/network/delete.py +++ b/riocli/network/delete.py @@ -27,14 +27,6 @@ from rapyuta_io import Client from riocli.network.model import Network - -def _apply_delete(client: Client, result: Queue, network: Network) -> None: - try: - client.delete_network(network_name=network.metadata.name) - result.put((network.metadata.name, True)) - except Exception: - result.put((network.metadata.name, False)) - @click.command( 'delete', cls=HelpColorsCommand, @@ -119,3 +111,11 @@ def delete_network( 'Failed to delete network(s): {}'.format(e), Colors.RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e + +def _apply_delete(client: Client, result: Queue, network: Network) -> None: + try: + client.delete_network(network_name=network.metadata.name) + result.put((network.metadata.name, True)) + except Exception as e: + click.secho() + result.put((network.metadata.name, False)) diff --git a/riocli/network/list.py b/riocli/network/list.py index c895691c..74b8d65a 100644 --- a/riocli/network/list.py +++ b/riocli/network/list.py @@ -41,7 +41,7 @@ def list_networks(network: str) -> None: if network in ("both", ""): networks = client.list_networks() else: - networks = client.list_networks(query={"network_type": network}) + networks = client.list_networks(query={"networkType": network}) networks = sorted(networks, key=lambda n: n.metadata.name.lower()) _display_network_list(networks, show_header=True) From bab4898b2d10de8f8ad80f460c741abf781de390 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Wed, 17 Jul 2024 10:04:54 +0530 Subject: [PATCH 10/54] fix(deployments): fix deployment wait command --- riocli/deployment/model.py | 1 + riocli/deployment/wait.py | 8 ++++++-- riocli/network/delete.py | 2 ++ riocli/v2client/client.py | 12 +++++------- riocli/v2client/error.py | 4 ++++ 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index f3a4ff45..693fa51c 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -16,6 +16,7 @@ from munch import unmunchify from rapyuta_io import Client from rapyuta_io.clients.catalog_client import Package +from rapyuta_io.clients.deployment import DeploymentNotRunningException from rapyuta_io.clients.package import ProvisionConfiguration, RestartPolicy from rapyuta_io.clients.rosbag import (OverrideOptions, ROSBagCompression, ROSBagJob, ROSBagOnDemandUploadOptions, ROSBagOptions, ROSBagTimeRange, ROSBagUploadTypes, TopicOverrideInfo, diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index 1a0c7d8c..5579ee2c 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -17,7 +17,7 @@ from riocli.config import new_v2_client from riocli.constants import Colors, Symbols from riocli.utils.spinner import with_spinner -from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning +from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError @click.command( @@ -38,7 +38,7 @@ def wait_for_deployment( try: client = new_v2_client() deployment = client.poll_deployment(deployment_name) - spinner.text = click.style('Deployment status: {}'.format(deployment.status.status), fg=Colors.GREEN) + spinner.text = click.style('Phase: Succeeded Status: {}'.format(deployment.status.status), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except RetriesExhausted as e: spinner.write(click.style(str(e), fg=Colors.RED)) @@ -48,6 +48,10 @@ def wait_for_deployment( spinner.text = click.style(str(e), fg=Colors.RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) + except ImagePullError as e: + spinner.text = click.style(str(e), fg=Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) except Exception as e: spinner.text = click.style(str(e), fg=Colors.RED) spinner.red.fail(Symbols.ERROR) diff --git a/riocli/network/delete.py b/riocli/network/delete.py index ed721fb2..2ea7fc68 100644 --- a/riocli/network/delete.py +++ b/riocli/network/delete.py @@ -27,6 +27,7 @@ from rapyuta_io import Client from riocli.network.model import Network + @click.command( 'delete', cls=HelpColorsCommand, @@ -112,6 +113,7 @@ def delete_network( spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e + def _apply_delete(client: Client, result: Queue, network: Network) -> None: try: client.delete_network(network_name=network.metadata.name) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 85b6bb04..a035aa66 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -27,7 +27,7 @@ from rapyuta_io.utils.rest_client import HttpMethod, RestClient from riocli.v2client.enums import DeploymentPhaseConstants -from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning +from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError class DeploymentNotFound(Exception): @@ -1048,15 +1048,13 @@ def poll_deployment( return deployment if status.phase == DeploymentPhaseConstants.DeploymentPhaseProvisioning.value: - errors = status.error_codes or [] + errors = status.get('error_codes', []) if 'DEP_E153' in errors: # DEP_E153 (image-pull error) will persist across retries - return deployment + raise ImagePullError('Deployment not running. Phase: Provisioning Status: {}'.format(status.phase)) elif status.phase == DeploymentPhaseConstants.DeploymentPhaseSucceeded.value: return deployment - elif status.phase in [DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate.value, - DeploymentPhaseConstants.DeploymentPhaseFailedToStart.value, - DeploymentPhaseConstants.DeploymentPhaseStopped.value]: - raise DeploymentNotRunning('Deployment not running. Deployment status: {}'.format(status.phase)) + elif status.phase == DeploymentPhaseConstants.DeploymentPhaseStopped.value: + raise DeploymentNotRunning('Deployment not running. Phase: Stopped Status: {}'.format(status.phase)) time.sleep(sleep_interval) deployment = self.get_deployment(name) diff --git a/riocli/v2client/error.py b/riocli/v2client/error.py index 98881e67..bef94764 100644 --- a/riocli/v2client/error.py +++ b/riocli/v2client/error.py @@ -5,5 +5,9 @@ def __init__(self, msg=None): class DeploymentNotRunning(Exception): + def __init__(self, msg=None): + Exception.__init__(self, msg) + +class ImagePullError(Exception): def __init__(self, msg=None): Exception.__init__(self, msg) \ No newline at end of file From 3e6d7b7d333cc037f1085f3580e5e45dd661d359 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Wed, 17 Jul 2024 14:50:50 +0530 Subject: [PATCH 11/54] fix(deployments): adds stopped at time for deployment list --- riocli/deployment/list.py | 10 ++-------- riocli/v2client/client.py | 13 ++++++------- riocli/v2client/enums.py | 2 -- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index f2b590ab..bfdc1cc9 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -28,16 +28,12 @@ DeploymentPhaseConstants.DeploymentPhaseProvisioning, DeploymentPhaseConstants.DeploymentPhaseSucceeded, DeploymentPhaseConstants.DeploymentPhaseStopped, - DeploymentPhaseConstants.DeploymentPhaseFailedToStart, - DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate, ] DEFAULT_PHASES = [ DeploymentPhaseConstants.DeploymentPhaseInProgress, DeploymentPhaseConstants.DeploymentPhaseProvisioning, DeploymentPhaseConstants.DeploymentPhaseSucceeded, - DeploymentPhaseConstants.DeploymentPhaseFailedToStart, - DeploymentPhaseConstants.DeploymentPhaseFailedToUpdate ] @@ -80,15 +76,13 @@ def display_deployment_list( ): headers = [] if show_header: - headers = ('Name', 'Status', 'Phase', 'Errors') - if wide: - headers += ('Package', 'Deployment ID',) + headers = ('Deployment ID', 'Name', 'Phase', 'Package', 'Creation Time (UTC)', 'Stopped Time (UTC)') data = [] for deployment in deployments: package_name_version = "{} ({})".format(deployment.metadata.depends.nameOrGUID, deployment.metadata.depends.version) phase = deployment.status.phase if deployment.status else "" data.append([deployment.metadata.guid, deployment.metadata.name, - phase, package_name_version]) + phase, package_name_version, deployment.metadata.createdAt, deployment.metadata.get('deletedAt')]) tabulate_data(data, headers=headers) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index a035aa66..44d5ac39 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -930,15 +930,13 @@ def list_deployments( url = "{}/v2/deployments/".format(self._host) headers = self._config.get_auth_header() - params = {} + params = { + "continue": 0, + "limit": 100, + } params.update(query or {}) - offset, result = 0, [] + result = [] while True: - params.update({ - "continue": offset, - "limit": 50, - "phases": ["Succeeded", "InProgress", "Provisioning", "FailedToUpdate", "FailedToStart"], - }) response = RestClient(url).method(HttpMethod.GET).query_param( params).headers(headers).execute() data = json.loads(response.text) @@ -949,6 +947,7 @@ def list_deployments( if not deployments: break offset = data['metadata']['continue'] + params.update({"continue": offset}) result.extend(deployments) return munchify(result) diff --git a/riocli/v2client/enums.py b/riocli/v2client/enums.py index 1a7435fa..90166f18 100644 --- a/riocli/v2client/enums.py +++ b/riocli/v2client/enums.py @@ -12,8 +12,6 @@ def __str__(self): DeploymentPhaseInProgress = "InProgress" DeploymentPhaseProvisioning = "Provisioning" DeploymentPhaseSucceeded = "Succeeded" - DeploymentPhaseFailedToUpdate = "FailedToUpdate" - DeploymentPhaseFailedToStart = "FailedToStart" DeploymentPhaseStopped = "Stopped" From 9101b01b087f8c8434c222dcc457b6669df00295 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Wed, 17 Jul 2024 16:26:12 +0530 Subject: [PATCH 12/54] fix(manifests): fix manifests of resources --- .../manifests/deployment-nonros-cloud.yaml | 5 ++-- .../apply/manifests/deployment-ros-cloud.yaml | 5 +--- .../deployment-ros-device-no-rosbag.yaml | 3 --- .../deployment-ros-device-rosbag.yaml | 5 ++-- riocli/apply/manifests/deployment.yaml | 23 ++++++++++--------- .../apply/manifests/network-native-cloud.yaml | 2 +- .../manifests/network-native-device.yaml | 3 ++- .../apply/manifests/network-routed-cloud.yaml | 2 +- .../manifests/network-routed-device.yaml | 3 ++- riocli/apply/manifests/network.yaml | 10 ++++---- .../apply/manifests/package-nonros-cloud.yaml | 6 ++--- .../manifests/package-nonros-device.yaml | 6 ++--- riocli/apply/manifests/package-ros-cloud.yaml | 7 +++--- .../package-ros-device-no-rosbag.yaml | 5 ++-- .../manifests/package-ros-device-rosbag.yaml | 6 ++--- riocli/apply/manifests/package.yaml | 21 ++++++++--------- riocli/apply/manifests/staticroute.yaml | 2 +- .../jsonschema/schemas/deployment-schema.yaml | 8 ------- 18 files changed, 55 insertions(+), 67 deletions(-) diff --git a/riocli/apply/manifests/deployment-nonros-cloud.yaml b/riocli/apply/manifests/deployment-nonros-cloud.yaml index 6d0b73d1..e8edb727 100644 --- a/riocli/apply/manifests/deployment-nonros-cloud.yaml +++ b/riocli/apply/manifests/deployment-nonros-cloud.yaml @@ -7,9 +7,9 @@ metadata: nameOrGUID: "package-nonros-cloud" version: "v1.0.0" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: runtime: cloud # Options: [device, cloud] depends: @@ -41,5 +41,4 @@ spec: enabled: true trees: - config01 - - config02 - disableSync: false \ No newline at end of file + - config02 \ No newline at end of file diff --git a/riocli/apply/manifests/deployment-ros-cloud.yaml b/riocli/apply/manifests/deployment-ros-cloud.yaml index 2e8115ed..3ea64ae9 100644 --- a/riocli/apply/manifests/deployment-ros-cloud.yaml +++ b/riocli/apply/manifests/deployment-ros-cloud.yaml @@ -8,9 +8,9 @@ metadata: nameOrGUID: "package-ros-cloud" version: "v1.0.0" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: runtime: cloud # Options: [device, cloud] depends: @@ -38,8 +38,6 @@ spec: - depends: kind: network nameOrGUID: "network-routed-cloud" - topics: - - "/telemetry" managedServices: - depends: kind: managedservice @@ -52,7 +50,6 @@ spec: trees: - config01 - config02 - disableSync: false rosBagJobs: - name: "testbag1" # Required recordOptions: # Required diff --git a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml index 42e989a0..de9c8210 100644 --- a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml @@ -22,7 +22,6 @@ spec: trees: - config01 - config02 - disableSync: false envArgs: - name: TEST_KEY value: test_value @@ -41,6 +40,4 @@ spec: - depends: kind: network nameOrGUID: "network-routed-device" - topics: - - "/telemetry" interface: eth0 \ No newline at end of file diff --git a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml index 3bc2f42e..e4bee65f 100644 --- a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml @@ -24,6 +24,9 @@ spec: - execName: exec-docker mountPath: "/tmp" # Path on container subPath: "/tmp" # Path on device + uid: 1000 # Optional userid for subpath + gid: 1000 # Optional groupid for subpath + perm: 755 # Optional permissions for subpath rosNetworks: - depends: kind: network @@ -32,8 +35,6 @@ spec: - depends: kind: network nameOrGUID: "network-routed-device" - topics: - - "/telemetry" interface: eth0 rosBagJobs: - name: "testbag1" # Required diff --git a/riocli/apply/manifests/deployment.yaml b/riocli/apply/manifests/deployment.yaml index 6cb67f3f..250b9008 100644 --- a/riocli/apply/manifests/deployment.yaml +++ b/riocli/apply/manifests/deployment.yaml @@ -8,9 +8,9 @@ metadata: nameOrGUID: "package-ros-cloud" version: "v1.0.0" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: runtime: cloud # Options: [device, cloud] depends: @@ -38,8 +38,6 @@ spec: - depends: kind: network nameOrGUID: "network-routed-cloud" - topics: - - "/telemetry" managedServices: - depends: kind: managedservice @@ -52,7 +50,6 @@ spec: trees: - config01 - config02 - disableSync: false rosBagJobs: - name: "testbag1" # Required recordOptions: # Required @@ -108,9 +105,9 @@ metadata: nameOrGUID: "package-nonros-cloud" version: "v1.0.0" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: runtime: cloud # Options: [device, cloud] depends: @@ -143,7 +140,6 @@ spec: trees: - config01 - config02 - disableSync: false --- apiVersion: apiextensions.rapyuta.io/v1 kind: Deployment @@ -170,6 +166,9 @@ spec: - execName: exec-docker mountPath: "/tmp" # Path on container subPath: "/tmp" # Path on device + uid: 1000 # Optional userid for subpath + gid: 1000 # Optional groupid for subpath + perm: 755 # Optional permissions for subpath rosNetworks: - depends: kind: network @@ -178,8 +177,6 @@ spec: - depends: kind: network nameOrGUID: "network-routed-device" - topics: - - "/telemetry" interface: eth0 --- apiVersion: apiextensions.rapyuta.io/v1 @@ -207,6 +204,9 @@ spec: - execName: exec-docker mountPath: "/tmp" # Path on container subPath: "/tmp" # Path on device + uid: 1000 # Optional userid for subpath + gid: 1000 # Optional groupid for subpath + perm: 755 # Optional permissions for subpath rosNetworks: - depends: kind: network @@ -215,8 +215,6 @@ spec: - depends: kind: network nameOrGUID: "network-routed-device" - topics: - - "/telemetry" interface: eth0 rosBagJobs: - name: "testbag1" # Required @@ -302,4 +300,7 @@ spec: volumes: - execName: exec-docker mountPath: "/tmp" # Path on container - subPath: "/tmp" # Path on device \ No newline at end of file + subPath: "/tmp" # Path on device + uid: 1000 # Optional userid for subpath + gid: 1000 # Optional groupid for subpath + perm: 755 # Optional permissions for subpath \ No newline at end of file diff --git a/riocli/apply/manifests/network-native-cloud.yaml b/riocli/apply/manifests/network-native-cloud.yaml index 2096dccd..dbc01bad 100644 --- a/riocli/apply/manifests/network-native-cloud.yaml +++ b/riocli/apply/manifests/network-native-cloud.yaml @@ -2,9 +2,9 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-native-cloud" + region: jp labels: app: test - #rapyuta.io/region: us spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] diff --git a/riocli/apply/manifests/network-native-device.yaml b/riocli/apply/manifests/network-native-device.yaml index 8452b908..4fe1d313 100644 --- a/riocli/apply/manifests/network-native-device.yaml +++ b/riocli/apply/manifests/network-native-device.yaml @@ -9,6 +9,7 @@ spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "device" # Required, Options: [cloud, device] - deviceGUID: "299a436f-9d7f-4bbf-adce-21be06bfbbce" # Required + depends: # Required + nameOrGuid: device-name networkInterface: "eth0" # Required restartPolicy: "always" # Options: [always, never, onFailure] diff --git a/riocli/apply/manifests/network-routed-cloud.yaml b/riocli/apply/manifests/network-routed-cloud.yaml index d15e6e94..c6b3ae14 100644 --- a/riocli/apply/manifests/network-routed-cloud.yaml +++ b/riocli/apply/manifests/network-routed-cloud.yaml @@ -3,9 +3,9 @@ kind: "Network" metadata: name: "network-routed-cloud" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] diff --git a/riocli/apply/manifests/network-routed-device.yaml b/riocli/apply/manifests/network-routed-device.yaml index 2fc29596..864b2a42 100644 --- a/riocli/apply/manifests/network-routed-device.yaml +++ b/riocli/apply/manifests/network-routed-device.yaml @@ -9,6 +9,7 @@ spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "device" # Required, Options: [cloud, device] - deviceGUID: "299a436f-9d7f-4bbf-adce-21be06bfbbce" # Required + depends: # Required + nameOrGuid: device-name networkInterface: "eth0" # Required restartPolicy: "always" # Options: [always, never, onFailure] diff --git a/riocli/apply/manifests/network.yaml b/riocli/apply/manifests/network.yaml index a5f2adee..395e6d7e 100644 --- a/riocli/apply/manifests/network.yaml +++ b/riocli/apply/manifests/network.yaml @@ -4,9 +4,9 @@ kind: "Network" metadata: name: "network-routed-cloud" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] @@ -26,7 +26,8 @@ spec: type: "routed" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "device" # Required, Options: [cloud, device] - deviceGUID: "0eac4320-b36e-4104-b3a9-6bdc2cd37aec" # Required + depends: # Required + nameOrGuid: device-name networkInterface: "eth0" # Required restartPolicy: "always" # Options: [always, never, onFailure] --- @@ -35,9 +36,9 @@ kind: "Network" metadata: name: "network-native-cloud" project: "project-guid" + region: jp labels: app: test - #rapyuta.io/region: us spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] @@ -57,6 +58,7 @@ spec: type: "native" # Required, Options: [routed, native] rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "device" # Required, Options: [cloud, device] - deviceGUID: "0eac4320-b36e-4104-b3a9-6bdc2cd37aec" # Required + depends: # Required + nameOrGuid: device-name networkInterface: "eth0" # Required restartPolicy: "always" # Options: [always, never, onFailure] diff --git a/riocli/apply/manifests/package-nonros-cloud.yaml b/riocli/apply/manifests/package-nonros-cloud.yaml index 8d01abe0..c6f35ecc 100644 --- a/riocli/apply/manifests/package-nonros-cloud.yaml +++ b/riocli/apply/manifests/package-nonros-cloud.yaml @@ -13,10 +13,10 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: - - "sleep infinity" + command: # Command supports both array and string + - "sleep infinity" + # command: "sleep infinity" runAsBash: True - simulation: False limits: cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] diff --git a/riocli/apply/manifests/package-nonros-device.yaml b/riocli/apply/manifests/package-nonros-device.yaml index 484b898e..ec19bb99 100644 --- a/riocli/apply/manifests/package-nonros-device.yaml +++ b/riocli/apply/manifests/package-nonros-device.yaml @@ -15,8 +15,9 @@ spec: executables: # Required - name: "exec-docker" type: preInstalled # Options: [docker (default), preInstalled] - command: - - "roslaunch talker talker.launch" + command: # Command supports both array and string + - "sleep infinity" + # command: "sleep infinity" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -30,7 +31,6 @@ spec: failureThreshold: 1 # Minimum consecutive failures for the probe to be considered failed after having succeeded. successThreshold: 3 # Minimum consecutive successes for the probe to be considered successful after having failed. timeoutSeconds: 10 # Number of seconds after which the probe times out. Minimun: 10 - docker: image: "busybox:latest" imagePullPolicy: "Always" # Always, Never, IfNotPresent(default) diff --git a/riocli/apply/manifests/package-ros-cloud.yaml b/riocli/apply/manifests/package-ros-cloud.yaml index 48f18e1a..1fcb8886 100644 --- a/riocli/apply/manifests/package-ros-cloud.yaml +++ b/riocli/apply/manifests/package-ros-cloud.yaml @@ -13,10 +13,10 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: - - "roslaunch talker talker.launch" + command: # Command supports both array and string + - "roslaunch talker talker.launch" + # command: "roslaunch talker talker.launch" runAsBash: True - simulation: False limits: cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] @@ -111,7 +111,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" diff --git a/riocli/apply/manifests/package-ros-device-no-rosbag.yaml b/riocli/apply/manifests/package-ros-device-no-rosbag.yaml index 9487ed32..b8586eef 100644 --- a/riocli/apply/manifests/package-ros-device-no-rosbag.yaml +++ b/riocli/apply/manifests/package-ros-device-no-rosbag.yaml @@ -15,7 +15,9 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: "roslaunch talker talker.launch" + command: # Command supports both array and string + - "roslaunch talker talker.launch" + # command: "roslaunch talker talker.launch" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -52,7 +54,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" diff --git a/riocli/apply/manifests/package-ros-device-rosbag.yaml b/riocli/apply/manifests/package-ros-device-rosbag.yaml index 445654c6..8f46e584 100644 --- a/riocli/apply/manifests/package-ros-device-rosbag.yaml +++ b/riocli/apply/manifests/package-ros-device-rosbag.yaml @@ -15,8 +15,9 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: - - "roslaunch talker talker.launch" + command: # Command supports both array and string + - "roslaunch talker talker.launch" + # command: "roslaunch talker talker.launch" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -109,7 +110,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" diff --git a/riocli/apply/manifests/package.yaml b/riocli/apply/manifests/package.yaml index 55346c5a..ec1b6ab6 100644 --- a/riocli/apply/manifests/package.yaml +++ b/riocli/apply/manifests/package.yaml @@ -13,10 +13,10 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: + command: # Command supports both array and string - "roslaunch talker talker.launch" + # command: "roslaunch talker talker.launch" runAsBash: True - simulation: False limits: cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] @@ -111,7 +111,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" @@ -148,8 +147,9 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: - - "shell-cmd" + command: # Command supports both array and string + - "shell-cmd" + # command: "shell-cmd" runAsBash: True limits: # Optional cpu: 0.025 # Unit: Core (Optional) @@ -241,7 +241,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" @@ -277,10 +276,10 @@ spec: executables: # Required - name: "exec-docker" type: docker # Options: [docker (default), preInstalled] - command: - - "sleep infinity" + command: # Command supports both array and string + - "sleep infinity" + # command: "sleep infinity" runAsBash: True - simulation: False limits: cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] @@ -344,7 +343,7 @@ spec: executables: # Required - name: "exec-docker-01" type: docker # Options: [docker (default), preInstalled] - command: + command: - "roslaunch talker talker.launch" runAsBash: True limits: # Optional @@ -453,7 +452,6 @@ spec: ros: enabled: True version: melodic # Required, Options: [ kinetic, melodic, noetic ] - inboundScopedTargeted: False rosEndpoints: - type: topic # Required, Options: [ topic, service, action ] name: "/telemetry" @@ -493,7 +491,6 @@ spec: type: docker # Options: [docker (default), build, preInstalled] command: - "roslaunch talker talker.launch" - simulation: False runAsBash: False limits: cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] diff --git a/riocli/apply/manifests/staticroute.yaml b/riocli/apply/manifests/staticroute.yaml index d1bb5f3b..24ff3fc9 100644 --- a/riocli/apply/manifests/staticroute.yaml +++ b/riocli/apply/manifests/staticroute.yaml @@ -3,7 +3,7 @@ kind: "StaticRoute" metadata: name: "staticroute" project: "project-guid" - # region: us + region: jp labels: app: test spec: diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index c38b8d0b..1d3fd435 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -57,19 +57,11 @@ definitions: "$ref": "#/definitions/networkDepends" interface: type: string - topics: - type: array - items: - type: string cloudNetworkAttachSpec: properties: depends: "$ref": "#/definitions/networkDepends" - topics: - type: array - items: - type: string cloudVolumeAttachSpec: type: object From 339c5d18616aaeb545bae9e456533e2c0b124524 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Wed, 17 Jul 2024 19:36:39 +0530 Subject: [PATCH 13/54] fix(v2client): fix v2 client org header --- riocli/v2client/client.py | 50 +++++++++++++++------------------------ riocli/v2client/error.py | 16 +++++++++++-- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 44d5ac39..dfbf1ced 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -20,20 +20,14 @@ from hashlib import md5 from typing import List, Optional, Dict, Any -import click import magic import requests from munch import munchify, Munch from rapyuta_io.utils.rest_client import HttpMethod, RestClient from riocli.v2client.enums import DeploymentPhaseConstants -from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError - - -class DeploymentNotFound(Exception): - def __init__(self, message='deployment not found!'): - self.message = message - super().__init__(self.message) +from riocli.v2client.error import (RetriesExhausted, DeploymentNotRunning, ImagePullError, + NetworkNotFound) def handle_server_errors(response: requests.Response): @@ -58,12 +52,6 @@ def handle_server_errors(response: requests.Response): raise Exception('unknown server error') -class NetworkNotFound(Exception): - def __init__(self, message='network not found!'): - self.message = message - super().__init__(self.message) - - class Client(object): """ v2 API Client @@ -738,7 +726,7 @@ def list_packages( List all packages in a project """ url = "{}/v2/packages/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -766,7 +754,7 @@ def create_package(self, payload: dict) -> Munch: Create a new package """ url = "{}/v2/packages/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.POST).headers( headers).execute(payload=payload) handle_server_errors(response) @@ -787,7 +775,7 @@ def get_package( List all packages in a project """ url = "{}/v2/packages/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -807,7 +795,7 @@ def delete_package(self, package_name: str, Delete a secret """ url = "{}/v2/packages/{}/".format(self._host, package_name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -831,7 +819,7 @@ def list_networks( List all networks in a project """ url = "{}/v2/networks/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -859,7 +847,7 @@ def create_network(self, payload: dict) -> Munch: Create a new network """ url = "{}/v2/networks/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.POST).headers( headers).execute(payload=payload) handle_server_errors(response) @@ -880,7 +868,7 @@ def get_network( get a network in a project """ url = "{}/v2/networks/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -904,7 +892,7 @@ def delete_network(self, network_name: str, Delete a secret """ url = "{}/v2/networks/{}/".format(self._host, network_name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -928,7 +916,7 @@ def list_deployments( List all deployments in a project """ url = "{}/v2/deployments/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = { "continue": 0, @@ -957,7 +945,7 @@ def create_deployment(self, deployment: dict) -> Munch: Create a new deployment """ url = "{}/v2/deployments/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() deployment["metadata"]["projectGUID"] = headers["project"] response = RestClient(url).method(HttpMethod.POST).headers( @@ -976,7 +964,7 @@ def get_deployment( query: dict = None ): url = "{}/v2/deployments/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -999,7 +987,7 @@ def update_deployment(self, name: str, dep: dict) -> Munch: Update a deployment """ url = "{}/v2/deployments/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.PATCH).headers( headers).execute(payload=dep) handle_server_errors(response) @@ -1015,7 +1003,7 @@ def delete_deployment(self, name: str, query: dict = None) -> Munch: Delete a deployment """ url = "{}/v2/deployments/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) response = RestClient(url).method( @@ -1070,7 +1058,7 @@ def list_disks( List all disks in a project """ url = "{}/v2/disks/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() params = {} params.update(query or {}) @@ -1098,7 +1086,7 @@ def get_disk(self, name: str) -> Munch: Get a Disk by its name """ url = "{}/v2/disks/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method( HttpMethod.GET).headers(headers).execute() @@ -1116,7 +1104,7 @@ def create_disk(self, disk: dict) -> Munch: Create a new disk """ url = "{}/v2/disks/".format(self._host) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.POST).headers( headers).execute(payload=disk) @@ -1134,7 +1122,7 @@ def delete_disk(self, name: str) -> Munch: Delete a disk by its name """ url = "{}/v2/disks/{}/".format(self._host, name) - headers = self._config.get_auth_header() + headers = self._get_auth_header() response = RestClient(url).method( HttpMethod.DELETE).headers(headers).execute() diff --git a/riocli/v2client/error.py b/riocli/v2client/error.py index bef94764..145b9214 100644 --- a/riocli/v2client/error.py +++ b/riocli/v2client/error.py @@ -1,4 +1,3 @@ - class RetriesExhausted(Exception): def __init__(self, msg=None): Exception.__init__(self, msg) @@ -8,6 +7,19 @@ class DeploymentNotRunning(Exception): def __init__(self, msg=None): Exception.__init__(self, msg) + class ImagePullError(Exception): def __init__(self, msg=None): - Exception.__init__(self, msg) \ No newline at end of file + Exception.__init__(self, msg) + + +class NetworkNotFound(Exception): + def __init__(self, message='network not found!'): + self.message = message + super().__init__(self.message) + + +class DeploymentNotFound(Exception): + def __init__(self, message='deployment not found!'): + self.message = message + super().__init__(self.message) From 34ac1c9d7d42d69d37529dc1f7a83f3780c3314c Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Wed, 17 Jul 2024 19:55:10 +0530 Subject: [PATCH 14/54] fix(networks): delete in use network --- riocli/network/delete.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/riocli/network/delete.py b/riocli/network/delete.py index 2ea7fc68..5ac84d6f 100644 --- a/riocli/network/delete.py +++ b/riocli/network/delete.py @@ -87,14 +87,14 @@ def delete_network( workers=workers, key=lambda x: x[0] ) data, statuses = [], [] - for name, status in result: + for name, status, msg in result: fg = Colors.GREEN if status else Colors.RED icon = Symbols.SUCCESS if status else Symbols.ERROR statuses.append(status) data.append([ click.style(name, fg), - click.style(icon, fg) + click.style('{} {}'.format(icon, msg), fg) ]) with spinner.hidden(): @@ -117,7 +117,7 @@ def delete_network( def _apply_delete(client: Client, result: Queue, network: Network) -> None: try: client.delete_network(network_name=network.metadata.name) - result.put((network.metadata.name, True)) + result.put((network.metadata.name, True, 'Network Deleted Successfully')) except Exception as e: click.secho() - result.put((network.metadata.name, False)) + result.put((network.metadata.name, False, str(e))) From 606c940af0785fba64e79f7629dd3b506dd14ad5 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:28:42 +0530 Subject: [PATCH 15/54] fix(packages): fix package commands --- riocli/package/delete.py | 19 ++++---- riocli/package/deployment.py | 21 +++++--- riocli/package/enum.py | 17 +++++++ riocli/package/inspect.py | 29 ++---------- riocli/package/list.py | 6 ++- riocli/package/model.py | 13 ++--- riocli/package/util.py | 92 ++++++++++++++++-------------------- 7 files changed, 95 insertions(+), 102 deletions(-) create mode 100644 riocli/package/enum.py diff --git a/riocli/package/delete.py b/riocli/package/delete.py index 7709fabd..efaa64b1 100644 --- a/riocli/package/delete.py +++ b/riocli/package/delete.py @@ -19,12 +19,12 @@ from riocli.config import new_v2_client from riocli.constants import Symbols, Colors +from riocli.package.model import Package from riocli.package.util import fetch_packages, print_packages_for_confirmation from riocli.utils import tabulate_data from riocli.utils.execute import apply_func_with_result from riocli.utils.spinner import with_spinner -from rapyuta_io import Client -from rapyuta_io.clients.package import Package +from riocli.v2client import Client @click.command('delete') @@ -56,7 +56,7 @@ def delete_package( return try: - packages = fetch_packages(client, package_name_or_regex, delete_all) + packages = fetch_packages(client, package_name_or_regex, package_version, delete_all) except Exception as e: spinner.text = click.style( 'Failed to find package(s): {}'.format(e), Colors.RED) @@ -85,13 +85,13 @@ def delete_package( ) data, statuses = [], [] - for name, status in result: + for name, status, msg in result: fg = Colors.GREEN if status else Colors.RED icon = Symbols.SUCCESS if status else Symbols.ERROR statuses.append(status) data.append([ click.style(name, fg), - click.style(icon, fg) + click.style('{} {}'.format(icon, msg), fg) ]) with spinner.hidden(): @@ -110,10 +110,11 @@ def delete_package( spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e -def _apply_delete(client: Client ,result: Queue, package: Package) -> None: + +def _apply_delete(client: Client, result: Queue, package: Package) -> None: name_version = "{}@{}".format(package.metadata.name, package.metadata.version) try: client.delete_package(package_name=package.metadata.name, query={"version": package.metadata.version}) - result.put((name_version, True)) - except Exception: - result.put((name_version, False)) \ No newline at end of file + result.put((name_version, True, 'Package deleted successfully')) + except Exception as e: + result.put((name_version, False, str(e))) diff --git a/riocli/package/deployment.py b/riocli/package/deployment.py index e3eacae6..79fa69fd 100644 --- a/riocli/package/deployment.py +++ b/riocli/package/deployment.py @@ -13,24 +13,31 @@ # limitations under the License. import click -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.deployment.list import display_deployment_list -from riocli.package.util import name_to_guid +from riocli.package.util import find_package +from riocli.utils.selector import show_selection @click.command('deployments') @click.option('--version', 'package_version', type=str, help='Semantic version of the Package, only used when name is used instead of GUID') @click.argument('package-name') -@name_to_guid -def list_package_deployments(package_name: str, package_guid: str) -> None: +def list_package_deployments(package_name: str, package_version: str) -> None: """ List the deployments of the package """ try: - client = new_client() - package = client.get_package(package_guid) - deployments = package.deployments() + client = new_v2_client() + + package_obj = find_package(client, package_name, package_version) + if not package_obj: + click.secho("package not found", fg='red') + raise SystemExit(1) + + deployments = client.list_deployments( + query={'packageName': package_obj.metadata.name, 'packageVersion': package_obj.metadata.version}) + display_deployment_list(deployments, show_header=True) except Exception as e: click.secho(str(e), fg='red') diff --git a/riocli/package/enum.py b/riocli/package/enum.py new file mode 100644 index 00000000..c80e5016 --- /dev/null +++ b/riocli/package/enum.py @@ -0,0 +1,17 @@ +import enum + + +class RestartPolicy(str, enum.Enum): + """ + Enumeration variables for the Restart Policy. Restart Policy may be 'Always', 'Never' or 'OnFailure' \n + RestartPolicy.Always \n + RestartPolicy.Never \n + RestartPolicy.OnFailure \n + """ + + def __str__(self): + return str(self.value) + + Always = "always" + Never = "no" + OnFailure = "on-failure" \ No newline at end of file diff --git a/riocli/package/inspect.py b/riocli/package/inspect.py index e13e97f1..ed329240 100644 --- a/riocli/package/inspect.py +++ b/riocli/package/inspect.py @@ -15,9 +15,11 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.package.util import find_package from riocli.utils import inspect_with_format from riocli.utils.selector import show_selection + @click.command('inspect') @click.option('--version', 'package_version', type=str, help='Semantic version of the Package, only used when name is used instead of GUID') @@ -30,32 +32,7 @@ def inspect_package(format_type: str, package_name: str, package_version: str) - """ try: client = new_v2_client() - package_obj = None - if package_name.startswith("pkg-"): - packages = client.list_packages(query = {"guid": package_name}) - if packages: - obj = packages[0] - package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) - - elif package_name and package_version: - package_obj = client.get_package(package_name, query = {"version": package_version}) - elif package_name: - packages = client.list_packages(query={"name": package_name}) - - if len(packages) == 0: - click.secho("package not found", fg='red') - raise SystemExit(1) - - options = {} - package_objs = {} - for pkg in packages: - options[pkg.metadata.guid] = '{} ({})'.format(pkg.metadata.name, pkg.metadata.version) - package_objs[pkg.metadata.guid] = pkg - choice = show_selection(options, header='Following packages were found with the same name') - obj = package_objs[choice] - package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) - - + package_obj = find_package(client, package_name, package_version) if not package_obj: click.secho("package not found", fg='red') raise SystemExit(1) diff --git a/riocli/package/list.py b/riocli/package/list.py index 3ccea526..6ad8fb0f 100644 --- a/riocli/package/list.py +++ b/riocli/package/list.py @@ -14,11 +14,12 @@ import typing import click -from rapyuta_io.clients.package import Package from riocli.config import new_v2_client +from riocli.package.model import Package from riocli.utils import tabulate_data + @click.command('list') @click.option('--filter', 'filter_word', type=str, default=None, help='A sub-string can be provided to filter down the package list') @@ -29,11 +30,12 @@ def list_packages(filter_word: str) -> None: try: client = new_v2_client(with_project=True) packages = client.list_packages() - _display_package_list(packages, filter_word,show_header=True) + _display_package_list(packages, filter_word, show_header=True) except Exception as e: click.secho(str(e), fg='red') raise SystemExit(1) + def _display_package_list( packages: typing.List[Package], filter_word: str, diff --git a/riocli/package/model.py b/riocli/package/model.py index d5ab9f4f..2a454089 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -11,17 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import typing -from munch import munchify, unmunchify - -from rapyuta_io import Client -from rapyuta_io.clients.package import RestartPolicy +from munch import unmunchify +from riocli.config import new_v2_client from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.config import new_v2_client +from riocli.package.enum import RestartPolicy +from riocli.v2client import Client + class Package(Model): RESTART_POLICY = { @@ -48,10 +47,8 @@ def create_object(self, client: Client, **kwargs) -> typing.Any: return unmunchify(r) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass - @staticmethod def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: v2_client = new_v2_client() v2_client.delete_package(obj.metadata.name, query={"version": obj.metadata.version}) diff --git a/riocli/package/util.py b/riocli/package/util.py index fc939353..b6527d35 100644 --- a/riocli/package/util.py +++ b/riocli/package/util.py @@ -11,75 +11,61 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import functools import re -import typing from typing import List import click -from rapyuta_io import Client -from rapyuta_io.clients.package import Package +from munch import Munch -from riocli.config import new_client +from riocli.package.model import Package from riocli.utils import tabulate_data from riocli.utils.selector import show_selection +from riocli.v2client import Client -def name_to_guid(f: typing.Callable) -> typing.Callable: - @functools.wraps(f) - def decorated(**kwargs: typing.Any): - try: - client = new_client() - except Exception as e: - click.secho(str(e), fg='red') - raise SystemExit(1) - - name = kwargs.pop('package_name') - guid = None - version = kwargs.pop('package_version') - - if name.startswith('pkg-') or name.startswith('io-'): - guid = name - name = None - - if name is None: - name = get_package_name(client, guid) - - if guid is None: - guid = find_package_guid(client, name, version) - - kwargs['package_name'] = name - kwargs['package_guid'] = guid - f(**kwargs) +def find_package(client: Client, + package_name: str, + package_version: str, + ) -> Munch | None: - return decorated + package_obj = None + if package_name.startswith("pkg-"): + packages = client.list_packages(query={"guid": package_name}) + if not packages: + raise Exception('Package not found') -def get_package_name(client: Client, guid: str) -> str: - pkg = client.get_package(guid) - return pkg.packageName + obj = packages[0] + package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) + elif package_name and package_version: + package_obj = client.get_package(package_name, query={"version": package_version}) + elif package_name: + packages = client.list_packages(query={"name": package_name}) + if len(packages) == 0: + click.secho("package not found", fg='red') + raise SystemExit(1) -def find_package_guid(client: Client, name: str, version: str = None) -> str: - packages = client.get_all_packages(name=name, version=version) - if len(packages) == 0: - click.secho("package not found", fg='red') - raise SystemExit(1) - - if len(packages) == 1: - return packages[0].packageId + if len(packages) == 1: + obj = packages[0] + package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) + else: + options = {} + package_objs = {} + for pkg in packages: + options[pkg.metadata.guid] = '{} ({})'.format(pkg.metadata.name, pkg.metadata.version) + package_objs[pkg.metadata.guid] = pkg + choice = show_selection(options, header='Following packages were found with the same name') + obj = package_objs[choice] + package_obj = client.get_package(obj.metadata.name, query={"version": obj.metadata.version}) - options = {} - for pkg in packages: - options[pkg.packageId] = '{} ({})'.format(pkg.packageName, pkg.packageVersion) - - choice = show_selection(options, header='Following packages were found with the same name') - return choice + return package_obj def fetch_packages( client: Client, package_name_or_regex: str, + package_version: str, include_all: bool, ) -> List[Package]: packages = client.list_packages() @@ -90,9 +76,15 @@ def fetch_packages( if 'io-public' in pkg.metadata.guid: continue - if include_all or re.search(package_name_or_regex, pkg.metadata.name): + if include_all: result.append(pkg) + if re.search(package_name_or_regex, pkg.metadata.name): + if package_version and package_version == pkg.metadata.version: + result.append(pkg) + elif not package_version: + result.append(pkg) + return result From 506426a2483b9d96401d599d63a5c08b7051ccb9 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:29:23 +0530 Subject: [PATCH 16/54] fix(network): fix network commands --- riocli/network/__init__.py | 2 - riocli/network/delete.py | 3 +- riocli/network/inspect.py | 1 + riocli/network/list.py | 9 ++-- riocli/network/logs.py | 46 -------------------- riocli/network/model.py | 66 +++++++++++------------------ riocli/network/util.py | 86 +++----------------------------------- 7 files changed, 35 insertions(+), 178 deletions(-) delete mode 100644 riocli/network/logs.py diff --git a/riocli/network/__init__.py b/riocli/network/__init__.py index 0c03b234..865de419 100644 --- a/riocli/network/__init__.py +++ b/riocli/network/__init__.py @@ -18,7 +18,6 @@ from riocli.network.delete import delete_network from riocli.network.inspect import inspect_network from riocli.network.list import list_networks -from riocli.network.logs import network_logs @click.group( @@ -37,4 +36,3 @@ def network() -> None: network.add_command(delete_network) network.add_command(list_networks) network.add_command(inspect_network) -network.add_command(network_logs) diff --git a/riocli/network/delete.py b/riocli/network/delete.py index 5ac84d6f..ecd6c295 100644 --- a/riocli/network/delete.py +++ b/riocli/network/delete.py @@ -24,8 +24,8 @@ from riocli.utils import tabulate_data from riocli.utils.execute import apply_func_with_result -from rapyuta_io import Client from riocli.network.model import Network +from riocli.v2client import Client @click.command( @@ -119,5 +119,4 @@ def _apply_delete(client: Client, result: Queue, network: Network) -> None: client.delete_network(network_name=network.metadata.name) result.put((network.metadata.name, True, 'Network Deleted Successfully')) except Exception as e: - click.secho() result.put((network.metadata.name, False, str(e))) diff --git a/riocli/network/inspect.py b/riocli/network/inspect.py index 3fdde41b..95692c84 100644 --- a/riocli/network/inspect.py +++ b/riocli/network/inspect.py @@ -19,6 +19,7 @@ from riocli.config import new_v2_client from munch import unmunchify + @click.command('inspect') @click.option('--format', '-f', 'format_type', default='yaml', type=click.Choice(['json', 'yaml'], case_sensitive=False)) diff --git a/riocli/network/list.py b/riocli/network/list.py index 74b8d65a..735b25e0 100644 --- a/riocli/network/list.py +++ b/riocli/network/list.py @@ -16,8 +16,6 @@ from click_help_colors import HelpColorsCommand from munch import Munch -from rapyuta_io import DeploymentPhaseConstants - from riocli.config import new_v2_client from riocli.constants import Colors from riocli.utils import tabulate_data @@ -56,15 +54,14 @@ def _display_network_list( ) -> None: headers = [] if show_header: - headers = ('Network ID', 'Network Name', 'Runtime', 'Type', 'Phase') + headers = ('Network ID', 'Network Name', 'Runtime', 'Type', 'Phase', 'Status') data = [] for network in networks: phase = network.status.phase if network.status else "" + status = network.status.status if network.status else "" network_type = network.spec.type - if phase and phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: - continue data.append( - [network.metadata.guid, network.metadata.name, network.spec.runtime, network_type, phase]) + [network.metadata.guid, network.metadata.name, network.spec.runtime, network_type, phase, status]) tabulate_data(data, headers) diff --git a/riocli/network/logs.py b/riocli/network/logs.py deleted file mode 100644 index 0789d4ce..00000000 --- a/riocli/network/logs.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click - -from riocli.config import new_client -from riocli.deployment.logs import stream_deployment_logs -from riocli.network.util import name_to_guid - - -@click.command('logs', hidden=True) -@click.option('--network', 'network_type', help='Type of Network', default=None, - type=click.Choice(['routed', 'native'])) -@click.argument('network-name') -@name_to_guid -def network_logs(network_name: str, network_guid: str, network_type: str) -> None: - if network_type == 'routed': - # FIXME: For routed network, it returns Pod not found error - click.secho('Not implemented yet!', fg='red') - raise SystemExit(1) - elif network_type == 'native': - native_network_logs(network_name, network_guid) - - -def native_network_logs(network_name: str, network_guid: str) -> None: - try: - client = new_client() - network = client.get_native_network(network_guid) - deployment = client.get_deployment(network.internal_deployment_guid) - comp_id = deployment.componentInfo[0].componentID - exec_id = deployment.componentInfo[0].executablesStatusInfo[0].id - pod_name = deployment.componentInfo[0].executablesStatusInfo[0].metadata[0].podName - stream_deployment_logs(deployment.deploymentId, comp_id, exec_id, pod_name) - except Exception as e: - click.secho(str(e), fg='red') - raise SystemExit(1) diff --git a/riocli/network/model.py b/riocli/network/model.py index 3994059b..5a35a873 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -12,23 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. import typing -from typing import Union, Any, Dict -from munch import munchify, unmunchify -import click -import time +from typing import Any, Dict -from rapyuta_io import Client -from rapyuta_io.clients.common_models import Limits -from rapyuta_io.clients.native_network import NativeNetwork, \ - Parameters as NativeNetworkParameters -from rapyuta_io.clients.routed_network import RoutedNetwork, \ - Parameters as RoutedNetworkParameters +import click +from munch import unmunchify +from riocli.config import new_v2_client +from riocli.constants import Colors, Symbols from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.v2client.client import NetworkNotFound -from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols, Status +from riocli.v2client.client import NetworkNotFound, Client + class Network(Model): def __init__(self, *args, **kwargs): @@ -37,7 +31,7 @@ def __init__(self, *args, **kwargs): def find_object(self, client: Client) -> bool: try: network, obj = self.rc.find_depends({"kind": self.kind.lower(), - "nameOrGUID": self.metadata.name}, self.spec.type) + "nameOrGUID": self.metadata.name}, self.spec.type) if not network: return False @@ -45,45 +39,26 @@ def find_object(self, client: Client) -> bool: except NetworkNotFound: return False - def poll_deployment_till_ready(self, client: Client, network: typing.Any, retry_count = 50, sleep_interval = 6): - for _ in range(retry_count): - if network.status and network.status.status == Status.RUNNING: - return network - - time.sleep(sleep_interval) - network = client.get_network(network.metadata.name) - return network - def create_object(self, client: Client, **kwargs) -> typing.Any: client = new_v2_client() - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - network = unmunchify(self) - network.pop("rc", None) - r = client.create_network(network) + r = client.create_network(self._sanitize_network()) retry_count = int(kwargs.get('retry_count')) retry_interval = int(kwargs.get('retry_interval')) try: - r = self.poll_deployment_till_ready( - client = client, - network = r, - retry_count=retry_count, - sleep_interval=retry_interval, - ) + r = client.poll_network(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) except Exception as e: click.secho(">> {}: Error polling for network ({}:{})".format( Symbols.WARNING, self.kind.lower(), self.metadata.name), fg=Colors.YELLOW) - + click.secho(str(e), fg=Colors.YELLOW) return unmunchify(r) - def update_object(self, client: Client, - obj: Union[RoutedNetwork, NativeNetwork]) -> Any: + def update_object(self, client: Client, obj: typing.Any) -> Any: pass def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: @@ -94,10 +69,6 @@ def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: def pre_process(cls, client: Client, d: Dict) -> None: pass - def _get_limits(self): - return Limits(self.spec.resourceLimits['cpu'], - self.spec.resourceLimits['memory']) - @staticmethod def validate(data): """ @@ -105,3 +76,16 @@ def validate(data): """ schema = load_schema('network') schema.validate(data) + + def _sanitize_network(self) -> typing.Dict: + # Unset createdAt and updatedAt to avoid timestamp parsing issue. + self.metadata.createdAt = None + self.metadata.updatedAt = None + + data = unmunchify(self) + + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + data.pop("rc", None) + + return data diff --git a/riocli/network/util.py b/riocli/network/util.py index 9d1e1065..6d64bf93 100644 --- a/riocli/network/util.py +++ b/riocli/network/util.py @@ -12,89 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import functools -from typing import Optional, Union, Tuple, Callable, Any, List +from typing import List from munch import Munch -import click import re -from rapyuta_io import DeploymentPhaseConstants, Client -from rapyuta_io.clients.native_network import NativeNetwork -from rapyuta_io.clients.routed_network import RoutedNetwork - -from riocli.config import new_client -from riocli.constants import Colors from riocli.utils import tabulate_data -from riocli.utils.selector import show_selection -from riocli.config import new_v2_client from riocli.network.model import Network - -def name_to_guid(f: Callable) -> Callable: - @functools.wraps(f) - def decorated(**kwargs: Any): - client = new_v2_client() - - name = kwargs.pop('network_name') - network_type = kwargs.pop('network', None) - guid = None - if name.startswith('net-'): - guid = name - name = None - - try: - if guid: - network = find_network_guid( - client, guid, network_type) - name = network.metadata.name - - else: - network = find_network_name( - client, name, network_type) - except Exception as e: - click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) from e - - kwargs['network_type'] = network_type - kwargs['network_name'] = name - kwargs['network_guid'] = guid - f(**kwargs) - - return decorated - - -def find_network_guid( - client: Client, - guid: str, - network_type: str, -) -> Munch: - - if network_type: - networks = client.list_networks(query={'network_type': network_type}) - else: - networks = client.list_networks() - - for network in networks: - if network.status and network.status.phase == DeploymentPhaseConstants.DEPLOYMENT_STOPPED.value: - continue - if guid == network.metadata.guid: - return network - - raise NetworkNotFound() - -def find_network_guid(client: Client, name: str, version: str = None) -> str: - packages = client.get_all_packages(name=name, version=version) - if len(packages) == 0: - click.secho("package not found", fg='red') - raise SystemExit(1) - - if len(packages) == 1: - return packages[0].packageId - - options = {} - for pkg in packages: - options[pkg.packageId] = '{} ({})'.format(pkg.packageName, pkg.packageVersion) - - choice = show_selection(options, header='Following packages were found with the same name') - return choice +from riocli.v2client import Client def fetch_networks( @@ -103,14 +27,13 @@ def fetch_networks( network_type: str, include_all: bool, ) -> List[Network]: - if network_type: networks = client.list_networks(query={'network_type': network_type}) else: networks = client.list_networks() if include_all: - return networks + return list(networks) result = [] for n in networks: @@ -119,7 +42,8 @@ def fetch_networks( return result + def print_networks_for_confirmation(networks: List[Munch]) -> None: headers = ['Name', 'Type'] data = [[n.metadata.name, n.spec.type] for n in networks] - tabulate_data(data, headers) \ No newline at end of file + tabulate_data(data, headers) From 6672b748208144b7579a0c030d20dcdda11ed6d9 Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:30:08 +0530 Subject: [PATCH 17/54] fix(disk): fix disk commands --- riocli/disk/create.py | 42 +++++++++++++++++-------------------- riocli/disk/delete.py | 35 ++++++++++++++++--------------- riocli/disk/enum.py | 27 ++++++++++++++++++++++++ riocli/disk/list.py | 3 ++- riocli/disk/model.py | 49 +++++++++++++++++-------------------------- riocli/disk/util.py | 40 +++++++++++++---------------------- 6 files changed, 100 insertions(+), 96 deletions(-) create mode 100644 riocli/disk/enum.py diff --git a/riocli/disk/create.py b/riocli/disk/create.py index d78f2c84..e5197564 100644 --- a/riocli/disk/create.py +++ b/riocli/disk/create.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import click -import time - from click_help_colors import HelpColorsCommand -from rapyuta_io.clients.persistent_volumes import DiskCapacity +from yaspin.api import Yaspin from riocli.constants import Colors, Symbols, Regions -from riocli.disk.util import is_disk_ready +from riocli.disk.enum import DiskCapacity from riocli.utils.spinner import with_spinner from riocli.config import new_v2_client @@ -38,6 +36,7 @@ Regions.US, ] + @click.command( 'create', cls=HelpColorsCommand, @@ -45,38 +44,35 @@ help_options_color=Colors.GREEN, ) @click.argument('disk-name', type=str) -@click.option('--capacity', 'capacity', type=click.Choice(SUPPORTED_CAPACITIES), - default=DiskCapacity.GiB_4.value, help='Disk size in GiB') +@click.option('--capacity', 'capacity', type=click.INT, + default=DiskCapacity.GiB_4.value, help='Disk size in GiB. [4|8|16|32|64|128|256|512]') @click.option('--region', 'region', type=click.Choice(SUPPORTED_REGIONS), default=Regions.JP, help='Region to create the disk in') - @with_spinner(text="Creating a new disk...") def create_disk( disk_name: str, capacity: int = 4, region: str = 'jp', - spinner=None, + spinner: Yaspin = None ) -> None: """ Creates a new disk """ - try: - client = new_v2_client() - payload = { - "metadata": { - "name": disk_name, - "region": region - }, - "spec": { - "capacity": capacity, - "runtime": "cloud" - } + client = new_v2_client() + payload = { + "metadata": { + "name": disk_name, + "region": region + }, + "spec": { + "capacity": capacity, + "runtime": "cloud" } - + } + + try: client.create_disk(payload) - while not is_disk_ready(client, disk_name): - time.sleep(3) - + client.poll_disk(disk_name) spinner.text = click.style( 'Disk {} created successfully.'. format(disk_name), fg=Colors.GREEN) diff --git a/riocli/disk/delete.py b/riocli/disk/delete.py index 7ea2b6aa..e16592e9 100644 --- a/riocli/disk/delete.py +++ b/riocli/disk/delete.py @@ -11,30 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools +from queue import Queue + import click from click_help_colors import HelpColorsCommand -from rapyuta_io.utils.rest_client import HttpMethod from yaspin.api import Yaspin -from queue import Queue -import functools from riocli.config import new_v2_client from riocli.constants import Symbols, Colors from riocli.disk.model import Disk from riocli.disk.util import fetch_disks, display_disk_list -from riocli.utils.spinner import with_spinner -from riocli.utils.execute import apply_func_with_result from riocli.utils import tabulate_data - -from rapyuta_io import Client - - -def _apply_delete(client: Client, result: Queue, disk: Disk) -> None: - try: - client.delete_disk(name = disk.metadata.name) - result.put((disk.metadata.name, True)) - except Exception as e: - result.put((disk.metadata.name, False)) +from riocli.utils.execute import apply_func_with_result +from riocli.utils.spinner import with_spinner +from riocli.v2client import Client @click.command( @@ -45,6 +36,8 @@ def _apply_delete(client: Client, result: Queue, disk: Disk) -> None: ) @click.option('--force', '-f', is_flag=True, default=False, help='Skip confirmation', type=bool) +@click.option('-a', '--all', 'delete_all', is_flag=True, default=False, + help='Deletes all deployments in the project') @click.option('--workers', '-w', help="Number of parallel workers while running deleting disks. Defaults to 10", type=int, default=10) @@ -96,14 +89,14 @@ def delete_disk( workers=workers, key=lambda x: x[0] ) data, statuses = [], [] - for name, status in result: + for name, status, msg in result: fg = Colors.GREEN if status else Colors.RED icon = Symbols.SUCCESS if status else Symbols.ERROR statuses.append(status) data.append([ click.style(name, fg), - click.style(icon, fg) + click.style('{} {}'.format(icon, msg), fg) ]) with spinner.hidden(): @@ -121,3 +114,11 @@ def delete_disk( 'Failed to delete disk(s): {}'.format(e), Colors.RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e + + +def _apply_delete(client: Client, result: Queue, disk: Disk) -> None: + try: + client.delete_disk(name=disk.metadata.name) + result.put((disk.metadata.name, True, 'Disk deleted successfully')) + except Exception as e: + result.put((disk.metadata.name, False, str(e))) diff --git a/riocli/disk/enum.py b/riocli/disk/enum.py new file mode 100644 index 00000000..2629ad25 --- /dev/null +++ b/riocli/disk/enum.py @@ -0,0 +1,27 @@ +import enum + + +class DiskCapacity(int, enum.Enum): + """ + Enumeration variables for disk capacity. The type may be one of the following \n + DiskCapacity.GiB_4 \n + DiskCapacity.GiB_8 \n + DiskCapacity.GiB_16 \n + DiskCapacity.GiB_32 \n + DiskCapacity.GiB_64 \n + DiskCapacity.GiB_128 \n + DiskCapacity.GiB_256 \n + DiskCapacity.GiB_512 \n + """ + + def __str__(self): + return str(self.value) + + GiB_4 = 4 + GiB_8 = 8 + GiB_16 = 16 + GiB_32 = 32 + GiB_64 = 64 + GiB_128 = 128 + GiB_256 = 256 + GiB_512 = 512 \ No newline at end of file diff --git a/riocli/disk/list.py b/riocli/disk/list.py index 7b5a8ff2..ad74ad31 100644 --- a/riocli/disk/list.py +++ b/riocli/disk/list.py @@ -17,6 +17,7 @@ from riocli.config import new_v2_client from riocli.disk.util import display_disk_list + @click.command('list') def list_disks() -> None: """ @@ -28,4 +29,4 @@ def list_disks() -> None: display_disk_list(disks, show_header=True) except Exception as e: click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) \ No newline at end of file + raise SystemExit(1) diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 411be2a2..85c13f48 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -11,22 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from os import stat import typing -from time import sleep -from munch import unmunchify -import time import click -from munch import munchify -from rapyuta_io import Client -from rapyuta_io.utils.rest_client import HttpMethod +from munch import unmunchify from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols, Status - +from riocli.constants import Colors, Symbols from riocli.jsonschema.validate import load_schema from riocli.model import Model +from riocli.v2client import Client class Disk(Model): @@ -47,38 +41,20 @@ def find_object(self, client: Client) -> typing.Any: def create_object(self, client: Client, **kwargs) -> typing.Any: v2_client = new_v2_client() - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - self.pop("rc", None) - disk = unmunchify(self) - r = v2_client.create_disk(disk) - + r = v2_client.create_disk(self._sanitize_disk()) retry_count = int(kwargs.get('retry_count')) retry_interval = int(kwargs.get('retry_interval')) try: - r = self.poll_deployment_till_ready( - client = v2_client, - disk = r, - retry_count=retry_count, - sleep_interval=retry_interval, - ) + v2_client.poll_disk(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) except Exception as e: click.secho(">> {}: Error polling for disk ({}:{})".format( Symbols.WARNING, self.kind.lower(), self.metadata.name), fg=Colors.YELLOW) + click.secho(str(e), fg=Colors.YELLOW) return unmunchify(r) - def poll_deployment_till_ready(self, client: Client, disk: typing.Any, retry_count = 50, sleep_interval = 6): - for _ in range(retry_count): - if disk.status.status == Status.AVAILABLE: - return disk - - time.sleep(sleep_interval) - disk = client.get_disk(disk.metadata.name) - return disk - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass @@ -94,3 +70,16 @@ def pre_process(cls, client: Client, d: typing.Dict) -> None: def validate(d): schema = load_schema('disk') schema.validate(d) + + def _sanitize_disk(self) -> typing.Dict: + # Unset createdAt and updatedAt to avoid timestamp parsing issue. + self.metadata.createdAt = None + self.metadata.updatedAt = None + + data = unmunchify(self) + + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + data.pop("rc", None) + + return data diff --git a/riocli/disk/util.py b/riocli/disk/util.py index c55bc881..1c6a7279 100644 --- a/riocli/disk/util.py +++ b/riocli/disk/util.py @@ -14,18 +14,9 @@ import typing import re -from rapyuta_io import Client -from rapyuta_io.clients.persistent_volumes import DiskCapacity, DiskType -from rapyuta_io.utils.rest_client import RestClient, HttpMethod - -from riocli.config import Configuration, new_client -from riocli.constants import Colors, Symbols from riocli.disk.model import Disk from riocli.utils import tabulate_data - -class DiskNotFound(Exception): - def __init__(self): - super().__init__('Disk not found') +from riocli.v2client import Client def fetch_disks( @@ -33,7 +24,6 @@ def fetch_disks( disk_name_or_regex: str, include_all: bool, ) -> typing.List[Disk]: - disks = client.list_disks() if include_all: @@ -46,25 +36,25 @@ def fetch_disks( return result -def is_disk_ready(client: Client , name: str) -> bool: - disk = client.get_disk(name) - return disk.status.get("status", "") == "Available" def display_disk_list(disks: typing.Any, show_header: bool = True): headers = [] if show_header: headers = ( - 'Disk ID', 'Name', 'Status', 'Capacity', - 'Used', 'Available', 'Used By', + 'Disk ID', 'Name', 'Status', 'Capacity (GB)', + 'Capacity Used (GB)', 'Used By', ) - data = [[d.metadata.guid, - d.metadata.name, - d.status.get("status"), - d.spec.capacity, - d.spec.get("CapacityUsed"), - d.spec.get("CapacityAvailable"), - d.get("diskBound", {}).get("DeploymentName")] - for d in disks] + data = [] + + for d in disks: + capacity = d.status.get("capacityUsed", 0) / (1024 * 1204 * 1024) # Bytes -> GB + + data.append([d.metadata.guid, + d.metadata.name, + d.status.get("status"), + d.spec.capacity, + capacity, + d.status.get("diskBound", {}).get("deployment_name")]) - tabulate_data(data, headers) \ No newline at end of file + tabulate_data(data, headers) From 53c1ce3e6c0af818cbf367743ccfb89dcb8320db Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:30:31 +0530 Subject: [PATCH 18/54] fix(deployment): fix deployment commands --- riocli/deployment/delete.py | 17 ++-- riocli/deployment/execute.py | 1 - riocli/deployment/inspect.py | 2 - riocli/deployment/list.py | 23 ++---- riocli/deployment/logs.py | 37 ++------- riocli/deployment/model.py | 150 ++++------------------------------- riocli/deployment/status.py | 1 - riocli/deployment/update.py | 45 ++--------- riocli/deployment/util.py | 32 +++++--- riocli/deployment/wait.py | 2 +- 10 files changed, 65 insertions(+), 245 deletions(-) diff --git a/riocli/deployment/delete.py b/riocli/deployment/delete.py index 88324e6a..515b703c 100644 --- a/riocli/deployment/delete.py +++ b/riocli/deployment/delete.py @@ -11,24 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from queue import Queue - import click from click_help_colors import HelpColorsCommand -from rapyuta_io.clients.deployment import Deployment import functools from queue import Queue - from riocli.config import new_v2_client from riocli.constants import Colors, Symbols +from riocli.deployment.model import Deployment from riocli.deployment.util import fetch_deployments from riocli.deployment.util import print_deployments_for_confirmation from riocli.utils import tabulate_data from riocli.utils.execute import apply_func_with_result from riocli.utils.spinner import with_spinner +from riocli.v2client import Client -from rapyuta_io import Client @click.command( 'delete', @@ -93,13 +90,13 @@ def delete_deployment( ) data, statuses = [], [] - for name, status in result: + for name, status, msg in result: fg = Colors.GREEN if status else Colors.RED icon = Symbols.SUCCESS if status else Symbols.ERROR statuses.append(status) data.append([ click.style(name, fg), - click.style(icon, fg) + click.style('{} {}'.format(icon, msg), fg) ]) with spinner.hidden(): @@ -123,6 +120,6 @@ def delete_deployment( def _apply_delete(client: Client, result: Queue, deployment: Deployment) -> None: try: client.delete_deployment(name=deployment.metadata.name) - result.put((deployment.metadata.name, True)) - except Exception: - result.put((deployment.metadata.name, False)) + result.put((deployment.metadata.name, True, 'Deployment Deleted Successfully')) + except Exception as e: + result.put((deployment.metadata.name, False, str(e))) diff --git a/riocli/deployment/execute.py b/riocli/deployment/execute.py index 32b9b55d..fee280a5 100644 --- a/riocli/deployment/execute.py +++ b/riocli/deployment/execute.py @@ -50,7 +50,6 @@ def execute_command( """ Execute commands on cloud deployment """ - # TODO(Romil): Move to V2 client try: comp_id, exec_id, pod_name = select_details(deployment_guid, component_name, exec_name) diff --git a/riocli/deployment/inspect.py b/riocli/deployment/inspect.py index e9e0f391..56e9a83a 100644 --- a/riocli/deployment/inspect.py +++ b/riocli/deployment/inspect.py @@ -13,12 +13,10 @@ # limitations under the License. import click from click_help_colors import HelpColorsCommand -from rapyuta_io.clients.deployment import Deployment from munch import unmunchify from riocli.config import new_v2_client from riocli.constants import Colors -from riocli.deployment.util import name_to_guid from riocli.utils import inspect_with_format diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index bfdc1cc9..cb79c0b9 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -15,27 +15,15 @@ import click from click_help_colors import HelpColorsCommand -from riocli.v2client.enums import DeploymentPhaseConstants -from rapyuta_io.clients.deployment import Deployment + +from riocli.deployment.model import Deployment +from riocli.deployment.util import ALL_PHASES, DEFAULT_PHASES from riocli.config import new_v2_client from riocli.constants import Colors from riocli.deployment.util import process_deployment_errors from riocli.utils import tabulate_data -ALL_PHASES = [ - DeploymentPhaseConstants.DeploymentPhaseInProgress, - DeploymentPhaseConstants.DeploymentPhaseProvisioning, - DeploymentPhaseConstants.DeploymentPhaseSucceeded, - DeploymentPhaseConstants.DeploymentPhaseStopped, -] - -DEFAULT_PHASES = [ - DeploymentPhaseConstants.DeploymentPhaseInProgress, - DeploymentPhaseConstants.DeploymentPhaseProvisioning, - DeploymentPhaseConstants.DeploymentPhaseSucceeded, -] - @click.command( 'list', @@ -60,7 +48,7 @@ def list_deployments( List the deployments in the selected project """ try: - client = new_v2_client(with_project=True) + client = new_v2_client() deployments = client.list_deployments(query={"phases": phase, "deviceName": device}) deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) display_deployment_list(deployments, show_header=True) @@ -80,7 +68,8 @@ def display_deployment_list( data = [] for deployment in deployments: - package_name_version = "{} ({})".format(deployment.metadata.depends.nameOrGUID, deployment.metadata.depends.version) + package_name_version = "{} ({})".format(deployment.metadata.depends.nameOrGUID, + deployment.metadata.depends.version) phase = deployment.status.phase if deployment.status else "" data.append([deployment.metadata.guid, deployment.metadata.name, phase, package_name_version, deployment.metadata.createdAt, deployment.metadata.get('deletedAt')]) diff --git a/riocli/deployment/logs.py b/riocli/deployment/logs.py index 2133b041..3dd33ee0 100644 --- a/riocli/deployment/logs.py +++ b/riocli/deployment/logs.py @@ -16,11 +16,8 @@ import click from click_help_colors import HelpColorsCommand -from riocli.config import Configuration +from riocli.config import new_v2_client from riocli.constants import Colors -from riocli.deployment.util import name_to_guid, select_details - -_LOG_URL_FORMAT = '{}/deployment/logstream?tailLines={}&deploymentId={}&componentId={}&executableId={}&podName={}' @click.command( @@ -29,44 +26,22 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -@click.option('--component', 'component_name', default=None, - help='Name of the component in the deployment') +@click.option('--replica', 'replica', default=None, + help='Replica identifier of the deployment') @click.option('--exec', 'exec_name', default=None, help='Name of a executable in the component') @click.argument('deployment-name', type=str) -@name_to_guid def deployment_logs( - component_name: str, + replica: str, exec_name: str, deployment_name: str, - deployment_guid: str, ) -> None: """ Stream live logs from cloud deployments (not supported for device deployments) """ - # TODO(Romil): Move to V2 client try: - comp_id, exec_id, pod_name = select_details(deployment_guid, component_name, exec_name) - stream_deployment_logs(deployment_guid, comp_id, exec_id, pod_name) + client = new_v2_client() + client.stream_deployment_logs(deployment_name, exec_name, replica) except Exception as e: click.secho(e, fg=Colors.RED) raise SystemExit(1) - - -def stream_deployment_logs(deployment_id, component_id, exec_id, pod_name=None): - # FIXME(ankit): The Upstream API ends up timing out when there is no log being written. - # IMO the correct behaviour should be to not timeout and keep the stream open. - config = Configuration() - - url = get_log_stream_url(config, deployment_id, component_id, exec_id, pod_name) - auth = config.get_auth_header() - curl = 'curl -H "project: {}" -H "Authorization: {}" "{}"'.format( - auth['project'], auth['Authorization'], url) - click.echo(click.style(curl, fg=Colors.BLUE, italic=True)) - - os.system(curl) - - -def get_log_stream_url(config, deployment_id, component_id, exec_id, pod_name=None, tail=50000): - catalog_host = config.data.get('catalog_host', 'https://gacatalog.apps.okd4v2.prod.rapyuta.io') - return _LOG_URL_FORMAT.format(catalog_host, tail, deployment_id, component_id, exec_id, pod_name) diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index 693fa51c..82998362 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -14,27 +14,14 @@ import typing from munch import unmunchify -from rapyuta_io import Client -from rapyuta_io.clients.catalog_client import Package -from rapyuta_io.clients.deployment import DeploymentNotRunningException -from rapyuta_io.clients.package import ProvisionConfiguration, RestartPolicy -from rapyuta_io.clients.rosbag import (OverrideOptions, ROSBagCompression, ROSBagJob, ROSBagOnDemandUploadOptions, - ROSBagOptions, ROSBagTimeRange, ROSBagUploadTypes, TopicOverrideInfo, - UploadOptions) from riocli.config import new_v2_client from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.static_route.util import find_static_route_guid +from riocli.v2client import Client class Deployment(Model): - RESTART_POLICY = { - 'always': RestartPolicy.Always, - 'never': RestartPolicy.Never, - 'onfailure': RestartPolicy.OnFailure - } - def __init__(self, *args, **kwargs): self.update(*args, **kwargs) @@ -46,11 +33,8 @@ def find_object(self, client: Client) -> typing.Any: def create_object(self, client: Client, **kwargs) -> typing.Any: client = new_v2_client() - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - deployment = unmunchify(self) - deployment.pop("rc", None) - r = client.create_deployment(deployment) + r = client.create_deployment(self._sanitize_deployment()) + return unmunchify(r) def update_object(self, client: Client, obj: typing.Any) -> typing.Any: pass @@ -63,121 +47,6 @@ def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: def pre_process(cls, client: Client, d: typing.Dict) -> None: pass - def _get_provision_config(self, client: Client, pkg: Package): - comp_name = pkg['plans']['components'][0]['name'] - prov_config = pkg.get_provision_configuration() - self._configure_static_routes(client, prov_config, comp_name) - - return prov_config - - def _configure_networks(self, client: Client, prov_config: ProvisionConfiguration): - if not self.spec.get('rosNetworks'): - return - - native_networks = client.list_native_networks() - routed_networks = client.get_all_routed_networks() - - def _configure_disks(self, client: Client, prov_config: ProvisionConfiguration, component: str): - if not self.spec.get('volumes'): - return - - # for volume in self.spec.volumes: - # # TODO: no support for the disk resource. - # # TODO: subpath is not there in the SDK. - # prov_config.mount_volume(component_name=component, volume='', mount_path=volume.mountPath) - - def _configure_static_routes(self, client: Client, prov_config: ProvisionConfiguration, component: str): - if not self.spec.get('staticRoutes'): - return - - # TODO: List instead of get calls again and again - - for route in self.spec.staticRoutes: - name = route.depends.nameOrGUID - if name.startswith('staticroute-'): - guid = name - else: - guid = find_static_route_guid(client, name) - static_route = client.get_static_route(route_guid=guid) - prov_config.add_static_route( - component_name=component, endpoint_name=route.name, static_route=static_route) - - def _form_rosbag_job(self, req_job): - rosbag_job_kw_args = { - 'name': req_job.name, - 'rosbag_options': ROSBagOptions( - all_topics=req_job.recordOptions.get('allTopics'), - topics=req_job.recordOptions.get('topics'), - topic_include_regex=req_job.recordOptions.get('topicIncludeRegex'), - topic_exclude_regex=req_job.recordOptions.get('topicExcludeRegex'), - max_message_count=req_job.recordOptions.get('maxMessageCount'), - node=req_job.recordOptions.get('node'), - compression=ROSBagCompression(req_job.recordOptions.compression) if hasattr( - req_job.recordOptions, 'compression' - ) else None, - max_splits=req_job.recordOptions.get('maxSplits'), - max_split_size=req_job.recordOptions.get('maxSplitSize'), - chunk_size=req_job.recordOptions.get('chunkSize'), - max_split_duration=req_job.recordOptions.get('maxSplitDuration') - )} - - if 'uploadOptions' in req_job: - rosbag_job_kw_args['upload_options'] = self._form_rosbag_upload_options(req_job.uploadOptions) - - if 'overrideOptions' in req_job: - rosbag_job_kw_args['override_options'] = self._form_rosbag_override_options( - req_job.overrideOptions) - - return ROSBagJob(**rosbag_job_kw_args) - - @staticmethod - def _form_rosbag_upload_options(upload_options): - upload_options_kw_args = { - 'max_upload_rate': upload_options.maxUploadRate, - 'upload_type': ROSBagUploadTypes(upload_options.uploadType), - } - - if 'purgeAfter' in upload_options: - upload_options_kw_args['purge_after'] = upload_options.purgeAfter - - if 'onDemandOpts' in upload_options: - time_range = ROSBagTimeRange( - from_time=upload_options.onDemandOpts.timeRange['from'], - to_time=upload_options.onDemandOpts.timeRange['to'] - ) - - upload_options_kw_args['on_demand_options'] = ROSBagOnDemandUploadOptions(time_range) - - return UploadOptions(**upload_options_kw_args) - - @staticmethod - def _form_rosbag_override_options(override_options): - override_options_kw_args = {} - - if 'topicOverrideInfo' in override_options: - override_infos = [] - for info in override_options.topicOverrideInfo: - topic_override_info_kw_args = { - 'topic_name': info.topicName - } - - if 'recordFrequency' in info: - topic_override_info_kw_args['record_frequency'] = info.recordFrequency - - if 'latched' in info: - topic_override_info_kw_args['latched'] = info.latched - - override_info = TopicOverrideInfo(**topic_override_info_kw_args) - - override_infos.append(override_info) - - override_options_kw_args['topic_override_info'] = override_infos - - if 'excludeTopics' in override_options: - override_options_kw_args['exclude_topics'] = override_options.excludeTopics - - return OverrideOptions(**override_options_kw_args) - @staticmethod def validate(data): """ @@ -185,3 +54,16 @@ def validate(data): """ schema = load_schema('deployment') schema.validate(data) + + def _sanitize_deployment(self) -> typing.Dict: + # Unset createdAt and updatedAt to avoid timestamp parsing issue. + self.metadata.createdAt = None + self.metadata.updatedAt = None + + data = unmunchify(self) + + # convert to a dict and remove the ResolverCache + # field since it's not JSON serializable + data.pop("rc", None) + + return data diff --git a/riocli/deployment/status.py b/riocli/deployment/status.py index d5636876..e41bacaa 100644 --- a/riocli/deployment/status.py +++ b/riocli/deployment/status.py @@ -16,7 +16,6 @@ from riocli.config import new_v2_client from riocli.constants import Colors -from riocli.deployment.util import name_to_guid @click.command( diff --git a/riocli/deployment/update.py b/riocli/deployment/update.py index 5f9a78ae..8a524e28 100644 --- a/riocli/deployment/update.py +++ b/riocli/deployment/update.py @@ -17,17 +17,17 @@ import click from click_help_colors import HelpColorsCommand -from rapyuta_io import Client -from rapyuta_io.clients.deployment import Deployment from yaspin.api import Yaspin from riocli.config import new_v2_client from riocli.constants import Symbols, Colors +from riocli.deployment.model import Deployment from riocli.deployment.util import fetch_deployments from riocli.deployment.util import print_deployments_for_confirmation from riocli.utils import tabulate_data from riocli.utils.execute import apply_func_with_result from riocli.utils.spinner import with_spinner +from riocli.v2client import Client @click.command( @@ -93,13 +93,13 @@ def update_deployment( ) data, fg, statuses = [], Colors.GREEN, [] - for name, status in result: + for name, status, msg in result: fg = Colors.GREEN if status else Colors.RED icon = Symbols.SUCCESS if status else Symbols.ERROR statuses.append(status) data.append([ click.style(name, fg), - click.style(icon, fg) + click.style('{} {}'.format(icon, msg), fg) ]) with spinner.hidden(): @@ -120,37 +120,6 @@ def update_deployment( raise SystemExit(1) from e -def get_component_context(component_info) -> dict: - result = {} - - for component in component_info: - comp = {} - executables = [] - exec_metadata = component.get("executableMetaData", []) or [] - - for e in exec_metadata: - # Component will be considered only if any of its executables is - # docker - if not (e.get("docker")): - continue - - executable = {} - - if e.get("docker"): - executable["docker"] = e["docker"] - - executable["id"] = e.get("id", "") - executable["name"] = e.get("name", "") - executables.append(executable) - - if len(executables) > 0: - result[component["componentID"]] = comp - comp["component"] = {"executables": executables} - comp["update_deployment"] = True - - return result - - def _apply_update( client: Client, result: Queue, @@ -162,6 +131,6 @@ def _apply_update( result.put((deployment.metadata.name, False)) return client.update_deployment(deployment.metadata.name, deployment) - result.put((deployment.metadata.name, True)) - except Exception: - result.put((deployment.metadata.name, False)) + result.put((deployment.metadata.name, True, 'Deployment Updated Successfully')) + except Exception as e: + result.put((deployment.metadata.name, False, str(e))) diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index 0781f0b9..c772f448 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -11,25 +11,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import copy import functools import re import typing from typing import List import click -from rapyuta_io import Client, DeploymentPhaseConstants -from rapyuta_io.clients import Device -from rapyuta_io.clients.deployment import Deployment -from rapyuta_io.clients.package import ExecutableMount -from rapyuta_io.utils import InvalidParameterException, OperationNotAllowedError -from rapyuta_io.utils.constants import DEVICE_ID from riocli.config import new_client from riocli.constants import Colors -from riocli.deployment.errors import ERRORS +from riocli.deployment.model import Deployment from riocli.utils import tabulate_data from riocli.utils.selector import show_selection +from riocli.v2client import Client +from riocli.v2client.enums import DeploymentPhaseConstants +from riocli.deployment.errors import ERRORS + +ALL_PHASES = [ + DeploymentPhaseConstants.DeploymentPhaseInProgress, + DeploymentPhaseConstants.DeploymentPhaseProvisioning, + DeploymentPhaseConstants.DeploymentPhaseSucceeded, + DeploymentPhaseConstants.DeploymentPhaseStopped, +] + +DEFAULT_PHASES = [ + DeploymentPhaseConstants.DeploymentPhaseInProgress, + DeploymentPhaseConstants.DeploymentPhaseProvisioning, + DeploymentPhaseConstants.DeploymentPhaseSucceeded, +] def name_to_guid(f: typing.Callable) -> typing.Callable: @@ -122,12 +131,13 @@ def __init__(self, message='deployment not found!'): self.message = message super().__init__(self.message) + def fetch_deployments( client: Client, deployment_name_or_regex: str, include_all: bool, ) -> List[Deployment]: - deployments = client.list_deployments() + deployments = client.list_deployments(query={"phases": DEFAULT_PHASES}) result = [] for deployment in deployments: if (include_all or deployment_name_or_regex == deployment.metadata.name or @@ -144,7 +154,9 @@ def print_deployments_for_confirmation(deployments: List[Deployment]): data = [] for deployment in deployments: - data.append([deployment.metadata.name, deployment.metadata.guid, deployment.status.phase, deployment.status.status]) + data.append( + [deployment.metadata.name, deployment.metadata.guid, deployment.status.phase, + deployment.status.aggregateStatus]) tabulate_data(data, headers) diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index 5579ee2c..5f9af5d6 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -38,7 +38,7 @@ def wait_for_deployment( try: client = new_v2_client() deployment = client.poll_deployment(deployment_name) - spinner.text = click.style('Phase: Succeeded Status: {}'.format(deployment.status.status), fg=Colors.GREEN) + spinner.text = click.style('Phase: Succeeded Status: {}'.format(deployment.status.aggregateStatus), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except RetriesExhausted as e: spinner.write(click.style(str(e), fg=Colors.RED)) From 1fd0546bbab958f76bc897f57fa43a4c93f30fea Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:32:18 +0530 Subject: [PATCH 19/54] feat(v2client): adds poll methods and stream deployment logs method --- riocli/v2client/client.py | 78 ++++++++++++++++++++++++++++++++++++++- riocli/v2client/enums.py | 18 ++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index dfbf1ced..2e82e234 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -20,12 +20,14 @@ from hashlib import md5 from typing import List, Optional, Dict, Any +import click import magic import requests from munch import munchify, Munch from rapyuta_io.utils.rest_client import HttpMethod, RestClient -from riocli.v2client.enums import DeploymentPhaseConstants +from riocli.constants import Colors +from riocli.v2client.enums import DeploymentPhaseConstants, DiskStatusConstants from riocli.v2client.error import (RetriesExhausted, DeploymentNotRunning, ImagePullError, NetworkNotFound) @@ -908,6 +910,40 @@ def delete_network(self, network_name: str, return munchify(data) + def poll_network( + self, + name: str, + retry_count: int = 50, + sleep_interval: int = 6, + ready_phases: List[str] = None, + ) -> Munch: + if ready_phases is None: + ready_phases = [] + + network = self.get_network(name) + + status = network.status + + for _ in range(retry_count): + if status.phase in ready_phases: + return network + + if status.phase == DeploymentPhaseConstants.DeploymentPhaseProvisioning.value: + errors = status.get('error_codes', []) + if 'DEP_E153' in errors: # DEP_E153 (image-pull error) will persist across retries + raise ImagePullError('Network not running. Phase: Provisioning Status: {}'.format(status.phase)) + elif status.phase == DeploymentPhaseConstants.DeploymentPhaseSucceeded.value: + return network + elif status.phase == DeploymentPhaseConstants.DeploymentPhaseStopped.value: + raise DeploymentNotRunning('Network not running. Phase: Stopped Status: {}'.format(status.phase)) + + time.sleep(sleep_interval) + network = self.get_network(name) + status = network.status + + raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Network status: {}'.format( + retry_count, sleep_interval, status.phase)) + def list_deployments( self, query: dict = None @@ -1050,6 +1086,21 @@ def poll_deployment( raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Deployment status: {}'.format( retry_count, sleep_interval, status.phase)) + def stream_deployment_logs( + self, + name: str, + executable: str, + replica: int = 0, + ): + url = "{}/v2/deployments/{}/logs/?replica={}&executable={}".format(self._host, name, replica, executable) + headers = self._get_auth_header() + + curl = 'curl -H "project: {}" -H "Authorization: {}" "{}"'.format( + headers['project'], headers['Authorization'], url) + click.echo(click.style(curl, fg=Colors.BLUE, italic=True)) + + os.system(curl) + def list_disks( self, query: dict = None @@ -1134,3 +1185,28 @@ def delete_disk(self, name: str) -> Munch: raise Exception("disks: {}".format(err_msg)) return munchify(data) + + def poll_disk( + self, + name: str, + retry_count: int = 50, + sleep_interval: int = 6, + ) -> Munch: + disk = self.get_disk(name) + + status = disk.status + + for _ in range(retry_count): + if status.status in [DiskStatusConstants.DiskStatusAvailable.value, + DiskStatusConstants.DiskStatusReleased.value, + DiskStatusConstants.DiskStatusBound.value]: + return disk + elif status.status == DiskStatusConstants.DiskStatusFailed.value: + raise DeploymentNotRunning('Disk not running. Status: {}'.format(status.status)) + + time.sleep(sleep_interval) + disk = self.get_disk(name) + status = disk.status + + raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Disk status: {}'.format( + retry_count, sleep_interval, status.status)) diff --git a/riocli/v2client/enums.py b/riocli/v2client/enums.py index 90166f18..6fe68ef9 100644 --- a/riocli/v2client/enums.py +++ b/riocli/v2client/enums.py @@ -28,4 +28,20 @@ def __str__(self): DeploymentStatusPending = "Pending" DeploymentStatusError = "Error" DeploymentStatusUnknown = "Unknown" - DeploymentStatusStopped = "Stopped" \ No newline at end of file + DeploymentStatusStopped = "Stopped" + + +class DiskStatusConstants(str, enum.Enum): + """ + Enumeration variables for the deployment status + + """ + + def __str__(self): + return str(self.value) + + DiskStatusAvailable = "Available" + DiskStatusBound = "Bound" + DiskStatusReleased = "Released" + DiskStatusFailed = "Failed" + DiskStatusPending = "Pending" From f35c2db0590ab1b88edbd91f341939c274007ead Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 02:32:52 +0530 Subject: [PATCH 20/54] fix: fixes resolver and resource models to work with apply This commit contains miscellaneous bug fixes in order make the apply command work properly for different resources. --- riocli/apply/parse.py | 5 --- riocli/apply/resolver.py | 2 +- riocli/deployment/execute.py | 4 +-- riocli/deployment/util.py | 60 ------------------------------------ riocli/disk/model.py | 7 ++--- riocli/network/model.py | 12 ++------ riocli/package/model.py | 5 +-- riocli/v2client/client.py | 7 ++--- 8 files changed, 12 insertions(+), 90 deletions(-) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 5a746880..ffa70cbf 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -319,13 +319,8 @@ def _resolve_dependency(self, dependent_key, dependency): self._add_remote_object_to_resolve_tree( dependent_key, obj_guid, dependency, obj) -<<<<<<< HEAD - if (name_or_guid == obj_name) and ( - 'version' in dependency and obj['packageVersion'] == dependency.get('version')): -======= if (name_or_guid == obj_name) and ('version' in dependency and obj.metadata.version == dependency.get('version')): ->>>>>>> 52515c9 (feat(deployment): uses v2 deployments APIs) self._add_remote_object_to_resolve_tree( dependent_key, obj_guid, dependency, obj) diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index f20ede66..4f54e89a 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -157,7 +157,7 @@ def _find_functors(self, kind): "network": lambda name, obj_list, network_type: filter( lambda x: name == x.metadata.name and network_type == x.spec.type, obj_list), "deployment": lambda name, obj_list: filter( - lambda x: name == x.metadata.name, obj_list), + lambda x: name == x.metadata.name and not x.metadata.get('deletedAt'), obj_list), "disk": lambda name, obj_list: filter( lambda x: name == x.metadata.name, obj_list), "device": self._generate_find_guid_functor(), diff --git a/riocli/deployment/execute.py b/riocli/deployment/execute.py index fee280a5..5efd2332 100644 --- a/riocli/deployment/execute.py +++ b/riocli/deployment/execute.py @@ -23,7 +23,7 @@ from riocli.utils.spinner import DummySpinner as Spinner from riocli.constants import Colors -from riocli.deployment.util import name_to_guid, select_details +from riocli.deployment.util import select_details from riocli.utils.execute import run_on_cloud @@ -39,7 +39,7 @@ help='Name of a executable in the component') @click.argument('deployment-name', type=str) @click.argument('command', nargs=-1) -@name_to_guid +# @name_to_guid def execute_command( component_name: str, exec_name: str, diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index c772f448..d060c7b5 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -13,13 +13,9 @@ # limitations under the License. import functools import re -import typing from typing import List -import click - from riocli.config import new_client -from riocli.constants import Colors from riocli.deployment.model import Deployment from riocli.utils import tabulate_data from riocli.utils.selector import show_selection @@ -41,56 +37,6 @@ ] -def name_to_guid(f: typing.Callable) -> typing.Callable: - @functools.wraps(f) - def decorated(**kwargs: typing.Any) -> None: - try: - client = new_client() - except Exception as e: - click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) from e - - name = kwargs.pop('deployment_name') - guid = None - - if name.startswith('dep-'): - guid = name - name = None - - try: - if name is None: - name = get_deployment_name(client, guid) - - if guid is None: - guid = find_deployment_guid(client, name) - except Exception as e: - click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) from e - - kwargs['deployment_name'] = name - kwargs['deployment_guid'] = guid - f(**kwargs) - - return decorated - - -def get_deployment_name(client: Client, guid: str) -> str: - deployment = client.get_deployment(guid) - return deployment.name - - -def find_deployment_guid(client: Client, name: str) -> str: - find_func = functools.partial(client.get_all_deployments, - phases=[DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.PROVISIONING]) - deployments = find_func() - for deployment in deployments: - if deployment.name == name: - return deployment.deploymentId - - raise DeploymentNotFound() - - def select_details(deployment_guid, component_name=None, exec_name=None) -> (str, str, str): client = new_client() deployment = client.get_deployment(deployment_guid) @@ -126,12 +72,6 @@ def select_details(deployment_guid, component_name=None, exec_name=None) -> (str return selected_component.componentID, exec_meta.id, pod_name -class DeploymentNotFound(Exception): - def __init__(self, message='deployment not found!'): - self.message = message - super().__init__(self.message) - - def fetch_deployments( client: Client, deployment_name_or_regex: str, diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 85c13f48..dc61fecc 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -28,15 +28,12 @@ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def find_object(self, client: Client) -> typing.Any: - _, disk = self.rc.find_depends({ + guid, disk = self.rc.find_depends({ 'kind': 'disk', 'nameOrGUID': self.metadata.name }) - if not disk: - return False - - return disk + return disk if guid else False def create_object(self, client: Client, **kwargs) -> typing.Any: v2_client = new_v2_client() diff --git a/riocli/network/model.py b/riocli/network/model.py index 5a35a873..4f41242e 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -29,15 +29,9 @@ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def find_object(self, client: Client) -> bool: - try: - network, obj = self.rc.find_depends({"kind": self.kind.lower(), - "nameOrGUID": self.metadata.name}, self.spec.type) - if not network: - return False - - return obj - except NetworkNotFound: - return False + network, obj = self.rc.find_depends({"kind": self.kind.lower(), + "nameOrGUID": self.metadata.name}, self.spec.type) + return obj if network else False def create_object(self, client: Client, **kwargs) -> typing.Any: client = new_v2_client() diff --git a/riocli/package/model.py b/riocli/package/model.py index 2a454089..912424e3 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -35,10 +35,7 @@ def __init__(self, *args, **kwargs): def find_object(self, client: Client): guid, obj = self.rc.find_depends({"kind": self.kind.lower(), "nameOrGUID": self.metadata.name}, self.metadata.version) - if not guid: - return False - - return obj + return obj if guid else False def create_object(self, client: Client, **kwargs) -> typing.Any: v2_client = new_v2_client() diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 2e82e234..75859f3c 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -29,7 +29,7 @@ from riocli.constants import Colors from riocli.v2client.enums import DeploymentPhaseConstants, DiskStatusConstants from riocli.v2client.error import (RetriesExhausted, DeploymentNotRunning, ImagePullError, - NetworkNotFound) + NetworkNotFound, DeploymentNotFound) def handle_server_errors(response: requests.Response): @@ -880,7 +880,7 @@ def get_network( data = json.loads(response.text) if response.status_code == http.HTTPStatus.NOT_FOUND: - raise NetworkNotFound() + raise NetworkNotFound("network: {} not found".format(name)) if not response.ok: err_msg = data.get('error') @@ -956,7 +956,6 @@ def list_deployments( params = { "continue": 0, - "limit": 100, } params.update(query or {}) result = [] @@ -1010,7 +1009,7 @@ def get_deployment( data = json.loads(response.text) if response.status_code == http.HTTPStatus.NOT_FOUND: - raise Exception("deployment: {} not found".format(name)) + raise DeploymentNotFound("deployment: {} not found".format(name)) if not response.ok: err_msg = data.get('error') From 966dbf2749ab0ef6227f81d3775ec13cf151a3de Mon Sep 17 00:00:00 2001 From: Smruti Ranjan Senapati Date: Mon, 22 Jul 2024 21:44:26 +0530 Subject: [PATCH 21/54] fix(v2client): fix retry exceeded error code description and action --- riocli/package/util.py | 2 +- riocli/rosbag/job.py | 10 +++---- riocli/v2client/client.py | 15 ++++++---- riocli/{deployment => v2client}/errors.py | 0 riocli/v2client/util.py | 36 +++++++++++++++++++++++ 5 files changed, 52 insertions(+), 11 deletions(-) rename riocli/{deployment => v2client}/errors.py (100%) create mode 100644 riocli/v2client/util.py diff --git a/riocli/package/util.py b/riocli/package/util.py index b6527d35..b959ce08 100644 --- a/riocli/package/util.py +++ b/riocli/package/util.py @@ -26,7 +26,7 @@ def find_package(client: Client, package_name: str, package_version: str, - ) -> Munch | None: + ) -> Munch: package_obj = None diff --git a/riocli/rosbag/job.py b/riocli/rosbag/job.py index 37e0a999..a53398f8 100644 --- a/riocli/rosbag/job.py +++ b/riocli/rosbag/job.py @@ -21,7 +21,7 @@ ROSBagOnDemandUploadOptions, ROSBagTimeRange from riocli.config import new_client -from riocli.deployment.util import name_to_guid as deployment_name_to_guid +# from riocli.deployment.util import name_to_guid as deployment_name_to_guid from riocli.rosbag.util import ROSBagJobNotFound from riocli.utils import inspect_with_format from riocli.utils import tabulate_data @@ -103,7 +103,7 @@ def job_inspect(job_guid: str, format_type: str) -> None: @click.argument('deployment-name') @click.option('--component-instance-ids', help='Filter by component instance ids ', multiple=True) @click.option('--guids', help='Filter by job guids', multiple=True) -@deployment_name_to_guid +# @deployment_name_to_guid def job_stop(deployment_guid: str, deployment_name: str, component_instance_ids: typing.List[str], guids: typing.List[str]) -> None: """ @@ -126,7 +126,7 @@ def job_stop(deployment_guid: str, deployment_name: str, component_instance_ids: @click.option('--guids', help='Filter by job guids ', multiple=True) @click.option('--statuses', help='Filter by rosbag job statuses ', multiple=True, default=['Starting', 'Running'], type=click.Choice(['Starting', 'Running', 'Error', 'Stopping', 'Stopped'], case_sensitive=True)) -@deployment_name_to_guid +# @deployment_name_to_guid def job_list(deployment_guid: str, deployment_name: str, component_instance_ids: typing.List[str], guids: typing.List[str], statuses: typing.List[str]) -> None: """ @@ -153,7 +153,7 @@ def job_list(deployment_guid: str, deployment_name: str, 'format (1985-04-12T23:20:50.52Z)', required=True) @click.option('--upload-to', help='Rosbags recorded before or at this time are uploaded. Specify time in RFC 3339 ' 'format (1985-04-12T23:20:50.52Z)', required=True) -@deployment_name_to_guid +# @deployment_name_to_guid def job_trigger_upload(deployment_guid: str, deployment_name: str, job_guid: str, upload_from: str, upload_to: str) -> None: """ @@ -224,7 +224,7 @@ def job_trigger_upload(deployment_guid: str, deployment_name: str, job_guid: str @click.argument('job-guid') @click.option('--upload-mode', help='Change upload mode', type=click.Choice([t for t in ROSBagUploadTypes]), required=True) -@deployment_name_to_guid +# @deployment_name_to_guid def update_job(deployment_guid: str, deployment_name: str, job_guid: str, upload_mode: str) -> None: """ Update the Rosbag Job diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 75859f3c..dcbd559d 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -30,6 +30,7 @@ from riocli.v2client.enums import DeploymentPhaseConstants, DiskStatusConstants from riocli.v2client.error import (RetriesExhausted, DeploymentNotRunning, ImagePullError, NetworkNotFound, DeploymentNotFound) +from riocli.v2client.util import process_errors def handle_server_errors(response: requests.Response): @@ -941,8 +942,10 @@ def poll_network( network = self.get_network(name) status = network.status - raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Network status: {}'.format( - retry_count, sleep_interval, status.phase)) + msg = 'Retries exhausted: Tried {} times with {}s interval. Network: phase={} status={} \n{}'.format( + retry_count, sleep_interval, status.phase, status.status, process_errors(status.get('error_codes', []))) + + raise RetriesExhausted(msg) def list_deployments( self, @@ -1082,8 +1085,10 @@ def poll_deployment( deployment = self.get_deployment(name) status = deployment.status - raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Deployment status: {}'.format( - retry_count, sleep_interval, status.phase)) + msg = 'Retries exhausted: Tried {} times with {}s interval. Deployment: phase={} status={} \n{}'.format( + retry_count, sleep_interval, status.phase, status.status, process_errors(status.get('error_codes', []))) + + raise RetriesExhausted(msg) def stream_deployment_logs( self, @@ -1207,5 +1212,5 @@ def poll_disk( disk = self.get_disk(name) status = disk.status - raise RetriesExhausted('Retried {} time done with an interval of {} seconds. Disk status: {}'.format( + raise RetriesExhausted('Retries exhausted: Tried {} times with {}s interval. Disk: status={}'.format( retry_count, sleep_interval, status.status)) diff --git a/riocli/deployment/errors.py b/riocli/v2client/errors.py similarity index 100% rename from riocli/deployment/errors.py rename to riocli/v2client/errors.py diff --git a/riocli/v2client/util.py b/riocli/v2client/util.py new file mode 100644 index 00000000..23a2aa64 --- /dev/null +++ b/riocli/v2client/util.py @@ -0,0 +1,36 @@ +import typing + +import click + +from riocli.constants import Colors +from riocli.v2client.errors import ERRORS + + +def process_errors(errors: typing.List[str]) -> str: + err_fmt = '[{}] {}\nAction: {}' + support_action = ('Report the issue together with the relevant' + ' details to the support team') + + action, description = '', '' + msgs = [] + for code in errors: + if code in ERRORS: + description = ERRORS[code]['description'] + action = ERRORS[code]['action'] + elif code.startswith('DEP_E2'): + description = 'Internal rapyuta.io error in the components deployed on cloud' + action = support_action + elif code.startswith('DEP_E3'): + description = 'Internal rapyuta.io error in the components deployed on a device' + action = support_action + elif code.startswith('DEP_E4'): + description = 'Internal rapyuta.io error' + action = support_action + + code = click.style(code, fg=Colors.YELLOW) + description = click.style(description, fg=Colors.RED) + action = click.style(action, fg=Colors.GREEN) + + msgs.append(err_fmt.format(code, description, action)) + + return '\n'.join(msgs) \ No newline at end of file From dbab2dac4279061f34a2ee4d1470ceb53ba1d2c8 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Wed, 24 Jul 2024 13:29:10 +0530 Subject: [PATCH 22/54] fix(jsonschema): remove interface from deployment rosNetworks Wrike Ticket: https://www.wrike.com/open.htm?id=1454117722 --- riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml | 4 +--- riocli/apply/manifests/deployment-ros-device-rosbag.yaml | 2 -- riocli/apply/manifests/deployment.yaml | 4 ---- riocli/jsonschema/schemas/deployment-schema.yaml | 2 -- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml index de9c8210..e334cba6 100644 --- a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml @@ -36,8 +36,6 @@ spec: - depends: kind: network nameOrGUID: "network-native-device" - interface: eth0 - depends: kind: network - nameOrGUID: "network-routed-device" - interface: eth0 \ No newline at end of file + nameOrGUID: "network-routed-device" \ No newline at end of file diff --git a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml index e4bee65f..3a9bdda1 100644 --- a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml @@ -31,11 +31,9 @@ spec: - depends: kind: network nameOrGUID: "network-native-device" - interface: eth0 - depends: kind: network nameOrGUID: "network-routed-device" - interface: eth0 rosBagJobs: - name: "testbag1" # Required recordOptions: # Required diff --git a/riocli/apply/manifests/deployment.yaml b/riocli/apply/manifests/deployment.yaml index 250b9008..3d63e754 100644 --- a/riocli/apply/manifests/deployment.yaml +++ b/riocli/apply/manifests/deployment.yaml @@ -173,11 +173,9 @@ spec: - depends: kind: network nameOrGUID: "network-native-device" - interface: eth0 - depends: kind: network nameOrGUID: "network-routed-device" - interface: eth0 --- apiVersion: apiextensions.rapyuta.io/v1 kind: Deployment @@ -211,11 +209,9 @@ spec: - depends: kind: network nameOrGUID: "network-native-device" - interface: eth0 - depends: kind: network nameOrGUID: "network-routed-device" - interface: eth0 rosBagJobs: - name: "testbag1" # Required recordOptions: # Required diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index 1d3fd435..9a161a8f 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -55,8 +55,6 @@ definitions: properties: depends: "$ref": "#/definitions/networkDepends" - interface: - type: string cloudNetworkAttachSpec: properties: From b14a72d64d0eea80aff8e9f77cf87748548d92ae Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 27 Jun 2024 23:13:53 +0530 Subject: [PATCH 23/54] refactor(v2client): update list* functions to use _walk_pages helper (cherry picked from commit 6812a24ce18269c78a3a7400ed4c3ab0f8f6471b) --- riocli/v2client/client.py | 99 +++++++++------------------------------ 1 file changed, 21 insertions(+), 78 deletions(-) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index dcbd559d..986819a7 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -82,7 +82,7 @@ def _get_auth_header(self: Client, with_project: bool = True) -> dict: def list_projects( self, organization_guid: str = None, - query: dict = None + query: dict = None, ) -> Munch: """ List all projects in an organization @@ -697,9 +697,11 @@ def delete_key_in_revision(self, tree_name: str, rev_id: str, key: str) -> Munch return munchify(data) - def _walk_pages(self, c: RestClient, params: dict = {}, limit: Optional[int] = None) -> Munch: + def _walk_pages(self, c: RestClient, params: dict = None, limit: Optional[int] = None) -> Munch: offset, result = 0, [] + params = params or {} + if limit is not None: params["limit"] = limit @@ -731,26 +733,8 @@ def list_packages( url = "{}/v2/packages/".format(self._host) headers = self._get_auth_header() - params = {} - params.update(query or {}) - offset, result = 0, [] - while True: - params.update({ - "continue": offset, - }) - response = RestClient(url).method(HttpMethod.GET).query_param( - params).headers(headers).execute() - data = json.loads(response.text) - if not response.ok: - err_msg = data.get('error') - raise Exception("packages: {}".format(err_msg)) - packages = data.get('items', []) - if not packages: - break - offset = data['metadata']['continue'] - result.extend(packages) - - return munchify(result) + client = RestClient(url).method(HttpMethod.GET).headers(headers) + return self._walk_pages(client, params=query) def create_package(self, payload: dict) -> Munch: """ @@ -772,7 +756,7 @@ def create_package(self, payload: dict) -> Munch: def get_package( self, name: str, - query: dict = None + query: dict = None, ) -> Munch: """ List all packages in a project @@ -816,7 +800,7 @@ def delete_package(self, package_name: str, def list_networks( self, - query: dict = None + query: dict = None, ) -> Munch: """ List all networks in a project @@ -865,7 +849,7 @@ def create_network(self, payload: dict) -> Munch: def get_network( self, name: str, - query: dict = None + query: dict = None, ) -> Munch: """ get a network in a project @@ -889,8 +873,11 @@ def get_network( return munchify(data) - def delete_network(self, network_name: str, - query: dict = None) -> Munch: + def delete_network( + self, + network_name: str, + query: dict = None, + ) -> Munch: """ Delete a secret """ @@ -957,26 +944,8 @@ def list_deployments( url = "{}/v2/deployments/".format(self._host) headers = self._get_auth_header() - params = { - "continue": 0, - } - params.update(query or {}) - result = [] - while True: - response = RestClient(url).method(HttpMethod.GET).query_param( - params).headers(headers).execute() - data = json.loads(response.text) - if not response.ok: - err_msg = data.get('error') - raise Exception("deployments: {}".format(err_msg)) - deployments = data.get('items', []) - if not deployments: - break - offset = data['metadata']['continue'] - params.update({"continue": offset}) - result.extend(deployments) - - return munchify(result) + client = RestClient(url).method(HttpMethod.GET).headers(headers) + return self._walk_pages(client, params=query) def create_deployment(self, deployment: dict) -> Munch: """ @@ -996,19 +965,11 @@ def create_deployment(self, deployment: dict) -> Munch: return munchify(data) - def get_deployment( - self, - name: str, - query: dict = None - ): + def get_deployment(self, name: str): url = "{}/v2/deployments/{}/".format(self._host, name) headers = self._get_auth_header() - params = {} - params.update(query or {}) - - response = RestClient(url).method(HttpMethod.GET).query_param( - params).headers(headers).execute() + response = RestClient(url).method(HttpMethod.GET).headers(headers).execute() data = json.loads(response.text) if response.status_code == http.HTTPStatus.NOT_FOUND: @@ -1107,7 +1068,7 @@ def stream_deployment_logs( def list_disks( self, - query: dict = None + query: dict = None, ) -> Munch: """ List all disks in a project @@ -1115,26 +1076,8 @@ def list_disks( url = "{}/v2/disks/".format(self._host) headers = self._get_auth_header() - params = {} - params.update(query or {}) - offset, result = 0, [] - while True: - params.update({ - "continue": offset, - }) - response = RestClient(url).method(HttpMethod.GET).query_param( - params).headers(headers).execute() - data = json.loads(response.text) - if not response.ok: - err_msg = data.get('error') - raise Exception("disks: {}".format(err_msg)) - disks = data.get('items', []) - if not disks: - break - offset = data['metadata']['continue'] - result.extend(disks) - - return munchify(result) + client = RestClient(url).method(HttpMethod.GET).headers(headers) + return self._walk_pages(client, params=query) def get_disk(self, name: str) -> Munch: """ From b71eae3c8b780340ec908b2f02d6c6e94cd78370 Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 27 Jun 2024 23:14:58 +0530 Subject: [PATCH 24/54] fix(rosbag): update default statuses for list functions (cherry picked from commit 9c77720ad021abfdee289867030dcb666a3e5785) --- riocli/rosbag/blob.py | 4 ++-- riocli/rosbag/job.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/riocli/rosbag/blob.py b/riocli/rosbag/blob.py index bc39ff86..dd27cc43 100644 --- a/riocli/rosbag/blob.py +++ b/riocli/rosbag/blob.py @@ -1,4 +1,4 @@ -# Copyright 2021 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -79,7 +79,7 @@ def blob_download(guid: str, filename: str, download_dir: str) -> None: @click.option('--device-ids', help='Filter by device ids ', multiple=True) @click.option('--statuses', help='Filter by rosbag blob statuses ', type=click.Choice(['Starting', 'Uploading', 'Uploaded', 'Error'], case_sensitive=True), - multiple=True, default=['Uploaded']) + multiple=True, default=['Uploaded', 'Uploading', 'Starting']) def blob_list(guids: typing.List[str], deployment_ids: typing.List[str], component_instance_ids: typing.List[str], job_ids: typing.List[str], device_ids: typing.List[str], statuses: typing.List[str]) -> None: """ diff --git a/riocli/rosbag/job.py b/riocli/rosbag/job.py index a53398f8..6bdfcf38 100644 --- a/riocli/rosbag/job.py +++ b/riocli/rosbag/job.py @@ -1,4 +1,4 @@ -# Copyright 2021 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -124,7 +124,8 @@ def job_stop(deployment_guid: str, deployment_name: str, component_instance_ids: @click.argument('deployment-name') @click.option('--component-instance-ids', help='Filter by component instance ids ', multiple=True) @click.option('--guids', help='Filter by job guids ', multiple=True) -@click.option('--statuses', help='Filter by rosbag job statuses ', multiple=True, default=['Starting', 'Running'], +@click.option('--statuses', help='Filter by rosbag job statuses ', multiple=True, + default=['Starting', 'Running', 'Stopped', 'Stopping', 'Error'], type=click.Choice(['Starting', 'Running', 'Error', 'Stopping', 'Stopped'], case_sensitive=True)) # @deployment_name_to_guid def job_list(deployment_guid: str, deployment_name: str, From f3852b0c9648aa3a4d3a1564064252a96dfe3517 Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 27 Jun 2024 23:13:27 +0530 Subject: [PATCH 25/54] fix(deployment): add phase filter for list_deployments (cherry picked from commit 9b8705f1fb56d1dc636f3e0a3fe9368c77176bd8) --- riocli/apply/resolver.py | 6 +++- riocli/deployment/list.py | 26 +++++++++++---- riocli/deployment/util.py | 70 ++++++++++++++++++++++++++++++++------- 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py index 4f54e89a..529e9bbd 100644 --- a/riocli/apply/resolver.py +++ b/riocli/apply/resolver.py @@ -138,7 +138,11 @@ def _list_functors(self, kind): "staticroute": self.v2client.list_static_routes, "disk": self.v2client.list_disks, "network": self.v2client.list_networks, - "deployment": functools.partial(self.v2client.list_deployments), + "deployment": functools.partial(self.v2client.list_deployments, + query={'phases': ['InProgress', + 'Succeeded', + 'FailedToStart', + 'Provisioning']}), "device": self.client.get_all_devices, "managedservice": self._list_managedservices, "usergroup": self.client.list_usergroups diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index cb79c0b9..7221aa8d 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,26 @@ import click from click_help_colors import HelpColorsCommand -from riocli.deployment.model import Deployment -from riocli.deployment.util import ALL_PHASES, DEFAULT_PHASES - from riocli.config import new_v2_client from riocli.constants import Colors -from riocli.deployment.util import process_deployment_errors +from riocli.deployment.model import Deployment from riocli.utils import tabulate_data +ALL_PHASES = [ + 'InProgress', + 'Provisioning', + 'Succeeded', + 'FailedToStart', + 'Stopped', +] + +DEFAULT_PHASES = [ + 'InProgress', + 'Provisioning', + 'Succeeded', + 'FailedToStart', +] + @click.command( 'list', @@ -48,8 +60,8 @@ def list_deployments( List the deployments in the selected project """ try: - client = new_v2_client() - deployments = client.list_deployments(query={"phases": phase, "deviceName": device}) + client = new_v2_client(with_project=True) + deployments = client.list_deployments(query={'phases': phase}) deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) display_deployment_list(deployments, show_header=True) except Exception as e: diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index d060c7b5..d2212b91 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,15 @@ # limitations under the License. import functools import re -from typing import List +import typing -from riocli.config import new_client -from riocli.deployment.model import Deployment +import click +from rapyuta_io import DeploymentPhaseConstants +from rapyuta_io.clients.deployment import Deployment + +from riocli.config import new_client, new_v2_client +from riocli.constants import Colors +from riocli.deployment.list import DEFAULT_PHASES from riocli.utils import tabulate_data from riocli.utils.selector import show_selection from riocli.v2client import Client @@ -30,11 +35,52 @@ DeploymentPhaseConstants.DeploymentPhaseStopped, ] -DEFAULT_PHASES = [ - DeploymentPhaseConstants.DeploymentPhaseInProgress, - DeploymentPhaseConstants.DeploymentPhaseProvisioning, - DeploymentPhaseConstants.DeploymentPhaseSucceeded, -] + +def name_to_guid(f: typing.Callable) -> typing.Callable: + @functools.wraps(f) + def decorated(**kwargs: typing.Any) -> None: + try: + client = new_v2_client() + except Exception as e: + click.secho(str(e), fg=Colors.RED) + raise SystemExit(1) from e + + name = kwargs.pop('deployment_name') + guid = None + + if name.startswith('dep-'): + guid = name + name = None + + try: + if name is None: + name = get_deployment_name(client, guid) + + if guid is None: + guid = get_deployment_guid(client, name) + + except Exception as e: + click.secho(str(e), fg=Colors.RED) + raise SystemExit(1) from e + + kwargs['deployment_name'] = name + kwargs['deployment_guid'] = guid + f(**kwargs) + + return decorated + + +def get_deployment_guid(client: Client, name: str) -> str: + deployment = client.get_deployment(name) + return deployment.metadata.guid + + +def get_deployment_name(client: Client, guid: str) -> str: + deployments = client.list_deployments(query={'guids': [guid]}) + if len(deployments) == 0: + raise DeploymentNotFound + + return deployments[0].metadata.name def select_details(deployment_guid, component_name=None, exec_name=None) -> (str, str, str): @@ -76,8 +122,8 @@ def fetch_deployments( client: Client, deployment_name_or_regex: str, include_all: bool, -) -> List[Deployment]: - deployments = client.list_deployments(query={"phases": DEFAULT_PHASES}) +) -> typing.List[Deployment]: + deployments = client.list_deployments(query={'phases': DEFAULT_PHASES}) result = [] for deployment in deployments: if (include_all or deployment_name_or_regex == deployment.metadata.name or @@ -89,7 +135,7 @@ def fetch_deployments( return result -def print_deployments_for_confirmation(deployments: List[Deployment]): +def print_deployments_for_confirmation(deployments: typing.List[Deployment]): headers = ['Name', 'GUID', 'Phase', 'Status'] data = [] From 9d65742a7ccc2bebd8b152175f2e723b258de915 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Tue, 30 Jul 2024 14:59:16 +0530 Subject: [PATCH 26/54] feat(config): store org short id in the cli config The organization short guid does change and is required in some resources. This commit updates the CLI to store the short guid in the config. It will help avoid API calls to fetch the value every time we need it. --- riocli/auth/logout.py | 11 +- riocli/auth/util.py | 24 +++- riocli/config/config.py | 11 ++ riocli/organization/inspect.py | 7 +- riocli/organization/select.py | 4 +- riocli/project/create.py | 3 +- riocli/project/list.py | 1 + riocli/project/util.py | 30 ++--- riocli/v2client/constants.py | 240 +++++++++++++++++++++++++++++++++ 9 files changed, 301 insertions(+), 30 deletions(-) create mode 100644 riocli/v2client/constants.py diff --git a/riocli/auth/logout.py b/riocli/auth/logout.py index ee47dd35..548a593e 100644 --- a/riocli/auth/logout.py +++ b/riocli/auth/logout.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,14 +28,17 @@ def logout(ctx: click.Context): """ Log out from the Rapyuta.io account using the CLI. """ - if not ctx.obj.exists: return - ctx.obj.data.pop('auth_token', None) - ctx.obj.data.pop('password', None) ctx.obj.data.pop('email_id', None) + ctx.obj.data.pop('auth_token', None) ctx.obj.data.pop('project_id', None) + ctx.obj.data.pop('project_name', None) + ctx.obj.data.pop('organization_id', None) + ctx.obj.data.pop('organization_name', None) + ctx.obj.data.pop('organization_short_id', None) + ctx.obj.save() click.secho('{} Logged out successfully.'.format(Symbols.SUCCESS), fg=Colors.GREEN) diff --git a/riocli/auth/util.py b/riocli/auth/util.py index e425886c..081b1a44 100644 --- a/riocli/auth/util.py +++ b/riocli/auth/util.py @@ -20,7 +20,7 @@ from riocli.config import Configuration from riocli.constants import Colors, Symbols -from riocli.project.util import find_project_guid, find_organization_guid +from riocli.project.util import find_project_guid, find_organization_guid, get_organization_name from riocli.utils.selector import show_selection from riocli.utils.spinner import with_spinner @@ -42,11 +42,20 @@ def select_organization( """ client = config.new_client(with_project=False) - org_guid = None + org_guid, org_name, short_id = None, None, None if organization: - org_guid = organization if organization.startswith( - 'org-') else find_organization_guid(client, name=organization) + if organization.startswith('org-'): + org_guid = organization + org_name, short_id = get_organization_name(client, org_guid) + else: + org_guid, short_id = find_organization_guid(client, name=organization) + + if org_guid and org_name and short_id: + config.data['organization_id'] = org_guid + config.data['organization_name'] = org_name + config.data['organization_short_id'] = short_id + return org_guid # fetch user organizations and sort them on their name organizations = client.get_user_organizations() @@ -56,7 +65,11 @@ def select_organization( click.secho("You are not a part of any organization", fg=Colors.BLACK, bg=Colors.WHITE) raise SystemExit(1) - org_map = {o.guid: o.name for o in organizations} + org_map, org_short_guids = {}, {} + + for o in organizations: + org_map[o.guid] = o.name + org_short_guids[o.guid] = o.short_guid if not org_guid: org_guid = show_selection(org_map, "Select an organization:") @@ -67,6 +80,7 @@ def select_organization( config.data['organization_id'] = org_guid config.data['organization_name'] = org_map[org_guid] + config.data['organization_short_id'] = org_short_guids[org_guid] return org_guid diff --git a/riocli/config/config.py b/riocli/config/config.py index db5d1545..0560c290 100644 --- a/riocli/config/config.py +++ b/riocli/config/config.py @@ -164,6 +164,17 @@ def organization_guid(self: Configuration) -> str: return guid + @property + def organization_short_id(self: Configuration) -> str: + if 'auth_token' not in self.data: + raise LoggedOut + + short_id = self.data.get('organization_short_id') + if short_id is None: + raise NoOrganizationSelected + + return short_id + @property def piping_server(self: Configuration): return self.data.get('piping_server', self.PIPING_SERVER) diff --git a/riocli/organization/inspect.py b/riocli/organization/inspect.py index b1a3b847..9ad297ff 100644 --- a/riocli/organization/inspect.py +++ b/riocli/organization/inspect.py @@ -32,7 +32,12 @@ type=click.Choice(['json', 'yaml'], case_sensitive=False)) @click.argument('organization-name', type=str) @name_to_organization_guid -def inspect_organization(format_type: str, organization_name: str, organization_guid: str) -> None: +def inspect_organization( + format_type: str, + organization_name: str, + organization_guid: str, + organization_short_id: str, +) -> None: """ Inspect an organization """ diff --git a/riocli/organization/select.py b/riocli/organization/select.py index 3b716c57..949bc6db 100644 --- a/riocli/organization/select.py +++ b/riocli/organization/select.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ def select_organization( ctx: click.Context, organization_name: str, organization_guid: str, + organization_short_id: str, interactive: bool, ) -> None: """ @@ -58,6 +59,7 @@ def select_organization( ctx.obj.data['organization_id'] = organization_guid ctx.obj.data['organization_name'] = organization_name + ctx.obj.data['organization_short_id'] = organization_short_id if sys.stdout.isatty() and interactive: select_project(ctx.obj, organization=organization_guid) diff --git a/riocli/project/create.py b/riocli/project/create.py index 491323ed..d5cdbbc5 100644 --- a/riocli/project/create.py +++ b/riocli/project/create.py @@ -15,7 +15,7 @@ from click_help_colors import HelpColorsCommand from riocli.config import new_v2_client -from riocli.constants import Symbols, Colors +from riocli.constants import Colors, Symbols from riocli.project.util import name_to_organization_guid from riocli.utils.spinner import with_spinner @@ -37,6 +37,7 @@ def create_project( project_name: str, organization_guid: str, organization_name: str, + organization_short_id: str, spinner=None, ) -> None: """ diff --git a/riocli/project/list.py b/riocli/project/list.py index 18b39a21..2f49cc86 100644 --- a/riocli/project/list.py +++ b/riocli/project/list.py @@ -40,6 +40,7 @@ def list_projects( ctx: click.Context = None, organization_guid: str = None, organization_name: str = None, + organization_short_id: str = None, wide: bool = False, ) -> None: """ diff --git a/riocli/project/util.py b/riocli/project/util.py index 7f565fcb..c5aee5dc 100644 --- a/riocli/project/util.py +++ b/riocli/project/util.py @@ -20,7 +20,6 @@ from riocli.config import new_client, new_v2_client from riocli.constants import Colors, Symbols from riocli.exceptions import LoggedOut -from riocli.utils.selector import show_selection from riocli.v2client import Client as v2Client @@ -77,31 +76,22 @@ def get_project_name(client: v2Client, guid: str) -> str: return project.metadata.name -def find_organization_guid(client: Client, name: str) -> str: +def find_organization_guid(client: Client, name: str) -> typing.Tuple[str, str]: organizations = client.get_user_organizations() - options = {} for organization in organizations: if organization.name == name: - options[organization.guid] = '{} ({})'.format(organization.name, - organization.url) + return organization.guid, organization.short_guid - if len(options) == 1: - return list(options.keys())[0] - - if len(options) == 0: - raise Exception("User is not part of organization: {}".format(name)) - - choice = show_selection(options, - header='Following packages were found with the same name') - return choice + raise OrganizationNotFound( + "User is not part of organization with guid: {}".format(guid)) -def get_organization_name(client: Client, guid: str) -> str: +def get_organization_name(client: Client, guid: str) -> typing.Tuple[str, str]: organizations = client.get_user_organizations() for organization in organizations: if organization.guid == guid: - return organization.name + return organization.name, organization.short_guid raise OrganizationNotFound( "User is not part of organization with guid: {}".format(guid)) @@ -113,19 +103,23 @@ def decorated(*args: typing.Any, **kwargs: typing.Any): client = new_client(with_project=False) name = kwargs.get('organization_name') guid = None + short_id = None if name: try: if name.startswith('org-'): guid = name - name = get_organization_name(client, guid) + name, short_id = get_organization_name(client, guid) else: - guid = find_organization_guid(client, name) + guid, short_id = find_organization_guid(client, name) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) + kwargs['organization_name'] = name kwargs['organization_guid'] = guid + kwargs['organization_short_id'] = short_id + f(*args, **kwargs) return decorated diff --git a/riocli/v2client/constants.py b/riocli/v2client/constants.py new file mode 100644 index 00000000..94321788 --- /dev/null +++ b/riocli/v2client/constants.py @@ -0,0 +1,240 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DEPLOYMENT_ERRORS = { + # Device related Error codes + "DEP_E151": { + "description": "Device is either offline or not reachable", + "action": "Ensure that the device is connected to the internet" + }, + "DEP_E302": { + "description": "Supervisord related error on device", + "action": ("Ensure supervisord is installed on rapyuta virtual environment and working\n" + "Re-onboard the device, if issue persists report to rapyuta.io support.") + }, + "DEP_E161": { + "description": "Docker image not found for executables of components deployed on device", + "action": "Ensure that the docker image path is valid and secrets have privileges" + }, + "DEP_E316": { + "description": "Docker image pull failed in device", + "action": "Ensure that the docker image path is valid and secrets have privileges" + }, + "DEP_E317": { + "description": "Missing subpath or empty mount path", + "action": ("Ensure the current device has a subpath\n" + "Ensure that the mount path isn’t empty") + }, + "DEP_E304": { + "description": "DeviceManager internal error", + "action": "Report to rapyuta.io support" + }, + "DEP_E306": { + "description": "DeviceEdge internal error", + "action": "Ensure deviceedge is up and running" + }, + "DEP_E307": { + "description": "DeviceEdge bad request", + "action": "Report to rapyuta.io support" + }, + + # Application related errors. + "DEP_E152": { + "description": "Executables of the component deployed on the device either exited too early or failed", + "action": "Executables of the component deployed on the device either exited too early or failed" + }, + "DEP_E153": { + "description": "Unable to pull the docker image for the component deployed on cloud", + "action": ("Ensure that the docker image provided while adding the package still " + "exists at the specified registry endpoint ") + }, + "DEP_E154": { + "description": "Executables of the component deployed on cloud exited too early", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E155": { + "description": "Executables of the component deployed on cloud failed", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E156": { + "description": "Dependent deployment is in error state", + "action": "Troubleshoot the dependent deployment that is in error state" + }, + "DEP_E162": { + "description": ("Validation error. Cases include:\nInconsistent values of ROS distro " + "and CPU architecture variables for the device and package being provisioned. " + "\nrapyuta.io docker images not present on docker device."), + "action": ("Create package with appropriate values for ROS distro and CPU architecture " + "variables.\nOnboard the device again.") + }, + "DEP_E163": { + "description": "Application has stopped and exited unexpectedly, and crashes continuously on device", + "action": "Debug the application using the corresponding deployment logs" + }, + + # CloudBridge related error codes. + "DEP_E171": { + "description": "Cloud bridge encountered duplicate alias on the device", + "action": ("Change the alias name during deployment and ensure that there" + " is no duplication of alias name under the same routed network") + }, + "DEP_E172": { + "description": "Compression library required for the cloud bridge is missing on the device", + "action": "Re-onboard the device" + }, + "DEP_E173": { + "description": "Transport libraries required for the cloud bridge are missing on the device", + "action": "Re-onboard the device" + }, + "DEP_E174": { + "description": "Cloud bridge on the device encountered multiple ROS service origins", + "action": ("Ensure that there aren’t multiple deployments with the same ROS service " + "endpoint under the same routed network") + }, + "DEP_E175": { + "description": "Python actionlib/msgs required for the cloud bridge is missing on the device", + "action": "Re-onboard the device" + }, + "DEP_E176": { + "description": "Cloud bridge encountered duplicate alias on the cloud component", + "action": ("Change the alias name during deployment and ensure that there is no" + " duplication of alias name under the same routed network") + }, + "DEP_E177": { + "description": "Cloud bridge on the cloud component encountered multiple ROS service origins", + "action": "Re-onboard the device" + }, + + # Docker image related Error codes + "DEP_E350": { + "description": "Get http error when pulling image from registry on device", + "action": "Ensure registry is accesible." + }, + "DEP_E351": { + "description": "Unable to parse the image name on device", + "action": "Ensure image name is correct and available in registry." + }, + "DEP_E352": { + "description": "Unable to inspect image on device", + "action": "Ensure valid image is present" + }, + "DEP_E353": { + "description": "Required Image is absent on device and PullPolicy is NeverPullImage", + "action": "Ensure pull policy is not set to never or image is present on device" + }, + + # Container related error codes + "DEP_E360": { + "description": "Failed to create container config on device", + "action": ("Ensure executable config is mounted correctly\n" + "If issue persists report to rapyuta.io support") + }, + "DEP_E361": { + "description": "Runtime failed to start any of pod's container on device", + "action": ("Ensure executable command is valid\n" + "If issue persists report to rapyuta.io support") + }, + "DEP_E362": { + "description": "Runtime failed to create container on device", + "action": ("Report to rapyuta.io support") + }, + "DEP_E363": { + "description": "Runtime failed to kill any of pod's containers on device", + "action": ("Report to rapyuta.io support") + }, + "DEP_E364": { + "description": "Runtime failed to create a sandbox for pod on device", + "action": ("Report to rapyuta.io support") + }, + "DEP_E365": { + "description": "Runtime failed to get pod sandbox config from pod on device", + "action": ("Report to rapyuta.io support") + }, + "DEP_E366": { + "description": "Runtime failed to stop pod's sandbox on device", + "action": ("Report to rapyuta.io support") + }, + "DEP_E399": { + "description": "Deployment failed for some unknown reason on device", + "action": ("Report to rapyuta.io support") + }, + + # ROS Comm Related Device Errors + "DEP_E303": { + "description": "Cloud Bridge executable not running on device", + "action": ("Troubleshoot Cloud Bridge container on device by analyzing logs") + }, + "DEP_E305": { + "description": "Native Network executable not running on device", + "action": ("Troubleshoot native network container on device by analyzing logs") + }, + "DEP_E313": { + "description": "ROS Master executable not running on device", + "action": ("Troubleshoot ROS Master container on device by analyzing logs") + }, + "DEP_E330": { + "description": "ROS2 Native Network executable not running on device", + "action": ("Troubleshoot ROS2 Routed Network container on device by analyzing logs") + }, + "DEP_E331": { + "description": "ROS2 Routed Network executable not running on device", + "action": ("Troubleshoot ROS2 Routed Network container on device by analyzing logs") + }, + + # Cloud error codes + "DEP_E201": { + "description": "Cloud component deployment pending", + "action": ("Report to rapyuta.io support") + }, + "DEP_E202": { + "description": "Cloud component status unknown", + "action": ("Report to rapyuta.io support") + }, + "DEP_E203": { + "description": "Cloud Bridge not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E204": { + "description": "ROS Master not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E205": { + "description": "Cloud Broker not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E208": { + "description": "Desired set of replicas not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E209": { + "description": "Native Network not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E210": { + "description": "Disk Not Running", + "action": ("Ensure disk is running") + }, + "DEP_E213": { + "description": "Broker running low on memory", + "action": ("Report to rapyuta.io support") + }, + "DEP_E214": { + "description": "ROS2 Native Network not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + }, + "DEP_E215": { + "description": "ROS2 Routed Network not running on cloud", + "action": "Troubleshoot the failed component by analyzing the deployment logs" + } +} From bae925da80b147390976b76b271e65bef9493c4f Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 29 Jul 2024 12:34:55 +0530 Subject: [PATCH 27/54] refactor(apply): removes resolver cache The purpose of the ResolverCache was to perform name to guid translation from the dependencies defined in the manifests. However, the overall implementation added an overhead of performing multiple API calls before the actual call of creating or deleting a resource is made. This led to a slowdown in the performance of the CLI when there are multiple resources to create. In this commit, we completely remove the resolver cache and refactor the implementation to remove the overhead of making additional API calls. We have also simplified to base Model class and now, the concrete classes will be responsible for doing everything relevant for a particular resource. Wrike Ticket: https://www.wrike.com/open.htm?id=1450590338 --- riocli/apply/__init__.py | 23 ++-- riocli/apply/parse.py | 198 ++++++++++----------------- riocli/apply/resolver.py | 209 ---------------------------- riocli/apply/util.py | 74 +++++++++- riocli/deployment/model.py | 53 +++----- riocli/deployment/util.py | 34 ----- riocli/device/model.py | 119 +++++++--------- riocli/device/util.py | 10 +- riocli/disk/model.py | 72 +++------- riocli/managedservice/model.py | 56 +++----- riocli/model/base.py | 197 ++++++--------------------- riocli/network/model.py | 67 +++------ riocli/package/model.py | 78 +++++------ riocli/project/model.py | 67 +++------ riocli/secret/model.py | 53 ++------ riocli/static_route/model.py | 58 +++----- riocli/usergroup/model.py | 95 +++++++------ riocli/v2client/client.py | 46 ++----- riocli/v2client/enums.py | 14 ++ riocli/v2client/error.py | 23 +++- riocli/v2client/errors.py | 240 --------------------------------- riocli/v2client/util.py | 61 ++++++++- 22 files changed, 556 insertions(+), 1291 deletions(-) delete mode 100644 riocli/apply/resolver.py delete mode 100644 riocli/v2client/errors.py diff --git a/riocli/apply/__init__.py b/riocli/apply/__init__.py index 088f9087..63956be2 100644 --- a/riocli/apply/__init__.py +++ b/riocli/apply/__init__.py @@ -20,6 +20,7 @@ from riocli.apply.explain import explain, list_examples from riocli.apply.parse import Applier from riocli.apply.template import template +from riocli.apply.util import print_resolved_objects from riocli.apply.util import process_files_values_secrets from riocli.constants import Colors from riocli.utils import print_centered_text @@ -79,8 +80,8 @@ def apply( for file in glob_files: click.secho(file, fg=Colors.YELLOW) - rc = Applier(glob_files, abs_values, abs_secrets) - rc.parse_dependencies() + applier = Applier(glob_files, abs_values, abs_secrets) + applier.parse_dependencies() if show_graph and dryrun: click.secho('You cannot dry run and launch the graph together.', @@ -88,14 +89,14 @@ def apply( return if show_graph: - rc.show_dependency_graph() + applier.show_dependency_graph() return if not silent and not dryrun: - click.confirm("Do you want to proceed?", default=True, abort=True) + click.confirm("\nDo you want to proceed?", default=True, abort=True) print_centered_text('Applying Manifests') - rc.apply(dryrun=dryrun, workers=workers, retry_count=retry_count, retry_interval=retry_interval) + applier.apply(dryrun=dryrun, workers=workers, retry_count=retry_count, retry_interval=retry_interval) @click.command( @@ -133,11 +134,15 @@ def delete( click.secho('no files specified', fg=Colors.RED) raise SystemExit(1) - rc = Applier(glob_files, abs_values, abs_secrets) - rc.parse_dependencies(check_missing=False, delete=True) + print_centered_text('Files Processed') + for file in glob_files: + click.secho(file, fg=Colors.YELLOW) + + applier = Applier(glob_files, abs_values, abs_secrets) + applier.parse_dependencies() if not silent and not dryrun: - click.confirm("Do you want to proceed?", default=True, abort=True) + click.confirm("\nDo you want to proceed?", default=True, abort=True) print_centered_text('Deleting Resources') - rc.delete(dryrun=dryrun) + applier.delete(dryrun=dryrun) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index ffa70cbf..427a6e92 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -20,14 +20,16 @@ import click import jinja2 import yaml +from munch import munchify -from riocli.apply.resolver import ResolverCache -from riocli.config import Configuration +from riocli.apply.util import get_model, print_resolved_objects, message_with_prompt from riocli.constants import Colors, Symbols -from riocli.utils import dump_all_yaml, run_bash +from riocli.utils import dump_all_yaml, run_bash, print_centered_text from riocli.utils.graph import Graphviz from riocli.utils.spinner import with_spinner +DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' + class Applier(object): DEFAULT_MAX_WORKERS = 6 @@ -35,14 +37,10 @@ class Applier(object): def __init__(self, files: typing.List, values, secrets): self.environment = None self.input_file_paths = files - self.config = Configuration() - self.client = self.config.new_client() - self.dependencies = {} self.objects = {} self.resolved_objects = {} self.files = {} self.graph = TopologicalSorter() - self.rc = ResolverCache(self.client) self.secrets = {} self.values = {} self.diagram = Graphviz(direction='LR', format='svg') @@ -59,15 +57,12 @@ def __init__(self, files: typing.List, values, secrets): self._process_file_list(files) - # Public Functions - def order(self): - return self.graph.static_order() - @with_spinner(text='Applying...', timer=True) def apply(self, *args, **kwargs): spinner = kwargs.get('spinner') kwargs['workers'] = int(kwargs.get('workers') or self.DEFAULT_MAX_WORKERS) + apply_func = self.apply_async if kwargs['workers'] == 1: apply_func = self.apply_sync @@ -83,26 +78,26 @@ def apply(self, *args, **kwargs): def apply_async(self, *args, **kwargs): workers = int(kwargs.get('workers') or self.DEFAULT_MAX_WORKERS) + task_queue = queue.Queue() done_queue = queue.Queue() def worker(): while True: o = task_queue.get() - if o in self.resolved_objects and 'manifest' in \ - self.resolved_objects[o]: + if o in self.resolved_objects and 'manifest' in self.resolved_objects[o]: try: self._apply_manifest(o, *args, **kwargs) except Exception as ex: - click.secho( - '[Err] Object "{}" apply failed. Apply will not progress further.'.format( - o, str(ex))) done_queue.put(ex) continue task_queue.task_done() done_queue.put(o) + # Start the workers that will accept tasks from the task_queue + # and process them. Upon completion, they will put the object + # in the done_queue. worker_list = [] for worker_id in range(workers): worker_list.append(threading.Thread(target=worker, daemon=True)) @@ -119,6 +114,7 @@ def worker(): else: raise Exception(done_obj) + # Block until the task_queue is empty. task_queue.join() def apply_sync(self, *args, **kwargs): @@ -150,49 +146,72 @@ def print_resolved_manifests(self): manifests = [o for _, o in self.objects.items()] dump_all_yaml(manifests) - def parse_dependencies( - self, - check_missing=True, - delete=False, - template=False - ): - number_of_objects = 0 - for f, data in self.files.items(): + def parse_dependencies(self): + for _, data in self.files.items(): for model in data: key = self._get_object_key(model) self._parse_dependency(key, model) self._add_graph_node(key) - number_of_objects = number_of_objects + 1 - if check_missing: - missing_resources = [] - for key, item in self.resolved_objects.items(): - if 'src' in item and item['src'] == 'missing': - missing_resources.append(key) + print_centered_text("Parsed Resources") + print_resolved_objects(self.resolved_objects) - if missing_resources: - click.secho( - "Missing resources found in yaml. Please ensure the " - "following are either available in your YAML or created" - " on the server. {}".format( - set(missing_resources)), fg=Colors.RED) + def show_dependency_graph(self): + """Lauches mermaid.live dependency graph""" + self.diagram.visualize() - raise SystemExit(1) + def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None: + """Instantiate and apply the object manifest""" + spinner = kwargs.get('spinner') + dryrun = kwargs.get("dryrun", False) - # Manifest Operations via base.py - def _apply_manifest(self, obj_key, *args, **kwargs): obj = self.objects[obj_key] - cls = ResolverCache.get_model(obj) - ist = cls.from_dict(self.client, obj) - setattr(ist, 'rc', ResolverCache(self.client)) - ist.apply(self.client, *args, **kwargs) + kls = get_model(obj) + kls.validate(obj) + ist = kls(munchify(obj)) + + message_with_prompt("{} Applying {}...".format( + Symbols.WAITING, obj_key), fg=Colors.CYAN, spinner=spinner) + + try: + if not dryrun: + ist.apply(*args, **kwargs) + except Exception as ex: + message_with_prompt("{} Failed to apply {}. Error: {}".format( + Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) + raise ex + + message_with_prompt("{} Applied {}".format( + Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) + + def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: + """Instantiate and delete the object manifest""" + spinner = kwargs.get('spinner') + dryrun = kwargs.get("dryrun", False) - def _delete_manifest(self, obj_key, *args, **kwargs): obj = self.objects[obj_key] - cls = ResolverCache.get_model(obj) - ist = cls.from_dict(self.client, obj) - setattr(ist, 'rc', ResolverCache(self.client)) - ist.delete(self.client, obj, *args, **kwargs) + kls = get_model(obj) + kls.validate(obj) + ist = kls(munchify(obj)) + + message_with_prompt("{} Deleting {}...".format( + Symbols.WAITING, obj_key), fg=Colors.CYAN, spinner=spinner) + + # If a resource has a label with DELETE_POLICY_LABEL set + # to 'retain', it should not be deleted. + labels = obj.get('metadata', {}).get('labels', {}) + can_delete = labels.get(DELETE_POLICY_LABEL) != 'retain' + + try: + if not dryrun and can_delete: + ist.delete(*args, **kwargs) + except Exception as ex: + message_with_prompt("{} Failed to delete {}. Error: {}".format( + Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) + raise ex + + message_with_prompt("{} Deleted {}.".format( + Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) # File Loading Operations @@ -281,6 +300,11 @@ def _add_graph_edge(self, dependent_key, key): # Dependency Resolution def _parse_dependency(self, dependent_key, model): + # TODO(pallab): let resources determine their own dependencies and return them + # kls = get_model(model) + # for dependency in kls.parse_dependencies(model): + # self._resolve_dependency(dependent_key, dependency) + for key, value in model.items(): if key == "depends": if 'kind' in value and value.get('kind'): @@ -306,86 +330,8 @@ def _resolve_dependency(self, dependent_key, dependency): name_or_guid = dependency.get('nameOrGUID') key = '{}:{}'.format(kind, name_or_guid) - self._initialize_kind_dependency(kind) - guid = ResolverCache._maybe_guid(kind, name_or_guid) - - obj_list = self.rc.list_objects(kind) - for obj in obj_list: - obj_guid = self._get_attr(obj, ResolverCache.GUID_KEYS) - obj_name = self._get_attr(obj, ResolverCache.NAME_KEYS) - - if kind == 'package': - if guid and obj_guid == guid: - self._add_remote_object_to_resolve_tree( - dependent_key, obj_guid, dependency, obj) - - if (name_or_guid == obj_name) and ('version' in dependency and - obj.metadata.version == dependency.get('version')): - self._add_remote_object_to_resolve_tree( - dependent_key, obj_guid, dependency, obj) - - # Special handling for Static route since it doesn't have a name field. - # StaticRoute sends a URLPrefix field with name being the prefix along with short org guid. - elif kind == 'staticroute' and name_or_guid in obj_name: - self._add_remote_object_to_resolve_tree( - dependent_key, obj_guid, dependency, obj) - - elif (guid and obj_guid == guid) or (name_or_guid == obj_name): - self._add_remote_object_to_resolve_tree( - dependent_key, obj_guid, dependency, obj) - - self.dependencies[kind][name_or_guid] = {'local': True} self._add_graph_edge(dependent_key, key) - if key not in self.resolved_objects: - self.resolved_objects[key] = {'src': 'missing'} - - def _add_remote_object_to_resolve_tree(self, dependent_key, guid, - dependency, obj): - kind = dependency.get('kind') - name_or_guid = dependency.get('nameOrGUID') - key = '{}:{}'.format(kind, name_or_guid) - self.dependencies[kind][name_or_guid] = { - 'guid': guid, 'raw': obj, 'local': False} - if key not in self.resolved_objects: - self.resolved_objects[key] = {} - self.resolved_objects[key]['guid'] = guid - self.resolved_objects[key]['raw'] = obj - self.resolved_objects[key]['src'] = 'remote' - - self._add_graph_edge(dependent_key, key) - - dependency['guid'] = guid - if kind.lower() == "disk": - dependency['depGuid'] = obj.metadata.guid - - if kind.lower() == "deployment": - dependency['guid'] = obj.metadata.guid - - def _initialize_kind_dependency(self, kind): - if not self.dependencies.get(kind): - self.dependencies[kind] = {} - - def show_dependency_graph(self): - """Lauches mermaid.live dependency graph""" - self.diagram.visualize() - - # Utils - @staticmethod - def _get_attr(obj, accept_keys): - metadata = None - - if hasattr(obj, 'metadata'): - metadata = getattr(obj, 'metadata') - - for key in accept_keys: - if hasattr(obj, key): - return getattr(obj, key) - if metadata is not None and hasattr(metadata, key): - return getattr(metadata, key) - - raise Exception('guid resolve failed') - @staticmethod def _get_object_key(obj: dict) -> str: kind = obj.get('kind').lower() diff --git a/riocli/apply/resolver.py b/riocli/apply/resolver.py deleted file mode 100644 index 529e9bbd..00000000 --- a/riocli/apply/resolver.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2024 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import functools -import json -import re -import typing - -from munch import munchify -from rapyuta_io import DeploymentPhaseConstants -from rapyuta_io.utils.rest_client import HttpMethod, RestClient - -from riocli.config import new_v2_client -from riocli.config.config import Configuration -from riocli.deployment.model import Deployment -from riocli.device.model import Device -from riocli.disk.model import Disk -from riocli.managedservice.model import ManagedService -from riocli.network.model import Network -from riocli.package.model import Package -from riocli.project.model import Project -from riocli.secret.model import Secret -from riocli.static_route.model import StaticRoute -from riocli.usergroup.model import UserGroup - - -class _Singleton(type): - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super( - _Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] - - -class ResolverCache(object, metaclass=_Singleton): - KIND_TO_CLASS = { - 'Project': Project, - 'Secret': Secret, - 'Device': Device, - 'Network': Network, - 'StaticRoute': StaticRoute, - 'Package': Package, - 'Disk': Disk, - 'Deployment': Deployment, - "ManagedService": ManagedService, - 'UserGroup': UserGroup, - } - - KIND_REGEX = { - "organization": "^org-[a-z]{24}$", - "project": "^project-[a-z]{24}$", - "secret": "^secret-[a-z]{24}$", - "package": "^pkg-[a-z0-9]{20}$", - "staticroute": "^staticroute-[a-z]{24}$", - "disk": "^disk-[a-z]{24}$", - "deployment": "^dep-[a-z]{24}$", - "network": "^net-[a-z]{24}$", - "device": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$", - "user": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$", - "managedservice": "^[a-zA-Z][a-zA-Z0-9_-]$" - } - - GUID_KEYS = ['guid', 'GUID', 'deploymentId', 'uuid', 'ID', 'Id', 'id'] - NAME_KEYS = ['name', 'urlPrefix'] - - def __init__(self, client): - self.client = client - self.v2client = new_v2_client() - - @functools.lru_cache() - def list_objects(self, kind): - return self._list_functors(kind)() - - @functools.lru_cache() - def find_guid(self, name, kind, *args): - obj_list = self.list_objects(kind) - obj_match = list(self._find_functors(kind)(name, obj_list, *args)) - if obj_match and isinstance(obj_match, list) and len(obj_match) > 0: - return obj_match[0] - else: - return None - - def find_depends(self, depends, *args): - if 'depGuid' in depends and depends['kind'] == 'disk': - return depends['depGuid'], None - elif 'guid' in depends and depends['kind'] not in ('network', 'managedservice'): - return depends['guid'], None - elif 'nameOrGUID' in depends: - if depends['kind'] == 'usergroup': - org_guid = depends['organization'] - obj_list = self._list_functors(depends['kind'])(org_guid) - else: - obj_list = self._list_functors(depends['kind'])() - - obj_match = list(self._find_functors(depends['kind'])( - depends['nameOrGUID'], obj_list, *args)) - if not obj_list or (isinstance(obj_list, list) and len(obj_list) == 0): - return None, None - if obj_match and isinstance(obj_match, list) and len(obj_match) > 0: - return self._guid_functor(depends['kind'])(obj_match[0]), obj_match[0] - else: - return None, None - return None, None - - def _guid_functor(self, kind): - mapping = { - 'secret': lambda x: munchify(x).metadata.guid, - "project": lambda x: munchify(x).metadata.guid, - "package": lambda x: munchify(x)['metadata']['guid'], - "staticroute": lambda x: munchify(x)['metadata']['guid'], - "deployment": lambda x: munchify(x)['metadata']['guid'], - "network": lambda x: munchify(x)['metadata']['guid'], - # This is only temporarily like this - "disk": lambda x: munchify(x)['metadata']['guid'], - "device": lambda x: munchify(x)['uuid'], - "managedservice": lambda x: munchify(x)['metadata']['name'], - "usergroup": lambda x: munchify(x).guid - } - return mapping[kind] - - def _list_functors(self, kind): - mapping = { - 'secret': self.v2client.list_secrets, - "project": self.v2client.list_projects, - "package": self.v2client.list_packages, - "staticroute": self.v2client.list_static_routes, - "disk": self.v2client.list_disks, - "network": self.v2client.list_networks, - "deployment": functools.partial(self.v2client.list_deployments, - query={'phases': ['InProgress', - 'Succeeded', - 'FailedToStart', - 'Provisioning']}), - "device": self.client.get_all_devices, - "managedservice": self._list_managedservices, - "usergroup": self.client.list_usergroups - } - - return mapping[kind] - - def _find_functors(self, kind): - mapping = { - 'secret': lambda name, secrets: filter(lambda i: i.metadata.name == name, secrets), - "project": lambda name, projects: filter(lambda i: i.metadata.name == name, projects), - "package": lambda name, obj_list, version: filter( - lambda x: name == x.metadata.name and version == x.metadata.version, obj_list), - "staticroute": lambda name, obj_list: filter( - lambda x: name == x.metadata.name.rsplit('-', 1)[0], obj_list), - "network": lambda name, obj_list, network_type: filter( - lambda x: name == x.metadata.name and network_type == x.spec.type, obj_list), - "deployment": lambda name, obj_list: filter( - lambda x: name == x.metadata.name and not x.metadata.get('deletedAt'), obj_list), - "disk": lambda name, obj_list: filter( - lambda x: name == x.metadata.name, obj_list), - "device": self._generate_find_guid_functor(), - "managedservice": lambda name, instances: filter(lambda i: i.metadata.name == name, instances), - "usergroup": lambda name, groups: filter(lambda i: i.name == name, groups), - } - - return mapping[kind] - - def _generate_find_guid_functor(self, name_field='name'): - return lambda name, obj_list: filter(lambda x: name == getattr(x, name_field), obj_list) - - def _list_networks(self): - native = self.client.list_native_networks() - routed = self.client.get_all_routed_networks() - - networks = [] - if native: - networks.extend(native) - - if routed: - networks.extend(routed) - - return networks - - def _list_managedservices(self): - instances = ManagedService.list_instances() - return munchify(instances) - - @classmethod - def _maybe_guid(cls, kind: str, name_or_guid: str) -> typing.Union[str, None]: - if re.fullmatch(cls.KIND_REGEX[kind], name_or_guid): - return name_or_guid - - @classmethod - def get_model(cls, data: dict) -> typing.Any: - kind = data.get('kind', None) - if not kind: - raise Exception('kind is missing') - - kind_cls = cls.KIND_TO_CLASS.get(kind, None) - if not kind_cls: - raise Exception('invalid kind {}'.format(kind)) - - return kind_cls diff --git a/riocli/apply/util.py b/riocli/apply/util.py index a4aabcef..d92b39db 100644 --- a/riocli/apply/util.py +++ b/riocli/apply/util.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,51 @@ import glob import os +import typing +from datetime import datetime +from shutil import get_terminal_size + +import click +from yaspin.api import Yaspin + +from riocli.constants import Colors +from riocli.deployment.model import Deployment +from riocli.device.model import Device +from riocli.disk.model import Disk +from riocli.managedservice.model import ManagedService +from riocli.network.model import Network +from riocli.package.model import Package +from riocli.project.model import Project +from riocli.secret.model import Secret +from riocli.static_route.model import StaticRoute +from riocli.usergroup.model import UserGroup +from riocli.utils import tabulate_data + +KIND_TO_CLASS = { + 'project': Project, + 'secret': Secret, + 'device': Device, + 'network': Network, + 'staticroute': StaticRoute, + 'package': Package, + 'disk': Disk, + 'deployment': Deployment, + "managedservice": ManagedService, + 'usergroup': UserGroup, +} + + +def get_model(data: dict) -> typing.Any: + """Get the model class based on the kind""" + kind = data.get('kind', None) + if kind is None: + raise Exception('kind is missing') + + klass = KIND_TO_CLASS.get(str(kind).lower(), None) + if klass is None: + raise Exception('invalid kind {}'.format(kind)) + + return klass def parse_variadic_path_args(path_item): @@ -59,3 +104,30 @@ def process_files_values_secrets(files, values, secrets): glob_files = sorted(list(set(glob_files))) return glob_files, abs_values, abs_secret + + +def message_with_prompt( + left_msg: str, + right_msg: str = '', + fg: str = Colors.WHITE, + spinner: Yaspin = None, +) -> None: + """Prints a message with a prompt and a timestamp. + + >> left_msg spacer right_msg time + """ + columns, _ = get_terminal_size() + t = datetime.now().isoformat('T') + spacer = ' ' * (int(columns) - len(left_msg + right_msg + t) - 12) + text = click.style(f">> {left_msg}{spacer}{right_msg} [{t}]", fg=fg) + printer = spinner.write if spinner else click.echo + printer(text) + + +def print_resolved_objects(objects: typing.Dict) -> None: + data = [] + for o in objects: + kind, name = o.split(':') + data.append([kind.title(), name]) + + tabulate_data(data, headers=['Kind', 'Name']) diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index 82998362..34fe9eed 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,54 +16,31 @@ from munch import unmunchify from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.v2client import Client +from riocli.v2client.error import HttpAlreadyExistsError +from riocli.v2client.error import HttpNotFoundError class Deployment(Model): def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> typing.Any: - guid, obj = self.rc.find_depends({"kind": "deployment", "nameOrGUID": self.metadata.name}) - - return obj if guid else False - - def create_object(self, client: Client, **kwargs) -> typing.Any: + def apply(self, *args, **kwargs) -> typing.Any: client = new_v2_client() - r = client.create_deployment(self._sanitize_deployment()) - return unmunchify(r) - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - client = new_v2_client() - client.delete_deployment(obj.metadata.name) - - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - - @staticmethod - def validate(data): - """ - Validates if deployment data is matching with its corresponding schema - """ - schema = load_schema('deployment') - schema.validate(data) - - def _sanitize_deployment(self) -> typing.Dict: - # Unset createdAt and updatedAt to avoid timestamp parsing issue. self.metadata.createdAt = None self.metadata.updatedAt = None - data = unmunchify(self) + try: + client.create_deployment(unmunchify(self)) + except HttpAlreadyExistsError: + pass - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - data.pop("rc", None) + def delete(self, *args, **kwargs): + client = new_v2_client() - return data + try: + client.delete_deployment(self.metadata.name) + except HttpNotFoundError: + pass diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index d2212b91..c00fa676 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -26,7 +26,6 @@ from riocli.utils.selector import show_selection from riocli.v2client import Client from riocli.v2client.enums import DeploymentPhaseConstants -from riocli.deployment.errors import ERRORS ALL_PHASES = [ DeploymentPhaseConstants.DeploymentPhaseInProgress, @@ -145,36 +144,3 @@ def print_deployments_for_confirmation(deployments: typing.List[Deployment]): deployment.status.aggregateStatus]) tabulate_data(data, headers) - - -def process_deployment_errors(errors: List, no_action: bool = False) -> str: - err_fmt = '[{}] {}\nAction: {}' - support_action = ('Report the issue together with the relevant' - ' details to the support team') - - action, description = '', '' - msgs = [] - for code in errors: - if code in ERRORS: - description = ERRORS[code]['description'] - action = ERRORS[code]['action'] - elif code.startswith('DEP_E2'): - description = 'Internal rapyuta.io error in the components deployed on cloud' - action = support_action - elif code.startswith('DEP_E3'): - description = 'Internal rapyuta.io error in the components deployed on a device' - action = support_action - elif code.startswith('DEP_E4'): - description = 'Internal rapyuta.io error' - action = support_action - - code = click.style(code, fg=Colors.YELLOW) - description = click.style(description, fg=Colors.RED) - action = click.style(action, fg=Colors.GREEN) - - if no_action: - msgs.append('{}: {}'.format(code, description, '')) - else: - msgs.append(err_fmt.format(code, description, action)) - - return '\n'.join(msgs) diff --git a/riocli/device/model.py b/riocli/device/model.py index daad4563..1d3a53ba 100644 --- a/riocli/device/model.py +++ b/riocli/device/model.py @@ -11,19 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing -from rapyuta_io import Client from rapyuta_io.clients.device import Device as v1Device, DevicePythonVersion +from riocli.config import new_client from riocli.device.util import ( create_hwil_device, delete_hwil_device, execute_onboard_command, - find_device_guid, - make_device_labels_from_hwil_device + find_device_by_name, + make_device_labels_from_hwil_device, + DeviceNotFound, ) -from riocli.jsonschema.validate import load_schema from riocli.model import Model @@ -31,81 +30,67 @@ class Device(Model): def __init__(self, *args, **kwargs): self.update(*args, **kwargs) - def find_object(self, client: Client) -> bool: - guid, obj = self.rc.find_depends({ - "kind": "device", - "nameOrGUID": self.metadata.name, - }) + def apply(self, *args, **kwargs) -> None: + client = new_client() - if not guid: - return False + device = None - return obj + try: + device = find_device_by_name(client, self.metadata.name) + except DeviceNotFound: + pass - def create_object(self, client: Client, **kwargs) -> v1Device: + # If the device is not virtual, create it and return. if not self.spec.get('virtual', {}).get('enabled', False): - device = client.create_device(self.to_v1()) - return device + if device is None: + client.create_device(self.to_v1()) + return - # Create the HWIL device. - hwil_response = create_hwil_device(self.spec, self.metadata) + # Return if the device is already online or initializing. + if device and device['status'] in ('ONLINE', 'INITIALIZING'): + return - # Generate labels to store HWIL metadata in rapyuta.io device. + # Create the HWIL (virtual) device and then generate the labels + # to store HWIL metadata in rapyuta.io device. + hwil_response = create_hwil_device(self.spec, self.metadata) labels = make_device_labels_from_hwil_device(hwil_response) - labels.update(self.metadata.get('labels', {})) - self.metadata.labels = labels - # Create the rapyuta.io device. - device = client.create_device(self.to_v1()) + # Create the rapyuta.io device with labels if the + # device does not exist. Else, update the labels. + if device is None: + labels.update(self.metadata.get('labels', {})) + self.metadata.labels = labels + device = client.create_device(self.to_v1()) + else: + device_labels = device.get('labels', {}) + # Convert list to dict for easy access. + device_labels = {l['key']: l for l in device_labels} + # Add or update labels in the device. + for k, v in labels.items(): + if k in device_labels: + device_labels[k]['value'] = v + device.update_label(device_labels[k]) + continue + + device.add_label(k, v) # On-board the HWIL device using the onboard script. onboard_script = device.onboard_script() onboard_command = onboard_script.full_command() execute_onboard_command(hwil_response.id, onboard_command) - return device - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - if not self.spec.get('virtual', {}).get('enabled', False): - return obj - - device_uuid = find_device_guid(client, self.metadata.name) - device = client.get_device(device_uuid) - - # Nothing to do if the device is already online or initializing. - if device['status'] in ('ONLINE', 'INITIALIZING'): - return device + def delete(self, *args, **kwargs) -> None: + client = new_client() - device_labels = device.get('labels', {}) - # Convert list to dict for easy access. - device_labels = {l['key']: l for l in device_labels} + try: + device = find_device_by_name(client, self.metadata.name) + except DeviceNotFound: + return - # Otherwise, re-onboard the device. - hwil_response = create_hwil_device(self.spec, self.metadata) - labels = make_device_labels_from_hwil_device(hwil_response) - - # Add or update labels in the device. - for k, v in labels.items(): - if k in device_labels: - device_labels[k]['value'] = v - device.update_label(device_labels[k]) - continue - - device.add_label(k, v) - - onboard_script = device.onboard_script() - onboard_command = onboard_script.full_command() - execute_onboard_command(hwil_response.id, onboard_command) - - return device - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: if self.spec.get('virtual', {}).get('enabled', False): - device_uuid = find_device_guid(client, self.metadata.name) - device = client.get_device(device_uuid) delete_hwil_device(device) - obj.delete() + device.delete() def to_v1(self) -> v1Device: python_version = DevicePythonVersion(self.spec.python) @@ -130,15 +115,3 @@ def to_v1(self) -> v1Device: rosbag_mount_path=rosbag_mount_path, ros_workspace=ros_workspace, config_variables=config_variables, labels=labels, ) - - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - - @staticmethod - def validate(data): - """ - Validates if device data is matching with its corresponding schema - """ - schema = load_schema('device') - schema.validate(data) diff --git a/riocli/device/util.py b/riocli/device/util.py index 4501d977..ed16fae1 100644 --- a/riocli/device/util.py +++ b/riocli/device/util.py @@ -78,13 +78,21 @@ def get_device_name(client: Client, guid: str) -> str: def find_device_guid(client: Client, name: str) -> str: - devices = client.get_all_devices() + devices = client.get_all_devices(device_name=name) for device in devices: if device.name == name: return device.uuid raise DeviceNotFound() +def find_device_by_name(client: Client, name: str) -> Device: + devices = client.get_all_devices(device_name=name) + for device in devices: + if device.name == name: + return device + + raise DeviceNotFound() + def name_to_request_id(f: typing.Callable) -> typing.Callable: @functools.wraps(f) diff --git a/riocli/disk/model.py b/riocli/disk/model.py index dc61fecc..8bbe15fe 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -11,72 +11,44 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing -import click from munch import unmunchify from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols -from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.v2client import Client +from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError class Disk(Model): def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> typing.Any: - guid, disk = self.rc.find_depends({ - 'kind': 'disk', - 'nameOrGUID': self.metadata.name - }) + def apply(self, *args, **kwargs) -> None: + client = new_v2_client() - return disk if guid else False - - def create_object(self, client: Client, **kwargs) -> typing.Any: - v2_client = new_v2_client() + self.metadata.createdAt = None + self.metadata.updatedAt = None - r = v2_client.create_disk(self._sanitize_disk()) retry_count = int(kwargs.get('retry_count')) retry_interval = int(kwargs.get('retry_interval')) + try: - v2_client.poll_disk(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) + r = client.create_disk(unmunchify(self)) + client.poll_disk( + r.metadata.name, + retry_count=retry_count, + sleep_interval=retry_interval, + ) + except HttpAlreadyExistsError: + pass except Exception as e: - click.secho(">> {}: Error polling for disk ({}:{})".format( - Symbols.WARNING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW) - click.secho(str(e), fg=Colors.YELLOW) - - return unmunchify(r) - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - v2_client = new_v2_client() - v2_client.delete_disk(obj.metadata.name) + raise e - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass + def delete(self, *args, **kwargs) -> None: + client = new_v2_client() - @staticmethod - def validate(d): - schema = load_schema('disk') - schema.validate(d) - - def _sanitize_disk(self) -> typing.Dict: - # Unset createdAt and updatedAt to avoid timestamp parsing issue. - self.metadata.createdAt = None - self.metadata.updatedAt = None - - data = unmunchify(self) - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - data.pop("rc", None) - - return data + try: + client.delete_disk(self.metadata.name) + except HttpNotFoundError: + pass diff --git a/riocli/managedservice/model.py b/riocli/managedservice/model.py index 54886874..72863589 100644 --- a/riocli/managedservice/model.py +++ b/riocli/managedservice/model.py @@ -1,4 +1,4 @@ -# Copyright 2022 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,55 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing -from munch import munchify, unmunchify -from rapyuta_io import Client +from munch import unmunchify from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema from riocli.model import Model +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class ManagedService(Model): - def find_object(self, client: Client) -> typing.Any: - name = self.metadata.name - client = new_v2_client() - - try: - instance = client.get_instance(name) - return munchify(instance) - except Exception: - return False + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.update(*args, **kwargs) - def create_object(self, client: Client, **kwargs) -> typing.Any: + def apply(self, *args, **kwargs) -> None: client = new_v2_client() - ms = unmunchify(self) - ms.pop('rc', None) - result = client.create_instance(ms) - return munchify(result) - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - client = new_v2_client() - client.delete_instance(obj.metadata.name) + try: + client.create_instance(unmunchify(self)) + except HttpAlreadyExistsError: + pass - @staticmethod - def list_instances(): + def delete(self, *args, **kwargs) -> None: client = new_v2_client() - return client.list_instances() - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - - @staticmethod - def validate(data): - """ - Validates if managedservice data is matching with its corresponding schema - """ - schema = load_schema('managedservice') - schema.validate(data) + try: + client.delete_instance(self.metadata.name) + except HttpNotFoundError: + pass diff --git a/riocli/model/base.py b/riocli/model/base.py index b5d4f71e..16245725 100644 --- a/riocli/model/base.py +++ b/riocli/model/base.py @@ -1,4 +1,4 @@ -# Copyright 2022 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,179 +13,62 @@ # limitations under the License. import typing from abc import ABC, abstractmethod -from datetime import datetime -from shutil import get_terminal_size -import click -from munch import Munch, munchify -from rapyuta_io import Client -from yaspin.api import Yaspin +from munch import Munch -from riocli.constants import Colors, Symbols -from riocli.project.util import find_project_guid - -prompt = ">> {}{}{} [{}]" # >> left_msg spacer right_msg time +from riocli.jsonschema.validate import load_schema DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' -def message_with_prompt( - msg, - right_msg='', - fg=Colors.WHITE, - with_time=True, - spinner: Yaspin = None, -) -> None: - columns, _ = get_terminal_size() - t = datetime.now().isoformat('T') - spacer = ' ' * (int(columns) - len(msg + right_msg + t) - 12) - msg = prompt.format(msg, spacer, right_msg, t) - msg = click.style(msg, fg=fg) - if spinner: - spinner.write(msg) - else: - click.echo(msg) - - class Model(ABC, Munch): + """Base class for all models. - def apply(self, client: Client, *args, **kwargs) -> typing.Any: - spinner = kwargs.get('spinner') - try: - self._set_project_in_client(client) - obj = self.find_object(client) - dryrun = kwargs.get("dryrun", False) - if not obj: - message_with_prompt("{} Create {}:{}".format( - Symbols.WAITING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW, spinner=spinner) - if not dryrun: - result = self.create_object(client, **kwargs) - message_with_prompt("{} Created {}:{}".format( - Symbols.SUCCESS, - self.kind.lower(), - self.metadata.name), fg=Colors.GREEN, spinner=spinner) - return result - else: - message_with_prompt('{} {}:{} exists. will be updated'.format( - Symbols.INFO, - self.kind.lower(), - self.metadata.name), spinner=spinner) - message_with_prompt("{} Update {}:{}".format( - Symbols.WAITING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW, spinner=spinner) - if not dryrun: - result = self.update_object(client, obj) - message_with_prompt("{} Updated {}:{}".format( - Symbols.SUCCESS, - self.kind.lower(), - self.metadata.name), fg=Colors.GREEN, spinner=spinner) - return result - except Exception as e: - message_with_prompt("{} {}:{}. {} ‼".format( - Symbols.ERROR, - self.kind.lower(), - self.metadata.name, - str(e)), fg=Colors.RED, spinner=spinner) - raise e - - def delete(self, client: Client, obj: typing.Any, *args, **kwargs): - spinner = kwargs.get('spinner') - dryrun = kwargs.get("dryrun", False) - try: - self._set_project_in_client(client) - obj = self.find_object(client) - - if not obj: - message_with_prompt( - '⁉ {}:{} does not exist'.format( - self.kind.lower(), self.metadata.name), - spinner=spinner) - return - - message_with_prompt("{} Delete {}:{}".format( - Symbols.WAITING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW, spinner=spinner) + This class provides the basic structure for all models. It + also provides the basic methods that should be implemented by + all models. The methods are: - if not dryrun: - labels = self.metadata.get('labels', {}) - if (DELETE_POLICY_LABEL in labels and - labels.get(DELETE_POLICY_LABEL) and - labels.get(DELETE_POLICY_LABEL).lower() == "retain"): - click.secho( - ">> {} Delete protection enabled on {}:{}. " - "Resource will be retained ".format( - Symbols.WARNING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW) - return - - self.delete_object(client, obj) - message_with_prompt("{} Deleted {}:{}".format( - Symbols.SUCCESS, - self.kind.lower(), - self.metadata.name), fg=Colors.GREEN, spinner=spinner) - except Exception as e: - message_with_prompt("{} {}:{}. {} ‼".format( - Symbols.ERROR, - self.kind.lower(), - self.metadata.name, - str(e)), fg=Colors.RED, spinner=spinner) - raise e - - @abstractmethod - def find_object(self, client: Client) -> typing.Any: + # Create or update the object + def apply(self, *args, **kwargs): pass - @abstractmethod - def create_object(self, client: Client, **kwargs) -> typing.Any: + # Delete the object + def delete(self, *args, **kwargs): pass + The validate method is a class method that need not be implemented + by the subclasses. It validates the model against the corresponding + schema that are defined in the schema files. + """ @abstractmethod - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass + def apply(self, *args, **kwargs) -> None: + """Create or update the object. - @staticmethod - @abstractmethod - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass + This method should be implemented by the subclasses. It should + create or update the object based on the values in the model. + """ + raise NotImplementedError - @classmethod @abstractmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass + def delete(self, *args, **kwargs) -> None: + """Delete the object. - @staticmethod - @abstractmethod - def validate(d): - pass + This method should be implemented by the subclasses. It should + delete the object based on the values in the model. + """ + raise NotImplementedError @classmethod - def from_dict(cls, client: Client, d: typing.Dict): - cls.pre_process(client, d) - cls.validate(d) - return cls(munchify(d)) - - def _set_project_in_client(self, client: Client) -> Client: - # If the Type is Project itself then no need to configure Client. - if self.kind == 'Project': - return client - - # If Project is not specified then no need to configure the Client. It - # will use the pre-configured Project by default. - # - # TODO(ankit): Move this to the pre-processing step, once implemented. - project = self.metadata.get('project', None) - if not project: - return client - - # This should work unless someone has a Project Name starting with - # 'project-' prefix - if not project.startswith('project-'): - project = find_project_guid(client, project) - - client.set_project(project_guid=project) - return client + def validate(cls, d: typing.Dict) -> None: + """Validate the model against the corresponding schema.""" + kind = d.get('kind') + if not kind: + raise ValueError('kind is required') + + # StaticRoute's schema file is named + # static_route-schema.yaml. + if kind == 'StaticRoute': + kind = 'static_route' + + schema = load_schema(kind.lower()) + schema.validate(d) diff --git a/riocli/network/model.py b/riocli/network/model.py index 4f41242e..877b3ce2 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,75 +11,40 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing -from typing import Any, Dict -import click from munch import unmunchify from riocli.config import new_v2_client -from riocli.constants import Colors, Symbols -from riocli.jsonschema.validate import load_schema from riocli.model import Model -from riocli.v2client.client import NetworkNotFound, Client +from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError class Network(Model): def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> bool: - network, obj = self.rc.find_depends({"kind": self.kind.lower(), - "nameOrGUID": self.metadata.name}, self.spec.type) - return obj if network else False - - def create_object(self, client: Client, **kwargs) -> typing.Any: + def apply(self, *args, **kwargs) -> None: client = new_v2_client() - r = client.create_network(self._sanitize_network()) + self.metadata.createdAt = None + self.metadata.updatedAt = None retry_count = int(kwargs.get('retry_count')) retry_interval = int(kwargs.get('retry_interval')) try: - r = client.poll_network(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) + r = client.create_network(unmunchify(self)) + client.poll_network(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) + except HttpAlreadyExistsError: + pass except Exception as e: - click.secho(">> {}: Error polling for network ({}:{})".format( - Symbols.WARNING, - self.kind.lower(), - self.metadata.name), fg=Colors.YELLOW) - click.secho(str(e), fg=Colors.YELLOW) - - return unmunchify(r) + raise e - def update_object(self, client: Client, obj: typing.Any) -> Any: - pass - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: + def delete(self, *args, **kwargs) -> None: client = new_v2_client() - client.delete_network(obj.metadata.name) - - @classmethod - def pre_process(cls, client: Client, d: Dict) -> None: - pass - - @staticmethod - def validate(data): - """ - Validates if network data is matching with its corresponding schema - """ - schema = load_schema('network') - schema.validate(data) - def _sanitize_network(self) -> typing.Dict: - # Unset createdAt and updatedAt to avoid timestamp parsing issue. - self.metadata.createdAt = None - self.metadata.updatedAt = None - - data = unmunchify(self) - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - data.pop("rc", None) - - return data + try: + client.delete_network(self.metadata.name) + except HttpNotFoundError: + pass \ No newline at end of file diff --git a/riocli/package/model.py b/riocli/package/model.py index 912424e3..a0cbe52f 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -16,10 +16,9 @@ from munch import unmunchify from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema from riocli.model import Model from riocli.package.enum import RestartPolicy -from riocli.v2client import Client +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class Package(Model): @@ -30,64 +29,53 @@ class Package(Model): } def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client): - guid, obj = self.rc.find_depends({"kind": self.kind.lower(), "nameOrGUID": self.metadata.name}, - self.metadata.version) - return obj if guid else False + def apply(self, *args, **kwargs) -> None: + client = new_v2_client() - def create_object(self, client: Client, **kwargs) -> typing.Any: - v2_client = new_v2_client() + package = self._sanitize_package() - r = v2_client.create_package(self._sanitize_package()) - return unmunchify(r) + try: + client.create_package(package) + except HttpAlreadyExistsError: + pass - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - pass + def delete(self, *args, **kwargs) -> None: + client = new_v2_client() - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - v2_client = new_v2_client() - v2_client.delete_package(obj.metadata.name, query={"version": obj.metadata.version}) - - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass + try: + client.delete_package( + self.metadata.name, + query={"version": self.metadata.version} + ) + except HttpNotFoundError: + pass def _sanitize_package(self) -> typing.Dict: # Unset createdAt and updatedAt to avoid timestamp parsing issue. self.metadata.createdAt = None self.metadata.updatedAt = None - self._convert_command() - - data = unmunchify(self) - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - data.pop("rc", None) + self._sanitize_command() - return data + return unmunchify(self) - def _convert_command(self): - for exec in self.spec.executables: - if exec.get('command') is not None: - c = [] + def _sanitize_command(self): + for e in self.spec.executables: + # Skip if command is not set. + if not e.get('command'): + continue - if exec.get('runAsBash'): - c = ['/bin/bash', '-c'] + c = [] - if isinstance(exec.command, list): - c.extend(exec.command) - else: - c.append(exec.command) + if e.get('runAsBash'): + c = ['/bin/bash', '-c'] - exec.command = c + if isinstance(e.command, list): + c.extend(e.command) + else: + c.append(e.command) - @staticmethod - def validate(data): - """ - Validates if package data is matching with its corresponding schema - """ - schema = load_schema('package') - schema.validate(data) + e.command = c diff --git a/riocli/project/model.py b/riocli/project/model.py index d7bd804f..c1883bc1 100644 --- a/riocli/project/model.py +++ b/riocli/project/model.py @@ -1,4 +1,4 @@ -# Copyright 2022 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,15 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing from munch import unmunchify -from rapyuta_io import Client -from waiting import wait, TimeoutExpired +from waiting import wait from riocli.config import new_v2_client, Configuration -from riocli.jsonschema.validate import load_schema from riocli.model import Model +from riocli.project.util import find_project_guid, ProjectNotFound +from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError PROJECT_READY_TIMEOUT = 150 @@ -30,65 +29,35 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> bool: - guid, obj = self.rc.find_depends( - {"kind": "project", "nameOrGUID": self.metadata.name}) - if not guid: - return False - - return obj - - def create_object(self, client: Client, **kwargs) -> typing.Any: + def apply(self, *args, **kwargs) -> None: client = new_v2_client() - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable project = unmunchify(self) - project.pop("rc", None) # set organizationGUID irrespective of it being present in the manifest - project['metadata']['organizationGUID'] = Configuration().data[ - 'organization_id'] - - r = client.create_project(project) + project['metadata']['organizationGUID'] = Configuration().organization_guid try: + r = client.create_project(project) wait(self.is_ready, timeout_seconds=PROJECT_READY_TIMEOUT, sleep_seconds=(1, 30, 2)) - except TimeoutExpired as e: + except HttpAlreadyExistsError: + guid = find_project_guid(client, self.metadata.name, Configuration().organization_guid) + client.update_project(guid, project) + return + except Exception as e: raise e - return unmunchify(r) - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: + def delete(self, *args, **kwargs) -> None: client = new_v2_client() - project = unmunchify(self) - project.pop("rc", None) - - # set organizationGUID irrespective of it being present in the manifest - project['metadata']['organizationGUID'] = Configuration().data[ - 'organization_id'] - - client.update_project(obj.metadata.guid, project) - - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - client = new_v2_client() - client.delete_project(obj.metadata.guid) + try: + guid = find_project_guid(client, self.metadata.name, Configuration().data['organization_id']) + client.delete_project(guid) + except (HttpNotFoundError, ProjectNotFound): + pass def is_ready(self) -> bool: client = new_v2_client() projects = client.list_projects(query={"name": self.metadata.name}) return projects[0].status.status == 'Success' - - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - - @staticmethod - def validate(data): - """ - Validates if project data is matching with its corresponding schema - """ - schema = load_schema('project') - schema.validate(data) diff --git a/riocli/secret/model.py b/riocli/secret/model.py index 2b416f54..30baf2ba 100644 --- a/riocli/secret/model.py +++ b/riocli/secret/model.py @@ -11,62 +11,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing from munch import unmunchify -from rapyuta_io import Client from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema from riocli.model import Model +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class Secret(Model): def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> bool: - _, secret = self.rc.find_depends({ - 'kind': 'secret', - 'nameOrGUID': self.metadata.name - }) - - if not secret: - return False - - return secret - - def create_object(self, client: Client, **kwargs) -> typing.Any: - client = new_v2_client() - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - secret = unmunchify(self) - secret.pop("rc", None) - r = client.create_secret(secret) - return unmunchify(r) - - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: + def apply(self, *args, **kwargs) -> None: client = new_v2_client() secret = unmunchify(self) - secret.pop("rc", None) - r = client.update_secret(obj.metadata.name, secret) - return unmunchify(r) + try: + client.create_secret(unmunchify(self)) + except HttpAlreadyExistsError: + client.update_secret(self.metadata.name, secret) - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: + def delete(self, *args, **kwargs) -> None: client = new_v2_client() - client.delete_secret(obj.metadata.name) - - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - @staticmethod - def validate(data): - """ - Validates if secret data is matching with its corresponding schema - """ - schema = load_schema('secret') - schema.validate(data) + try: + client.delete_secret(self.metadata.name) + except HttpNotFoundError: + pass diff --git a/riocli/static_route/model.py b/riocli/static_route/model.py index 53d9e225..79c23112 100644 --- a/riocli/static_route/model.py +++ b/riocli/static_route/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,62 +11,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import typing from munch import unmunchify -from rapyuta_io import Client - -from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema +from riocli.config import new_v2_client, Configuration from riocli.model import Model +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class StaticRoute(Model): def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def find_object(self, client: Client) -> bool: - _, static_route = self.rc.find_depends({ - 'kind': 'staticroute', - 'nameOrGUID': self.metadata.name - }) - if not static_route: - return False - - return static_route - - def create_object(self, client: Client, **kwargs) -> typing.Any: - client = new_v2_client() - - # convert to a dict and remove the ResolverCache - # field since it's not JSON serializable - self.pop("rc", None) - static_route = unmunchify(self) - r = client.create_static_route(static_route) - return unmunchify(r) - - def update_object(self, client: Client, obj: typing.Any) -> None: + def apply(self, *args, **kwargs) -> None: client = new_v2_client() - self.pop("rc", None) - static_route = unmunchify(self) - r = client.update_static_route(obj.metadata.name, static_route) - return unmunchify(r) + try: + client.create_static_route(static_route) + except HttpAlreadyExistsError: + client.update_static_route(self.metadata.name, static_route) - def delete_object(self, client: Client, obj: typing.Any): + def delete(self, *args, **kwargs) -> None: client = new_v2_client() - client.delete_static_route(obj.metadata.name) - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass + short_id = Configuration().organization_short_id - @staticmethod - def validate(data): - """ - Validates if static route data is matching with its corresponding schema - """ - schema = load_schema('static_route') - schema.validate(data) + try: + client.delete_static_route(f'{self.metadata.name}-{short_id}') + except HttpNotFoundError: + pass diff --git a/riocli/usergroup/model.py b/riocli/usergroup/model.py index 73b49a28..53a4fb81 100644 --- a/riocli/usergroup/model.py +++ b/riocli/usergroup/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ import typing from munch import unmunchify -from rapyuta_io import Client -from riocli.config import new_v2_client -from riocli.jsonschema.validate import load_schema +from riocli.config import new_client, new_v2_client, Configuration from riocli.model import Model from riocli.organization.utils import get_organization_details +from riocli.usergroup.util import find_usergroup_guid, UserGroupNotFound USER_GUID = 'guid' USER_EMAIL = 'emailID' @@ -28,48 +27,65 @@ class UserGroup(Model): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.update(*args, **kwargs) + + self.user_email_to_guid_map = {} + self.project_name_to_guid_map = {} + + def apply(self, *args, **kwargs) -> None: + v1client = new_client() v2client = new_v2_client() - organization_details = get_organization_details(self.metadata.organization) - user_projects = v2client.list_projects(self.metadata.organization) + + organization_id = self.metadata.organization or Configuration().organization_guid + existing = None + + try: + guid = find_usergroup_guid(v1client, organization_id, self.metadata.name) + existing = v1client.get_usergroup(organization_id, guid) + except UserGroupNotFound: + pass + except Exception as e: + raise e + + organization_details = get_organization_details(organization_id) + user_projects = v2client.list_projects(organization_id) self.project_name_to_guid_map = {p['metadata']['name']: p['metadata']['guid'] for p in user_projects} self.user_email_to_guid_map = {user[USER_EMAIL]: user[USER_GUID] for user in organization_details['users']} - def unmunchify(self) -> typing.Dict: - """Unmuchify self""" - usergroup = unmunchify(self) - usergroup.pop('rc', None) - usergroup.pop('project_name_to_guid_map', None) - usergroup.pop('user_email_to_guid_map', None) + sanitized = self._sanitize() + payload = self._modify_payload(sanitized) - return usergroup + if not existing: + try: + payload['spec']['name'] = sanitized['metadata']['name'] + v1client.create_usergroup(self.metadata.organization, payload['spec']) + return + except Exception as e: + raise e - def find_object(self, client: Client) -> typing.Any: - group_guid, usergroup = self.rc.find_depends({ - 'kind': self.kind.lower(), - 'nameOrGUID': self.metadata.name, - 'organization': self.metadata.organization - }) + payload = self._generate_update_payload(existing, payload) + v1client.update_usergroup(organization_id, existing.guid, payload) - if not usergroup: - return False + def delete(self, *args, **kwargs) -> None: + v1client = new_client() - return usergroup + organization_id = self.metadata.organization or Configuration().organization_guid - def create_object(self, client: Client, **kwargs) -> typing.Any: - usergroup = self.unmunchify() - payload = self._modify_payload(usergroup) - # Inject the user group name in the payload - payload['spec']['name'] = usergroup['metadata']['name'] - return client.create_usergroup(self.metadata.organization, payload['spec']) + try: + guid = find_usergroup_guid(v1client, organization_id, self.metadata.name) + v1client.delete_usergroup(organization_id, guid) + except UserGroupNotFound: + pass + except Exception as e: + raise e - def update_object(self, client: Client, obj: typing.Any) -> typing.Any: - payload = self._modify_payload(self.unmunchify()) - payload = self._create_update_payload(obj, payload) - return client.update_usergroup(self.metadata.organization, obj.guid, payload) + def _sanitize(self) -> typing.Dict: + u = unmunchify(self) + u.pop('project_name_to_guid_map', None) + u.pop('user_email_to_guid_map', None) - def delete_object(self, client: Client, obj: typing.Any) -> typing.Any: - return client.delete_usergroup(self.metadata.organization, obj.guid) + return u def _modify_payload(self, group: typing.Dict) -> typing.Dict: group['spec']['userGroupRoleInProjects'] = [] @@ -94,17 +110,8 @@ def _modify_payload(self, group: typing.Dict) -> typing.Dict: return group - @classmethod - def pre_process(cls, client: Client, d: typing.Dict) -> None: - pass - - @staticmethod - def validate(data): - schema = load_schema('usergroup') - schema.validate(data) - @staticmethod - def _create_update_payload(old: typing.Any, new: typing.Dict) -> typing.Dict: + def _generate_update_payload(old: typing.Any, new: typing.Dict) -> typing.Dict: payload = { 'name': old.name, 'guid': old.guid, diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 986819a7..07527b99 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import annotations -import http import json import os import time @@ -22,37 +21,13 @@ import click import magic -import requests from munch import munchify, Munch from rapyuta_io.utils.rest_client import HttpMethod, RestClient from riocli.constants import Colors from riocli.v2client.enums import DeploymentPhaseConstants, DiskStatusConstants -from riocli.v2client.error import (RetriesExhausted, DeploymentNotRunning, ImagePullError, - NetworkNotFound, DeploymentNotFound) -from riocli.v2client.util import process_errors - - -def handle_server_errors(response: requests.Response): - status_code = response.status_code - # 500 Internal Server Error - if status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: - raise Exception('internal server error') - # 501 Not Implemented - if status_code == http.HTTPStatus.NOT_IMPLEMENTED: - raise Exception('not implemented') - # 502 Bad Gateway - if status_code == http.HTTPStatus.BAD_GATEWAY: - raise Exception('bad gateway') - # 503 Service Unavailable - if status_code == http.HTTPStatus.SERVICE_UNAVAILABLE: - raise Exception('service unavailable') - # 504 Gateway Timeout - if status_code == http.HTTPStatus.GATEWAY_TIMEOUT: - raise Exception('gateway timeout') - # Anything else that is not known - if status_code > 504: - raise Exception('unknown server error') +from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError +from riocli.v2client.util import process_errors, handle_server_errors class Client(object): @@ -709,7 +684,7 @@ def _walk_pages(self, c: RestClient, params: dict = None, limit: Optional[int] = params["continue"] = offset response = c.query_param(params).execute() - data = json.loads(response.text) + data = response.json() if not response.ok: err_msg = data.get('error') raise Exception("listing: {}".format(err_msg)) @@ -837,6 +812,7 @@ def create_network(self, payload: dict) -> Munch: headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.POST).headers( headers).execute(payload=payload) + handle_server_errors(response) data = json.loads(response.text) @@ -862,10 +838,10 @@ def get_network( response = RestClient(url).method(HttpMethod.GET).query_param( params).headers(headers).execute() - data = json.loads(response.text) - if response.status_code == http.HTTPStatus.NOT_FOUND: - raise NetworkNotFound("network: {} not found".format(name)) + handle_server_errors(response) + + data = json.loads(response.text) if not response.ok: err_msg = data.get('error') @@ -957,7 +933,9 @@ def create_deployment(self, deployment: dict) -> Munch: deployment["metadata"]["projectGUID"] = headers["project"] response = RestClient(url).method(HttpMethod.POST).headers( headers).execute(payload=deployment) + handle_server_errors(response) + data = json.loads(response.text) if not response.ok: err_msg = data.get('error') @@ -970,10 +948,10 @@ def get_deployment(self, name: str): headers = self._get_auth_header() response = RestClient(url).method(HttpMethod.GET).headers(headers).execute() - data = json.loads(response.text) - if response.status_code == http.HTTPStatus.NOT_FOUND: - raise DeploymentNotFound("deployment: {} not found".format(name)) + handle_server_errors(response) + + data = json.loads(response.text) if not response.ok: err_msg = data.get('error') diff --git a/riocli/v2client/enums.py b/riocli/v2client/enums.py index 6fe68ef9..917aea8f 100644 --- a/riocli/v2client/enums.py +++ b/riocli/v2client/enums.py @@ -1,3 +1,17 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import enum diff --git a/riocli/v2client/error.py b/riocli/v2client/error.py index 145b9214..b8724c19 100644 --- a/riocli/v2client/error.py +++ b/riocli/v2client/error.py @@ -1,3 +1,17 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class RetriesExhausted(Exception): def __init__(self, msg=None): Exception.__init__(self, msg) @@ -13,13 +27,14 @@ def __init__(self, msg=None): Exception.__init__(self, msg) -class NetworkNotFound(Exception): - def __init__(self, message='network not found!'): +class HttpAlreadyExistsError(Exception): + def __init__(self, message='resource already exists'): self.message = message super().__init__(self.message) -class DeploymentNotFound(Exception): - def __init__(self, message='deployment not found!'): +class HttpNotFoundError(Exception): + def __init__(self, message='resource not found'): self.message = message super().__init__(self.message) + diff --git a/riocli/v2client/errors.py b/riocli/v2client/errors.py deleted file mode 100644 index 9576e24f..00000000 --- a/riocli/v2client/errors.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2021 Rapyuta Robotics -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ERRORS = { - # Device related Error codes - "DEP_E151": { - "description": "Device is either offline or not reachable", - "action": "Ensure that the device is connected to the internet" - }, - "DEP_E302": { - "description": "Supervisord related error on device", - "action": ("Ensure supervisord is installed on rapyuta virtual environment and working\n" - "Re-onboard the device, if issue persists report to rapyuta.io support.") - }, - "DEP_E161": { - "description": "Docker image not found for executables of components deployed on device", - "action": "Ensure that the docker image path is valid and secrets have privileges" - }, - "DEP_E316": { - "description": "Docker image pull failed in device", - "action": "Ensure that the docker image path is valid and secrets have privileges" - }, - "DEP_E317": { - "description": "Missing subpath or empty mount path", - "action": ("Ensure the current device has a subpath\n" - "Ensure that the mount path isn’t empty") - }, - "DEP_E304": { - "description": "DeviceManager internal error", - "action": "Report to rapyuta.io support" - }, - "DEP_E306": { - "description": "DeviceEdge internal error", - "action": "Ensure deviceedge is up and running" - }, - "DEP_E307": { - "description": "DeviceEdge bad request", - "action": "Report to rapyuta.io support" - }, - - # Application related errors. - "DEP_E152": { - "description": "Executables of the component deployed on the device either exited too early or failed", - "action": "Executables of the component deployed on the device either exited too early or failed" - }, - "DEP_E153": { - "description": "Unable to pull the docker image for the component deployed on cloud", - "action": ("Ensure that the docker image provided while adding the package still " - "exists at the specified registry endpoint ") - }, - "DEP_E154": { - "description": "Executables of the component deployed on cloud exited too early", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E155": { - "description": "Executables of the component deployed on cloud failed", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E156": { - "description": "Dependent deployment is in error state", - "action": "Troubleshoot the dependent deployment that is in error state" - }, - "DEP_E162": { - "description": ("Validation error. Cases include:\nInconsistent values of ROS distro " - "and CPU architecture variables for the device and package being provisioned. " - "\nrapyuta.io docker images not present on docker device."), - "action": ("Create package with appropriate values for ROS distro and CPU architecture " - "variables.\nOnboard the device again.") - }, - "DEP_E163": { - "description": "Application has stopped and exited unexpectedly, and crashes continuously on device", - "action": "Debug the application using the corresponding deployment logs" - }, - - # CloudBridge related error codes. - "DEP_E171": { - "description": "Cloud bridge encountered duplicate alias on the device", - "action": ("Change the alias name during deployment and ensure that there" - " is no duplication of alias name under the same routed network") - }, - "DEP_E172": { - "description": "Compression library required for the cloud bridge is missing on the device", - "action": "Re-onboard the device" - }, - "DEP_E173": { - "description": "Transport libraries required for the cloud bridge are missing on the device", - "action": "Re-onboard the device" - }, - "DEP_E174": { - "description": "Cloud bridge on the device encountered multiple ROS service origins", - "action": ("Ensure that there aren’t multiple deployments with the same ROS service " - "endpoint under the same routed network") - }, - "DEP_E175": { - "description": "Python actionlib/msgs required for the cloud bridge is missing on the device", - "action": "Re-onboard the device" - }, - "DEP_E176": { - "description": "Cloud bridge encountered duplicate alias on the cloud component", - "action": ("Change the alias name during deployment and ensure that there is no" - " duplication of alias name under the same routed network") - }, - "DEP_E177": { - "description": "Cloud bridge on the cloud component encountered multiple ROS service origins", - "action": "Re-onboard the device" - }, - - # Docker image related Error codes - "DEP_E350": { - "description": "Get http error when pulling image from registry on device", - "action": "Ensure registry is accesible." - }, - "DEP_E351": { - "description": "Unable to parse the image name on device", - "action": "Ensure image name is correct and available in registry." - }, - "DEP_E352": { - "description": "Unable to inspect image on device", - "action": "Ensure valid image is present" - }, - "DEP_E353": { - "description": "Required Image is absent on device and PullPolicy is NeverPullImage", - "action": "Ensure pull policy is not set to never or image is present on device" - }, - - # Container related error codes - "DEP_E360": { - "description": "Failed to create container config on device", - "action": ("Ensure executable config is mounted correctly\n" - "If issue persists report to rapyuta.io support") - }, - "DEP_E361": { - "description": "Runtime failed to start any of pod's container on device", - "action": ("Ensure executable command is valid\n" - "If issue persists report to rapyuta.io support") - }, - "DEP_E362": { - "description": "Runtime failed to create container on device", - "action": ("Report to rapyuta.io support") - }, - "DEP_E363": { - "description": "Runtime failed to kill any of pod's containers on device", - "action": ("Report to rapyuta.io support") - }, - "DEP_E364": { - "description": "Runtime failed to create a sandbox for pod on device", - "action": ("Report to rapyuta.io support") - }, - "DEP_E365": { - "description": "Runtime failed to get pod sandbox config from pod on device", - "action": ("Report to rapyuta.io support") - }, - "DEP_E366": { - "description": "Runtime failed to stop pod's sandbox on device", - "action": ("Report to rapyuta.io support") - }, - "DEP_E399": { - "description": "Deployment failed for some unknown reason on device", - "action": ("Report to rapyuta.io support") - }, - - # ROS Comm Related Device Errors - "DEP_E303": { - "description": "Cloud Bridge executable not running on device", - "action": ("Troubleshoot Cloud Bridge container on device by analyzing logs") - }, - "DEP_E305": { - "description": "Native Network executable not running on device", - "action": ("Troubleshoot native network container on device by analyzing logs") - }, - "DEP_E313": { - "description": "ROS Master executable not running on device", - "action": ("Troubleshoot ROS Master container on device by analyzing logs") - }, - "DEP_E330": { - "description": "ROS2 Native Network executable not running on device", - "action": ("Troubleshoot ROS2 Routed Network container on device by analyzing logs") - }, - "DEP_E331": { - "description": "ROS2 Routed Network executable not running on device", - "action": ("Troubleshoot ROS2 Routed Network container on device by analyzing logs") - }, - - # Cloud error codes - "DEP_E201": { - "description": "Cloud component deployment pending", - "action": ("Report to rapyuta.io support") - }, - "DEP_E202": { - "description": "Cloud component status unknown", - "action": ("Report to rapyuta.io support") - }, - "DEP_E203": { - "description": "Cloud Bridge not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E204": { - "description": "ROS Master not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E205": { - "description": "Cloud Broker not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E208": { - "description": "Desired set of replicas not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E209": { - "description": "Native Network not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E210": { - "description": "Disk Not Running", - "action": ("Ensure disk is running") - }, - "DEP_E213": { - "description": "Broker running low on memory", - "action": ("Report to rapyuta.io support") - }, - "DEP_E214": { - "description": "ROS2 Native Network not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - }, - "DEP_E215": { - "description": "ROS2 Routed Network not running on cloud", - "action": "Troubleshoot the failed component by analyzing the deployment logs" - } -} diff --git a/riocli/v2client/util.py b/riocli/v2client/util.py index 23a2aa64..dca89237 100644 --- a/riocli/v2client/util.py +++ b/riocli/v2client/util.py @@ -1,12 +1,31 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import http +import json import typing import click +import requests from riocli.constants import Colors -from riocli.v2client.errors import ERRORS +from riocli.v2client.constants import DEPLOYMENT_ERRORS as ERRORS +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError def process_errors(errors: typing.List[str]) -> str: + """Process the deployment errors and return the formatted error message""" err_fmt = '[{}] {}\nAction: {}' support_action = ('Report the issue together with the relevant' ' details to the support team') @@ -33,4 +52,42 @@ def process_errors(errors: typing.List[str]) -> str: msgs.append(err_fmt.format(code, description, action)) - return '\n'.join(msgs) \ No newline at end of file + return '\n'.join(msgs) + + +def handle_server_errors(response: requests.Response): + status_code = response.status_code + + if status_code < 400: + return + + err = '' + try: + err = response.json().get('error') + except json.JSONDecodeError: + err = response.text + + # 404 Not Found + if status_code == http.HTTPStatus.NOT_FOUND: + raise HttpNotFoundError(err) + # 409 Conflict + if status_code == http.HTTPStatus.CONFLICT: + raise HttpAlreadyExistsError() + # 500 Internal Server Error + if status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: + raise Exception('internal server error') + # 501 Not Implemented + if status_code == http.HTTPStatus.NOT_IMPLEMENTED: + raise Exception('not implemented') + # 502 Bad Gateway + if status_code == http.HTTPStatus.BAD_GATEWAY: + raise Exception('bad gateway') + # 503 Service Unavailable + if status_code == http.HTTPStatus.SERVICE_UNAVAILABLE: + raise Exception('service unavailable') + # 504 Gateway Timeout + if status_code == http.HTTPStatus.GATEWAY_TIMEOUT: + raise Exception('gateway timeout') + # Anything else that is not known + if status_code > 504: + raise Exception('unknown server error') From f99518b6b3b675070caa130178d8c694663b760c Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 1 Aug 2024 00:04:42 +0530 Subject: [PATCH 28/54] refactor(explain): updates example manifests Removes the unncessary fields from the example manifests and updates them per the latest specs. Wrike Ticket: https://www.wrike.com/open.htm?id=1465945304 --- riocli/apply/manifests/deployment-nonros-cloud.yaml | 1 - riocli/apply/manifests/deployment-nonros-device.yaml | 1 - riocli/apply/manifests/deployment-ros-cloud.yaml | 1 - .../apply/manifests/deployment-ros-device-no-rosbag.yaml | 1 - riocli/apply/manifests/deployment-ros-device-rosbag.yaml | 1 - riocli/apply/manifests/deployment.yaml | 5 ----- riocli/apply/manifests/device-docker.yaml | 1 - riocli/apply/manifests/device-hybrid.yaml | 1 - riocli/apply/manifests/device-preinstalled.yaml | 1 - riocli/apply/manifests/device.yaml | 5 +---- riocli/apply/manifests/disk.yaml | 3 +-- riocli/apply/manifests/managedservice.yaml | 1 - riocli/apply/manifests/network-native-cloud.yaml | 4 ++-- riocli/apply/manifests/network-native-device.yaml | 1 - riocli/apply/manifests/network-routed-cloud.yaml | 5 ++--- riocli/apply/manifests/network-routed-device.yaml | 1 - riocli/apply/manifests/network.yaml | 7 +++---- riocli/apply/manifests/package-nonros-device.yaml | 2 +- riocli/apply/manifests/project.yaml | 1 - riocli/apply/manifests/secret-docker.yaml | 1 - riocli/apply/manifests/secret.yaml | 1 - riocli/apply/manifests/staticroute.yaml | 1 - riocli/apply/manifests/usergroup.yaml | 1 - 23 files changed, 10 insertions(+), 37 deletions(-) diff --git a/riocli/apply/manifests/deployment-nonros-cloud.yaml b/riocli/apply/manifests/deployment-nonros-cloud.yaml index e8edb727..b96c02bb 100644 --- a/riocli/apply/manifests/deployment-nonros-cloud.yaml +++ b/riocli/apply/manifests/deployment-nonros-cloud.yaml @@ -6,7 +6,6 @@ metadata: kind: package nameOrGUID: "package-nonros-cloud" version: "v1.0.0" - project: "project-guid" region: jp labels: app: test diff --git a/riocli/apply/manifests/deployment-nonros-device.yaml b/riocli/apply/manifests/deployment-nonros-device.yaml index ae42bb94..f7a4e581 100644 --- a/riocli/apply/manifests/deployment-nonros-device.yaml +++ b/riocli/apply/manifests/deployment-nonros-device.yaml @@ -7,7 +7,6 @@ metadata: kind: package nameOrGUID: "package-nonros-device" version: "v1.0.0" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/deployment-ros-cloud.yaml b/riocli/apply/manifests/deployment-ros-cloud.yaml index 3ea64ae9..a6b17b6e 100644 --- a/riocli/apply/manifests/deployment-ros-cloud.yaml +++ b/riocli/apply/manifests/deployment-ros-cloud.yaml @@ -7,7 +7,6 @@ metadata: kind: package nameOrGUID: "package-ros-cloud" version: "v1.0.0" - project: "project-guid" region: jp labels: app: test diff --git a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml index e334cba6..b07ef1c1 100644 --- a/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-no-rosbag.yaml @@ -6,7 +6,6 @@ metadata: kind: package nameOrGUID: "package-ros-device-no-rosbag" version: "v1.0.0" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml index 3a9bdda1..97783505 100644 --- a/riocli/apply/manifests/deployment-ros-device-rosbag.yaml +++ b/riocli/apply/manifests/deployment-ros-device-rosbag.yaml @@ -7,7 +7,6 @@ metadata: kind: package nameOrGUID: "package-ros-device-rosbag" version: "v1.0.0" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/deployment.yaml b/riocli/apply/manifests/deployment.yaml index 3d63e754..ecd6ae7e 100644 --- a/riocli/apply/manifests/deployment.yaml +++ b/riocli/apply/manifests/deployment.yaml @@ -7,7 +7,6 @@ metadata: kind: package nameOrGUID: "package-ros-cloud" version: "v1.0.0" - project: "project-guid" region: jp labels: app: test @@ -104,7 +103,6 @@ metadata: kind: package nameOrGUID: "package-nonros-cloud" version: "v1.0.0" - project: "project-guid" region: jp labels: app: test @@ -149,7 +147,6 @@ metadata: kind: package nameOrGUID: "package-ros-device-no-rosbag" version: "v1.0.0" - project: "project-guid" labels: app: test spec: @@ -185,7 +182,6 @@ metadata: kind: package nameOrGUID: "package-ros-device-rosbag" version: "v1.0.0" - project: "project-guid" labels: app: test spec: @@ -280,7 +276,6 @@ metadata: kind: package nameOrGUID: "package-nonros-device" version: "v1.0.0" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/device-docker.yaml b/riocli/apply/manifests/device-docker.yaml index 5901ed2d..0b56165a 100644 --- a/riocli/apply/manifests/device-docker.yaml +++ b/riocli/apply/manifests/device-docker.yaml @@ -3,7 +3,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-docker" # Required - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/device-hybrid.yaml b/riocli/apply/manifests/device-hybrid.yaml index aa4f9cc7..44bcd2b3 100644 --- a/riocli/apply/manifests/device-hybrid.yaml +++ b/riocli/apply/manifests/device-hybrid.yaml @@ -3,7 +3,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-hybrid" # Required - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/device-preinstalled.yaml b/riocli/apply/manifests/device-preinstalled.yaml index ad32aaed..9861525b 100644 --- a/riocli/apply/manifests/device-preinstalled.yaml +++ b/riocli/apply/manifests/device-preinstalled.yaml @@ -3,7 +3,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-preinstalled" # Required - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/device.yaml b/riocli/apply/manifests/device.yaml index be977db3..5a824266 100644 --- a/riocli/apply/manifests/device.yaml +++ b/riocli/apply/manifests/device.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-docker" # Required - project: "project-guid" labels: custom_label_1: label1 custom_label_2: label2 @@ -23,7 +22,7 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-preinstalled" # Required - project: "project-guid" + labels: app: test spec: @@ -38,7 +37,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "device-hybrid" # Required - project: "project-guid" labels: app: test spec: @@ -56,7 +54,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" metadata: name: "virtual-device-docker" # Required - project: "project-guid" labels: custom_label_1: label1 custom_label_2: label2 diff --git a/riocli/apply/manifests/disk.yaml b/riocli/apply/manifests/disk.yaml index dfbd4ce8..4ac37835 100644 --- a/riocli/apply/manifests/disk.yaml +++ b/riocli/apply/manifests/disk.yaml @@ -1,8 +1,7 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Disk" metadata: - name: "disk" - project: "project-guid" + name: "disk-name" labels: app: test spec: diff --git a/riocli/apply/manifests/managedservice.yaml b/riocli/apply/manifests/managedservice.yaml index 3d6ca4a9..89bd60d9 100644 --- a/riocli/apply/manifests/managedservice.yaml +++ b/riocli/apply/manifests/managedservice.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: ManagedService metadata: name: "managedservice" - project: "project-guid" labels: creator: riocli spec: diff --git a/riocli/apply/manifests/network-native-cloud.yaml b/riocli/apply/manifests/network-native-cloud.yaml index dbc01bad..5a076ff1 100644 --- a/riocli/apply/manifests/network-native-cloud.yaml +++ b/riocli/apply/manifests/network-native-cloud.yaml @@ -10,5 +10,5 @@ spec: rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "cloud" # Required, Options: [cloud, device] resourceLimits: - cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] - memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] + cpu: 0.1 # Unit: Core Options: [Multiple of 0.025, <= 8] + memory: 512 # Unit: MB Options: [Multiple of 128, <= 32768] diff --git a/riocli/apply/manifests/network-native-device.yaml b/riocli/apply/manifests/network-native-device.yaml index 4fe1d313..54a044d5 100644 --- a/riocli/apply/manifests/network-native-device.yaml +++ b/riocli/apply/manifests/network-native-device.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-native-device" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/network-routed-cloud.yaml b/riocli/apply/manifests/network-routed-cloud.yaml index c6b3ae14..fa9b7262 100644 --- a/riocli/apply/manifests/network-routed-cloud.yaml +++ b/riocli/apply/manifests/network-routed-cloud.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-routed-cloud" - project: "project-guid" region: jp labels: app: test @@ -11,5 +10,5 @@ spec: rosDistro: "melodic" # Required, Options: [kinetic, melodic, noetic] runtime: "cloud" # Required, Options: [cloud, device] resourceLimits: - cpu: 0.025 # Unit: Core Options: [Multiple of 0.025, <= 8] - memory: 128 # Unit: MB Options: [Multiple of 128, <= 32768] \ No newline at end of file + cpu: 0.5 # Unit: Core Options: [Multiple of 0.025, <= 8] + memory: 1024 # Unit: MB Options: [Multiple of 128, <= 32768] \ No newline at end of file diff --git a/riocli/apply/manifests/network-routed-device.yaml b/riocli/apply/manifests/network-routed-device.yaml index 864b2a42..c3317333 100644 --- a/riocli/apply/manifests/network-routed-device.yaml +++ b/riocli/apply/manifests/network-routed-device.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-routed-device" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/network.yaml b/riocli/apply/manifests/network.yaml index 395e6d7e..7950f971 100644 --- a/riocli/apply/manifests/network.yaml +++ b/riocli/apply/manifests/network.yaml @@ -3,7 +3,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-routed-cloud" - project: "project-guid" region: jp labels: app: test @@ -19,7 +18,7 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-routed-device" - project: "project-guid" + labels: app: test spec: @@ -35,7 +34,7 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-native-cloud" - project: "project-guid" + region: jp labels: app: test @@ -51,7 +50,7 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Network" metadata: name: "network-native-device" - project: "project-guid" + labels: app: test spec: diff --git a/riocli/apply/manifests/package-nonros-device.yaml b/riocli/apply/manifests/package-nonros-device.yaml index ec19bb99..cd5fc51c 100644 --- a/riocli/apply/manifests/package-nonros-device.yaml +++ b/riocli/apply/manifests/package-nonros-device.yaml @@ -4,7 +4,7 @@ metadata: name: "package-nonros-device" # Required version: "v1.0.0" # Required description: "A RIO non ROS device package" - #project: "project-guid" + # labels: app: test spec: diff --git a/riocli/apply/manifests/project.yaml b/riocli/apply/manifests/project.yaml index 6b95f3e6..6f148c6e 100644 --- a/riocli/apply/manifests/project.yaml +++ b/riocli/apply/manifests/project.yaml @@ -2,7 +2,6 @@ apiVersion: api.rapyuta.io/v2 kind: Project metadata: name: test-project-name - organizationGUID: org-guid labels: purpose: testing version: 1.0 diff --git a/riocli/apply/manifests/secret-docker.yaml b/riocli/apply/manifests/secret-docker.yaml index 146a281f..a2a5bd59 100644 --- a/riocli/apply/manifests/secret-docker.yaml +++ b/riocli/apply/manifests/secret-docker.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Secret" metadata: name: "secret-docker" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/secret.yaml b/riocli/apply/manifests/secret.yaml index d5b6af93..8fa2e5bf 100644 --- a/riocli/apply/manifests/secret.yaml +++ b/riocli/apply/manifests/secret.yaml @@ -3,7 +3,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "Secret" metadata: name: "secret-docker" - project: "project-guid" labels: app: test spec: diff --git a/riocli/apply/manifests/staticroute.yaml b/riocli/apply/manifests/staticroute.yaml index 24ff3fc9..3da6789d 100644 --- a/riocli/apply/manifests/staticroute.yaml +++ b/riocli/apply/manifests/staticroute.yaml @@ -2,7 +2,6 @@ apiVersion: "apiextensions.rapyuta.io/v1" kind: "StaticRoute" metadata: name: "staticroute" - project: "project-guid" region: jp labels: app: test diff --git a/riocli/apply/manifests/usergroup.yaml b/riocli/apply/manifests/usergroup.yaml index 100cded6..41fe46e9 100644 --- a/riocli/apply/manifests/usergroup.yaml +++ b/riocli/apply/manifests/usergroup.yaml @@ -3,7 +3,6 @@ apiVersion: api.rapyuta.io/v2 kind: UserGroup metadata: name: usergroup_name - organization: org-bqgpmsafgnvnawlkuvxtxohs labels: key: value spec: From 49b328b884fdf24b88c1b5b522b7d4ea76f3d036 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 2 Aug 2024 22:05:08 +0530 Subject: [PATCH 29/54] refactor: removes examples directory The examples directory has been lying around for some time now. It is outdated and serves no purpose in this repo. --- examples/cvat/01-disks.yaml | 23 - examples/cvat/02-deployment.yaml | 64 -- examples/cvat/03-extra-tools.yaml | 31 - examples/demo/all-in-one.yaml | 292 --------- examples/demo/values.yaml | 1 - examples/kiba-robots/00_disk.yaml | 7 - examples/kiba-robots/01_secret.yaml | 12 - examples/kiba-robots/02_routednetwork.yaml | 9 - examples/kiba-robots/03_edge_packages.yaml | 207 ------ examples/kiba-robots/03_packages.yaml | 174 ----- examples/kiba-robots/04_ims.yaml | 144 ----- examples/kiba-robots/05_wcs.yaml | 40 -- examples/kiba-robots/06_onlydb.yaml | 84 --- examples/kiba-robots/all-in-one.yaml | 712 --------------------- examples/kiba-robots/secrets.staging.yaml | 15 - examples/kiba-robots/values.yaml | 3 - examples/manifests | 1 - examples/marketplace/dependencies.yaml | 9 - 18 files changed, 1828 deletions(-) delete mode 100644 examples/cvat/01-disks.yaml delete mode 100644 examples/cvat/02-deployment.yaml delete mode 100644 examples/cvat/03-extra-tools.yaml delete mode 100644 examples/demo/all-in-one.yaml delete mode 100644 examples/demo/values.yaml delete mode 100644 examples/kiba-robots/00_disk.yaml delete mode 100644 examples/kiba-robots/01_secret.yaml delete mode 100644 examples/kiba-robots/02_routednetwork.yaml delete mode 100644 examples/kiba-robots/03_edge_packages.yaml delete mode 100644 examples/kiba-robots/03_packages.yaml delete mode 100644 examples/kiba-robots/04_ims.yaml delete mode 100644 examples/kiba-robots/05_wcs.yaml delete mode 100644 examples/kiba-robots/06_onlydb.yaml delete mode 100644 examples/kiba-robots/all-in-one.yaml delete mode 100644 examples/kiba-robots/secrets.staging.yaml delete mode 100644 examples/kiba-robots/values.yaml delete mode 120000 examples/manifests delete mode 100644 examples/marketplace/dependencies.yaml diff --git a/examples/cvat/01-disks.yaml b/examples/cvat/01-disks.yaml deleted file mode 100644 index 508644ba..00000000 --- a/examples/cvat/01-disks.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "cvat_minio" -spec: - runtime: cloud - capacity: 256 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "cvat_data_128gb" -spec: - runtime: cloud - capacity: 256 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "cvat_db" -spec: - runtime: cloud - capacity: 32 \ No newline at end of file diff --git a/examples/cvat/02-deployment.yaml b/examples/cvat/02-deployment.yaml deleted file mode 100644 index 2b9e646d..00000000 --- a/examples/cvat/02-deployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: cvat_db - depends: - kind: package - nameOrGUID: cvat_db_redis - version: "v1.0.2" - labels: - app: cvat -spec: - runtime: cloud - envArgs: - - name: PGDATA - value: "/var/lib/postgresql/data/pgdata" - - name: POSTGRES_DB - value: "cvat" - - name: POSTGRES_USER - value: root - - name: POSTGRES_HOST_AUTH_METHOD - value: "trust" - volumes: - - execName: postgres - mountPath: "/var/lib/postgresql/data" - depends: - kind: disk - nameOrGUID: "cvat_data_128gb" - staticRoutes: - - name: CVAT_UI - depends: - kind: staticroute - nameOrGUID: cvat-prod ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: cvat_prod - depends: - kind: package - nameOrGUID: cvat_custom - version: "v1.3.0" - labels: - app: cvat -spec: - depends: - - kind: deployment - nameOrGUID: cvat_db - runtime: cloud - envArgs: - - name: DJANGO_MODWSGI_EXTRA_ARGS - value: " " - - name: ALLOWED_HOSTS - value: "*" - volumes: - - execName: cvat_server - mountPath: "/home/django/user_data" - depends: - kind: disk - nameOrGUID: "cvat_data_128gb" - staticRoutes: - - name: CVAT_UI - depends: - kind: staticroute - nameOrGUID: cvat-prod \ No newline at end of file diff --git a/examples/cvat/03-extra-tools.yaml b/examples/cvat/03-extra-tools.yaml deleted file mode 100644 index 0c371fc1..00000000 --- a/examples/cvat/03-extra-tools.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: pgAdmin - depends: - kind: package - nameOrGUID: pgAdmin - version: "v1.0.1" - labels: - app: cvat -spec: - depends: - - kind: deployment - nameOrGUID: cvat_db - runtime: cloud - envArgs: - - name: PGADMIN_DEFAULT_PASSWORD - value: "pgadmin" - - name: PGADMIN_DEFAULT_EMAIL - value: "pgadmin@rapyuta-robotics.com" - volumes: - - execName: cvat_server - mountPath: "/home/django/user_data" - depends: - kind: disk - nameOrGUID: "cvat_data_128gb" - staticRoutes: - - name: endpoint - depends: - kind: staticroute - nameOrGUID: cvat-prod \ No newline at end of file diff --git a/examples/demo/all-in-one.yaml b/examples/demo/all-in-one.yaml deleted file mode 100644 index 8c0aacfc..00000000 --- a/examples/demo/all-in-one.yaml +++ /dev/null @@ -1,292 +0,0 @@ ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_ims - depends: - kind: package - nameOrGUID: "sootballs_ims" - version: "1.13.0" - labels: - app: ims -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_ims_pgbouncer - # TODO: Come back once Minio and DB is deployed - # - kind: deployment - # nameOrGUID: sootballs_minio - envArgs: - - name: ACC_KEY - value: "rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==" - - name: ACC_NAME - value: "rrkibdem" - - name: AUTH_ENABLED - value: "True" - - name: AZURE_CONNECTION_STRING - value: "DefaultEndpointsProtocol=https;AccountName=rrkibdem;AccountKey=rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==;EndpointSuffix=core.windows.net" - - name: TIMEZONE - value: "Asia/Tokyo" - - name: CONTAINER_NAME - value: "warehouse" - - name: DAYS_BEFORE_ARCHIVE - value: "15" - - name: DEFAULT_FILE_STORAGE - value: "storages.backends.gcloud.GoogleCloudStorage" - - name: DEPLOY_ON_CLOUD - value: "True" - - name: EODTIME - value: "19:00" - - name: GCP_CREDS - value: "U2FsdGVkX19WrtHt0rhAAhj61mBc8S43+r1Hp61wGeAM3HOksh893DzhmuBMNfrzISGwyU5EOove2VSlwhXVzBp9TapdlDS1LZ2GEmAMsthuHtsoRv/F6wt61pKQ6D0dTQME8mia/weqjzy4UKQPv/acUsSMvAsk6MyDTNwMuz0SuGTvbJwZfIXEVgkC7qXsX8h7yjczbol2vxlXTTNC8QC0iJGT923MoxGt6Xd1XUZjB/stp3y1DLXG7i9EdPr4U4C36ywpv13gDKSzRetWnNamhPVreMyNIwX9xs7c5eWRvfZXn70ayuLWizJajVSljOWDf+atgyUMHtvBKPrvXnhegOXmMwkyDmvKxhTb+lEYlcQCDzERqZB/Co0SbqQUJhs9O3/XnmLmxgY1T83EuSBgiT3Gbt9TQXI2p8l21QapMJ4uvItLD3dXvEMa/VlmLDRc7+thlhpP6z8UFOMdK84j15IFa4oEtPpPA6FKkZPZq8nGLSTVp1WAAf54yMb3H5W1tVz6ICva+lk35Lm8Zd7VxocZxoDnq36oOB9/y6VajU6IVDPP8iuIq3sHQD8JhbGvoqh9wRdcKWUwZsc3TNK3fxHtS0wTxa9TTuoX34A3FKFri2GskiEv55ICeCu733yL9KnR5bOHs11s0ECBH40M7a8W0XAxMIe5qeDOJwhjOXlO6tDaCsFjHAIGDs8NRjgdJJI13+8Vn1XqNj4LMxaLFm2oRG+krHCyyC2PoUWBUixu+6jpLpcIhi9G3JM98RUPLy6qX4Te9eQFuODMi9tPXLyBQAAFPXBxTbnGTJY1xyl+m+P8piT1t67V4FTK1JGollEEn7eL+V5tYWwPG/TVS9AKTcg7lBi/dEdwVZ/HH8PG6LUs1DKUmHm7DhX7KRv4tNqTyLlPNzUsFwTPzkWovdy8EeS6O6++sTS+DWPTTP1cVbckNcztyB1lgkhq0iK50WPkB5NWG5EDdih8yZpRdPXSGxNnap73HmxTfeEfIqDFnHm8avsZjc13qM2rYJhSM2wwRlQxWyoPbL3DcsYO5UlcRycxa1fwAY/rrXguaH1L84LvSDjDSzbmbgFw4zFD78q6wUUgXIidpxLtloxSoBFgqRJGokTdMPK6o7cfdBrvBPcOpIXOhpHe/z+wikzCRB3f/gRpc44Dj8JJLsjpL9XATrAF4FDT5P2h945RX4bqOfrX/EqFyaLMvlQAieUgvcyQu+XDP+VEMmq4lM9rQTTCygEIQqKEkjsVGvuJZyud/z1VwgeDIFs2Cad7eXPu+vjw6h993w+qj3Yc2rYgI1cXk6J67qYt6xziEc+8gFMMF7Y15qX9Cu2S47ERN3yg1lfw6sEEgHdlcCLSdp77SyzvYww/SdyLzNeWNyp/EbtnkgoXip+OvsKzNO6wMJ6cXfkSvn2K8xwyVLb1nqOemEsn/dhl7qMaQ5TqNkRLuZrNPhEotoDCLQROJJaophrsC0w2jtxhqtzF3hv3foEMDnBwY88Lr5YP72+JN2+ELa1ryb7gC48mdWRWQq8Am07b2shJMXa5Kc/0R7AuGJd9schQc6kRM6b2B1xTp8iC0bAwDB1m6osx5KCMkDNwyLOUdmVhOL4PcZ57Ns+jsBrU6oK2lVa4H+Ns2Ud4ei2DzFAqB4smsQ50ZMAyeKst9NJYmoKg4Fl7oUIAaQsrn85uUlDOXPnJHCO93WUzNzF+ZKsVW9FP0KwBzcX/lHpy0VbLHPK7ZfNMGuW+LGlk1QvE8L5temzSWFygZ4QbLiu2AlXo5bFKPZ05/Zotb41+EmD+lRlIZxPUBLI6niH2wpjtdpkSopOlB6VHeHdLQ1aAk1+DpX7E5AAAtCsR9EvbfJxriY9QxQumw/C0Z+XQ7R5qyveYTuC3sjRXf82qT5agrc8swt0g0tikbWP3vUT3gSPmYF0TzXSo/VvyOxHyIrSAhftljiS25UPP+e4KZk5cLUVxkmtP9t3ePu+aAM4LmeY+R41O60yaEKv4KEkKo0ORKdhjGwhrgk8D2NaUfnrPCPfykHC6ginfYVMK9UTps/jfbZjTKr1hbg1F4pYYshq9dn2K4IUjFiOeIZeA1xzY8Xt/TnLOTKWM5wI/CPZ71nU0VrdSi8qvA+vv2odhkB0P92+mjdPcEMMh3igrrRowjwAheGxMq8k34x2kBzMoe7mH6tOTMDYRHUxjxYNpVlfPDDQ2dV0+4wm76GBPesSwwOmh2T2TOlNRlaZGyVN79TIz5jy2iTJ4dLuTeHdh5JMXMS+OLZ8TftPZuRCEVri6I/h8lG5WhAhLnJS9aBNEhstAZX+cUnLxs61D80XVRd+izU6GXuOH/INpYV/9zPwlkl5CNpz0kneBhJTMEgWPaAJ3vERMkLzFe57mEeIOCz9Jba9AQ8mqKZBpIkMGfo9da06wWDRW1eVo+FPmlSzhomCYRX30uI+813KPQRF2l9orfTJvrDiuE8RqVXBXu/ap1FKbgyRmMp8PNrU3JU5HWRKY3ONNQAnLFoBcB0DUxxY1KzR4IwjJ4Vp+ugKcWN8CHH3knuyuJu+jHXkLBGeOExLM1IlL+yc2iBx2fasGZz/DlXkeCrjPdjcMCUmuhPRvAKi2C+nlnuBEITV0pjMkBq3M/o/Rob+lDZy/ravlz/0l/3nr1pExSMqQT70sSsUWMO4m/VkJYL5b86ZecjlJb8bFvVkbgAyFXCy3Iwpp+0CwFSAhcwAhh6cLR7P0bJE6/N16vHHKgc70pDx4Lu7Q4IL4xS5HqkhMwOq8eDmdcalMtYz3O8O+i4i4+FzLqK7SInj/Jr/yBquA78N7jYTNnCtN5k3WoNJOum1vohAXWax9nYPTdtPLyczpYVfG+dUZTKhnCLkoOacjKbe1mdTRXgR3jKIEoiW5eIXZKBbcMyh6V5uGOsiXYUqzPaa/pbDnk6r9c/fPUeVPqnsMydl1q/ry+fs0IIWHmIUS3ir/ZaBpsI3aI7uQ1/aUJ70Qtz6GGKPQoSA7tSrwnns4EJG4Qy8J9fVYcHhY8xavaLFaNL5LsEAKoFaCuAXjQd9Hrnv+IA2R17GqUC2J8Nq9Z2za14w/klBncOUPHRv4NJ1fTA==" - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/code/creds.json" - - name: GS_BUCKET_NAME - value: "rrkibdem" - - name: IMS_DB - value: "rrkibdem_ims" - - name: CELERY_BROKER_URL - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: CELERY_RESULT_BACKEND - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: SECRET_KEY - value: "igy0moow=_qbsbiw4ln&wnpp5+ocvy*y(ov_9a$7j^1k4ccn86" - - name: SODTIME - value: "20:00" - staticRoutes: - - name: IMS_URL - depends: - kind: staticroute - nameOrGUID: ims-{{ routePrefix }} ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_ims_worker - depends: - kind: package - nameOrGUID: "sootballs_ims_worker" - version: "1.13.0" -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_ims_pgbouncer - # TODO: Come back once Minio and DB is deployed - # - kind: deployment - # nameOrGUID: sootballs_minio - envArgs: - - name: ACC_KEY - value: "rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==" - - name: ACC_NAME - value: "rrkibdem" - - name: AUTH_ENABLED - value: "True" - - name: AZURE_CONNECTION_STRING - value: "DefaultEndpointsProtocol=https;AccountName=rrkibdem;AccountKey=rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==;EndpointSuffix=core.windows.net" - - name: TIMEZONE - value: "Asia/Tokyo" - - name: CONTAINER_NAME - value: "warehouse" - - name: DAYS_BEFORE_ARCHIVE - value: "15" - - name: DEFAULT_FILE_STORAGE - value: "storages.backends.gcloud.GoogleCloudStorage" - - name: DEPLOY_ON_CLOUD - value: "True" - - name: EODTIME - value: "19:00" - - name: GCP_CREDS - value: "U2FsdGVkX19WrtHt0rhAAhj61mBc8S43+r1Hp61wGeAM3HOksh893DzhmuBMNfrzISGwyU5EOove2VSlwhXVzBp9TapdlDS1LZ2GEmAMsthuHtsoRv/F6wt61pKQ6D0dTQME8mia/weqjzy4UKQPv/acUsSMvAsk6MyDTNwMuz0SuGTvbJwZfIXEVgkC7qXsX8h7yjczbol2vxlXTTNC8QC0iJGT923MoxGt6Xd1XUZjB/stp3y1DLXG7i9EdPr4U4C36ywpv13gDKSzRetWnNamhPVreMyNIwX9xs7c5eWRvfZXn70ayuLWizJajVSljOWDf+atgyUMHtvBKPrvXnhegOXmMwkyDmvKxhTb+lEYlcQCDzERqZB/Co0SbqQUJhs9O3/XnmLmxgY1T83EuSBgiT3Gbt9TQXI2p8l21QapMJ4uvItLD3dXvEMa/VlmLDRc7+thlhpP6z8UFOMdK84j15IFa4oEtPpPA6FKkZPZq8nGLSTVp1WAAf54yMb3H5W1tVz6ICva+lk35Lm8Zd7VxocZxoDnq36oOB9/y6VajU6IVDPP8iuIq3sHQD8JhbGvoqh9wRdcKWUwZsc3TNK3fxHtS0wTxa9TTuoX34A3FKFri2GskiEv55ICeCu733yL9KnR5bOHs11s0ECBH40M7a8W0XAxMIe5qeDOJwhjOXlO6tDaCsFjHAIGDs8NRjgdJJI13+8Vn1XqNj4LMxaLFm2oRG+krHCyyC2PoUWBUixu+6jpLpcIhi9G3JM98RUPLy6qX4Te9eQFuODMi9tPXLyBQAAFPXBxTbnGTJY1xyl+m+P8piT1t67V4FTK1JGollEEn7eL+V5tYWwPG/TVS9AKTcg7lBi/dEdwVZ/HH8PG6LUs1DKUmHm7DhX7KRv4tNqTyLlPNzUsFwTPzkWovdy8EeS6O6++sTS+DWPTTP1cVbckNcztyB1lgkhq0iK50WPkB5NWG5EDdih8yZpRdPXSGxNnap73HmxTfeEfIqDFnHm8avsZjc13qM2rYJhSM2wwRlQxWyoPbL3DcsYO5UlcRycxa1fwAY/rrXguaH1L84LvSDjDSzbmbgFw4zFD78q6wUUgXIidpxLtloxSoBFgqRJGokTdMPK6o7cfdBrvBPcOpIXOhpHe/z+wikzCRB3f/gRpc44Dj8JJLsjpL9XATrAF4FDT5P2h945RX4bqOfrX/EqFyaLMvlQAieUgvcyQu+XDP+VEMmq4lM9rQTTCygEIQqKEkjsVGvuJZyud/z1VwgeDIFs2Cad7eXPu+vjw6h993w+qj3Yc2rYgI1cXk6J67qYt6xziEc+8gFMMF7Y15qX9Cu2S47ERN3yg1lfw6sEEgHdlcCLSdp77SyzvYww/SdyLzNeWNyp/EbtnkgoXip+OvsKzNO6wMJ6cXfkSvn2K8xwyVLb1nqOemEsn/dhl7qMaQ5TqNkRLuZrNPhEotoDCLQROJJaophrsC0w2jtxhqtzF3hv3foEMDnBwY88Lr5YP72+JN2+ELa1ryb7gC48mdWRWQq8Am07b2shJMXa5Kc/0R7AuGJd9schQc6kRM6b2B1xTp8iC0bAwDB1m6osx5KCMkDNwyLOUdmVhOL4PcZ57Ns+jsBrU6oK2lVa4H+Ns2Ud4ei2DzFAqB4smsQ50ZMAyeKst9NJYmoKg4Fl7oUIAaQsrn85uUlDOXPnJHCO93WUzNzF+ZKsVW9FP0KwBzcX/lHpy0VbLHPK7ZfNMGuW+LGlk1QvE8L5temzSWFygZ4QbLiu2AlXo5bFKPZ05/Zotb41+EmD+lRlIZxPUBLI6niH2wpjtdpkSopOlB6VHeHdLQ1aAk1+DpX7E5AAAtCsR9EvbfJxriY9QxQumw/C0Z+XQ7R5qyveYTuC3sjRXf82qT5agrc8swt0g0tikbWP3vUT3gSPmYF0TzXSo/VvyOxHyIrSAhftljiS25UPP+e4KZk5cLUVxkmtP9t3ePu+aAM4LmeY+R41O60yaEKv4KEkKo0ORKdhjGwhrgk8D2NaUfnrPCPfykHC6ginfYVMK9UTps/jfbZjTKr1hbg1F4pYYshq9dn2K4IUjFiOeIZeA1xzY8Xt/TnLOTKWM5wI/CPZ71nU0VrdSi8qvA+vv2odhkB0P92+mjdPcEMMh3igrrRowjwAheGxMq8k34x2kBzMoe7mH6tOTMDYRHUxjxYNpVlfPDDQ2dV0+4wm76GBPesSwwOmh2T2TOlNRlaZGyVN79TIz5jy2iTJ4dLuTeHdh5JMXMS+OLZ8TftPZuRCEVri6I/h8lG5WhAhLnJS9aBNEhstAZX+cUnLxs61D80XVRd+izU6GXuOH/INpYV/9zPwlkl5CNpz0kneBhJTMEgWPaAJ3vERMkLzFe57mEeIOCz9Jba9AQ8mqKZBpIkMGfo9da06wWDRW1eVo+FPmlSzhomCYRX30uI+813KPQRF2l9orfTJvrDiuE8RqVXBXu/ap1FKbgyRmMp8PNrU3JU5HWRKY3ONNQAnLFoBcB0DUxxY1KzR4IwjJ4Vp+ugKcWN8CHH3knuyuJu+jHXkLBGeOExLM1IlL+yc2iBx2fasGZz/DlXkeCrjPdjcMCUmuhPRvAKi2C+nlnuBEITV0pjMkBq3M/o/Rob+lDZy/ravlz/0l/3nr1pExSMqQT70sSsUWMO4m/VkJYL5b86ZecjlJb8bFvVkbgAyFXCy3Iwpp+0CwFSAhcwAhh6cLR7P0bJE6/N16vHHKgc70pDx4Lu7Q4IL4xS5HqkhMwOq8eDmdcalMtYz3O8O+i4i4+FzLqK7SInj/Jr/yBquA78N7jYTNnCtN5k3WoNJOum1vohAXWax9nYPTdtPLyczpYVfG+dUZTKhnCLkoOacjKbe1mdTRXgR3jKIEoiW5eIXZKBbcMyh6V5uGOsiXYUqzPaa/pbDnk6r9c/fPUeVPqnsMydl1q/ry+fs0IIWHmIUS3ir/ZaBpsI3aI7uQ1/aUJ70Qtz6GGKPQoSA7tSrwnns4EJG4Qy8J9fVYcHhY8xavaLFaNL5LsEAKoFaCuAXjQd9Hrnv+IA2R17GqUC2J8Nq9Z2za14w/klBncOUPHRv4NJ1fTA==" - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/code/creds.json" - - name: GS_BUCKET_NAME - value: "rrkibdem" - - name: IMS_DB - value: "rrkibdem_ims" - - name: CELERY_BROKER_URL - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: CELERY_RESULT_BACKEND - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: SECRET_KEY - value: "igy0moow=_qbsbiw4ln&wnpp5+ocvy*y(ov_9a$7j^1k4ccn86" - - name: SODTIME - value: "20:00" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_ims_beat - depends: - kind: package - nameOrGUID: "sootballs_ims_beat" - version: "1.13.0" -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_ims_pgbouncer - # TODO: Come back once Minio and DB is deployed - # - kind: deployment - # nameOrGUID: sootballs_minio - envArgs: - - name: ACC_KEY - value: "rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==" - - name: ACC_NAME - value: "rrkibdem" - - name: AUTH_ENABLED - value: "True" - - name: AZURE_CONNECTION_STRING - value: "DefaultEndpointsProtocol=https;AccountName=rrkibdem;AccountKey=rooir+ucWD0ObAHVGdco3T7qFskEpZfgZm0lQdIxbsowLkct2uXi3zKYi+EKJmTpmtqsNGYVMswsp719klvl7Q==;EndpointSuffix=core.windows.net" - - name: TIMEZONE - value: "Asia/Tokyo" - - name: CONTAINER_NAME - value: "warehouse" - - name: DAYS_BEFORE_ARCHIVE - value: "15" - - name: DEFAULT_FILE_STORAGE - value: "storages.backends.gcloud.GoogleCloudStorage" - - name: DEPLOY_ON_CLOUD - value: "True" - - name: EODTIME - value: "19:00" - - name: GCP_CREDS - value: "U2FsdGVkX19WrtHt0rhAAhj61mBc8S43+r1Hp61wGeAM3HOksh893DzhmuBMNfrzISGwyU5EOove2VSlwhXVzBp9TapdlDS1LZ2GEmAMsthuHtsoRv/F6wt61pKQ6D0dTQME8mia/weqjzy4UKQPv/acUsSMvAsk6MyDTNwMuz0SuGTvbJwZfIXEVgkC7qXsX8h7yjczbol2vxlXTTNC8QC0iJGT923MoxGt6Xd1XUZjB/stp3y1DLXG7i9EdPr4U4C36ywpv13gDKSzRetWnNamhPVreMyNIwX9xs7c5eWRvfZXn70ayuLWizJajVSljOWDf+atgyUMHtvBKPrvXnhegOXmMwkyDmvKxhTb+lEYlcQCDzERqZB/Co0SbqQUJhs9O3/XnmLmxgY1T83EuSBgiT3Gbt9TQXI2p8l21QapMJ4uvItLD3dXvEMa/VlmLDRc7+thlhpP6z8UFOMdK84j15IFa4oEtPpPA6FKkZPZq8nGLSTVp1WAAf54yMb3H5W1tVz6ICva+lk35Lm8Zd7VxocZxoDnq36oOB9/y6VajU6IVDPP8iuIq3sHQD8JhbGvoqh9wRdcKWUwZsc3TNK3fxHtS0wTxa9TTuoX34A3FKFri2GskiEv55ICeCu733yL9KnR5bOHs11s0ECBH40M7a8W0XAxMIe5qeDOJwhjOXlO6tDaCsFjHAIGDs8NRjgdJJI13+8Vn1XqNj4LMxaLFm2oRG+krHCyyC2PoUWBUixu+6jpLpcIhi9G3JM98RUPLy6qX4Te9eQFuODMi9tPXLyBQAAFPXBxTbnGTJY1xyl+m+P8piT1t67V4FTK1JGollEEn7eL+V5tYWwPG/TVS9AKTcg7lBi/dEdwVZ/HH8PG6LUs1DKUmHm7DhX7KRv4tNqTyLlPNzUsFwTPzkWovdy8EeS6O6++sTS+DWPTTP1cVbckNcztyB1lgkhq0iK50WPkB5NWG5EDdih8yZpRdPXSGxNnap73HmxTfeEfIqDFnHm8avsZjc13qM2rYJhSM2wwRlQxWyoPbL3DcsYO5UlcRycxa1fwAY/rrXguaH1L84LvSDjDSzbmbgFw4zFD78q6wUUgXIidpxLtloxSoBFgqRJGokTdMPK6o7cfdBrvBPcOpIXOhpHe/z+wikzCRB3f/gRpc44Dj8JJLsjpL9XATrAF4FDT5P2h945RX4bqOfrX/EqFyaLMvlQAieUgvcyQu+XDP+VEMmq4lM9rQTTCygEIQqKEkjsVGvuJZyud/z1VwgeDIFs2Cad7eXPu+vjw6h993w+qj3Yc2rYgI1cXk6J67qYt6xziEc+8gFMMF7Y15qX9Cu2S47ERN3yg1lfw6sEEgHdlcCLSdp77SyzvYww/SdyLzNeWNyp/EbtnkgoXip+OvsKzNO6wMJ6cXfkSvn2K8xwyVLb1nqOemEsn/dhl7qMaQ5TqNkRLuZrNPhEotoDCLQROJJaophrsC0w2jtxhqtzF3hv3foEMDnBwY88Lr5YP72+JN2+ELa1ryb7gC48mdWRWQq8Am07b2shJMXa5Kc/0R7AuGJd9schQc6kRM6b2B1xTp8iC0bAwDB1m6osx5KCMkDNwyLOUdmVhOL4PcZ57Ns+jsBrU6oK2lVa4H+Ns2Ud4ei2DzFAqB4smsQ50ZMAyeKst9NJYmoKg4Fl7oUIAaQsrn85uUlDOXPnJHCO93WUzNzF+ZKsVW9FP0KwBzcX/lHpy0VbLHPK7ZfNMGuW+LGlk1QvE8L5temzSWFygZ4QbLiu2AlXo5bFKPZ05/Zotb41+EmD+lRlIZxPUBLI6niH2wpjtdpkSopOlB6VHeHdLQ1aAk1+DpX7E5AAAtCsR9EvbfJxriY9QxQumw/C0Z+XQ7R5qyveYTuC3sjRXf82qT5agrc8swt0g0tikbWP3vUT3gSPmYF0TzXSo/VvyOxHyIrSAhftljiS25UPP+e4KZk5cLUVxkmtP9t3ePu+aAM4LmeY+R41O60yaEKv4KEkKo0ORKdhjGwhrgk8D2NaUfnrPCPfykHC6ginfYVMK9UTps/jfbZjTKr1hbg1F4pYYshq9dn2K4IUjFiOeIZeA1xzY8Xt/TnLOTKWM5wI/CPZ71nU0VrdSi8qvA+vv2odhkB0P92+mjdPcEMMh3igrrRowjwAheGxMq8k34x2kBzMoe7mH6tOTMDYRHUxjxYNpVlfPDDQ2dV0+4wm76GBPesSwwOmh2T2TOlNRlaZGyVN79TIz5jy2iTJ4dLuTeHdh5JMXMS+OLZ8TftPZuRCEVri6I/h8lG5WhAhLnJS9aBNEhstAZX+cUnLxs61D80XVRd+izU6GXuOH/INpYV/9zPwlkl5CNpz0kneBhJTMEgWPaAJ3vERMkLzFe57mEeIOCz9Jba9AQ8mqKZBpIkMGfo9da06wWDRW1eVo+FPmlSzhomCYRX30uI+813KPQRF2l9orfTJvrDiuE8RqVXBXu/ap1FKbgyRmMp8PNrU3JU5HWRKY3ONNQAnLFoBcB0DUxxY1KzR4IwjJ4Vp+ugKcWN8CHH3knuyuJu+jHXkLBGeOExLM1IlL+yc2iBx2fasGZz/DlXkeCrjPdjcMCUmuhPRvAKi2C+nlnuBEITV0pjMkBq3M/o/Rob+lDZy/ravlz/0l/3nr1pExSMqQT70sSsUWMO4m/VkJYL5b86ZecjlJb8bFvVkbgAyFXCy3Iwpp+0CwFSAhcwAhh6cLR7P0bJE6/N16vHHKgc70pDx4Lu7Q4IL4xS5HqkhMwOq8eDmdcalMtYz3O8O+i4i4+FzLqK7SInj/Jr/yBquA78N7jYTNnCtN5k3WoNJOum1vohAXWax9nYPTdtPLyczpYVfG+dUZTKhnCLkoOacjKbe1mdTRXgR3jKIEoiW5eIXZKBbcMyh6V5uGOsiXYUqzPaa/pbDnk6r9c/fPUeVPqnsMydl1q/ry+fs0IIWHmIUS3ir/ZaBpsI3aI7uQ1/aUJ70Qtz6GGKPQoSA7tSrwnns4EJG4Qy8J9fVYcHhY8xavaLFaNL5LsEAKoFaCuAXjQd9Hrnv+IA2R17GqUC2J8Nq9Z2za14w/klBncOUPHRv4NJ1fTA==" - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/code/creds.json" - - name: GS_BUCKET_NAME - value: "rrkibdem" - - name: IMS_DB - value: "rrkibdem_ims" - - name: CELERY_BROKER_URL - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: CELERY_RESULT_BACKEND - value: "rediss://:QYTYXHGQHAdzx27jlxuJyqfnnaks61klHvPhD1vPx+Q=@sootballsdev.redis.cache.windows.net:6380/0" - - name: SECRET_KEY - value: "igy0moow=_qbsbiw4ln&wnpp5+ocvy*y(ov_9a$7j^1k4ccn86" - - name: SODTIME - value: "20:00" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_systemui - depends: - kind: package - nameOrGUID: "sootballs_systemui" - version: "1.13.0" -spec: - runtime: cloud - envArgs: - - name: DOCKER_STDOUT - value: "true" - - name: IMS_AUTH_PASSWORD - value: "airborne_rr" - - name: IMS_AUTH_USERNAME - value: "root" - - name: IMS_URL - value: "https://ims-rrkibdem-buonj.ep-r.io" - - name: SENTRY_DSN - value: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: SOOTBALLS_MAP - value: "rrkibdem" - - name: SYSTEM_UI_REMOTE_MODE - value: "true" - - name: USE_LOCAL_MAP - value: "false" - - name: WS_EXTERNAL_PORT - value: "80" - rosNetworks: - - depends: - kind: network - nameOrGUID: "sootballs" - staticRoutes: - - name: "SYSTEM_UI" - depends: - kind: staticroute - nameOrGUID: ui-{{ routePrefix }} ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_amr_06 - depends: - kind: package - nameOrGUID: "sootballs_robot" - version: "1.13.0" -spec: - runtime: device - # TODO: Come back when IMS is bindable - # depends: - # - kind: deployment - # nameOrGUID: sootballs_ims - rosNetworks: - - depends: - kind: network - nameOrGUID: sootballs - device: - depends: - kind: device - nameOrGUID: amr06 - envArgs: - - name: IMS_AUTH_USERNAME - value: root - - name: IMS_AUTH_PASSWORD - value: airborne_rr - - name: IMS_URL - value: "https://ims-rrkibdem-buonj.ep-r.io" - - name: "SOOTBALLS_MAP" - value: "rrkibdem" - - name: "SENTRY_DSN" - value: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - value: "true" - - name: "DISABLE_MULTICAST" - value: "false" - - name: "ROS_DOMAIN_ID" - value: "5" - - name: "USE_PARAMS_IO" - value: "true" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_edge_edge02 - depends: - kind: package - nameOrGUID: "sootballs_edge" - version: "1.13.0" -spec: - runtime: device - # TODO: Come back when IMS is bindable - # depends: - # - kind: deployment - # nameOrGUID: sootballs_ims - rosNetworks: - - depends: - kind: network - nameOrGUID: sootballs - device: - depends: - kind: device - nameOrGUID: edge02 - envArgs: - - name: IMS_AUTH_USERNAME - value: root - - name: IMS_AUTH_PASSWORD - value: airborne_rr - - name: IMS_URL - value: "https://ims-rrkibdem-buonj.ep-r.io" - - name: "SOOTBALLS_MAP" - value: "rrkibdem" - - name: "SENTRY_DSN" - value: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - value: "true" - - name: "DISABLE_MULTICAST" - value: "false" - - name: "ROS_DOMAIN_ID" - value: "5" - - name: "USE_PARAMS_IO" - value: "true" diff --git a/examples/demo/values.yaml b/examples/demo/values.yaml deleted file mode 100644 index 01cb50ae..00000000 --- a/examples/demo/values.yaml +++ /dev/null @@ -1 +0,0 @@ -routePrefix: "rrkibdem" diff --git a/examples/kiba-robots/00_disk.yaml b/examples/kiba-robots/00_disk.yaml deleted file mode 100644 index 4549163a..00000000 --- a/examples/kiba-robots/00_disk.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "small-pvc-4" -spec: - runtime: cloud - capacity: 4 diff --git a/examples/kiba-robots/01_secret.yaml b/examples/kiba-robots/01_secret.yaml deleted file mode 100644 index c0ee3a6f..00000000 --- a/examples/kiba-robots/01_secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Secret" -metadata: - name: "sootballs" - guid: secret-vebkjobybhhwmyiwkwvndagu -spec: - type: Docker - docker: - username: shaishavrapyuta - password: asdfg123$ - email: shaishavrapyuta - # registry: https://index.docker.io/v1/ diff --git a/examples/kiba-robots/02_routednetwork.yaml b/examples/kiba-robots/02_routednetwork.yaml deleted file mode 100644 index 67fea6da..00000000 --- a/examples/kiba-robots/02_routednetwork.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Network" -metadata: - name: "sootballs" -spec: - runtime: "cloud" - type: "routed" - rosDistro: "melodic" - resourceLimits: "small" \ No newline at end of file diff --git a/examples/kiba-robots/03_edge_packages.yaml b/examples/kiba-robots/03_edge_packages.yaml deleted file mode 100644 index c58ac062..00000000 --- a/examples/kiba-robots/03_edge_packages.yaml +++ /dev/null @@ -1,207 +0,0 @@ ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs SUI2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "cloud" - ros: - enabled: True - version: "melodic" - rosEndpoints: - - name: "/cmd_global_charge" - type: "topic" - qos: "hi" - - name: "/cmd_global_in" - type: "topic" - qos: "hi" - - name: "/cmd_move_to" - type: "topic" - qos: "hi" - - name: "/dispatcher/active_tote_unload_request" - type: "topic" - qos: "low" - - name: "/dispatcher/control_request" - type: "topic" - qos: "low" - - name: "/dispatcher/modify_order_request" - type: "topic" - qos: "low" - - name: "/dispatcher/productivity_configs_request" - type: "topic" - qos: "low" - - name: "/edge/emergency_released_request" - type: "topic" - qos: "low" - - name: "/manual_order_recovery" - type: "topic" - qos: "hi" - - name: "/reservation_request" - type: "topic" - qos: "hi" - cloud: - replicas: 1 - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:1.14.0-rc4" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_edge remote_ui.launch" - limits: - cpu: 2 - memory: 8192 - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "SOOTBALLS_MAP" - defaultValue: "rrkibstg" - - name: "SYSTEM_UI_REMOTE_MODE" - defaultValue: "true" - - name: "WS_EXTERNAL_PORT" - defaultValue: "80" - - name: "DOCKER_STDOUT" - defaultValue: "true" - endpoints: - - name: "SYSTEM_UI" - type: external-https - port: 443 - targetPort: 7099 - - name: "SYSTEM_UI_ROSBRIDGE_URL" - type: external-https - port: 443 - targetPort: 9092 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs Edge2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "device" - device: - arch: amd64 - restart: always - ros: - enabled: True - version: "melodic" - rosEndpoints: - - name: "/dispatcher/control_response" - type: "topic" - qos: "low" - - name: "/dispatcher/modify_order_response" - type: "topic" - qos: "low" - - name: "/dispatcher/productivity_configs_response" - type: "topic" - qos: "low" - - name: "/dispatcher/active_tote_unload_response" - type: "topic" - qos: "low" - - name: "/edge/emergency_released_response" - type: "topic" - qos: "low" - - name: "/robot_reservation" - type: "topic" - qos: "low" - - name: "/sui/status" - type: "topic" - qos: "low" - - name: "/sui/overview" - type: "topic" - qos: "low" - - name: "/sui/main" - type: "topic" - qos: "low" - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:1.14.0-rc4" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_edge default.launch --wait" - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - defaultValue: "true" - - name: "DISABLE_MULTICAST" - defaultValue: "false" - - name: "ROS_DOMAIN_ID" - defaultValue: "10" - - name: "USE_PARAMS_IO" - defaultValue: "true" - - name: "EDGE_ALARM_MONITOR" - defaultValue: "true" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs Robot2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "device" - device: - arch: amd64 - restart: always - ros: - enabled: True - version: "melodic" - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:1.14.0-rc4" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_edge default.launch --wait" - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - defaultValue: "true" - - name: "DISABLE_MULTICAST" - defaultValue: "false" - - name: "ROS_DOMAIN_ID" - defaultValue: "10" - - name: "USE_PARAMS_IO" - defaultValue: "true" \ No newline at end of file diff --git a/examples/kiba-robots/03_packages.yaml b/examples/kiba-robots/03_packages.yaml deleted file mode 100644 index f471f6e6..00000000 --- a/examples/kiba-robots/03_packages.yaml +++ /dev/null @@ -1,174 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_ims - version: v1.14.0-rc4-6 - description: Sootballs IMS Package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: Django - docker: - image: rrdockerhub/sootballs_ims:1.14.0-rc4 - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - limits: - cpu: 0.5 - memory: 2048 - environmentVars: - - name: SECRET_KEY - - name: AUTH_ENABLED - default: "False" - - name: TIMEZONE - default: "Asia/Tokyo" - - name: CONTAINER_NAME - default: "warehouse" - - name: DAYS_BEFORE_ARCHIVE - default: "15" - - name: DEPLOY_ON_CLOUD - default: "False" - - name: SODTIME - default: "09:00" - - name: EODTIME - default: "19:00" - - name: AWS_STORAGE_BUCKET_NAME - default: "kiba-robots" - - name: IMS_DB - default: ims_db - - name: CELERY_BROKER_URL - - name: CELERY_RESULT_BACKEND - - endpoints: - - name: IMS_URL - type: external-https - port: 443 - targetPort: 8002 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_db - version: v1.0.11 - description: Sootballs DB package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: redis - docker: - image: redis:4.0-alpine - limits: - cpu: 0.5 - memory: 2048 - - type: docker - name: postgres - docker: - image: postgis/postgis:9.6-3.2 - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: POSTGRES_MULTIPLE_DATABASES - default: ims_db,wcs_db - - name: PGDATA - default: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - default: trust - - name: POSTGRES_USER - default: postgres - exposed: true - exposedName: POSTGRES_USER - - name: POSTGRES_PASSWORD - default: password - exposed: true - exposedName: POSTGRES_PASSWORD - endpoints: - - name: POSTGRES - type: internal-tcp - port: 5432 - targetPort: 5432 - - name: REDIS - type: internal-tcp - port: 6379 - targetPort: 6379 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: MinIO File Server - version: v1.0.11 - description: Sootballs File Server package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: minio_executable - docker: - image: rrdockerhub/minio-server - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: MINIO_ACCESS_KEY - - name: MINIO_SECRET_KEY - endpoints: - - name: MINIO - type: external-https - port: 443 - targetPort: 9000 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_wcs - version: v1.14.0-rc4-2 - description: Sootballs WCS package - labels: - app: wcs -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: django - command: - - /code/docker/entrypoint.sh - docker: - image: rrdockerhub/sootballs_wcs:1.14.0-rc4 - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: TIMEZONE - default: Asia/Tokyo - - name: WCS_DB - default: wcs_db - - name: CELERY_BROKER_URL - - name: CELERY_RESULT_BACKEND - - name: LOCAL_PRINT_SERVER_URL - - name: SECRET_KEY - endpoints: - - name: WCS_URL - type: external-https - port: 443 - targetPort: 8003 \ No newline at end of file diff --git a/examples/kiba-robots/04_ims.yaml b/examples/kiba-robots/04_ims.yaml deleted file mode 100644 index 4c8ee8e5..00000000 --- a/examples/kiba-robots/04_ims.yaml +++ /dev/null @@ -1,144 +0,0 @@ ---- -#sootballs_staticroutes -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "ims-kibarobots-apply" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "minio-kibarobots-apply" ---- -#sootballs_minio -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_minio - depends: - kind: package - nameOrGUID: "MinIO File Server" - version: "v1.0.11" - labels: - app: ims -spec: - runtime: cloud - envArgs: - - name: MINIO_ACCESS_KEY - value: access - - name: MINIO_SECRET_KEY - value: secret_key - staticRoutes: - - name: MINIO - depends: - kind: staticroute - nameOrGUID: minio-kibarobots-apply - volumes: - - execName: minio_executable - mountPath: "/data" - subPath: "data" - depends: - kind: disk - nameOrGUID: "minio-pvc" ---- -#sootballs_db -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_db - depends: - kind: package - nameOrGUID: sootballs_db - version: "v1.0.11" - labels: - app: ims -spec: - runtime: cloud - envArgs: - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - - name: POSTGRES_MULTIPLE_DATABASES - value: ims_db,wcs_db - - name: POSTGRES_PASSWORD - value: sootballs - - name: POSTGRES_USER - value: postgres - volumes: - - execName: postgres - mountPath: "/var/lib/postgresql/data/pgdata" - subPath: "pgdata" - depends: - kind: disk - nameOrGUID: "postgres-pvc" - -#sootballs_ims ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_ims - depends: - kind: package - nameOrGUID: "sootballs_ims" - version: "v1.14.0-rc4-6" - labels: - app: ims -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_db - - kind: deployment - nameOrGUID: sootballs_minio - envArgs: - - name: AUTH_ENABLED - value: "True" - - name: AWS_STORAGE_BUCKET_NAME - value: kiba-robots - #TODO this should be parsed from redis url in the docker container. - - name: CELERY_BROKER_URL - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - #TODO this should be parsed from redis url in the docker container. - - name: CELERY_RESULT_BACKEND - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: CONTAINER_NAME - value: warehouse - - name: DAYS_BEFORE_ARCHIVE - value: "15" - - name: DEPLOY_ON_CLOUD - value: "False" - - name: EODTIME - value: "19:00" - - name: IMS_DB - value: ims_db - - name: SECRET_KEY - value: asdasd - - name: SODTIME - value: 09:00 - - name: TIMEZONE - value: Asia/Tokyo - - name: TEST_ENV - value: asdsad - staticRoutes: - - name: IMS_URL - depends: - kind: staticroute - nameOrGUID: ims-kibarobots-apply ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "postgres-pvc" -spec: - runtime: cloud - capacity: 4 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "minio-pvc" -spec: - runtime: cloud - capacity: 4 diff --git a/examples/kiba-robots/05_wcs.yaml b/examples/kiba-robots/05_wcs.yaml deleted file mode 100644 index 70c9fed0..00000000 --- a/examples/kiba-robots/05_wcs.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "wcs-kibarobots-apply" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_wcs - depends: - kind: package - nameOrGUID: sootballs_wcs - version: "v1.14.0-rc4-2" - labels: - app: wcs -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_db - - kind: deployment - nameOrGUID: sootballs_ims - envArgs: - - name: CELERY_BROKER_URL - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: CELERY_RESULT_BACKEND - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: LOCAL_PRINT_SERVER_URL - value: " " - - name: SECRET_KEY - value: asdasd - - name: TIMEZONE - value: Asia/Tokyo - - name: WCS_DB - value: wcs_db - - - - \ No newline at end of file diff --git a/examples/kiba-robots/06_onlydb.yaml b/examples/kiba-robots/06_onlydb.yaml deleted file mode 100644 index b60dc17c..00000000 --- a/examples/kiba-robots/06_onlydb.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -#sootballs_db -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_db - depends: - kind: package - nameOrGUID: sootballs_db - version: "v1.0.11" - labels: - app: ims -spec: - runtime: cloud - envArgs: - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - - name: POSTGRES_MULTIPLE_DATABASES - value: ims_db,wcs_db - - name: POSTGRES_PASSWORD - value: sootballs - - name: POSTGRES_USER - value: postgres - volumes: - - execName: postgres - mountPath: "/var/lib/postgresql/data/pgdata" - subPath: "pgdata" - depends: - kind: disk - nameOrGUID: "postgres-pvc" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_db - version: v1.0.11 - description: Sootballs DB package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: redis - docker: - image: redis:4.0-alpine - limits: - cpu: 0.5 - memory: 2048 - - type: docker - name: postgres - docker: - image: postgis/postgis:9.6-3.2 - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: POSTGRES_MULTIPLE_DATABASES - default: ims_db,wcs_db - - name: PGDATA - default: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - default: trust - - name: POSTGRES_USER - default: postgres - exposed: true - exposedName: POSTGRES_USER - - name: POSTGRES_PASSWORD - default: password - exposed: true - exposedName: POSTGRES_PASSWORD - endpoints: - - name: POSTGRES - type: internal-tcp - port: 5432 - targetPort: 5432 - - name: REDIS - type: internal-tcp - port: 6379 - targetPort: 6379 diff --git a/examples/kiba-robots/all-in-one.yaml b/examples/kiba-robots/all-in-one.yaml deleted file mode 100644 index 3384be50..00000000 --- a/examples/kiba-robots/all-in-one.yaml +++ /dev/null @@ -1,712 +0,0 @@ -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Secret" -metadata: - name: "sootballs" - guid: secret-vebkjobybhhwmyiwkwvndagu -spec: - type: Docker - docker: - username: shaishavrapyuta - password: asdfg123$ - email: shaishavrapyuta - # registry: https://index.docker.io/v1/ ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs SUI2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "cloud" - ros: - enabled: True - version: "melodic" - rosEndpoints: - - name: "/cmd_global_charge" - type: "topic" - qos: "hi" - - name: "/cmd_global_in" - type: "topic" - qos: "hi" - - name: "/cmd_move_to" - type: "topic" - qos: "hi" - - name: "/dispatcher/active_tote_unload_request" - type: "topic" - qos: "low" - - name: "/dispatcher/control_request" - type: "topic" - qos: "low" - - name: "/dispatcher/modify_order_request" - type: "topic" - qos: "low" - - name: "/dispatcher/productivity_configs_request" - type: "topic" - qos: "low" - - name: "/edge/emergency_released_request" - type: "topic" - qos: "low" - - name: "/manual_order_recovery" - type: "topic" - qos: "hi" - - name: "/reservation_request" - type: "topic" - qos: "hi" - cloud: - replicas: 1 - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:{{ testlabel }}" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_edge remote_ui.launch" - runAsBash: False - limits: - cpu: 2 - memory: 8192 - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "SOOTBALLS_MAP" - defaultValue: "rrkibstg" - - name: "SYSTEM_UI_REMOTE_MODE" - defaultValue: "true" - - name: "WS_EXTERNAL_PORT" - defaultValue: "80" - - name: "DOCKER_STDOUT" - defaultValue: "true" - endpoints: - - name: "SYSTEM_UI" - type: external-https - port: 443 - targetPort: 7099 - - name: "SYSTEM_UI_ROSBRIDGE_URL" - type: external-https - port: 443 - targetPort: 9092 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs Edge2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "device" - device: - arch: amd64 - restart: always - ros: - enabled: True - version: "melodic" - rosEndpoints: - - name: "/dispatcher/control_response" - type: "topic" - qos: "low" - - name: "/dispatcher/modify_order_response" - type: "topic" - qos: "low" - - name: "/dispatcher/productivity_configs_response" - type: "topic" - qos: "low" - - name: "/dispatcher/active_tote_unload_response" - type: "topic" - qos: "low" - - name: "/edge/emergency_released_response" - type: "topic" - qos: "low" - - name: "/robot_reservation" - type: "topic" - qos: "low" - - name: "/sui/status" - type: "topic" - qos: "low" - - name: "/sui/overview" - type: "topic" - qos: "low" - - name: "/sui/main" - type: "topic" - qos: "low" - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:{{ testlabel }}" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_edge default.launch --wait" - runAsBash: False - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - defaultValue: "true" - - name: "DISABLE_MULTICAST" - defaultValue: "false" - - name: "ROS_DOMAIN_ID" - defaultValue: "10" - - name: "USE_PARAMS_IO" - defaultValue: "true" - - name: "EDGE_ALARM_MONITOR" - defaultValue: "true" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: "Sootballs Robot2" - version: "1.0.0" - labels: - app: sootballs -spec: - runtime: "device" - device: - arch: amd64 - restart: always - ros: - enabled: True - version: "melodic" - executables: - - name: "systemui" - type: docker - docker: - image: "rrdockerhub/sootballs:{{ testlabel }}" - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - command: "roslaunch sootballs_applications_robot default.launch --wait" - runAsBash: False - - environmentVars: - - name: "IMS_AUTH_USERNAME" - description: "Username to authenticate to IMS" - defaultValue: "root" - - name: "IMS_AUTH_PASSWORD" - description: "Password to authenticate to IMS" - defaultValue: "airborne_rr" - - name: "SENTRY_DSN" - description: "Password to authenticate to IMS" - defaultValue: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - defaultValue: "true" - - name: "DISABLE_MULTICAST" - defaultValue: "false" - - name: "ROS_DOMAIN_ID" - defaultValue: "10" - - name: "USE_PARAMS_IO" - defaultValue: "true" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_ims - version: v1.14.0-rc4-6 - description: Sootballs IMS Package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: Django - docker: - image: rrdockerhub/sootballs_ims:{{ testlabel }} - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - limits: - cpu: 0.5 - memory: 2048 - command: "/code/docker/entrypoint.sh" - environmentVars: - - name: SECRET_KEY - - name: AUTH_ENABLED - default: "False" - - name: TIMEZONE - default: "Asia/Tokyo" - - name: CONTAINER_NAME - default: "warehouse" - - name: DAYS_BEFORE_ARCHIVE - default: "15" - - name: DEPLOY_ON_CLOUD - default: "False" - - name: SODTIME - default: "09:00" - - name: EODTIME - default: "19:00" - - name: AWS_STORAGE_BUCKET_NAME - default: "kiba-robots" - - name: IMS_DB - default: ims_db - - name: CELERY_BROKER_URL - - name: CELERY_RESULT_BACKEND - - endpoints: - - name: IMS_URL - type: external-https - port: 443 - targetPort: 8002 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_db - version: v1.0.11 - description: Sootballs DB package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: redis - docker: - image: redis:4.0-alpine - limits: - cpu: 0.5 - memory: 2048 - - type: docker - name: postgres - docker: - image: postgis/postgis:9.6-3.2 - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: POSTGRES_MULTIPLE_DATABASES - default: ims_db,wcs_db - - name: PGDATA - default: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - default: trust - - name: POSTGRES_USER - default: postgres - exposed: true - exposedName: POSTGRES_USER - - name: POSTGRES_PASSWORD - default: password - exposed: true - exposedName: POSTGRES_PASSWORD - endpoints: - - name: POSTGRES - type: internal-tcp - port: 5432 - targetPort: 5432 - - name: REDIS - type: internal-tcp - port: 6379 - targetPort: 6379 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: MinIO File Server - version: v1.0.11 - description: Sootballs File Server package - labels: - app: ims -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: minio_executable - docker: - image: rrdockerhub/minio-server - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: MINIO_ACCESS_KEY - - name: MINIO_SECRET_KEY - endpoints: - - name: MINIO - type: external-https - port: 443 - targetPort: 9000 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Package" -metadata: - name: sootballs_wcs - version: v1.14.0-rc4-2 - description: Sootballs WCS package - labels: - app: wcs -spec: - runtime: cloud - cloud: - replicas: 1 - executables: - - type: docker - name: django - command: /code/docker/entrypoint.sh - docker: - image: rrdockerhub/sootballs_wcs:{{ testlabel}} - pullSecret: - depends: - kind: secret - nameOrGUID: sootballs - limits: - cpu: 1 - memory: 4096 - environmentVars: - - name: TIMEZONE - default: Asia/Tokyo - - name: WCS_DB - default: wcs_db - - name: CELERY_BROKER_URL - - name: CELERY_RESULT_BACKEND - - name: LOCAL_PRINT_SERVER_URL - - name: SECRET_KEY - endpoints: - - name: WCS_URL - type: external-https - port: 443 - targetPort: 8003 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "postgres-pvc" -spec: - runtime: cloud - capacity: 4 ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Disk" -metadata: - name: "minio-pvc" -spec: - runtime: cloud - capacity: 4 ---- -#sootballs_staticroutes -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "ims-{{ routePrefix }}" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "minio-{{ routePrefix }}" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "wcs-{{ routePrefix }}" ---- -#sootballs_minio -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_minio - depends: - kind: package - nameOrGUID: "MinIO File Server" - version: "v1.0.11" - labels: - app: ims -spec: - runtime: cloud - envArgs: - - name: MINIO_ACCESS_KEY - value: access - - name: MINIO_SECRET_KEY - value: secret_key - staticRoutes: - - name: MINIO - depends: - kind: staticroute - nameOrGUID: minio-{{ routePrefix }} - volumes: - - execName: minio_executable - mountPath: "/data" - subPath: "data" - depends: - kind: disk - nameOrGUID: "minio-pvc" ---- -#sootballs_db -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_db - depends: - kind: package - nameOrGUID: sootballs_db - version: "v1.0.11" - labels: - app: ims -spec: - runtime: cloud - envArgs: - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - - name: POSTGRES_MULTIPLE_DATABASES - value: ims_db,wcs_db - - name: POSTGRES_PASSWORD - value: sootballs - - name: POSTGRES_USER - value: postgres - volumes: - - execName: postgres - mountPath: "/var/lib/postgresql/data/pgdata" - subPath: "pgdata" - depends: - kind: disk - nameOrGUID: "postgres-pvc" - -#sootballs_ims ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_ims - depends: - kind: package - nameOrGUID: "sootballs_ims" - version: "v1.14.0-rc4-6" - labels: - app: ims -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_db - - kind: deployment - nameOrGUID: sootballs_minio - envArgs: - - name: AUTH_ENABLED - value: "True" - - name: AWS_STORAGE_BUCKET_NAME - value: kiba-robots - #TODO this should be parsed from redis url in the docker container. - - name: CELERY_BROKER_URL - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - #TODO this should be parsed from redis url in the docker container. - - name: CELERY_RESULT_BACKEND - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: CONTAINER_NAME - value: warehouse - - name: DAYS_BEFORE_ARCHIVE - value: "15" - - name: DEPLOY_ON_CLOUD - value: "False" - - name: EODTIME - value: "19:00" - - name: IMS_DB - value: ims_db - - name: SECRET_KEY - value: asdasd - - name: SODTIME - value: 09:00 - - name: TIMEZONE - value: Asia/Tokyo - - name: TEST_ENV - value: asdsad - staticRoutes: - - name: IMS_URL - depends: - kind: staticroute - nameOrGUID: ims-{{ routePrefix }} ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_wcs - depends: - kind: package - nameOrGUID: sootballs_wcs - version: "v1.14.0-rc4-2" - labels: - app: wcs -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_db - - kind: deployment - nameOrGUID: sootballs_ims - envArgs: - - name: CELERY_BROKER_URL - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: CELERY_RESULT_BACKEND - value: rediss://inst-lrxokslpvkctnstujixsczsw-redis-srv.dep-ns-inst-lrxokslpvkctnstujixsczsw.svc:6379/5 - - name: LOCAL_PRINT_SERVER_URL - value: " " - - name: SECRET_KEY - value: asdasd - - name: TIMEZONE - value: Asia/Tokyo - - name: WCS_DB - value: wcs_db ---- -#sootballs_staticroutes -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "StaticRoute" -metadata: - name: "ui-{{ routePrefix }}" ---- -apiVersion: "apiextensions.rapyuta.io/v1" -kind: "Network" -metadata: - name: "sootballs" -spec: - runtime: "cloud" - type: "routed" - rosDistro: "melodic" - resourceLimits: "small" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_systemui - depends: - kind: package - nameOrGUID: Sootballs SUI2 - version: "1.0.0" - labels: - app: systemui -spec: - runtime: cloud - depends: - - kind: deployment - nameOrGUID: sootballs_ims - rosNetworks: - - depends: - kind: network - nameOrGUID: sootballs - envArgs: - - name: DOCKER_STDOUT - value: "true" - - name: IMS_AUTH_PASSWORD - value: airborne_rr - - name: IMS_AUTH_USERNAME - value: root - - name: SENTRY_DSN - value: https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18 - - name: SOOTBALLS_MAP - value: rrkibstg - - name: SYSTEM_UI_REMOTE_MODE - value: "true" - - name: USE_LOCAL_MAP - value: "false" - - name: WS_EXTERNAL_PORT - value: "80" - staticRoutes: - - name: SYSTEM_UI - depends: - kind: staticroute - nameOrGUID: ui-{{ routePrefix }} - ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_edge_01 - depends: - kind: package - nameOrGUID: "Sootballs Edge2" - version: "1.0.0" -spec: - runtime: device - depends: - - kind: deployment - nameOrGUID: sootballs_ims - rosNetworks: - - depends: - kind: network - nameOrGUID: sootballs - device: - depends: - kind: device - nameOrGUID: kibstg-edge-1 - envArgs: - - name: IMS_AUTH_USERNAME - value: root - - name: IMS_AUTH_PASSWORD - value: airborne_rr - - name: "SENTRY_DSN" - value: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - value: "true" - - name: "DISABLE_MULTICAST" - value: "false" - - name: "ROS_DOMAIN_ID" - value: "10" - - name: "USE_PARAMS_IO" - value: "true" - - name: "EDGE_ALARM_MONITOR" - value: "true" ---- -apiVersion: apiextensions.rapyuta.io/v1 -kind: Deployment -metadata: - name: sootballs_amr_01 - depends: - kind: package - nameOrGUID: "Sootballs Robot2" - version: "1.0.0" -spec: - runtime: device - depends: - - kind: deployment - nameOrGUID: sootballs_ims - rosNetworks: - - depends: - kind: network - nameOrGUID: sootballs - device: - depends: - kind: device - nameOrGUID: kibstg-amr-1 - envArgs: - - name: IMS_AUTH_USERNAME - value: root - - name: IMS_AUTH_PASSWORD - value: airborne_rr - - name: "SENTRY_DSN" - value: "https://a83904fb2f394a768557eaea8ec39db4:7d8a02b629994bf884534917dfb08511@sentry.svc.rapyuta.io/18" - - name: "DOCKER_STDOUT" - value: "true" - - name: "DISABLE_MULTICAST" - value: "false" - - name: "ROS_DOMAIN_ID" - value: "10" - - name: "USE_PARAMS_IO" - value: "true" diff --git a/examples/kiba-robots/secrets.staging.yaml b/examples/kiba-robots/secrets.staging.yaml deleted file mode 100644 index acbc06a4..00000000 --- a/examples/kiba-robots/secrets.staging.yaml +++ /dev/null @@ -1,15 +0,0 @@ -global: - dbHost: ENC[AES256_GCM,data:+7cvT+9J09CigDcvxm837Hfz0+WFwBzA1TcQvI75SFs5PmIs7A==,iv:km6KEq0AYSHBxgL1a1WlIa8gvguU4dEaB9oogaVTU4o=,tag:G+fxpM7FB7TQYsWeekrJ0g==,type:str] - dbUser: ENC[AES256_GCM,data:4j9zoiRsJhMJ0MWW0XEQCHt4,iv:Q4NboLJnT4oA6Z6uxp7ans1V4sjgYl/quemOXNEulco=,tag:AxLp7Xj75MN+JU0+x5YfSA==,type:str] - dbPass: ENC[AES256_GCM,data:YyfHySZ4bcAvR3qnIAa2BMLMFQuUkg==,iv:FJJTQ+TQcjnzo7iNXg/a7P5do8b8ButaaPALvf4Q+FQ=,tag:OZIrjBt0CGAhVbWpzvfAYg==,type:str] -sops: - kms: [ ] - gcp_kms: - - resource_id: projects/rapyuta-io/locations/global/keyRings/sops_key/cryptoKeys/sops-key - created_at: '2022-04-12T19:53:13Z' - enc: CiQAJwkHSPloHZMqFLMdQRRActkGdVWHzMZOwLw/YrTDyS/ZE3gSSQD+fVSX3pLI/l0n8INc9C6nY+bGWBIxbuXWELl3d5jYVurpxGqXeikLhKsJZo6C2v1gknKv734uZgcPXVG7Y5dLDMGaMYeBpzA= - lastmodified: '2022-04-12T19:53:14Z' - mac: ENC[AES256_GCM,data:t/lH23kZ3lZymEpuiTwVxICJacw5jgRSXJ1REuCxcoN2GT9Dz76CsS5g9RYJpawx7LtdTU8SqRN17eZ/7f4gctuqvVWFOqu/vh/ui1SDYonkclh4JVxU4wTGAkXKnDJG5qGF/1z0d2zHr/I4CfWHmJFBWbvD1MxMCpJaU0T11n4=,iv:q62pfYu3Ku9EAKwvR0k+69A9goJ4KOz4FBO6DqRKN0U=,tag:ICqGK8A6+bHWnGam4ECIsg==,type:str] - pgp: [ ] - unencrypted_suffix: _unencrypted - version: 3.0.5 diff --git a/examples/kiba-robots/values.yaml b/examples/kiba-robots/values.yaml deleted file mode 100644 index 90983848..00000000 --- a/examples/kiba-robots/values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -testlabel: 1.14.0-rc4 -routePrefix: rc4 -# this is a simple yaml file which can be used to abstract out frequently changed values from your resource yamls. diff --git a/examples/manifests b/examples/manifests deleted file mode 120000 index 91067c47..00000000 --- a/examples/manifests +++ /dev/null @@ -1 +0,0 @@ -../riocli/apply/manifests/ \ No newline at end of file diff --git a/examples/marketplace/dependencies.yaml b/examples/marketplace/dependencies.yaml deleted file mode 100644 index 673da5f6..00000000 --- a/examples/marketplace/dependencies.yaml +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: - - name: org.unyscape.wordpress - version: '<1.0.5,>1.0.0' - - name: rio.db.postgres - version: '^11.0.1' - - name: rio.ml.cvat.psql-redis-aio55555 - version: '*' - - name: rio.db.redis - version: '*' From 4c5c040291ac5b7aa779fbb365d3161dd503b63a Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 1 Aug 2024 00:12:13 +0530 Subject: [PATCH 30/54] fix(template): validate manifests against jsonschema The rio template command doesn't validate the manifest against the jsonschema before printing them. It helps in catching missing values in the templates, but doesn't help with incorrect manifests. This commit fixes that and should help users validate the manifests before applying them. Wrike Ticket: https://www.wrike.com/open.htm?id=1464064144 --- riocli/apply/parse.py | 8 +++++++- riocli/apply/template.py | 6 +++--- riocli/apply/util.py | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 427a6e92..f861c59f 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -143,7 +143,13 @@ def delete(self, *args, **kwargs): spinner.red.fail(Symbols.ERROR) def print_resolved_manifests(self): - manifests = [o for _, o in self.objects.items()] + """Validates and prints the resolved manifests""" + manifests = [] + for _, o in self.objects.items(): + kls = get_model(o) + kls.validate(o) + manifests.append(o) + dump_all_yaml(manifests) def parse_dependencies(self): diff --git a/riocli/apply/template.py b/riocli/apply/template.py index 82909219..417c45ad 100644 --- a/riocli/apply/template.py +++ b/riocli/apply/template.py @@ -1,4 +1,4 @@ -# Copyright 2022 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,5 +46,5 @@ def template(values: str, secrets: str, files: Iterable[str]) -> None: click.secho('No files specified', fg=Colors.RED) raise SystemExit(1) - rc = Applier(glob_files, abs_values, abs_secrets) - rc.print_resolved_manifests() + applier = Applier(glob_files, abs_values, abs_secrets) + applier.print_resolved_manifests() diff --git a/riocli/apply/util.py b/riocli/apply/util.py index d92b39db..654139f4 100644 --- a/riocli/apply/util.py +++ b/riocli/apply/util.py @@ -33,6 +33,7 @@ from riocli.static_route.model import StaticRoute from riocli.usergroup.model import UserGroup from riocli.utils import tabulate_data +from riocli.model import Model KIND_TO_CLASS = { 'project': Project, @@ -48,7 +49,7 @@ } -def get_model(data: dict) -> typing.Any: +def get_model(data: dict) -> Model: """Get the model class based on the kind""" kind = data.get('kind', None) if kind is None: From 50b7f220d5927362c020b391389c63670c65d50e Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 1 Aug 2024 11:30:18 +0530 Subject: [PATCH 31/54] chore: bumps rapyuta_io version to 1.17.0 --- Pipfile | 2 +- Pipfile.lock | 261 +++++++++++++++++++++++++-------------------------- setup.py | 2 +- 3 files changed, 131 insertions(+), 134 deletions(-) diff --git a/Pipfile b/Pipfile index 3e9e9482..9ae24758 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,7 @@ graphlib-backport = ">=1.0.3" jinja2 = ">=3.0.1" munch = ">=2.4.0" pyyaml = ">=5.4.1" -rapyuta-io = ">=1.16.0" +rapyuta-io = ">=1.17.0" tabulate = ">=0.8.0" pyrfc3339 = ">=1.1" directory-tree = ">=0.0.3.1" diff --git a/Pipfile.lock b/Pipfile.lock index 5d59cc3d..f05874f3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "57e69626a9e7256c116eaa747186cc795e94f4a20a99586c0c83966abbc82f5a" + "sha256": "9b06f5701b4180a801cbc4a21ebe5a6ca594716a58a0baf7dc00dac3075d5c2a" }, "pipfile-spec": 6, "requires": { @@ -34,11 +34,11 @@ }, "certifi": { "hashes": [ - "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", - "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" + "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", + "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" ], "markers": "python_version >= '3.6'", - "version": "==2024.6.2" + "version": "==2024.7.4" }, "charset-normalizer": { "hashes": [ @@ -133,7 +133,7 @@ "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.3.2" }, "click": { @@ -224,15 +224,11 @@ }, "etcd3gw": { "hashes": [ - "sha256:128ea7255247e18dfee73dbd30fcfea5819a6bb15ab1b4b7260016afc86904ba", - "sha256:6a3f765f01a245921db2543a2804283b30dd9d42dc2f70feeb88e9137605f697" + "sha256:47c89d496c146fe3abc28e5229c7a7f39b74c9a6e6268dc7c143a768b1e625c1", + "sha256:9442510be21ba692150f953d36a185c77416de01d2d412e0856eb2a5c608c44e" ], "index": "pypi", - "version": "==2.4.0" - }, - "flake8": { - "path": ".", - "version": "==7.2.1" + "version": "==2.4.1" }, "futurist": { "hashes": [ @@ -377,7 +373,7 @@ "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10", "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.0.47" }, "pyrfc3339": { @@ -531,11 +527,15 @@ }, "rapyuta-io": { "hashes": [ - "sha256:8c946e6561c88d176013ab197bbb46797653a4e4e5ed223972071a110e36e682", - "sha256:e77f5e42a9ad6892c6de516896e914d7f147e6aa904857bff5627bf83effbe6c" + "sha256:27c60d1721a84efdc05128407d7f0c3548bc655f0e76073a40228f08aa7ef7d0", + "sha256:a04268d5c7345e3182947272c79519ce522b4ace7195d2947fd25204b422df11" ], "index": "pypi", - "version": "==1.16.0" + "version": "==1.17.0" + }, + "rapyuta-io-cli": { + "path": ".", + "version": "==7.2.1" }, "requests": { "hashes": [ @@ -545,9 +545,6 @@ "markers": "python_version >= '3.8'", "version": "==2.32.3" }, - "riocli": { - "path": "." - }, "semver": { "hashes": [ "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc", @@ -558,11 +555,11 @@ }, "setuptools": { "hashes": [ - "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650", - "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95" + "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1", + "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec" ], "markers": "python_version >= '3.8'", - "version": "==70.1.1" + "version": "==72.1.0" }, "shellingham": { "hashes": [ @@ -671,7 +668,7 @@ "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051", "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.12.3" }, "black": { @@ -698,11 +695,11 @@ }, "certifi": { "hashes": [ - "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", - "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" + "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", + "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" ], "markers": "python_version >= '3.6'", - "version": "==2024.6.2" + "version": "==2024.7.4" }, "charset-normalizer": { "hashes": [ @@ -797,7 +794,7 @@ "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.3.2" }, "click": { @@ -962,11 +959,11 @@ }, "pip": { "hashes": [ - "sha256:5aa64f65e1952733ee0a9a9b1f52496ebdb3f3077cc46f80a16d983b58d1180a", - "sha256:efca15145a95e95c00608afeab66311d40bfb73bb2266a855befd705e6bb15a0" + "sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2", + "sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8" ], "markers": "python_version >= '3.8'", - "version": "==24.1.1" + "version": "==24.2" }, "pip-shims": { "hashes": [ @@ -1035,88 +1032,88 @@ }, "regex": { "hashes": [ - "sha256:0721931ad5fe0dda45d07f9820b90b2148ccdd8e45bb9e9b42a146cb4f695649", - "sha256:10002e86e6068d9e1c91eae8295ef690f02f913c57db120b58fdd35a6bb1af35", - "sha256:10e4ce0dca9ae7a66e6089bb29355d4432caed736acae36fef0fdd7879f0b0cb", - "sha256:119af6e56dce35e8dfb5222573b50c89e5508d94d55713c75126b753f834de68", - "sha256:1337b7dbef9b2f71121cdbf1e97e40de33ff114801263b275aafd75303bd62b5", - "sha256:13cdaf31bed30a1e1c2453ef6015aa0983e1366fad2667657dbcac7b02f67133", - "sha256:1595f2d10dff3d805e054ebdc41c124753631b6a471b976963c7b28543cf13b0", - "sha256:16093f563098448ff6b1fa68170e4acbef94e6b6a4e25e10eae8598bb1694b5d", - "sha256:1878b8301ed011704aea4c806a3cadbd76f84dece1ec09cc9e4dc934cfa5d4da", - "sha256:19068a6a79cf99a19ccefa44610491e9ca02c2be3305c7760d3831d38a467a6f", - "sha256:19dfb1c504781a136a80ecd1fff9f16dddf5bb43cec6871778c8a907a085bb3d", - "sha256:1b5269484f6126eee5e687785e83c6b60aad7663dafe842b34691157e5083e53", - "sha256:1c1c174d6ec38d6c8a7504087358ce9213d4332f6293a94fbf5249992ba54efa", - "sha256:2431b9e263af1953c55abbd3e2efca67ca80a3de8a0437cb58e2421f8184717a", - "sha256:287eb7f54fc81546346207c533ad3c2c51a8d61075127d7f6d79aaf96cdee890", - "sha256:2b4c884767504c0e2401babe8b5b7aea9148680d2e157fa28f01529d1f7fcf67", - "sha256:35cb514e137cb3488bce23352af3e12fb0dbedd1ee6e60da053c69fb1b29cc6c", - "sha256:391d7f7f1e409d192dba8bcd42d3e4cf9e598f3979cdaed6ab11288da88cb9f2", - "sha256:3ad070b823ca5890cab606c940522d05d3d22395d432f4aaaf9d5b1653e47ced", - "sha256:3cd7874d57f13bf70078f1ff02b8b0aa48d5b9ed25fc48547516c6aba36f5741", - "sha256:3e507ff1e74373c4d3038195fdd2af30d297b4f0950eeda6f515ae3d84a1770f", - "sha256:455705d34b4154a80ead722f4f185b04c4237e8e8e33f265cd0798d0e44825fa", - "sha256:4a605586358893b483976cffc1723fb0f83e526e8f14c6e6614e75919d9862cf", - "sha256:4babf07ad476aaf7830d77000874d7611704a7fcf68c9c2ad151f5d94ae4bfc4", - "sha256:4eee78a04e6c67e8391edd4dad3279828dd66ac4b79570ec998e2155d2e59fd5", - "sha256:5397de3219a8b08ae9540c48f602996aa6b0b65d5a61683e233af8605c42b0f2", - "sha256:5b5467acbfc153847d5adb21e21e29847bcb5870e65c94c9206d20eb4e99a384", - "sha256:5eaa7ddaf517aa095fa8da0b5015c44d03da83f5bd49c87961e3c997daed0de7", - "sha256:632b01153e5248c134007209b5c6348a544ce96c46005d8456de1d552455b014", - "sha256:64c65783e96e563103d641760664125e91bd85d8e49566ee560ded4da0d3e704", - "sha256:64f18a9a3513a99c4bef0e3efd4c4a5b11228b48aa80743be822b71e132ae4f5", - "sha256:673b5a6da4557b975c6c90198588181029c60793835ce02f497ea817ff647cb2", - "sha256:68811ab14087b2f6e0fc0c2bae9ad689ea3584cad6917fc57be6a48bbd012c49", - "sha256:6e8d717bca3a6e2064fc3a08df5cbe366369f4b052dcd21b7416e6d71620dca1", - "sha256:71a455a3c584a88f654b64feccc1e25876066c4f5ef26cd6dd711308aa538694", - "sha256:72d7a99cd6b8f958e85fc6ca5b37c4303294954eac1376535b03c2a43eb72629", - "sha256:7b59138b219ffa8979013be7bc85bb60c6f7b7575df3d56dc1e403a438c7a3f6", - "sha256:7dbe2467273b875ea2de38ded4eba86cbcbc9a1a6d0aa11dcf7bd2e67859c435", - "sha256:833616ddc75ad595dee848ad984d067f2f31be645d603e4d158bba656bbf516c", - "sha256:87e2a9c29e672fc65523fb47a90d429b70ef72b901b4e4b1bd42387caf0d6835", - "sha256:8fe45aa3f4aa57faabbc9cb46a93363edd6197cbc43523daea044e9ff2fea83e", - "sha256:9e717956dcfd656f5055cc70996ee2cc82ac5149517fc8e1b60261b907740201", - "sha256:9efa1a32ad3a3ea112224897cdaeb6aa00381627f567179c0314f7b65d354c62", - "sha256:9ff11639a8d98969c863d4617595eb5425fd12f7c5ef6621a4b74b71ed8726d5", - "sha256:a094801d379ab20c2135529948cb84d417a2169b9bdceda2a36f5f10977ebc16", - "sha256:a0981022dccabca811e8171f913de05720590c915b033b7e601f35ce4ea7019f", - "sha256:a0bd000c6e266927cb7a1bc39d55be95c4b4f65c5be53e659537537e019232b1", - "sha256:a32b96f15c8ab2e7d27655969a23895eb799de3665fa94349f3b2fbfd547236f", - "sha256:a81e3cfbae20378d75185171587cbf756015ccb14840702944f014e0d93ea09f", - "sha256:ac394ff680fc46b97487941f5e6ae49a9f30ea41c6c6804832063f14b2a5a145", - "sha256:ada150c5adfa8fbcbf321c30c751dc67d2f12f15bd183ffe4ec7cde351d945b3", - "sha256:b2b6f1b3bb6f640c1a92be3bbfbcb18657b125b99ecf141fb3310b5282c7d4ed", - "sha256:b802512f3e1f480f41ab5f2cfc0e2f761f08a1f41092d6718868082fc0d27143", - "sha256:ba68168daedb2c0bab7fd7e00ced5ba90aebf91024dea3c88ad5063c2a562cca", - "sha256:bfc4f82cabe54f1e7f206fd3d30fda143f84a63fe7d64a81558d6e5f2e5aaba9", - "sha256:c0c18345010870e58238790a6779a1219b4d97bd2e77e1140e8ee5d14df071aa", - "sha256:c3bea0ba8b73b71b37ac833a7f3fd53825924165da6a924aec78c13032f20850", - "sha256:c486b4106066d502495b3025a0a7251bf37ea9540433940a23419461ab9f2a80", - "sha256:c49e15eac7c149f3670b3e27f1f28a2c1ddeccd3a2812cba953e01be2ab9b5fe", - "sha256:c6a2b494a76983df8e3d3feea9b9ffdd558b247e60b92f877f93a1ff43d26656", - "sha256:cab12877a9bdafde5500206d1020a584355a97884dfd388af3699e9137bf7388", - "sha256:cac27dcaa821ca271855a32188aa61d12decb6fe45ffe3e722401fe61e323cd1", - "sha256:cdd09d47c0b2efee9378679f8510ee6955d329424c659ab3c5e3a6edea696294", - "sha256:cf2430df4148b08fb4324b848672514b1385ae3807651f3567871f130a728cc3", - "sha256:d0a3d8d6acf0c78a1fff0e210d224b821081330b8524e3e2bc5a68ef6ab5803d", - "sha256:d0c0c0003c10f54a591d220997dd27d953cd9ccc1a7294b40a4be5312be8797b", - "sha256:d1f059a4d795e646e1c37665b9d06062c62d0e8cc3c511fe01315973a6542e40", - "sha256:d347a741ea871c2e278fde6c48f85136c96b8659b632fb57a7d1ce1872547600", - "sha256:d3ee02d9e5f482cc8309134a91eeaacbdd2261ba111b0fef3748eeb4913e6a2c", - "sha256:d99ceffa25ac45d150e30bd9ed14ec6039f2aad0ffa6bb87a5936f5782fc1569", - "sha256:e38a7d4e8f633a33b4c7350fbd8bad3b70bf81439ac67ac38916c4a86b465456", - "sha256:e4682f5ba31f475d58884045c1a97a860a007d44938c4c0895f41d64481edbc9", - "sha256:e5bb9425fe881d578aeca0b2b4b3d314ec88738706f66f219c194d67179337cb", - "sha256:e64198f6b856d48192bf921421fdd8ad8eb35e179086e99e99f711957ffedd6e", - "sha256:e6662686aeb633ad65be2a42b4cb00178b3fbf7b91878f9446075c404ada552f", - "sha256:ec54d5afa89c19c6dd8541a133be51ee1017a38b412b1321ccb8d6ddbeb4cf7d", - "sha256:f5b1dff3ad008dccf18e652283f5e5339d70bf8ba7c98bf848ac33db10f7bc7a", - "sha256:f8ec0c2fea1e886a19c3bee0cd19d862b3aa75dcdfb42ebe8ed30708df64687a", - "sha256:f9ebd0a36102fcad2f03696e8af4ae682793a5d30b46c647eaf280d6cfb32796" + "sha256:01b689e887f612610c869421241e075c02f2e3d1ae93a037cb14f88ab6a8934c", + "sha256:04ce29e2c5fedf296b1a1b0acc1724ba93a36fb14031f3abfb7abda2806c1535", + "sha256:0ffe3f9d430cd37d8fa5632ff6fb36d5b24818c5c986893063b4e5bdb84cdf24", + "sha256:18300a1d78cf1290fa583cd8b7cde26ecb73e9f5916690cf9d42de569c89b1ce", + "sha256:185e029368d6f89f36e526764cf12bf8d6f0e3a2a7737da625a76f594bdfcbfc", + "sha256:19c65b00d42804e3fbea9708f0937d157e53429a39b7c61253ff15670ff62cb5", + "sha256:228b0d3f567fafa0633aee87f08b9276c7062da9616931382993c03808bb68ce", + "sha256:23acc72f0f4e1a9e6e9843d6328177ae3074b4182167e34119ec7233dfeccf53", + "sha256:25419b70ba00a16abc90ee5fce061228206173231f004437730b67ac77323f0d", + "sha256:2dfbb8baf8ba2c2b9aa2807f44ed272f0913eeeba002478c4577b8d29cde215c", + "sha256:2f1baff13cc2521bea83ab2528e7a80cbe0ebb2c6f0bfad15be7da3aed443908", + "sha256:33e2614a7ce627f0cdf2ad104797d1f68342d967de3695678c0cb84f530709f8", + "sha256:3426de3b91d1bc73249042742f45c2148803c111d1175b283270177fdf669024", + "sha256:382281306e3adaaa7b8b9ebbb3ffb43358a7bbf585fa93821300a418bb975281", + "sha256:3d974d24edb231446f708c455fd08f94c41c1ff4f04bcf06e5f36df5ef50b95a", + "sha256:3f3b6ca8eae6d6c75a6cff525c8530c60e909a71a15e1b731723233331de4169", + "sha256:3fac296f99283ac232d8125be932c5cd7644084a30748fda013028c815ba3364", + "sha256:416c0e4f56308f34cdb18c3f59849479dde5b19febdcd6e6fa4d04b6c31c9faa", + "sha256:438d9f0f4bc64e8dea78274caa5af971ceff0f8771e1a2333620969936ba10be", + "sha256:43affe33137fcd679bdae93fb25924979517e011f9dea99163f80b82eadc7e53", + "sha256:44fc61b99035fd9b3b9453f1713234e5a7c92a04f3577252b45feefe1b327759", + "sha256:45104baae8b9f67569f0f1dca5e1f1ed77a54ae1cd8b0b07aba89272710db61e", + "sha256:4fdd1384619f406ad9037fe6b6eaa3de2749e2e12084abc80169e8e075377d3b", + "sha256:538d30cd96ed7d1416d3956f94d54e426a8daf7c14527f6e0d6d425fcb4cca52", + "sha256:558a57cfc32adcf19d3f791f62b5ff564922942e389e3cfdb538a23d65a6b610", + "sha256:5eefee9bfe23f6df09ffb6dfb23809f4d74a78acef004aa904dc7c88b9944b05", + "sha256:64bd50cf16bcc54b274e20235bf8edbb64184a30e1e53873ff8d444e7ac656b2", + "sha256:65fd3d2e228cae024c411c5ccdffae4c315271eee4a8b839291f84f796b34eca", + "sha256:66b4c0731a5c81921e938dcf1a88e978264e26e6ac4ec96a4d21ae0354581ae0", + "sha256:68a8f8c046c6466ac61a36b65bb2395c74451df2ffb8458492ef49900efed293", + "sha256:6a1141a1dcc32904c47f6846b040275c6e5de0bf73f17d7a409035d55b76f289", + "sha256:6b9fc7e9cc983e75e2518496ba1afc524227c163e43d706688a6bb9eca41617e", + "sha256:6f51f9556785e5a203713f5efd9c085b4a45aecd2a42573e2b5041881b588d1f", + "sha256:7214477bf9bd195894cf24005b1e7b496f46833337b5dedb7b2a6e33f66d962c", + "sha256:731fcd76bbdbf225e2eb85b7c38da9633ad3073822f5ab32379381e8c3c12e94", + "sha256:74007a5b25b7a678459f06559504f1eec2f0f17bca218c9d56f6a0a12bfffdad", + "sha256:7a5486ca56c8869070a966321d5ab416ff0f83f30e0e2da1ab48815c8d165d46", + "sha256:7c479f5ae937ec9985ecaf42e2e10631551d909f203e31308c12d703922742f9", + "sha256:7df9ea48641da022c2a3c9c641650cd09f0cd15e8908bf931ad538f5ca7919c9", + "sha256:7e37e809b9303ec3a179085415cb5f418ecf65ec98cdfe34f6a078b46ef823ee", + "sha256:80c811cfcb5c331237d9bad3bea2c391114588cf4131707e84d9493064d267f9", + "sha256:836d3cc225b3e8a943d0b02633fb2f28a66e281290302a79df0e1eaa984ff7c1", + "sha256:84c312cdf839e8b579f504afcd7b65f35d60b6285d892b19adea16355e8343c9", + "sha256:86b17ba823ea76256b1885652e3a141a99a5c4422f4a869189db328321b73799", + "sha256:871e3ab2838fbcb4e0865a6e01233975df3a15e6fce93b6f99d75cacbd9862d1", + "sha256:88ecc3afd7e776967fa16c80f974cb79399ee8dc6c96423321d6f7d4b881c92b", + "sha256:8bc593dcce679206b60a538c302d03c29b18e3d862609317cb560e18b66d10cf", + "sha256:8fd5afd101dcf86a270d254364e0e8dddedebe6bd1ab9d5f732f274fa00499a5", + "sha256:945352286a541406f99b2655c973852da7911b3f4264e010218bbc1cc73168f2", + "sha256:973335b1624859cb0e52f96062a28aa18f3a5fc77a96e4a3d6d76e29811a0e6e", + "sha256:994448ee01864501912abf2bad9203bffc34158e80fe8bfb5b031f4f8e16da51", + "sha256:9cfd009eed1a46b27c14039ad5bbc5e71b6367c5b2e6d5f5da0ea91600817506", + "sha256:a2ec4419a3fe6cf8a4795752596dfe0adb4aea40d3683a132bae9c30b81e8d73", + "sha256:a4997716674d36a82eab3e86f8fa77080a5d8d96a389a61ea1d0e3a94a582cf7", + "sha256:a512eed9dfd4117110b1881ba9a59b31433caed0c4101b361f768e7bcbaf93c5", + "sha256:a82465ebbc9b1c5c50738536fdfa7cab639a261a99b469c9d4c7dcbb2b3f1e57", + "sha256:ae2757ace61bc4061b69af19e4689fa4416e1a04840f33b441034202b5cd02d4", + "sha256:b16582783f44fbca6fcf46f61347340c787d7530d88b4d590a397a47583f31dd", + "sha256:ba2537ef2163db9e6ccdbeb6f6424282ae4dea43177402152c67ef869cf3978b", + "sha256:bf7a89eef64b5455835f5ed30254ec19bf41f7541cd94f266ab7cbd463f00c41", + "sha256:c0abb5e4e8ce71a61d9446040c1e86d4e6d23f9097275c5bd49ed978755ff0fe", + "sha256:c414cbda77dbf13c3bc88b073a1a9f375c7b0cb5e115e15d4b73ec3a2fbc6f59", + "sha256:c51edc3541e11fbe83f0c4d9412ef6c79f664a3745fab261457e84465ec9d5a8", + "sha256:c5e69fd3eb0b409432b537fe3c6f44ac089c458ab6b78dcec14478422879ec5f", + "sha256:c918b7a1e26b4ab40409820ddccc5d49871a82329640f5005f73572d5eaa9b5e", + "sha256:c9bb87fdf2ab2370f21e4d5636e5317775e5d51ff32ebff2cf389f71b9b13750", + "sha256:ca5b2028c2f7af4e13fb9fc29b28d0ce767c38c7facdf64f6c2cd040413055f1", + "sha256:d0a07763776188b4db4c9c7fb1b8c494049f84659bb387b71c73bbc07f189e96", + "sha256:d33a0021893ede5969876052796165bab6006559ab845fd7b515a30abdd990dc", + "sha256:d55588cba7553f0b6ec33130bc3e114b355570b45785cebdc9daed8c637dd440", + "sha256:dac8e84fff5d27420f3c1e879ce9929108e873667ec87e0c8eeb413a5311adfe", + "sha256:eaef80eac3b4cfbdd6de53c6e108b4c534c21ae055d1dbea2de6b3b8ff3def38", + "sha256:eb462f0e346fcf41a901a126b50f8781e9a474d3927930f3490f38a6e73b6950", + "sha256:eb563dd3aea54c797adf513eeec819c4213d7dbfc311874eb4fd28d10f2ff0f2", + "sha256:f273674b445bcb6e4409bf8d1be67bc4b58e8b46fd0d560055d515b8830063cd", + "sha256:f6442f0f0ff81775eaa5b05af8a0ffa1dda36e9cf6ec1e0d3d245e8564b684ce", + "sha256:fb168b5924bef397b5ba13aabd8cf5df7d3d93f10218d7b925e360d436863f66", + "sha256:fbf8c2f00904eaf63ff37718eb13acf8e178cb940520e47b2f05027f5bb34ce3", + "sha256:fe4ebef608553aff8deb845c7f4f1d0740ff76fa672c011cc0bacb2a00fbde86" ], "markers": "python_version >= '3.8'", - "version": "==2024.5.15" + "version": "==2024.7.24" }, "requests": { "hashes": [ @@ -1136,11 +1133,11 @@ }, "setuptools": { "hashes": [ - "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650", - "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95" + "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1", + "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec" ], "markers": "python_version >= '3.8'", - "version": "==70.1.1" + "version": "==72.1.0" }, "six": { "hashes": [ @@ -1191,27 +1188,27 @@ }, "sphinxcontrib-applehelp": { "hashes": [ - "sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619", - "sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4" + "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", + "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" ], "markers": "python_version >= '3.9'", - "version": "==1.0.8" + "version": "==2.0.0" }, "sphinxcontrib-devhelp": { "hashes": [ - "sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f", - "sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3" + "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", + "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" ], "markers": "python_version >= '3.9'", - "version": "==1.0.6" + "version": "==2.0.0" }, "sphinxcontrib-htmlhelp": { "hashes": [ - "sha256:0dc87637d5de53dd5eec3a6a01753b1ccf99494bd756aafecd74b4fa9e729015", - "sha256:393f04f112b4d2f53d93448d4bce35842f62b307ccdc549ec1585e950bc35e04" + "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", + "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" ], "markers": "python_version >= '3.9'", - "version": "==2.0.5" + "version": "==2.1.0" }, "sphinxcontrib-jsmath": { "hashes": [ @@ -1223,19 +1220,19 @@ }, "sphinxcontrib-qthelp": { "hashes": [ - "sha256:053dedc38823a80a7209a80860b16b722e9e0209e32fea98c90e4e6624588ed6", - "sha256:e2ae3b5c492d58fcbd73281fbd27e34b8393ec34a073c792642cd8e529288182" + "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", + "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" ], "markers": "python_version >= '3.9'", - "version": "==1.0.7" + "version": "==2.0.0" }, "sphinxcontrib-serializinghtml": { "hashes": [ - "sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7", - "sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f" + "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", + "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" ], "markers": "python_version >= '3.9'", - "version": "==1.1.10" + "version": "==2.0.0" }, "toml": { "hashes": [ @@ -1255,11 +1252,11 @@ }, "tomlkit": { "hashes": [ - "sha256:af914f5a9c59ed9d0762c7b64d3b5d5df007448eb9cd2edc8a46b1eafead172f", - "sha256:eef34fba39834d4d6b73c9ba7f3e4d1c417a4e56f89a7e96e090dd0d24b8fb3c" + "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72", + "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264" ], - "markers": "python_version >= '3.7'", - "version": "==0.12.5" + "markers": "python_version >= '3.8'", + "version": "==0.13.0" }, "typed-ast": { "hashes": [ diff --git a/setup.py b/setup.py index 25a54dad..3087c6bc 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ "python-dateutil>=2.8.2", "pytz", "pyyaml>=5.4.1", - "rapyuta-io>=1.16.0", + "rapyuta-io>=1.17.0", "requests>=2.20.0", "setuptools", "six>=1.13.0", From e14ae5cbe8da0b38eace4fbeee1bcd3f3c2805ea Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 2 Aug 2024 00:00:44 +0530 Subject: [PATCH 32/54] perf(delete): implements multi-threaded delete operation The delete operation has been sequential, till date. For large number of deployments, it takes quite a lot of time to perform a cleanup. The problem was performing the deletes in the right order and the TopologicalSorter does not provide a way to do that while maintaining concurrency. This commit implements the required logic to get the right delete order while processing them concurrently. Wrike Ticket: https://www.wrike.com/open.htm?id=1467788260 --- riocli/apply/__init__.py | 13 ++- riocli/apply/parse.py | 205 ++++++++++++++++++++++----------- riocli/deployment/model.py | 6 +- riocli/device/model.py | 13 +-- riocli/disk/model.py | 5 +- riocli/exceptions/__init__.py | 6 + riocli/managedservice/model.py | 3 +- riocli/network/model.py | 5 +- riocli/package/model.py | 3 +- riocli/project/model.py | 9 +- riocli/secret/model.py | 3 +- riocli/static_route/model.py | 5 +- riocli/usergroup/model.py | 7 +- 13 files changed, 188 insertions(+), 95 deletions(-) diff --git a/riocli/apply/__init__.py b/riocli/apply/__init__.py index 63956be2..86401a92 100644 --- a/riocli/apply/__init__.py +++ b/riocli/apply/__init__.py @@ -20,7 +20,6 @@ from riocli.apply.explain import explain, list_examples from riocli.apply.parse import Applier from riocli.apply.template import template -from riocli.apply.util import print_resolved_objects from riocli.apply.util import process_files_values_secrets from riocli.constants import Colors from riocli.utils import print_centered_text @@ -116,12 +115,22 @@ def apply( @click.option('-f', '--force', '--silent', 'silent', is_flag=True, type=click.BOOL, default=False, help="Skip confirmation") +@click.option('--workers', '-w', + help="number of parallel workers while running apply " + "command. defaults to 6.", type=int) +@click.option('--retry-count', '-rc', type=int, default=50, + help="Number of retries before a resource creation times out status, defaults to 50") +@click.option('--retry-interval', '-ri', type=int, default=6, + help="Interval between retries defaults to 6") @click.argument('files', nargs=-1) def delete( values: str, secrets: str, files: Iterable[str], + retry_count: int = 50, + retry_interval: int = 6, dryrun: bool = False, + workers: int = 6, silent: bool = False ) -> None: """ @@ -145,4 +154,4 @@ def delete( click.confirm("\nDo you want to proceed?", default=True, abort=True) print_centered_text('Deleting Resources') - applier.delete(dryrun=dryrun) + applier.delete(dryrun=dryrun, workers=workers, retry_count=retry_count, retry_interval=retry_interval) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index f861c59f..00a684f3 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -1,4 +1,4 @@ -# Copyright 2022 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,17 +22,17 @@ import yaml from munch import munchify -from riocli.apply.util import get_model, print_resolved_objects, message_with_prompt +from riocli.apply.util import get_model, message_with_prompt, print_resolved_objects from riocli.constants import Colors, Symbols -from riocli.utils import dump_all_yaml, run_bash, print_centered_text +from riocli.exceptions import ResourceNotFound +from riocli.utils import dump_all_yaml, print_centered_text, run_bash from riocli.utils.graph import Graphviz from riocli.utils.spinner import with_spinner -DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' - class Applier(object): DEFAULT_MAX_WORKERS = 6 + DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' def __init__(self, files: typing.List, values, secrets): self.environment = None @@ -57,8 +57,19 @@ def __init__(self, files: typing.List, values, secrets): self._process_file_list(files) + def print_resolved_manifests(self): + """Validates and prints the resolved manifests""" + manifests = [] + for _, o in self.objects.items(): + kls = get_model(o) + kls.validate(o) + manifests.append(o) + + dump_all_yaml(manifests) + @with_spinner(text='Applying...', timer=True) def apply(self, *args, **kwargs): + """Apply the resources defined in the manifest files""" spinner = kwargs.get('spinner') kwargs['workers'] = int(kwargs.get('workers') or self.DEFAULT_MAX_WORKERS) @@ -76,32 +87,25 @@ def apply(self, *args, **kwargs): spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e - def apply_async(self, *args, **kwargs): - workers = int(kwargs.get('workers') or self.DEFAULT_MAX_WORKERS) + def apply_sync(self, *args, **kwargs): + self.graph.prepare() + while self.graph.is_active(): + for obj in self.graph.get_ready(): + if (obj in self.resolved_objects and + 'manifest' in self.resolved_objects[obj]): + self._apply_manifest(obj, *args, **kwargs) + self.graph.done(obj) + def apply_async(self, *args, **kwargs): task_queue = queue.Queue() done_queue = queue.Queue() - def worker(): - while True: - o = task_queue.get() - if o in self.resolved_objects and 'manifest' in self.resolved_objects[o]: - try: - self._apply_manifest(o, *args, **kwargs) - except Exception as ex: - done_queue.put(ex) - continue - - task_queue.task_done() - done_queue.put(o) - - # Start the workers that will accept tasks from the task_queue - # and process them. Upon completion, they will put the object - # in the done_queue. - worker_list = [] - for worker_id in range(workers): - worker_list.append(threading.Thread(target=worker, daemon=True)) - worker_list[worker_id].start() + self._start_apply_workers( + self._apply_manifest, + task_queue, + done_queue, + *args, **kwargs + ) self.graph.prepare() while self.graph.is_active(): @@ -109,48 +113,120 @@ def worker(): task_queue.put(obj) done_obj = done_queue.get() - if not isinstance(done_obj, Exception): - self.graph.done(done_obj) - else: + if isinstance(done_obj, Exception): raise Exception(done_obj) + self.graph.done(done_obj) + # Block until the task_queue is empty. task_queue.join() - def apply_sync(self, *args, **kwargs): - self.graph.prepare() - while self.graph.is_active(): - for obj in self.graph.get_ready(): - if (obj in self.resolved_objects and - 'manifest' in self.resolved_objects[obj]): - self._apply_manifest(obj, *args, **kwargs) - self.graph.done(obj) - @with_spinner(text='Deleting...', timer=True) def delete(self, *args, **kwargs): + """Delete resources defined in manifests.""" spinner = kwargs.get('spinner') - delete_order = list(self.graph.static_order()) - delete_order.reverse() + kwargs['workers'] = int(kwargs.get('workers') + or self.DEFAULT_MAX_WORKERS) + + delete_func = self.delete_async + if kwargs['workers'] == 1: + delete_func = self.delete_sync + try: - for obj in delete_order: - if (obj in self.resolved_objects and - 'manifest' in self.resolved_objects[obj]): - self._delete_manifest(obj, *args, **kwargs) + delete_func(*args, **kwargs) spinner.text = 'Delete successful.' spinner.green.ok(Symbols.SUCCESS) except Exception as e: spinner.text = 'Delete failed. Error: {}'.format(e) spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) from e - def print_resolved_manifests(self): - """Validates and prints the resolved manifests""" - manifests = [] - for _, o in self.objects.items(): - kls = get_model(o) - kls.validate(o) - manifests.append(o) + def delete_sync(self, *args, **kwargs) -> None: + delete_order = list(self.graph.static_order()) + delete_order.reverse() + for o in delete_order: + if o in self.resolved_objects and 'manifest' in self.resolved_objects[o]: + self._delete_manifest(o, *args, **kwargs) - dump_all_yaml(manifests) + def delete_async(self, *args, **kwargs) -> None: + task_queue = queue.Queue() + done_queue = queue.Queue() + + self._start_apply_workers( + self._delete_manifest, + task_queue, + done_queue, + *args, **kwargs + ) + + for nodes in self._get_async_delete_order(): + for node in nodes: + task_queue.put(node) + + done_obj = done_queue.get() + if isinstance(done_obj, Exception): + raise Exception(done_obj) + + task_queue.join() + + def _start_apply_workers( + self, + func: typing.Callable, + tasks: queue.Queue, + done: queue.Queue, + *args, **kwargs, + ) -> None: + """A helper method to start workers for apply/delete operations + + The `func` should have the following signature: + func(obj_key: str, *args, **kwargs) -> None + + The `tasks` queue is used to pass objects to the workers for processing. + The `done` queue is used to pass the processed objects back to the main. + """ + def _worker(): + while True: + o = tasks.get() + if o in self.resolved_objects and 'manifest' in self.resolved_objects[o]: + try: + func(o, *args, **kwargs) + except Exception as ex: + tasks.task_done() + done.put(ex) + continue + + tasks.task_done() + done.put(o) + + # Start the workers that will accept tasks from the task_queue + # and process them. Upon completion, they will put the object + # in the done_queue. The main thread will wait for the task_queue + # to be empty before exiting. The daemon threads will die with the + # main process. + n = int(kwargs.get('workers') or self.DEFAULT_MAX_WORKERS) + for i in range(n): + threading.Thread( + target=_worker, + daemon=True, + name=f'worker-{i}' + ).start() + + def _get_async_delete_order(self): + """Returns the delete order for async delete operation + + This method returns the delete order in a way that the + resources that are dependent on other resources are deleted + first while also ensuring that they can be processed concurrently. + """ + stack = [] + self.graph.prepare() + while self.graph.is_active(): + nodes = self.graph.get_ready() + stack.append(nodes) + self.graph.done(*nodes) + + while stack: + yield stack.pop() def parse_dependencies(self): for _, data in self.files.items(): @@ -182,14 +258,14 @@ def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None: try: if not dryrun: ist.apply(*args, **kwargs) + + message_with_prompt("{} Applied {}".format( + Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) except Exception as ex: message_with_prompt("{} Failed to apply {}. Error: {}".format( Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) raise ex - message_with_prompt("{} Applied {}".format( - Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) - def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: """Instantiate and delete the object manifest""" spinner = kwargs.get('spinner') @@ -206,21 +282,23 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: # If a resource has a label with DELETE_POLICY_LABEL set # to 'retain', it should not be deleted. labels = obj.get('metadata', {}).get('labels', {}) - can_delete = labels.get(DELETE_POLICY_LABEL) != 'retain' + can_delete = labels.get(self.DELETE_POLICY_LABEL) != 'retain' try: if not dryrun and can_delete: ist.delete(*args, **kwargs) + + message_with_prompt("{} Deleted {}.".format( + Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) + except ResourceNotFound: + message_with_prompt("{} {} not found".format( + Symbols.WARNING, obj_key), fg=Colors.YELLOW, spinner=spinner) + return except Exception as ex: message_with_prompt("{} Failed to delete {}. Error: {}".format( Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) raise ex - message_with_prompt("{} Deleted {}.".format( - Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) - - # File Loading Operations - def _process_file_list(self, files): for f in files: data = self._load_file_content(f) @@ -294,8 +372,6 @@ def _load_file_content(self, file_name, is_value=False, is_secret=False): return loaded_data - # Graph Operations - def _add_graph_node(self, key): self.graph.add(key) self.diagram.node(key) @@ -304,7 +380,6 @@ def _add_graph_edge(self, dependent_key, key): self.graph.add(dependent_key, key) self.diagram.edge(key, dependent_key) - # Dependency Resolution def _parse_dependency(self, dependent_key, model): # TODO(pallab): let resources determine their own dependencies and return them # kls = get_model(model) diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index 34fe9eed..b4af5254 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -16,9 +16,9 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model -from riocli.v2client.error import HttpAlreadyExistsError -from riocli.v2client.error import HttpNotFoundError +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class Deployment(Model): @@ -43,4 +43,4 @@ def delete(self, *args, **kwargs): try: client.delete_deployment(self.metadata.name) except HttpNotFoundError: - pass + raise ResourceNotFound diff --git a/riocli/device/model.py b/riocli/device/model.py index 1d3a53ba..5b950fad 100644 --- a/riocli/device/model.py +++ b/riocli/device/model.py @@ -15,14 +15,9 @@ from rapyuta_io.clients.device import Device as v1Device, DevicePythonVersion from riocli.config import new_client -from riocli.device.util import ( - create_hwil_device, - delete_hwil_device, - execute_onboard_command, - find_device_by_name, - make_device_labels_from_hwil_device, - DeviceNotFound, -) +from riocli.device.util import (DeviceNotFound, create_hwil_device, delete_hwil_device, execute_onboard_command, + find_device_by_name, make_device_labels_from_hwil_device) +from riocli.exceptions import ResourceNotFound from riocli.model import Model @@ -85,7 +80,7 @@ def delete(self, *args, **kwargs) -> None: try: device = find_device_by_name(client, self.metadata.name) except DeviceNotFound: - return + raise ResourceNotFound if self.spec.get('virtual', {}).get('enabled', False): delete_hwil_device(device) diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 8bbe15fe..197e03c2 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -15,8 +15,9 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model -from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class Disk(Model): @@ -51,4 +52,4 @@ def delete(self, *args, **kwargs) -> None: try: client.delete_disk(self.metadata.name) except HttpNotFoundError: - pass + raise ResourceNotFound diff --git a/riocli/exceptions/__init__.py b/riocli/exceptions/__init__.py index 677eb6b7..fb53854d 100644 --- a/riocli/exceptions/__init__.py +++ b/riocli/exceptions/__init__.py @@ -48,3 +48,9 @@ class DeviceNotFound(Exception): def __init__(self, message='device not found'): self.message = message super().__init__(self.message) + + +class ResourceNotFound(Exception): + def __init__(self, message='resource not found'): + self.message = message + super().__init__(self.message) diff --git a/riocli/managedservice/model.py b/riocli/managedservice/model.py index 72863589..b8b49164 100644 --- a/riocli/managedservice/model.py +++ b/riocli/managedservice/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -38,4 +39,4 @@ def delete(self, *args, **kwargs) -> None: try: client.delete_instance(self.metadata.name) except HttpNotFoundError: - pass + raise ResourceNotFound diff --git a/riocli/network/model.py b/riocli/network/model.py index 877b3ce2..4255d63f 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -15,8 +15,9 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model -from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError class Network(Model): @@ -47,4 +48,4 @@ def delete(self, *args, **kwargs) -> None: try: client.delete_network(self.metadata.name) except HttpNotFoundError: - pass \ No newline at end of file + raise ResourceNotFound diff --git a/riocli/package/model.py b/riocli/package/model.py index a0cbe52f..35bb7384 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -16,6 +16,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.package.enum import RestartPolicy from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -51,7 +52,7 @@ def delete(self, *args, **kwargs) -> None: query={"version": self.metadata.version} ) except HttpNotFoundError: - pass + raise ResourceNotFound def _sanitize_package(self) -> typing.Dict: # Unset createdAt and updatedAt to avoid timestamp parsing issue. diff --git a/riocli/project/model.py b/riocli/project/model.py index c1883bc1..62830cf0 100644 --- a/riocli/project/model.py +++ b/riocli/project/model.py @@ -15,10 +15,11 @@ from munch import unmunchify from waiting import wait -from riocli.config import new_v2_client, Configuration +from riocli.config import Configuration, new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model -from riocli.project.util import find_project_guid, ProjectNotFound -from riocli.v2client.error import HttpNotFoundError, HttpAlreadyExistsError +from riocli.project.util import ProjectNotFound, find_project_guid +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError PROJECT_READY_TIMEOUT = 150 @@ -55,7 +56,7 @@ def delete(self, *args, **kwargs) -> None: guid = find_project_guid(client, self.metadata.name, Configuration().data['organization_id']) client.delete_project(guid) except (HttpNotFoundError, ProjectNotFound): - pass + raise ResourceNotFound def is_ready(self) -> bool: client = new_v2_client() diff --git a/riocli/secret/model.py b/riocli/secret/model.py index 30baf2ba..46dfe7e3 100644 --- a/riocli/secret/model.py +++ b/riocli/secret/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -40,4 +41,4 @@ def delete(self, *args, **kwargs) -> None: try: client.delete_secret(self.metadata.name) except HttpNotFoundError: - pass + raise ResourceNotFound diff --git a/riocli/static_route/model.py b/riocli/static_route/model.py index 79c23112..13f3a1ad 100644 --- a/riocli/static_route/model.py +++ b/riocli/static_route/model.py @@ -13,7 +13,8 @@ # limitations under the License. from munch import unmunchify -from riocli.config import new_v2_client, Configuration +from riocli.config import Configuration, new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -41,4 +42,4 @@ def delete(self, *args, **kwargs) -> None: try: client.delete_static_route(f'{self.metadata.name}-{short_id}') except HttpNotFoundError: - pass + raise ResourceNotFound diff --git a/riocli/usergroup/model.py b/riocli/usergroup/model.py index 53a4fb81..ae1035dc 100644 --- a/riocli/usergroup/model.py +++ b/riocli/usergroup/model.py @@ -15,10 +15,11 @@ from munch import unmunchify -from riocli.config import new_client, new_v2_client, Configuration +from riocli.config import Configuration, new_client, new_v2_client +from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.organization.utils import get_organization_details -from riocli.usergroup.util import find_usergroup_guid, UserGroupNotFound +from riocli.usergroup.util import UserGroupNotFound, find_usergroup_guid USER_GUID = 'guid' USER_EMAIL = 'emailID' @@ -76,7 +77,7 @@ def delete(self, *args, **kwargs) -> None: guid = find_usergroup_guid(v1client, organization_id, self.metadata.name) v1client.delete_usergroup(organization_id, guid) except UserGroupNotFound: - pass + raise ResourceNotFound except Exception as e: raise e From 3c739774cddf1d8706d23c194528d5627a3de5ea Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 2 Aug 2024 19:55:17 +0530 Subject: [PATCH 33/54] fix(deployment): replace aggregateStatus with status The aggregateStatus field is no longer sent in the API response. This commit removes all usages of aggregateStatus and replaces it with status. --- riocli/deployment/status.py | 2 +- riocli/deployment/util.py | 2 +- riocli/deployment/wait.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/riocli/deployment/status.py b/riocli/deployment/status.py index e41bacaa..6c248521 100644 --- a/riocli/deployment/status.py +++ b/riocli/deployment/status.py @@ -32,7 +32,7 @@ def status(deployment_name: str) -> None: try: client = new_v2_client() deployment = client.get_deployment(deployment_name) - click.secho(deployment.status.aggregateStatus) + click.secho(deployment.status.status) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index c00fa676..60be54a2 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -141,6 +141,6 @@ def print_deployments_for_confirmation(deployments: typing.List[Deployment]): for deployment in deployments: data.append( [deployment.metadata.name, deployment.metadata.guid, deployment.status.phase, - deployment.status.aggregateStatus]) + deployment.status.status]) tabulate_data(data, headers) diff --git a/riocli/deployment/wait.py b/riocli/deployment/wait.py index 5f9af5d6..92b010f4 100644 --- a/riocli/deployment/wait.py +++ b/riocli/deployment/wait.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import click - from click_help_colors import HelpColorsCommand + from riocli.config import new_v2_client from riocli.constants import Colors, Symbols from riocli.utils.spinner import with_spinner -from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError +from riocli.v2client.error import DeploymentNotRunning, ImagePullError, RetriesExhausted @click.command( @@ -38,7 +38,7 @@ def wait_for_deployment( try: client = new_v2_client() deployment = client.poll_deployment(deployment_name) - spinner.text = click.style('Phase: Succeeded Status: {}'.format(deployment.status.aggregateStatus), fg=Colors.GREEN) + spinner.text = click.style('Phase: Succeeded Status: {}'.format(deployment.status.status), fg=Colors.GREEN) spinner.green.ok(Symbols.SUCCESS) except RetriesExhausted as e: spinner.write(click.style(str(e), fg=Colors.RED)) From 018e118f0fcbcb0dae694ee408224afc007791ec Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 2 Aug 2024 20:26:29 +0530 Subject: [PATCH 34/54] feat(apply): init Jinja environment with filters and rio namespace This commit introduces applying custom filters in the Jinja2 environment used for templated manifests. It also injects the rio namespace from the current context that includes project, organization and user email info. Wrike Ticket: https://www.wrike.com/open.htm?id=1446734697 --- riocli/apply/filters.py | 41 +++++++++++++++++++++++++++++++++++++++++ riocli/apply/parse.py | 37 ++++++++++++++++++++++++++++--------- riocli/apply/util.py | 13 ++++++++++++- 3 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 riocli/apply/filters.py diff --git a/riocli/apply/filters.py b/riocli/apply/filters.py new file mode 100644 index 00000000..b5c472c3 --- /dev/null +++ b/riocli/apply/filters.py @@ -0,0 +1,41 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Filters to use in the manifests. +""" + +import os + + +def getenv(default: str, env_var: str) -> str: + """ + Get the value of an environment variable. + + Usage: + "foo" : {{ "bar" | getenv('FOO') }} + + Args: + env_var: The environment variable to get the value of. + default: The default value to return if the environment variable is not set. + + Returns: + The value of the environment variable. + """ + return os.getenv(env_var, default) + + +FILTERS = { + 'getenv': getenv, +} diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 00a684f3..8b614395 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -18,11 +18,11 @@ from graphlib import TopologicalSorter import click -import jinja2 import yaml from munch import munchify -from riocli.apply.util import get_model, message_with_prompt, print_resolved_objects +from riocli.apply.util import (get_model, init_jinja_environment, message_with_prompt, print_resolved_objects) +from riocli.config import Configuration from riocli.constants import Colors, Symbols from riocli.exceptions import ResourceNotFound from riocli.utils import dump_all_yaml, print_centered_text, run_bash @@ -35,22 +35,23 @@ class Applier(object): DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' def __init__(self, files: typing.List, values, secrets): - self.environment = None - self.input_file_paths = files + self.files = {} + self.values = {} + self.secrets = {} self.objects = {} self.resolved_objects = {} - self.files = {} + self.input_file_paths = files + self.config = Configuration() self.graph = TopologicalSorter() - self.secrets = {} - self.values = {} + self.environment = init_jinja_environment() self.diagram = Graphviz(direction='LR', format='svg') - if values or secrets: - self.environment = jinja2.Environment() if values: self.values = self._load_file_content( values, is_value=True, is_secret=False)[0] + self.values = self._inject_rio_namespace(self.values) + if secrets: self.secrets = self._load_file_content( secrets, is_value=True, is_secret=True)[0] @@ -422,3 +423,21 @@ def _get_object_key(obj: dict) -> str: raise ValueError('[kind:{}] name is required.'.format(kind)) return '{}:{}'.format(kind, name_or_guid) + + def _inject_rio_namespace(self, values: typing.Optional[dict] = None) -> dict: + values = values or {} + + values['rio'] = { + 'project': { + 'name': self.config.data.get('project_name'), + 'guid': self.config.project_guid, + }, + 'organization': { + 'name': self.config.data.get('organization_name'), + 'guid': self.config.organization_guid, + 'short_id': self.config.organization_short_id, + }, + 'email_id': self.config.data.get('email_id'), + } + + return values diff --git a/riocli/apply/util.py b/riocli/apply/util.py index 654139f4..cf0fc9ec 100644 --- a/riocli/apply/util.py +++ b/riocli/apply/util.py @@ -19,13 +19,16 @@ from shutil import get_terminal_size import click +import jinja2 from yaspin.api import Yaspin +from riocli.apply.filters import FILTERS from riocli.constants import Colors from riocli.deployment.model import Deployment from riocli.device.model import Device from riocli.disk.model import Disk from riocli.managedservice.model import ManagedService +from riocli.model import Model from riocli.network.model import Network from riocli.package.model import Package from riocli.project.model import Project @@ -33,7 +36,6 @@ from riocli.static_route.model import StaticRoute from riocli.usergroup.model import UserGroup from riocli.utils import tabulate_data -from riocli.model import Model KIND_TO_CLASS = { 'project': Project, @@ -132,3 +134,12 @@ def print_resolved_objects(objects: typing.Dict) -> None: data.append([kind.title(), name]) tabulate_data(data, headers=['Kind', 'Name']) + + +def init_jinja_environment(): + """Initialize Jinja2 environment with custom filters""" + environment = jinja2.Environment() + for name, func in FILTERS.items(): + environment.filters[name] = func + + return environment From e092f2fa07a78e95e612b907e050b1f2de4f8c5a Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 2 Aug 2024 22:01:47 +0530 Subject: [PATCH 35/54] refactor(deployment): updates deployment list output This commit updates the printed output of the deployment list command. Some columns are re-arranged and it also implemented thw -w (wide) option to show extra columns. --- riocli/deployment/list.py | 32 ++++++++++++++++++++++++-------- riocli/v2client/util.py | 7 +++++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index 7221aa8d..0f040dee 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -20,6 +20,7 @@ from riocli.constants import Colors from riocli.deployment.model import Deployment from riocli.utils import tabulate_data +from riocli.v2client.util import process_errors ALL_PHASES = [ 'InProgress', @@ -63,7 +64,7 @@ def list_deployments( client = new_v2_client(with_project=True) deployments = client.list_deployments(query={'phases': phase}) deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) - display_deployment_list(deployments, show_header=True) + display_deployment_list(deployments, show_header=True, wide=wide) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) @@ -76,14 +77,29 @@ def display_deployment_list( ): headers = [] if show_header: - headers = ('Deployment ID', 'Name', 'Phase', 'Package', 'Creation Time (UTC)', 'Stopped Time (UTC)') + headers = ['Name', 'Package', 'Creation Time (UTC)', 'Phase', 'Status'] + + if show_header and wide: + headers.extend(['Deployment ID', 'Stopped Time (UTC)']) data = [] - for deployment in deployments: - package_name_version = "{} ({})".format(deployment.metadata.depends.nameOrGUID, - deployment.metadata.depends.version) - phase = deployment.status.phase if deployment.status else "" - data.append([deployment.metadata.guid, deployment.metadata.name, - phase, package_name_version, deployment.metadata.createdAt, deployment.metadata.get('deletedAt')]) + for d in deployments: + package_name_version = f'{d.metadata.depends.nameOrGUID} ({d.metadata.depends.version})' + phase = d.get('status', {}).get('phase', '') + + status = '' + + if d.status: + if d.status.get('error_codes'): + status = click.style(process_errors(d.status.error_codes, no_action=True), fg=Colors.RED) + else: + status = d.status.status + + row = [d.metadata.name, package_name_version, d.metadata.createdAt, phase, status] + + if wide: + row.extend([d.metadata.guid, d.metadata.get('deletedAt')]) + + data.append(row) tabulate_data(data, headers=headers) diff --git a/riocli/v2client/util.py b/riocli/v2client/util.py index dca89237..cf30c881 100644 --- a/riocli/v2client/util.py +++ b/riocli/v2client/util.py @@ -24,7 +24,7 @@ from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError -def process_errors(errors: typing.List[str]) -> str: +def process_errors(errors: typing.List[str], no_action: bool = False) -> str: """Process the deployment errors and return the formatted error message""" err_fmt = '[{}] {}\nAction: {}' support_action = ('Report the issue together with the relevant' @@ -50,7 +50,10 @@ def process_errors(errors: typing.List[str]) -> str: description = click.style(description, fg=Colors.RED) action = click.style(action, fg=Colors.GREEN) - msgs.append(err_fmt.format(code, description, action)) + if no_action: + msgs.append(f'[{code}]: {description}') + else: + msgs.append(err_fmt.format(code, description, action)) return '\n'.join(msgs) From cb9e2feb643c2f6f7782f7d02534f0e4a9bbe56e Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Sun, 4 Aug 2024 18:51:57 +0530 Subject: [PATCH 36/54] feat: adds filter by label option in list commands This commit adds the support for filtering supported resources by labels. Wrike Ticket: https://www.wrike.com/open.htm?id=1279835486 --- riocli/deployment/list.py | 10 +++++++++- riocli/disk/list.py | 20 +++++++++++++++----- riocli/network/list.py | 13 ++++++++----- riocli/project/list.py | 7 ++++++- riocli/secret/list.py | 11 +++++++---- riocli/static_route/list.py | 9 +++++++-- riocli/v2client/client.py | 9 ++++----- 7 files changed, 56 insertions(+), 23 deletions(-) diff --git a/riocli/deployment/list.py b/riocli/deployment/list.py index 0f040dee..e9b087dc 100644 --- a/riocli/deployment/list.py +++ b/riocli/deployment/list.py @@ -50,19 +50,27 @@ type=click.Choice(ALL_PHASES), default=DEFAULT_PHASES, help='Filter the Deployment list by Phases') +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') @click.option('--wide', '-w', is_flag=True, default=False, help='Print more details', type=bool) def list_deployments( device: str, phase: typing.List[str], + labels: typing.List[str], wide: bool = False, ) -> None: """ List the deployments in the selected project """ + query = { + 'phases': phase, + 'labelSelector': labels, + } + try: client = new_v2_client(with_project=True) - deployments = client.list_deployments(query={'phases': phase}) + deployments = client.list_deployments(query=query) deployments = sorted(deployments, key=lambda d: d.metadata.name.lower()) display_deployment_list(deployments, show_header=True, wide=wide) except Exception as e: diff --git a/riocli/disk/list.py b/riocli/disk/list.py index ad74ad31..20a8892b 100644 --- a/riocli/disk/list.py +++ b/riocli/disk/list.py @@ -11,22 +11,32 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import typing + import click +from click_help_colors import HelpColorsCommand -from riocli.constants import Colors from riocli.config import new_v2_client +from riocli.constants import Colors from riocli.disk.util import display_disk_list -@click.command('list') -def list_disks() -> None: +@click.command( + 'list', + cls=HelpColorsCommand, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') +def list_disks(labels: typing.List[str]) -> None: """ List the disks in the selected project """ try: client = new_v2_client(with_project=True) - disks = client.list_disks() + disks = client.list_disks(query={'labelSelector': labels}) display_disk_list(disks, show_header=True) except Exception as e: click.secho(str(e), fg=Colors.RED) - raise SystemExit(1) + raise SystemExit(1) from e diff --git a/riocli/network/list.py b/riocli/network/list.py index 735b25e0..4a312e8f 100644 --- a/riocli/network/list.py +++ b/riocli/network/list.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import typing + import click from click_help_colors import HelpColorsCommand from munch import Munch @@ -29,18 +30,20 @@ ) @click.option('--network', help='Type of Network', type=click.Choice(['routed', 'native', 'both']), default='both') -def list_networks(network: str) -> None: +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') +def list_networks(network: str, labels: typing.List[str]) -> None: """ List the networks in the selected project """ try: client = new_v2_client(with_project=True) - if network in ("both", ""): - networks = client.list_networks() - else: - networks = client.list_networks(query={"networkType": network}) + query = {'labelSelector': labels} + if network not in ("both", ""): + query.update({"networkType": network}) + networks = client.list_networks(query=query) networks = sorted(networks, key=lambda n: n.metadata.name.lower()) _display_network_list(networks, show_header=True) except Exception as e: diff --git a/riocli/project/list.py b/riocli/project/list.py index 2f49cc86..cd0b16cb 100644 --- a/riocli/project/list.py +++ b/riocli/project/list.py @@ -32,6 +32,8 @@ ) @click.option('--organization', 'organization_name', help='List projects for an organization') +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') @click.option('--wide', '-w', is_flag=True, default=False, help='Print more details', type=bool) @click.pass_context @@ -41,6 +43,7 @@ def list_projects( organization_guid: str = None, organization_name: str = None, organization_short_id: str = None, + labels: typing.List[str] = (), wide: bool = False, ) -> None: """ @@ -54,9 +57,11 @@ def list_projects( click.secho('{} {}'.format(Symbols.ERROR, err_msg), fg=Colors.RED) raise SystemExit(1) + query = {'labelSelector': labels} + try: client = new_v2_client(with_project=False) - projects = client.list_projects(organization_guid=organization_guid) + projects = client.list_projects(organization_guid=organization_guid, query=query) projects = sorted(projects, key=lambda p: p.metadata.name.lower()) current = ctx.obj.data.get('project_id', None) _display_project_list(projects, current, show_header=True, wide=wide) diff --git a/riocli/secret/list.py b/riocli/secret/list.py index 4e312a68..b08e177a 100644 --- a/riocli/secret/list.py +++ b/riocli/secret/list.py @@ -14,12 +14,13 @@ import typing import click +from click_help_colors import HelpColorsCommand from rapyuta_io import Secret from riocli.config import new_v2_client -from riocli.utils import tabulate_data from riocli.constants import Colors -from click_help_colors import HelpColorsCommand +from riocli.utils import tabulate_data + @click.command( 'list', @@ -27,13 +28,15 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -def list_secrets() -> None: +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') +def list_secrets(labels: typing.List[str]) -> None: """ List the secrets in the selected project """ try: client = new_v2_client(with_project=True) - secrets = client.list_secrets() + secrets = client.list_secrets(query={'labelSelector': labels}) secrets = sorted(secrets, key=lambda s: s.metadata.name.lower()) _display_secret_list(secrets, show_header=True) except Exception as e: diff --git a/riocli/static_route/list.py b/riocli/static_route/list.py index 35f886fd..b4097177 100644 --- a/riocli/static_route/list.py +++ b/riocli/static_route/list.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import typing from typing import List import click @@ -21,24 +22,28 @@ from riocli.constants import Colors from riocli.utils import tabulate_data + @click.command( 'list', cls=HelpColorsCommand, help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -def list_static_routes() -> None: +@click.option('--label', '-l', 'labels', multiple=True, type=click.STRING, + default=(), help='Filter the deployment list by labels') +def list_static_routes(labels: typing.List[str]) -> None: """ List the static routes in the selected project """ try: client = new_v2_client(with_project=True) - routes = client.list_static_routes() + routes = client.list_static_routes(query={'labelSelector': labels}) _display_routes_list(routes) except Exception as e: click.secho(str(e), fg=Colors.RED) raise SystemExit(1) from e + def _display_routes_list(routes: List[StaticRoute]) -> None: headers = ['Route ID', 'Name', 'URL', 'Creator', 'CreatedAt'] diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 07527b99..27d640b1 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -17,17 +17,17 @@ import os import time from hashlib import md5 -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List, Optional import click import magic -from munch import munchify, Munch +from munch import Munch, munchify from rapyuta_io.utils.rest_client import HttpMethod, RestClient from riocli.constants import Colors from riocli.v2client.enums import DeploymentPhaseConstants, DiskStatusConstants -from riocli.v2client.error import RetriesExhausted, DeploymentNotRunning, ImagePullError -from riocli.v2client.util import process_errors, handle_server_errors +from riocli.v2client.error import DeploymentNotRunning, ImagePullError, RetriesExhausted +from riocli.v2client.util import handle_server_errors, process_errors class Client(object): @@ -919,7 +919,6 @@ def list_deployments( """ url = "{}/v2/deployments/".format(self._host) headers = self._get_auth_header() - client = RestClient(url).method(HttpMethod.GET).headers(headers) return self._walk_pages(client, params=query) From c00b6f4f04d07994bfc337e045d3677ec9cdb600 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 5 Aug 2024 17:43:04 +0530 Subject: [PATCH 37/54] fix(jsonschema): corrects resource GUID regex --- riocli/jsonschema/schemas/deployment-schema.yaml | 10 +++++----- riocli/jsonschema/schemas/disk-schema.yaml | 4 ++-- riocli/jsonschema/schemas/network-schema.yaml | 2 +- riocli/jsonschema/schemas/package-schema.yaml | 14 +------------- riocli/jsonschema/schemas/secret-schema.yaml | 2 +- riocli/jsonschema/schemas/static_route-schema.yaml | 2 +- 6 files changed, 11 insertions(+), 23 deletions(-) diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index 9a161a8f..f1549889 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -389,19 +389,19 @@ definitions: pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" secretGUID: type: string - pattern: "^secret-[a-z]{24}$" + pattern: "^secret-([a-z0-9]{20}|[a-z]{24})$" diskGUID: type: string - pattern: "^disk-[a-z]{24}$" + pattern: "^disk-([a-z0-9]{20}|[a-z]{24})$" packageGUID: type: string - pattern: "^pkg-[a-z0-9]{20}$" + pattern: "^pkg-([a-z0-9]{20}|[a-z]{24})$" deploymentGUID: type: string - pattern: "^dep-[a-z]{24}$" + pattern: "^dep-([a-z0-9]{20}|[a-z]{24})$" networkGUID: type: string - pattern: "^network-[a-z]{24}$" + pattern: "^network-([a-z0-9]{20}|[a-z]{24})$" uuid: type: string pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" diff --git a/riocli/jsonschema/schemas/disk-schema.yaml b/riocli/jsonschema/schemas/disk-schema.yaml index 8b8fec2a..e0e29a75 100644 --- a/riocli/jsonschema/schemas/disk-schema.yaml +++ b/riocli/jsonschema/schemas/disk-schema.yaml @@ -41,10 +41,10 @@ definitions: pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" diskGUID: type: string - pattern: "^disk-[a-z]{24}$" + pattern: "^disk-([a-z0-9]{20}|[a-z]{24})$" packageGUID: type: string - pattern: "^pkg-[a-z0-9]{20}$" + pattern: "^pkg-([a-z0-9]{20}|[a-z]{24})$" stringMap: type: object additionalProperties: diff --git a/riocli/jsonschema/schemas/network-schema.yaml b/riocli/jsonschema/schemas/network-schema.yaml index 366b94d9..d2175ff3 100644 --- a/riocli/jsonschema/schemas/network-schema.yaml +++ b/riocli/jsonschema/schemas/network-schema.yaml @@ -114,7 +114,7 @@ definitions: - name networkGUID: type: string - pattern: "^network-[a-z]{24}$" + pattern: "^network-([a-z0-9]{20}|[a-z]{24})$" projectGUID: type: string pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" diff --git a/riocli/jsonschema/schemas/package-schema.yaml b/riocli/jsonschema/schemas/package-schema.yaml index 1535e03f..8581c282 100644 --- a/riocli/jsonschema/schemas/package-schema.yaml +++ b/riocli/jsonschema/schemas/package-schema.yaml @@ -628,21 +628,9 @@ definitions: projectGUID: type: string pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" - secretGUID: - type: string - pattern: "^secret-[a-z]{24}$" - diskGUID: - type: string - pattern: "^disk-[a-z]{24}$" packageGUID: type: string - pattern: "^pkg-[a-z0-9]{20}$" - deploymentGUID: - type: string - pattern: "^dep-[a-z]{24}$" - networkGUID: - type: string - pattern: "^network-[a-z]{24}$" + pattern: "^pkg-([a-z0-9]{20}|[a-z]{24})$" uuid: type: string pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" diff --git a/riocli/jsonschema/schemas/secret-schema.yaml b/riocli/jsonschema/schemas/secret-schema.yaml index 7570db36..f9015ebb 100644 --- a/riocli/jsonschema/schemas/secret-schema.yaml +++ b/riocli/jsonschema/schemas/secret-schema.yaml @@ -43,7 +43,7 @@ definitions: pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" secretGUID: type: string - pattern: "^secret-[a-z]{24}$" + pattern: "^secret-([a-z0-9]{20}|[a-z]{24})$" stringMap: type: object additionalProperties: diff --git a/riocli/jsonschema/schemas/static_route-schema.yaml b/riocli/jsonschema/schemas/static_route-schema.yaml index 9c22d467..76eb78de 100644 --- a/riocli/jsonschema/schemas/static_route-schema.yaml +++ b/riocli/jsonschema/schemas/static_route-schema.yaml @@ -43,7 +43,7 @@ definitions: pattern: "^project-([a-z0-9]{20}|[a-z]{24})$" staticRouteGUID: type: string - pattern: "^staticroute-[a-z]{24}$" + pattern: "^staticroute-([a-z0-9]{20}|[a-z]{24})$" stringMap: type: object additionalProperties: From 286db3dc06c83fe6d208102bbd4f7f1e481d07cb Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Tue, 6 Aug 2024 18:15:53 +0530 Subject: [PATCH 38/54] fix(project): fixes raised exception in find_organization_guid helper The find_organzation_guid helper raises OrganizationNotFound exception when the provided organization is not found. However, the exception message had an invalid variable in the string format function. This commit fixes that. Wrike Ticket: https://www.wrike.com/open.htm?id=1469984282 --- riocli/project/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riocli/project/util.py b/riocli/project/util.py index c5aee5dc..469a1594 100644 --- a/riocli/project/util.py +++ b/riocli/project/util.py @@ -84,7 +84,7 @@ def find_organization_guid(client: Client, name: str) -> typing.Tuple[str, str]: return organization.guid, organization.short_guid raise OrganizationNotFound( - "User is not part of organization with guid: {}".format(guid)) + "User is not part of organization: {}".format(name)) def get_organization_name(client: Client, guid: str) -> typing.Tuple[str, str]: @@ -94,7 +94,7 @@ def get_organization_name(client: Client, guid: str) -> typing.Tuple[str, str]: return organization.name, organization.short_guid raise OrganizationNotFound( - "User is not part of organization with guid: {}".format(guid)) + "User is not part of organization: {}".format(guid)) def name_to_organization_guid(f: typing.Callable) -> typing.Callable: From 96fbb71ed11256d1a0d39848f93fc37122fd5d44 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 8 Aug 2024 00:16:07 +0530 Subject: [PATCH 39/54] fix(jsonschema): accepts null value in deployment features When features is null, the jsonschema validation fails for deployment manifests. This commit fixes the issue by extending the valid types for features. Wrike Ticket: https://www.wrike.com/open.htm?id=1466707323 --- riocli/jsonschema/schemas/deployment-schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index f1549889..ad7bddd2 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -161,7 +161,7 @@ definitions: items: "$ref": "#/definitions/deviceROSBagJobSpec" features: - type: object + type: ["object", "null"] properties: params: type: object @@ -189,7 +189,7 @@ definitions: items: "$ref": "#/definitions/deploymentDepends" features: - type: object + type: ["object", "null"] properties: vpn: type: object From ca4b68800954d09a8f78d18f2e6cdbb8d4cf2613 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 8 Aug 2024 23:59:32 +0530 Subject: [PATCH 40/54] feat(deployment): implement restart alias for rio deployment update The rio deployment update command essentially restarts the deployment and serves no other purpose. Hence, it makes sense to rename the command to restart. However, there may be scripts that are currently using the update command and hence, in order to maintain compatibility, we implement an alias in this commit and mark the update command as deprecated. Wrike Ticket: https://www.wrike.com/open.htm?id=1470267514 --- riocli/deployment/__init__.py | 6 ++-- riocli/deployment/update.py | 62 ++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/riocli/deployment/__init__.py b/riocli/deployment/__init__.py index c53f4964..246fbe06 100644 --- a/riocli/deployment/__init__.py +++ b/riocli/deployment/__init__.py @@ -21,9 +21,10 @@ from riocli.deployment.list import list_deployments from riocli.deployment.logs import deployment_logs from riocli.deployment.status import status -from riocli.deployment.update import update_deployment +from riocli.deployment.update import update_deployment, restart_deployment from riocli.deployment.wait import wait_for_deployment + @click.group( invoke_without_command=False, cls=HelpColorsGroup, @@ -32,7 +33,7 @@ ) def deployment(): """ - Deployment of the packages + Deployed instances of a Package. """ pass @@ -45,3 +46,4 @@ def deployment(): deployment.add_command(status) deployment.add_command(execute_command) deployment.add_command(update_deployment) +deployment.add_command(restart_deployment) diff --git a/riocli/deployment/update.py b/riocli/deployment/update.py index 8a524e28..049ffc1f 100644 --- a/riocli/deployment/update.py +++ b/riocli/deployment/update.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ cls=HelpColorsCommand, help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, + deprecated=True, ) @click.option('--force', '-f', '--silent', is_flag=True, default=False, help='Skip confirmation') @@ -52,9 +53,58 @@ def update_deployment( update_all: bool = False, spinner: Yaspin = None, ) -> None: + """Updates one or more deployments""" + _update(force, workers, deployment_name_or_regex, update_all, spinner) + + +@click.command( + 'restart', + cls=HelpColorsCommand, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +@click.option('--force', '-f', '--silent', is_flag=True, default=False, + help='Skip confirmation') +@click.option('-a', '--all', 'update_all', is_flag=True, default=False, + help='Deletes all deployments in the project') +@click.option('--workers', '-w', + help="number of parallel workers while running update deployment " + "command. defaults to 10.", type=int, default=10) +@click.argument('deployment-name-or-regex', type=str, default="") +@with_spinner(text="Updating...") +def restart_deployment( + force: bool, + workers: int, + deployment_name_or_regex: str, + update_all: bool = False, + spinner: Yaspin = None, +) -> None: + """Restarts one or more deployments by name or regex. + + Examples: + + Restart a specific deployment + + >> rio deployment restart amr01 + + Restart all deployments in the project + + >> rio deployment restart --all + + Restart deployments matching a regex. + + >> rio deployment restart amr.* """ - Updates one more deployments - """ + _update(force, workers, deployment_name_or_regex, update_all, spinner) + + +def _update( + force: bool, + workers: int, + deployment_name_or_regex: str, + update_all: bool = False, + spinner: Yaspin = None, +) -> None: client = new_v2_client() if not (deployment_name_or_regex or update_all): spinner.text = "Nothing to update" @@ -126,11 +176,7 @@ def _apply_update( deployment: Deployment, ) -> None: try: - dep = client.get_deployment(deployment.metadata.name) - if not dep: - result.put((deployment.metadata.name, False)) - return client.update_deployment(deployment.metadata.name, deployment) - result.put((deployment.metadata.name, True, 'Deployment Updated Successfully')) + result.put((deployment.metadata.name, True, 'Restarted')) except Exception as e: result.put((deployment.metadata.name, False, str(e))) From 971e76fa38445e5cfa2a001c3b502958e26f67c4 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 12 Aug 2024 16:31:49 +0530 Subject: [PATCH 41/54] feat(context): add command to view cli config This commit introduces a sub-command to manage the CLI context. --- riocli/bootstrap.py | 5 ++- riocli/config/context.py | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 riocli/config/context.py diff --git a/riocli/bootstrap.py b/riocli/bootstrap.py index 658471e1..371c280c 100644 --- a/riocli/bootstrap.py +++ b/riocli/bootstrap.py @@ -16,7 +16,6 @@ __version__ = "7.2.1" import os -import pretty_traceback import click import rapyuta_io.version @@ -30,6 +29,7 @@ from riocli.chart import chart from riocli.completion import completion from riocli.config import Configuration +from riocli.config.context import cli_context from riocli.configtree import config_trees from riocli.constants import Colors, Symbols from riocli.deployment import deployment @@ -153,4 +153,5 @@ def update(silent: bool) -> None: cli.add_command(vpn) cli.add_command(usergroup) cli.add_command(config_trees) -cli.add_command(hwildevice) \ No newline at end of file +cli.add_command(hwildevice) +cli.add_command(cli_context) diff --git a/riocli/config/context.py b/riocli/config/context.py new file mode 100644 index 00000000..95add3f5 --- /dev/null +++ b/riocli/config/context.py @@ -0,0 +1,87 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from pathlib import Path +from typing import Optional + +import click +import yaml +from click_help_colors import HelpColorsGroup, HelpColorsCommand + +from riocli.config import Configuration +from riocli.constants import Symbols, Colors +from riocli.utils import inspect_with_format + + +@click.group( + name='context', + invoke_without_command=False, + cls=HelpColorsGroup, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +def cli_context() -> None: + """Manage the CLI's context. + + The CLI maintains a context that is used to store the configuration that + is used for all CLI operations. It contains essential information like the + user's auth token, the project ID, and the organization ID and other details. + """ + pass + + +@cli_context.command( + name='view', + cls=HelpColorsCommand, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +@click.option('--format', '-f', 'format_type', default='yaml', + type=click.Choice(['json', 'yaml'], case_sensitive=False)) +def view_cli_context(format_type: Optional[str]) -> None: + """View the current CLI context. + + This command prints the current CLI context to the console. + """ + data = Configuration().data + + inspect_with_format(data, format_type) + + +@cli_context.command( + hidden=True, + name='set', + cls=HelpColorsCommand, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +@click.argument('file', type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True, path_type=Path)) +def set_cli_context(file: Path) -> None: + """Set the CLI context. + + This command sets the CLI context to the values in the specified file. + The supported file formats are JSON and YAML only. + """ + with file.open(mode='r') as fp: + if file.suffix == '.json': + data = json.load(fp) + elif file.suffix == '.yaml': + data = yaml.safe_load(fp) + else: + raise Exception('unsupported file format') + + Configuration().data = data + Configuration().save() + + click.secho(f'{Symbols.SUCCESS} Context has been updated.', fg=Colors.GREEN) From 9fa4435df5814d0b6a6ecc9abd8cd6c18cfd01a2 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 19 Aug 2024 16:27:39 +0530 Subject: [PATCH 42/54] fix(device): updates client in list device deployments command The rio device deployment command was still using the old client to invoke the list deployments API. This commit updates the command to use the v2 client with the right query parameters. --- riocli/device/deployment.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/riocli/device/deployment.py b/riocli/device/deployment.py index 6f0238a0..1926a229 100644 --- a/riocli/device/deployment.py +++ b/riocli/device/deployment.py @@ -13,19 +13,17 @@ # limitations under the License. import click from click_help_colors import HelpColorsCommand -from rapyuta_io.clients.deployment import DeploymentPhaseConstants -from riocli.config import new_client +from riocli.config import new_v2_client from riocli.constants import Colors from riocli.deployment.list import display_deployment_list from riocli.device.util import name_to_guid -PHASES = [ - DeploymentPhaseConstants.INPROGRESS, - DeploymentPhaseConstants.PROVISIONING, - DeploymentPhaseConstants.SUCCEEDED, - DeploymentPhaseConstants.FAILED_TO_START, - DeploymentPhaseConstants.PARTIALLY_DEPROVISIONED, +DEFAULT_PHASES = [ + 'InProgress', + 'Provisioning', + 'Succeeded', + 'FailedToStart', ] @@ -38,12 +36,10 @@ @click.argument('device-name', type=str) @name_to_guid def list_deployments(device_name: str, device_guid: str) -> None: - """ - Lists all the deployments running on the Device - """ + """Lists all the deployments running on the device.""" try: - client = new_client() - deployments = client.get_all_deployments(device_id=device_guid, phases=PHASES) + client = new_v2_client() + deployments = client.list_deployments(query={'deviceName': device_name, 'phases': DEFAULT_PHASES}) display_deployment_list(deployments, show_header=True) except Exception as e: click.secho(str(e), fg=Colors.RED) From 985e984e6bcc4fc3ba242918632b33e50985312f Mon Sep 17 00:00:00 2001 From: RomilShah Date: Mon, 19 Aug 2024 18:35:19 +0530 Subject: [PATCH 43/54] feat(deployment): execute commands on device deployments (#345) This commit updates the rio deployment execute command to run commands on device deployments. BREAKING CHANGE: The rio deployment execute command no longer supports cloud deployments. --- riocli/deployment/execute.py | 60 ++++++++++++++++++++++--------- riocli/deployment/util.py | 36 ------------------- riocli/device/util.py | 5 ++- riocli/utils/execute.py | 69 +++++++++++++++--------------------- 4 files changed, 74 insertions(+), 96 deletions(-) diff --git a/riocli/deployment/execute.py b/riocli/deployment/execute.py index 5efd2332..32d6aeb2 100644 --- a/riocli/deployment/execute.py +++ b/riocli/deployment/execute.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,9 +22,10 @@ else: from riocli.utils.spinner import DummySpinner as Spinner -from riocli.constants import Colors -from riocli.deployment.util import select_details -from riocli.utils.execute import run_on_cloud +from riocli.config import new_v2_client +from riocli.constants import Colors, Status, Symbols +from riocli.utils.execute import run_on_device +from riocli.utils.selector import show_selection @click.command( @@ -33,33 +34,58 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -@click.option('--component', 'component_name', default=None, - help='Name of the component in the deployment') +@click.option('--user', default='root') +@click.option('--shell', default='/bin/bash') @click.option('--exec', 'exec_name', default=None, help='Name of a executable in the component') @click.argument('deployment-name', type=str) @click.argument('command', nargs=-1) -# @name_to_guid def execute_command( - component_name: str, + user: str, + shell: str, exec_name: str, deployment_name: str, - deployment_guid: str, command: typing.List[str] ) -> None: """ - Execute commands on cloud deployment + Execute commands on a device deployment """ try: - comp_id, exec_id, pod_name = select_details(deployment_guid, component_name, exec_name) + client = new_v2_client() - with Spinner(text='Executing command `{}`...'.format(command)): - stdout, stderr = run_on_cloud(deployment_guid, comp_id, exec_id, pod_name, command) + deployment = client.get_deployment(deployment_name) + if not deployment: + click.secho(f'{Symbols.ERROR} Deployment `{deployment_name}` not found', fg=Colors.RED) + raise SystemExit(1) + + if deployment.status.status != Status.RUNNING: + click.secho(f'{Symbols.ERROR} Deployment `{deployment_name}` is not running', fg=Colors.RED) + raise SystemExit(1) + + if deployment.spec.runtime != 'device': + click.secho(f'Only device runtime is supported.', fg=Colors.RED) + raise SystemExit(1) - if stderr: - click.secho(stderr, fg=Colors.RED) - if stdout: - click.secho(stdout, fg=Colors.YELLOW) + if exec_name is None: + package = client.get_package(deployment.metadata.depends.nameOrGUID, + query={"version": deployment.metadata.depends.version}) + executables = [e.name for e in package.spec.executables] + if len(executables) == 1: + exec_name = executables[0] + else: + exec_name = show_selection(executables, '\nSelect executable') + + with Spinner(text='Executing command `{}`...'.format(command)): + response = run_on_device( + user=user, + shell=shell, + command=command, + background=False, + deployment=deployment, + exec_name=exec_name, + device_name=deployment.spec.device.depends.nameOrGUID + ) + click.echo(response) except Exception as e: click.secho(e, fg=Colors.RED) raise SystemExit(1) diff --git a/riocli/deployment/util.py b/riocli/deployment/util.py index 60be54a2..a45e255f 100644 --- a/riocli/deployment/util.py +++ b/riocli/deployment/util.py @@ -81,42 +81,6 @@ def get_deployment_name(client: Client, guid: str) -> str: return deployments[0].metadata.name - -def select_details(deployment_guid, component_name=None, exec_name=None) -> (str, str, str): - client = new_client() - deployment = client.get_deployment(deployment_guid) - if deployment.phase != DeploymentPhaseConstants.SUCCEEDED.value: - raise Exception('Deployment is not in succeeded phase') - - if component_name is None: - components = [c.name for c in deployment.componentInfo] - component_name = show_selection(components, 'Choose the component') - - for component in deployment.componentInfo: - if component.name == component_name: - selected_component = component - - if exec_name is None: - executables = [e.name for e in selected_component.executableMetaData] - exec_name = show_selection(executables, 'Choose the executable') - - for executable in selected_component.executableMetaData: - if executable.name == exec_name: - exec_meta = executable - - for executable in selected_component.executablesStatusInfo: - if executable.id == exec_meta.id: - exec_status = executable - - if len(exec_status.metadata) == 1: # If there is a single pod - pod_name = exec_status.metadata[0].podName - else: - pods = [p.podName for p in exec_status.metadata[0]] - pod_name = show_selection(pods, 'Choose the pod') - - return selected_component.componentID, exec_meta.id, pod_name - - def fetch_deployments( client: Client, deployment_name_or_regex: str, diff --git a/riocli/device/util.py b/riocli/device/util.py index ed16fae1..3e7a5723 100644 --- a/riocli/device/util.py +++ b/riocli/device/util.py @@ -87,9 +87,8 @@ def find_device_guid(client: Client, name: str) -> str: def find_device_by_name(client: Client, name: str) -> Device: devices = client.get_all_devices(device_name=name) - for device in devices: - if device.name == name: - return device + if devices: + return devices[0] raise DeviceNotFound() diff --git a/riocli/utils/execute.py b/riocli/utils/execute.py index 28a95efe..83f58eef 100644 --- a/riocli/utils/execute.py +++ b/riocli/utils/execute.py @@ -18,56 +18,45 @@ from queue import Queue from rapyuta_io import Command -from rapyuta_io.utils import RestClient -from rapyuta_io.utils.rest_client import HttpMethod - -from riocli.config import Configuration, new_client - -_CLOUD_RUN_REMOTE_COMMAND = '{}/serviceinstance/{}/cmd' - - -def run_on_cloud(deployment_guid: str, comp_id: str, exec_id: str, pod_name: str, command: typing.List[str]) -> ( - str, str): - """ - run_on_cloud uses the RunCommand API of the IOBroker to execute arbitrary commands on the cloud deployment - containers. - """ - config = Configuration() - rest = RestClient(_run_cloud_url(config, deployment_guid)).headers(config.get_auth_header()).method(HttpMethod.PUT) - resp = rest.execute(payload=_run_cloud_data(comp_id, exec_id, pod_name, command)) - data = json.loads(resp.text) - if 'err' in data and data['err']: - raise Exception(data['err']) - - return data['stdout'], data['stderr'] - - -def _run_cloud_data(comp_id: str, exec_id: str, pod_name: str, command: typing.List[str]) -> dict: - return { - 'componentId': comp_id, - 'executableId': exec_id, - 'podName': pod_name, - 'command': command, - } - - -def _run_cloud_url(config: Configuration, deployment_guid: str) -> str: - host = config.data.get('catalog_host', 'https://gacatalog.apps.okd4v2.prod.rapyuta.io') - return _CLOUD_RUN_REMOTE_COMMAND.format(host, deployment_guid) +from riocli.config import new_client def run_on_device( - device_guid: str, - command: typing.List[str], + device_guid: str = None, + command: typing.List[str] = None, user: str = 'root', shell: str = '/bin/bash', background: bool = False, + deployment: str = None, + exec_name: str = None, + device_name: str = None ) -> str: client = new_client() - device = client.get_device(device_id=device_guid) + + device = None + if device_guid: + device = client.get_device(device_id=device_guid) + elif device_name: + devices = client.get_all_devices(device_name=device_name) + if devices: + device = devices[0] + else: + raise ValueError('Either `device_guid` or `device_name` must be specified') + + if not device: + raise ValueError('Device not found or is not online') + + if deployment and exec_name is None: + raise ValueError('The `exec_name` argument is required when `deployment` is specified') + + if not command: + raise ValueError('The `command` argument is required') + cmd = ' '.join(command) - return device.execute_command(Command(cmd, shell=shell, bg=background, runas=user)) + if deployment: + cmd = 'script -q -c "dectl exec {} -- {}"'.format(exec_name, cmd) + return device.execute_command(Command(cmd, shell=shell, bg=background, runas=user)) def apply_func( f: typing.Callable, From 369a070c8c740703353a3811d6d5d62525409c08 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Wed, 21 Aug 2024 23:29:08 +0530 Subject: [PATCH 44/54] fix(v2client): corrects status check in poll_disk The poll_disk method has 'Bound' in the list of statuses which caused the apply command to fail when a deployment depends on a disk. The poll_disk is supposed to wait until the status changes to 'Available' before returning. This commit fixes the issue. --- riocli/v2client/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index 27d640b1..cd06b945 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -1039,7 +1039,6 @@ def stream_deployment_logs( curl = 'curl -H "project: {}" -H "Authorization: {}" "{}"'.format( headers['project'], headers['Authorization'], url) - click.echo(click.style(curl, fg=Colors.BLUE, italic=True)) os.system(curl) @@ -1122,8 +1121,7 @@ def poll_disk( for _ in range(retry_count): if status.status in [DiskStatusConstants.DiskStatusAvailable.value, - DiskStatusConstants.DiskStatusReleased.value, - DiskStatusConstants.DiskStatusBound.value]: + DiskStatusConstants.DiskStatusReleased.value]: return disk elif status.status == DiskStatusConstants.DiskStatusFailed.value: raise DeploymentNotRunning('Disk not running. Status: {}'.format(status.status)) From 860d56d6356a1bddd64f0dfd3cb00fe70dc04b54 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Wed, 21 Aug 2024 23:36:01 +0530 Subject: [PATCH 45/54] fix(deployment): set replica=0 as default in deployment logs command The replica value defaults to None in the existing implementation which results in a wrong value sent in the query parameter. This commit fixes the issue by setting the default to 0. Wrike Ticket: https://www.wrike.com/open.htm?id=1478463257 --- riocli/deployment/logs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/riocli/deployment/logs.py b/riocli/deployment/logs.py index 3dd33ee0..480113c6 100644 --- a/riocli/deployment/logs.py +++ b/riocli/deployment/logs.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import click from click_help_colors import HelpColorsCommand @@ -26,13 +25,13 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -@click.option('--replica', 'replica', default=None, +@click.option('--replica', 'replica', default=0, help='Replica identifier of the deployment') @click.option('--exec', 'exec_name', default=None, help='Name of a executable in the component') @click.argument('deployment-name', type=str) def deployment_logs( - replica: str, + replica: int, exec_name: str, deployment_name: str, ) -> None: @@ -40,6 +39,7 @@ def deployment_logs( Stream live logs from cloud deployments (not supported for device deployments) """ try: + # TODO(pallab): when no exec name is given, implement the logic to set default or prompt a selection. client = new_v2_client() client.stream_deployment_logs(deployment_name, exec_name, replica) except Exception as e: From f548d8c8b6c9b36e64815ed8d3c075deefe57c7f Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 22 Aug 2024 00:30:53 +0530 Subject: [PATCH 46/54] feat(deployment): wait until all dependecies are running This commit adds a field in the deployment schema to wait for dependencies to come to a running state before a deployment is created. --- .../manifests/deployment-nonros-cloud.yaml | 1 + .../apply/manifests/deployment-ros-cloud.yaml | 1 + riocli/apply/manifests/deployment.yaml | 2 ++ riocli/deployment/model.py | 32 ++++++++++++++++++- .../jsonschema/schemas/deployment-schema.yaml | 2 ++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/riocli/apply/manifests/deployment-nonros-cloud.yaml b/riocli/apply/manifests/deployment-nonros-cloud.yaml index b96c02bb..12a39ba3 100644 --- a/riocli/apply/manifests/deployment-nonros-cloud.yaml +++ b/riocli/apply/manifests/deployment-nonros-cloud.yaml @@ -14,6 +14,7 @@ spec: depends: - kind: deployment nameOrGUID: "deployment-ros-cloud" + wait: true # Options: [true, false] Wait until dependency is ready envArgs: - name: TEST_KEY value: test_value diff --git a/riocli/apply/manifests/deployment-ros-cloud.yaml b/riocli/apply/manifests/deployment-ros-cloud.yaml index a6b17b6e..9f5c21af 100644 --- a/riocli/apply/manifests/deployment-ros-cloud.yaml +++ b/riocli/apply/manifests/deployment-ros-cloud.yaml @@ -15,6 +15,7 @@ spec: depends: - kind: deployment nameOrGUID: "dep-guid" + wait: false # Options: [true, false] Wait until dependency is ready envArgs: - name: TEST_KEY value: test_value diff --git a/riocli/apply/manifests/deployment.yaml b/riocli/apply/manifests/deployment.yaml index ecd6ae7e..d4d21f26 100644 --- a/riocli/apply/manifests/deployment.yaml +++ b/riocli/apply/manifests/deployment.yaml @@ -15,6 +15,7 @@ spec: depends: - kind: deployment nameOrGUID: "dep-guid" + wait: false # Options: [true, false] Wait until dependency is ready envArgs: - name: TEST_KEY value: test_value @@ -111,6 +112,7 @@ spec: depends: - kind: deployment nameOrGUID: "deployment-ros-cloud" + wait: true # Options: [true, false] Wait until dependency is ready envArgs: - name: TEST_KEY value: test_value diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index b4af5254..2d69db58 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import time import typing from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import Status from riocli.exceptions import ResourceNotFound from riocli.model import Model -from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError +from riocli.v2client import Client +from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError, RetriesExhausted class Deployment(Model): @@ -27,8 +30,14 @@ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def apply(self, *args, **kwargs) -> typing.Any: + hard_dependencies = [d.nameOrGUID for d in self.spec.get('depends', []) if d.get('wait', False)] + client = new_v2_client() + # Block until all hard dependencies are in RUNNING state + if hard_dependencies: + wait_for_dependencies(client, hard_dependencies) + self.metadata.createdAt = None self.metadata.updatedAt = None @@ -44,3 +53,24 @@ def delete(self, *args, **kwargs): client.delete_deployment(self.metadata.name) except HttpNotFoundError: raise ResourceNotFound + + +def wait_for_dependencies( + client: Client, + deployment_names: typing.List[str], + retry_count: int = 50, + retry_interval: int = 6, +) -> None: + """Waits until all deployment_names are in RUNNING state.""" + for _ in range(retry_count): + deployments = client.list_deployments(query={ + 'names': deployment_names, + 'phases': ['InProgress', 'Provisioning', 'Succeeded'] + }) + + if all(d.status.status == Status.RUNNING for d in deployments): + return + + time.sleep(retry_interval) + + raise RetriesExhausted(f'Retries exhausted waiting for dependencies: {deployment_names}') diff --git a/riocli/jsonschema/schemas/deployment-schema.yaml b/riocli/jsonschema/schemas/deployment-schema.yaml index ad7bddd2..a0659565 100755 --- a/riocli/jsonschema/schemas/deployment-schema.yaml +++ b/riocli/jsonschema/schemas/deployment-schema.yaml @@ -471,6 +471,8 @@ definitions: type: string guid: type: string + wait: + type: boolean managedServiceDepends: properties: kind: From 5fac718040750fa63337523edb920b5eabcdac48 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 22 Aug 2024 01:45:09 +0530 Subject: [PATCH 47/54] fix(disk): prints a better message when no disks are deleted When no disks are deleted using the rio disk delete command, the existing implementation returns a message that's not really accurate. Ideally, it should say that no disk was deleted. This commit fixes the issue. Wrike Ticket: https://www.wrike.com/open.htm?id=1478463819 --- riocli/disk/delete.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riocli/disk/delete.py b/riocli/disk/delete.py index e16592e9..097e87a4 100644 --- a/riocli/disk/delete.py +++ b/riocli/disk/delete.py @@ -102,6 +102,13 @@ def delete_disk( with spinner.hidden(): tabulate_data(data, headers=['Name', 'Status']) + # When no disk is deleted, raise an exception. + if not any(statuses): + spinner.write('') + spinner.text = click.style('Failed to delete disk(s).', Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) + icon = Symbols.SUCCESS if all(statuses) else Symbols.WARNING fg = Colors.GREEN if all(statuses) else Colors.YELLOW text = "successfully" if all(statuses) else "partially" From 65e3b7e096bf0e6567d3a51c098b02578792b18c Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 22 Aug 2024 15:18:56 +0530 Subject: [PATCH 48/54] refactor(apply): prints specific status messages --- riocli/apply/parse.py | 29 ++++++++++++++++++++--------- riocli/constants/__init__.py | 6 +++--- riocli/constants/status.py | 24 ++++++++++++++++++++++++ riocli/deployment/model.py | 7 ++++--- riocli/device/model.py | 13 ++++++++++--- riocli/disk/model.py | 6 ++++-- riocli/managedservice/model.py | 6 ++++-- riocli/model/base.py | 3 ++- riocli/network/model.py | 4 +++- riocli/package/model.py | 6 ++++-- riocli/project/model.py | 6 ++++-- riocli/secret/model.py | 3 +++ riocli/static_route/model.py | 5 ++++- riocli/usergroup/model.py | 6 ++++-- 14 files changed, 93 insertions(+), 31 deletions(-) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 8b614395..c39dbf3c 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -23,7 +23,7 @@ from riocli.apply.util import (get_model, init_jinja_environment, message_with_prompt, print_resolved_objects) from riocli.config import Configuration -from riocli.constants import Colors, Symbols +from riocli.constants import Colors, Symbols, ApplyResult from riocli.exceptions import ResourceNotFound from riocli.utils import dump_all_yaml, print_centered_text, run_bash from riocli.utils.graph import Graphviz @@ -81,10 +81,10 @@ def apply(self, *args, **kwargs): try: apply_func(*args, **kwargs) - spinner.text = 'Apply successful.' + spinner.text = click.style('Apply successful.', fg=Colors.BRIGHT_GREEN) spinner.green.ok(Symbols.SUCCESS) except Exception as e: - spinner.text = 'Apply failed. Error: {}'.format(e) + spinner.text = click.style('Apply failed. Error: {}'.format(e), fg=Colors.BRIGHT_RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e @@ -135,10 +135,10 @@ def delete(self, *args, **kwargs): try: delete_func(*args, **kwargs) - spinner.text = 'Delete successful.' + spinner.text = click.style('Delete successful.', fg=Colors.BRIGHT_GREEN) spinner.green.ok(Symbols.SUCCESS) except Exception as e: - spinner.text = 'Delete failed. Error: {}'.format(e) + spinner.text = click.style('Delete failed. Error: {}'.format(e), fg=Colors.BRIGHT_RED) spinner.red.fail(Symbols.ERROR) raise SystemExit(1) from e @@ -253,15 +253,24 @@ def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None: kls.validate(obj) ist = kls(munchify(obj)) + obj_key = click.style(obj_key, bold=True) + message_with_prompt("{} Applying {}...".format( Symbols.WAITING, obj_key), fg=Colors.CYAN, spinner=spinner) try: + result = ApplyResult.CREATED if not dryrun: - ist.apply(*args, **kwargs) + result = ist.apply(*args, **kwargs) - message_with_prompt("{} Applied {}".format( - Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) + if result == ApplyResult.EXISTS: + message_with_prompt("{} {} already exists".format( + Symbols.INFO, obj_key), fg=Colors.WHITE, spinner=spinner) + return + + message_with_prompt("{} {} {}".format( + Symbols.SUCCESS, result, obj_key), + fg=Colors.GREEN, spinner=spinner) except Exception as ex: message_with_prompt("{} Failed to apply {}. Error: {}".format( Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) @@ -277,6 +286,8 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: kls.validate(obj) ist = kls(munchify(obj)) + obj_key = click.style(obj_key, bold=True) + message_with_prompt("{} Deleting {}...".format( Symbols.WAITING, obj_key), fg=Colors.CYAN, spinner=spinner) @@ -289,7 +300,7 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: if not dryrun and can_delete: ist.delete(*args, **kwargs) - message_with_prompt("{} Deleted {}.".format( + message_with_prompt("{} Deleted {}".format( Symbols.SUCCESS, obj_key), fg=Colors.GREEN, spinner=spinner) except ResourceNotFound: message_with_prompt("{} {} not found".format( diff --git a/riocli/constants/__init__.py b/riocli/constants/__init__.py index b905f913..31975eff 100644 --- a/riocli/constants/__init__.py +++ b/riocli/constants/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from riocli.constants.colors import Colors -from riocli.constants.symbols import Symbols from riocli.constants.regions import Regions -from riocli.constants.status import Status +from riocli.constants.status import Status, ApplyResult +from riocli.constants.symbols import Symbols -__all__ = [Colors, Symbols, Regions, Status] +__all__ = [Colors, Symbols, Regions, Status, ApplyResult] diff --git a/riocli/constants/status.py b/riocli/constants/status.py index c3822076..8c66a560 100644 --- a/riocli/constants/status.py +++ b/riocli/constants/status.py @@ -1,3 +1,17 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from enum import Enum @@ -8,3 +22,13 @@ def __str__(self): RUNNING = 'Running' AVAILABLE = 'Available' + + +class ApplyResult(str, Enum): + + def __str__(self): + return str(self.value) + + CREATED = 'Created' + UPDATED = 'Updated' + EXISTS = 'Exists' diff --git a/riocli/deployment/model.py b/riocli/deployment/model.py index 2d69db58..22a9c97b 100644 --- a/riocli/deployment/model.py +++ b/riocli/deployment/model.py @@ -17,7 +17,7 @@ from munch import unmunchify from riocli.config import new_v2_client -from riocli.constants import Status +from riocli.constants import Status, ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client import Client @@ -29,7 +29,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> typing.Any: + def apply(self, *args, **kwargs) -> ApplyResult: hard_dependencies = [d.nameOrGUID for d in self.spec.get('depends', []) if d.get('wait', False)] client = new_v2_client() @@ -43,8 +43,9 @@ def apply(self, *args, **kwargs) -> typing.Any: try: client.create_deployment(unmunchify(self)) + return ApplyResult.CREATED except HttpAlreadyExistsError: - pass + return ApplyResult.EXISTS def delete(self, *args, **kwargs): client = new_v2_client() diff --git a/riocli/device/model.py b/riocli/device/model.py index 5b950fad..7c1a0ece 100644 --- a/riocli/device/model.py +++ b/riocli/device/model.py @@ -15,6 +15,7 @@ from rapyuta_io.clients.device import Device as v1Device, DevicePythonVersion from riocli.config import new_client +from riocli.constants import ApplyResult from riocli.device.util import (DeviceNotFound, create_hwil_device, delete_hwil_device, execute_onboard_command, find_device_by_name, make_device_labels_from_hwil_device) from riocli.exceptions import ResourceNotFound @@ -25,7 +26,7 @@ class Device(Model): def __init__(self, *args, **kwargs): self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_client() device = None @@ -39,11 +40,15 @@ def apply(self, *args, **kwargs) -> None: if not self.spec.get('virtual', {}).get('enabled', False): if device is None: client.create_device(self.to_v1()) - return + return ApplyResult.CREATED + + return ApplyResult.EXISTS # Return if the device is already online or initializing. if device and device['status'] in ('ONLINE', 'INITIALIZING'): - return + return ApplyResult.EXISTS + + result = ApplyResult.CREATED if device is None else ApplyResult.UPDATED # Create the HWIL (virtual) device and then generate the labels # to store HWIL metadata in rapyuta.io device. @@ -74,6 +79,8 @@ def apply(self, *args, **kwargs) -> None: onboard_command = onboard_script.full_command() execute_onboard_command(hwil_response.id, onboard_command) + return result + def delete(self, *args, **kwargs) -> None: client = new_client() diff --git a/riocli/disk/model.py b/riocli/disk/model.py index 197e03c2..e4937a6b 100644 --- a/riocli/disk/model.py +++ b/riocli/disk/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -25,7 +26,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_v2_client() self.metadata.createdAt = None @@ -41,8 +42,9 @@ def apply(self, *args, **kwargs) -> None: retry_count=retry_count, sleep_interval=retry_interval, ) + return ApplyResult.CREATED except HttpAlreadyExistsError: - pass + return ApplyResult.EXISTS except Exception as e: raise e diff --git a/riocli/managedservice/model.py b/riocli/managedservice/model.py index b8b49164..c273089c 100644 --- a/riocli/managedservice/model.py +++ b/riocli/managedservice/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -25,13 +26,14 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_v2_client() try: client.create_instance(unmunchify(self)) + return ApplyResult.CREATED except HttpAlreadyExistsError: - pass + return ApplyResult.EXISTS def delete(self, *args, **kwargs) -> None: client = new_v2_client() diff --git a/riocli/model/base.py b/riocli/model/base.py index 16245725..42fbd07d 100644 --- a/riocli/model/base.py +++ b/riocli/model/base.py @@ -16,6 +16,7 @@ from munch import Munch +from riocli.constants import ApplyResult from riocli.jsonschema.validate import load_schema DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' @@ -41,7 +42,7 @@ def delete(self, *args, **kwargs): schema that are defined in the schema files. """ @abstractmethod - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: """Create or update the object. This method should be implemented by the subclasses. It should diff --git a/riocli/network/model.py b/riocli/network/model.py index 4255d63f..afb82fd1 100644 --- a/riocli/network/model.py +++ b/riocli/network/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -37,8 +38,9 @@ def apply(self, *args, **kwargs) -> None: try: r = client.create_network(unmunchify(self)) client.poll_network(r.metadata.name, retry_count=retry_count, sleep_interval=retry_interval) + return ApplyResult.CREATED except HttpAlreadyExistsError: - pass + return ApplyResult.EXISTS except Exception as e: raise e diff --git a/riocli/package/model.py b/riocli/package/model.py index 35bb7384..9ec4b204 100644 --- a/riocli/package/model.py +++ b/riocli/package/model.py @@ -16,6 +16,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.package.enum import RestartPolicy @@ -33,15 +34,16 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_v2_client() package = self._sanitize_package() try: client.create_package(package) + return ApplyResult.CREATED except HttpAlreadyExistsError: - pass + return ApplyResult.EXISTS def delete(self, *args, **kwargs) -> None: client = new_v2_client() diff --git a/riocli/project/model.py b/riocli/project/model.py index 62830cf0..d8b41de5 100644 --- a/riocli/project/model.py +++ b/riocli/project/model.py @@ -16,6 +16,7 @@ from waiting import wait from riocli.config import Configuration, new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.project.util import ProjectNotFound, find_project_guid @@ -30,7 +31,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_v2_client() project = unmunchify(self) @@ -42,10 +43,11 @@ def apply(self, *args, **kwargs) -> None: r = client.create_project(project) wait(self.is_ready, timeout_seconds=PROJECT_READY_TIMEOUT, sleep_seconds=(1, 30, 2)) + return ApplyResult.CREATED except HttpAlreadyExistsError: guid = find_project_guid(client, self.metadata.name, Configuration().organization_guid) client.update_project(guid, project) - return + return ApplyResult.UPDATED except Exception as e: raise e diff --git a/riocli/secret/model.py b/riocli/secret/model.py index 46dfe7e3..dd02111e 100644 --- a/riocli/secret/model.py +++ b/riocli/secret/model.py @@ -15,6 +15,7 @@ from munch import unmunchify from riocli.config import new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -32,8 +33,10 @@ def apply(self, *args, **kwargs) -> None: try: client.create_secret(unmunchify(self)) + return ApplyResult.CREATED except HttpAlreadyExistsError: client.update_secret(self.metadata.name, secret) + return ApplyResult.UPDATED def delete(self, *args, **kwargs) -> None: client = new_v2_client() diff --git a/riocli/static_route/model.py b/riocli/static_route/model.py index 13f3a1ad..af84a603 100644 --- a/riocli/static_route/model.py +++ b/riocli/static_route/model.py @@ -14,6 +14,7 @@ from munch import unmunchify from riocli.config import Configuration, new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.v2client.error import HttpAlreadyExistsError, HttpNotFoundError @@ -24,15 +25,17 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.update(*args, **kwargs) - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: client = new_v2_client() static_route = unmunchify(self) try: client.create_static_route(static_route) + return ApplyResult.CREATED except HttpAlreadyExistsError: client.update_static_route(self.metadata.name, static_route) + return ApplyResult.UPDATED def delete(self, *args, **kwargs) -> None: client = new_v2_client() diff --git a/riocli/usergroup/model.py b/riocli/usergroup/model.py index ae1035dc..ff0b0a14 100644 --- a/riocli/usergroup/model.py +++ b/riocli/usergroup/model.py @@ -16,6 +16,7 @@ from munch import unmunchify from riocli.config import Configuration, new_client, new_v2_client +from riocli.constants import ApplyResult from riocli.exceptions import ResourceNotFound from riocli.model import Model from riocli.organization.utils import get_organization_details @@ -33,7 +34,7 @@ def __init__(self, *args, **kwargs): self.user_email_to_guid_map = {} self.project_name_to_guid_map = {} - def apply(self, *args, **kwargs) -> None: + def apply(self, *args, **kwargs) -> ApplyResult: v1client = new_client() v2client = new_v2_client() @@ -61,12 +62,13 @@ def apply(self, *args, **kwargs) -> None: try: payload['spec']['name'] = sanitized['metadata']['name'] v1client.create_usergroup(self.metadata.organization, payload['spec']) - return + return ApplyResult.CREATED except Exception as e: raise e payload = self._generate_update_payload(existing, payload) v1client.update_usergroup(organization_id, existing.guid, payload) + return ApplyResult.UPDATED def delete(self, *args, **kwargs) -> None: v1client = new_client() From 9038bd878c6127ecafe670f3121108964c0323fa Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Thu, 22 Aug 2024 17:29:35 +0530 Subject: [PATCH 49/54] ci: updates workflow names --- .github/workflows/conventional-commits.yml | 2 +- .github/workflows/gh-pages.yml | 2 +- .github/workflows/pypi.yml | 2 +- .github/workflows/python-compatibility.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/upload-appimage.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index d9a0cce5..ff932755 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -1,4 +1,4 @@ -name: Check Commit Hygiene 💬 +name: 💬 Check Commit Hygiene on: pull_request: diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index fa09cf58..e72e1929 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,4 +1,4 @@ -name: Github Pages for Documentation +name: 📝 Github Pages for Documentation on: push: branches: diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 072b3737..e1962ee9 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,4 +1,4 @@ -name: Upload to PyPi +name: 📦️ Upload to PyPi on: release: types: diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml index a873146c..3733c8da 100644 --- a/.github/workflows/python-compatibility.yml +++ b/.github/workflows/python-compatibility.yml @@ -1,9 +1,9 @@ -name: Python Compatibility Check 🐍 +name: 🐍 Python Compatibility Check on: [ push ] jobs: python-compatibility: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: python-version: [ '3.8', '3.9', '3.10', '3.11' ] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b19e6b14..07857fd0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: 🎉 Release on: push: branches: diff --git a/.github/workflows/upload-appimage.yml b/.github/workflows/upload-appimage.yml index 2496e6f5..c6b0508e 100644 --- a/.github/workflows/upload-appimage.yml +++ b/.github/workflows/upload-appimage.yml @@ -1,4 +1,4 @@ -name: Upload AppImage +name: ⬆️ Upload AppImage on: push: branches: From 6098fffc43cca17c5246b3af82a45438a55eea6f Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Fri, 23 Aug 2024 16:05:55 +0530 Subject: [PATCH 50/54] fix(configtree): returns error when no files are provided in import The files argument in the import command is variadic. If you forget specifying a tree name and pass only file to import, it will assume the file as the tree name and the files tuple will be empty. It has to be handled explicitly per the documentation here: https://click.palletsprojects.com/en/8.1.x/arguments/ --- riocli/configtree/import_keys.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/riocli/configtree/import_keys.py b/riocli/configtree/import_keys.py index 97c12dd3..56a3fee4 100644 --- a/riocli/configtree/import_keys.py +++ b/riocli/configtree/import_keys.py @@ -100,6 +100,11 @@ def import_keys( Note: If --etcd-endpoint is provided, the keys are imported to the local etcd cluster instead of the rapyuta.io cloud. """ + if not files: + spinner.text = click.style('No files provided.', fg=Colors.RED) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) + data, metadata = _process_files_with_overrides(files, overrides, spinner) if export_directory is not None: From 55cb7b4016b1398ad7497eb74ece88391f7494bc Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Tue, 27 Aug 2024 00:30:12 +0530 Subject: [PATCH 51/54] fix(apply): prints resource name when apply or delete fails When the apply or delete command fails, it doesn't clearly mention which resource failed. This commit fixes the issue by printing the resource name along with the eror. Wrike Ticket: https://www.wrike.com/open.htm?id=1480954550 --- riocli/apply/parse.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index c39dbf3c..f20cc8fe 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -250,7 +250,12 @@ def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None: obj = self.objects[obj_key] kls = get_model(obj) - kls.validate(obj) + + try: + kls.validate(obj) + except Exception as ex: + raise Exception(f'invalid manifest {obj_key}: {str(ex)}') + ist = kls(munchify(obj)) obj_key = click.style(obj_key, bold=True) @@ -274,7 +279,7 @@ def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None: except Exception as ex: message_with_prompt("{} Failed to apply {}. Error: {}".format( Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) - raise ex + raise Exception(f'{obj_key}: {str(ex)}') def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: """Instantiate and delete the object manifest""" @@ -283,7 +288,12 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: obj = self.objects[obj_key] kls = get_model(obj) - kls.validate(obj) + + try: + kls.validate(obj) + except Exception as ex: + raise Exception(f'invalid manifest {obj_key}: {str(ex)}') + ist = kls(munchify(obj)) obj_key = click.style(obj_key, bold=True) @@ -309,7 +319,7 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: except Exception as ex: message_with_prompt("{} Failed to delete {}. Error: {}".format( Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner) - raise ex + raise Exception(f'{obj_key}: {str(ex)}') def _process_file_list(self, files): for f in files: From acdbd87163439e75a0d62c737c8827260ff48355 Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 2 Sep 2024 16:00:15 +0530 Subject: [PATCH 52/54] fix(delete): prints appropriate message when deletionPolicy = retain When deletion policy for a resource is set to `retain` in the manifest, the delete command should print an appropriate error message. --- riocli/apply/parse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index f20cc8fe..8439f30a 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -306,6 +306,11 @@ def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None: labels = obj.get('metadata', {}).get('labels', {}) can_delete = labels.get(self.DELETE_POLICY_LABEL) != 'retain' + if not can_delete: + message_with_prompt("{} {} cannot be deleted since deletion policy is set to 'retain'".format( + Symbols.INFO, obj_key), fg=Colors.WHITE, spinner=spinner) + return + try: if not dryrun and can_delete: ist.delete(*args, **kwargs) From 4ca439af979ab075840118f2d9ef0652c65362ab Mon Sep 17 00:00:00 2001 From: Dev Gupta <72784479+guptadev21@users.noreply.github.com> Date: Mon, 2 Sep 2024 16:20:37 +0530 Subject: [PATCH 53/54] ci: deletes old comments when new appimage is uploaded This commit updates the AppImage workflow to delete older comments that publish the link to the artifacts before adding the new comment ensuring that there's always one latest comment with the link to the last successful AppImage build on a PR. Wrike Ticket: https://www.wrike.com/open.htm?id=1130198233 --- .github/workflows/upload-appimage.yml | 38 ++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/upload-appimage.yml b/.github/workflows/upload-appimage.yml index c6b0508e..e198c191 100644 --- a/.github/workflows/upload-appimage.yml +++ b/.github/workflows/upload-appimage.yml @@ -34,8 +34,44 @@ jobs: if-no-files-found: error retention-days: 15 - - name: Artifact Comment on PR + - name: Install GitHub CLI + run: | + sudo apt-get update + sudo apt-get install gh + + - name: Edit the last bot comment + id: edit-comment if: github.event_name == 'pull_request' + run: | + set -e + gh pr comment $PR_NUMBER --edit-last -b " + **🤖 Pull Request [Artifacts](https://github.com/rapyuta-robotics/rapyuta-io-cli/actions/runs/"$RUN_ID") (#"$RUN_ID") 🎉** + " + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.run_id }} + PR_NUMBER: ${{ github.event.number }} + continue-on-error: true + + + - name: Delete existing bot comments + if: steps.edit-comment.outcome == 'failure' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: | + # Get all comments on the PR where the author is the bot + COMMENTS=$(gh api repos/${{ github.repository }}/issues/$PR_NUMBER/comments --jq '.[] | select(.user.login == "github-actions[bot]") | .id') + + # Loop through each bot comment and delete it + for COMMENT_ID in $COMMENTS; do + gh api repos/${{ github.repository }}/issues/comments/$COMMENT_ID -X DELETE + done + + + - name: Artifact Comment on PR + # if: github.event_name == 'pull_request' + if: steps.edit-comment.outcome == 'failure' run: | gh pr comment $PR_NUMBER -b " **🤖 Pull Request [Artifacts](https://github.com/rapyuta-robotics/rapyuta-io-cli/actions/runs/"$RUN_ID") (#"$RUN_ID") 🎉** From e6f59271fb611f5b31217411fc5ee2912d9efc0c Mon Sep 17 00:00:00 2001 From: Yaswanth Kumar Date: Wed, 4 Sep 2024 13:07:13 +0530 Subject: [PATCH 54/54] feat(v2client): inject X-Request-ID in headers if REQUEST_ID in env If REQUEST_ID is set in the system env, then all v2 API calls will have X-Request-ID header set to the value of REQUEST_ID. This is particularly helpful in debugging. --- riocli/v2client/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/riocli/v2client/client.py b/riocli/v2client/client.py index cd06b945..e29c64cf 100644 --- a/riocli/v2client/client.py +++ b/riocli/v2client/client.py @@ -50,6 +50,10 @@ def _get_auth_header(self: Client, with_project: bool = True) -> dict: if with_project and self._project is not None: headers['project'] = self._project + custom_client_request_id = os.getenv('REQUEST_ID') + if custom_client_request_id: + headers['X-Request-ID'] = custom_client_request_id + return headers # Project APIs