diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cad6a..57bad49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +**28/09/2023** [Version 3.6.3] + - Re-enable original endpoints (deprecated) to retain compatibility with Portal + +**21/09/2023** [Version 3.6.2] + - Add a prefix (`/elcm/api/v1/`) to all endpoints + - Allow defining and return a more complete description of the KPIs in the `/elcm/api/v1//kpis` endpoint + **09/11/2022** [Version 3.6.1] - Allow defining a set of KPIs per TestCase - Implement `/execution//kpis` endpoint diff --git a/Executor/Tasks/Run/run_script.py b/Executor/Tasks/Run/run_script.py new file mode 100644 index 0000000..da71b65 --- /dev/null +++ b/Executor/Tasks/Run/run_script.py @@ -0,0 +1,20 @@ +from Task import Task +from Helper import Cli +import platform + + +class RunScript(Task): + def __init__(self, logMethod, parent, params): + super().__init__("CLI Execute", parent, params, logMethod, None) + self.paramRules = { + 'Parameters': (None, True), + 'CWD': (None, True) + } + + def Run(self): + parameters = self.params['Parameters'] + if platform.system() == 'Windows': + pass + + cli = Cli(parameters, self.params['CWD'], self.Log) + cli.Execute() diff --git a/Experiment/experiment_run.py b/Experiment/experiment_run.py index 1535254..8c9b6be 100644 --- a/Experiment/experiment_run.py +++ b/Experiment/experiment_run.py @@ -1,3 +1,7 @@ +import subprocess +import re +import platform + from Executor import PreRunner, Executor, PostRunner, ExecutorBase, Verdict from Data import ExperimentDescriptor from typing import Dict, Optional, List @@ -9,6 +13,7 @@ from Interfaces import PortalApi from Composer import Composer, PlatformConfiguration from os.path import join, abspath +from Helper import Cli @unique @@ -147,6 +152,9 @@ def Cancel(self): if current is not None: current.RequestStop() self.CoarseStatus = CoarseStatus.Cancelled + self.AppEviction(self.Params["DeviceId"]) + self.TapEviction() + self.PostRunner.Start() # Temporal fix for the release of the resources after the cancellation. def PreRun(self): self.CoarseStatus = CoarseStatus.PreRun @@ -267,3 +275,28 @@ def Save(self): @classmethod def Digest(cls, id: str) -> Dict: return Serialize.Load(Serialize.Path('Execution', id)) + + @staticmethod + def AppEviction(device_id): + commands = f'adb -s {device_id} shell pm list packages' + pattern = r"(.*)(com.uma.(.*))" + process = subprocess.Popen(commands.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd='.') + pipe = process.stdout + + for line in iter(pipe.readline, b''): + try: + result = re.search(pattern, line.decode('utf-8')) + if result: + app_stop_command = f'adb -s {device_id} shell am force-stop {result.group(2)}' + subprocess.Popen(app_stop_command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd='.') + Log.I(f"Package {result.group(2)} stopped.") + except Exception as e: + Log.E(f"DECODING EXCEPTION: {e}") + + @staticmethod + def TapEviction(): + if platform.system() == 'Linux': + pass + elif platform.system() == 'Windows': + tap_stop_command = f'taskkill /IM "tap.exe" /F' + subprocess.run(tap_stop_command, shell=True) diff --git a/Facility/Loader/testcase_loader.py b/Facility/Loader/testcase_loader.py index b65e16c..20d723d 100644 --- a/Facility/Loader/testcase_loader.py +++ b/Facility/Loader/testcase_loader.py @@ -25,7 +25,7 @@ class TestCaseLoader(Loader): testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} - kpis: Dict[str, List[Tuple[str, str]]] = {} + kpis: Dict[str, List[Tuple[str, str, str, str]]] = {} # (Measurement, KPI, Type, Description) parameters: Dict[str, Tuple[str, str]] = {} # For use only while processing data, not necessary afterwards @classmethod @@ -100,10 +100,22 @@ def validateKPIs(cls, key: str, defs: TestCaseData) -> [(Level, str)]: (Level.ERROR, f"KPIs for '{measurement}' ({key}) are not a list. Found '{kpiList}'")) elif len(kpiList) == 0: validation.append( - (Level.ERROR, f"'{measurement}' ({key}) defines an empty listf of KPIs")) + (Level.ERROR, f"'{measurement}' ({key}) defines an empty list of KPIs")) else: - for kpi in sorted(kpiList): - kpis.append((measurement, kpi)) + for kpi in sorted(kpiList, key=lambda x: x if isinstance(x, str) else x.get('Name', '')): + description = kind = "" + if isinstance(kpi, str): + name = kpi + elif isinstance(kpi, dict): + name = kpi.get('Name', '') + description = kpi.get('Description', '') + kind = kpi.get('Type', '') + else: + validation.append( + (Level.ERROR, f"KPI definitions for '{measurement}' must either be str or a " + f"dictionary (keys ['Name', 'Type', 'Description']). Found '{kpi}'")) + continue + kpis.append((measurement, name, kind, description)) except Exception as e: validation.append((Level.ERROR, f"Could not read KPIs dictionary for testcase '{key}': {e}")) diff --git a/Facility/facility.py b/Facility/facility.py index 72df8a4..07d5d23 100644 --- a/Facility/facility.py +++ b/Facility/facility.py @@ -24,7 +24,7 @@ class Facility: testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} - kpis: Dict[str, List[Tuple[str, str]]] = {} + kpis: Dict[str, List[Tuple[str, str, str, str]]] = {} resources: Dict[str, Resource] = {} scenarios: Dict[str, Dict] = {} @@ -98,7 +98,7 @@ def GetTestCaseExtra(cls, id: str) -> Dict[str, object]: return cls.extra.get(id, {}) @classmethod - def GetTestCaseKPIs(cls, id: str) -> List[Tuple[str, str]]: + def GetTestCaseKPIs(cls, id: str) -> List[Tuple[str, str, str, str]]: return cls.kpis.get(id, []) @classmethod diff --git a/Helper/cli_executor.py b/Helper/cli_executor.py index ea807b3..38712c5 100644 --- a/Helper/cli_executor.py +++ b/Helper/cli_executor.py @@ -4,14 +4,15 @@ class Cli: - def __init__(self, parameters: List[str], cwd: str, logger: Callable): + def __init__(self, parameters, cwd: str, logger: Callable): + self.parameters = parameters self.cwd = cwd self.logger = logger def Execute(self) -> int: process = subprocess.Popen(self.parameters, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, cwd=self.cwd) + stderr=subprocess.STDOUT, cwd=self.cwd, shell=True) self.stdout(process) return process.wait() @@ -19,7 +20,9 @@ def stdout(self, process: subprocess.Popen): pipe = process.stdout for line in iter(pipe.readline, b''): - try: line = line.decode('utf-8').rstrip() - except Exception as e: line = f"DECODING EXCEPTION: {e}" + try: + line = line.decode('utf-8').rstrip() + except Exception as e: + line = f"DECODING EXCEPTION: {e}" self.logger(Level.INFO, f"[CLI]{line}") diff --git a/Interfaces/management.py b/Interfaces/management.py index be293c6..95ce3dd 100644 --- a/Interfaces/management.py +++ b/Interfaces/management.py @@ -6,6 +6,7 @@ from Facility import Facility + class Management: sliceManager = None @@ -16,7 +17,7 @@ def HasResources(cls, owner: 'ExecutorBase', localResources: List[str], - Available indicates that the required local resources are locked and can be used, and there are enough on all VIMs to fit the network services. - A feasible value of False indicates that the network services can never fit on the VIMs due to - their total resoutces. + their total resources. """ if len(networkServices) != 0: @@ -47,7 +48,6 @@ def HasResources(cls, owner: 'ExecutorBase', localResources: List[str], return False, True # Execution possible, but not enough resources at the moment return Facility.TryLockResources(localResources, owner, exclusive), True - @classmethod def ReleaseLocalResources(cls, owner: 'ExecutorBase', localResources: List[str]): Facility.ReleaseResources(localResources, owner) @@ -144,7 +144,8 @@ def GetNsdData(self, nsd: str) -> Tuple[Optional[str], Optional[str], Optional[M if isinstance(data, list): if len(data) != 0: data = data[0] - else: raise RuntimeError("Received an empty list") + else: + raise RuntimeError("Received an empty list") try: flavor = data["flavor"] return data['nsd-name'], data['nsd-id'], Metal(cpu=flavor["vcpu-count"], diff --git a/Scheduler/__init__.py b/Scheduler/__init__.py index 129c0eb..08fb7d4 100644 --- a/Scheduler/__init__.py +++ b/Scheduler/__init__.py @@ -37,18 +37,21 @@ def _showValidation(name, validation): HeartBeat.Initialize() from Scheduler.execution import bp as ExecutionBp -app.register_blueprint(ExecutionBp, url_prefix='/execution') +app.register_blueprint(ExecutionBp, url_prefix='/execution', name='deprecatedExecutionApi') +app.register_blueprint(ExecutionBp, url_prefix='/elcm/api/v1/execution') from Scheduler.dispatcher import bp as DispatcherBp app.register_blueprint(DispatcherBp, url_prefix='/api/v0', name='deprecatedDispatcherApi') -app.register_blueprint(DispatcherBp, url_prefix='/experiment') +app.register_blueprint(DispatcherBp, url_prefix='/elcm/api/v1/experiment') from Scheduler.facility import bp as FacilityBp -app.register_blueprint(FacilityBp, url_prefix='/facility') +app.register_blueprint(FacilityBp, url_prefix='/facility', name='deprecatedFacilityApi') +app.register_blueprint(FacilityBp, url_prefix='/elcm/api/v1/facility') if config.EastWest.Enabled: from Scheduler.east_west import bp as EastwestBp - app.register_blueprint(EastwestBp, url_prefix='/distributed') + app.register_blueprint(EastwestBp, url_prefix='/distributed', name='deprecatedEastWestApi') + app.register_blueprint(EastwestBp, url_prefix='/elcm/api/v1/distributed') Log.I(f'Optional East/West interface is {Log.State(config.EastWest.Enabled)}') diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 82b4f27..014b5da 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -10,7 +10,7 @@ @bp.route('/cancel') # Deprecated -@bp.route('', methods=["DELETE"]) +# @bp.route('', methods=["DELETE"]) def cancel(executionId: int): ExecutionQueue.Cancel(executionId) flash(f'Cancelled execution {executionId}', 'info') @@ -138,7 +138,14 @@ def kpis(executionId: int): for testcase in descriptor.TestCases: kpis.update(Facility.GetTestCaseKPIs(testcase)) - return jsonify({"KPIs": sorted(kpis)}) + res = [] + for kpi in sorted(kpis): + measurement, name, kind, description = kpi + res.append({ + 'Measurement': measurement, 'KPI': name, 'Type': kind, 'Description': description + }) + + return jsonify({"KPIs": res}) else: return f"Execution {executionId} not found", 404 diff --git a/docs/A1_ENDPOINTS.md b/docs/A1_ENDPOINTS.md index 3563e63..bcd6582 100644 --- a/docs/A1_ENDPOINTS.md +++ b/docs/A1_ENDPOINTS.md @@ -1,11 +1,11 @@ # REST Endpoints > ⚠ The endpoints of the ELCM are not expected to be exposed to the Internet and may leak information. For user -> management and authorization always use a different front-end, such as the Dispatcher. +> management and authorization always use a different front-end, such as the Dispatcher. ## Experiment management endpoints -### [POST] `/experiment/run` +### [POST] `/elcm/api/v1/experiment/run` > *[POST] `/api/v0/run` (Deprecated)* Creates and queues a new experiment execution, based on the contents of the received Experiment Descriptor (JSON). @@ -15,8 +15,8 @@ Replies with the following response JSON: ``` Where is a unique execution identification that can be used as input in other endpoints. -### [GET] `/execution//status` -> *[GET] `/execution//json` (Deprecated)* +### [GET] `/elcm/api/v1/execution//status` +> *[GET] `/elcm/api/v1/execution//json` (Deprecated)* Returns a JSON that contains general information about the status of the selected execution id, with the following format: @@ -28,7 +28,7 @@ format: “Verdict”: } ``` -### [GET] `/execution//logs` +### [GET] `/elcm/api/v1/execution//logs` Returns a JSON that contains all the log messages generated by the execution, separated by stage: ```text @@ -38,37 +38,46 @@ Returns a JSON that contains all the log messages generated by the execution, se “PostRun”: } ``` -### [GET] `/execution//results` +### [GET] `/elcm/api/v1/execution//results` Returns a compressed file that includes the logs and all files generated by the experiment execution. -### [GET] `/execution//descriptor` +### [GET] `/elcm/api/v1/execution//descriptor` Returns a copy of the Experiment Descriptor that was used to define the execution. -### [GET] '/execution//kpis' +### [GET] `/elcm/api/v1/execution//kpis` -Returns a dictionary with a single `KPIs` key, containing a list of pairs (`measurement`, `kpi`) that are considered of -interest. +Returns a dictionary with a single `KPIs` key, containing a list of objects that describe KPIs that are considered of +interest. The objects have the following format: + +```text +{ + "Measurement": , + "KPI": , + "Type": , + "Description": +} +``` > These values can be used as part of queries to the [Analytics Module](https://github.com/5genesis/Analytics), in order > to extract a sub-set of important KPIs from all the generated measurements. -### [DELETE] `/execution/` -> *[GET] `/execution//cancel` (Deprecated)* +### [DELETE] `/elcm/api/v1/execution/` +> *[GET] `/elcm/api/v1/execution//cancel` (Deprecated)* Marks the selected execution for cancellation. The execution will be cancelled after finalization of the current task. ## Facility information -### [GET] `/facility/baseSliceDescriptors` +### [GET] `/elcm/api/v1/facility/baseSliceDescriptors` Returns a list of available Base Slice Descriptors, with the following format: ```json { "SliceDescriptors": [] } ``` -### [GET] `/facility/testcases` +### [GET] `/elcm/api/v1/facility/testcases` Returns a list of available UEs, with the following format: ```text @@ -86,14 +95,14 @@ Returns a list of available UEs, with the following format: } ``` -### [GET] `/facility/ues` +### [GET] `/elcm/api/v1/facility/ues` Returns a list of available UEs, with the following format: ```json { "UEs": [] } ``` -### [GET] `/facility/resource_status` +### [GET] `/elcm/api/v1/facility/resource_status` Returns a list of available Resources, separated by current usage status: ```json @@ -101,7 +110,7 @@ Returns a list of available Resources, separated by current usage status: "Idle": [] } ``` -### [GET] `/facility/scenarios` +### [GET] `/elcm/api/v1/facility/scenarios` Returns a list of available Scenarios, with the following format: ```json diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index a3988fa..da1e413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -Flask>=1.0.2 -Flask-Bootstrap>=3.3.7.1 -Flask-Moment>=0.6.0 -flask-paginate>=0.5.2 -influxdb>=5.2.2 -psutil>=5.6.1 -python-dotenv>=0.10.1 -PyYAML>=3.12 +Flask~=2.3.0 +Flask-Bootstrap~=3.3.7.1 +Flask-Moment~=1.0.5 +flask-paginate~=2022.1.8 +influxdb~=5.3.1 +psutil~=5.9.5 +python-dotenv~=1.0.0 +PyYAML~=6.0 diff --git a/start.sh b/start.sh old mode 100644 new mode 100755 index 791057d..f72596e --- a/start.sh +++ b/start.sh @@ -8,6 +8,6 @@ fi echo Starting ELCM on port $port source ./venv/bin/activate -export SECRET_KEY='' +export SECRET_KEY='super secret' flask run --host 0.0.0.0 --port $port deactivate diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..fce7212 --- /dev/null +++ b/test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo "Hola Mundo !!!"