From 039d2f081c86ba0d7853ed1e92e81b6fab38394a Mon Sep 17 00:00:00 2001 From: "sandipsamal117@gmail.com" Date: Wed, 26 Jul 2023 19:16:28 -0400 Subject: [PATCH 1/4] List failed plugin names in analysis errors --- app/controllers/subprocesses/status.py | 6 +++++- app/controllers/subprocesses/wf_manager.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/controllers/subprocesses/status.py b/app/controllers/subprocesses/status.py index 16a29e4..bc9a6b1 100644 --- a/app/controllers/subprocesses/status.py +++ b/app/controllers/subprocesses/status.py @@ -202,6 +202,10 @@ def _get_feed_status(request: WorkflowRequestSchema, feed_id: str) -> dict: # search for feed resp = cl.getFeed({"id": feed_id, "name_exact": feed_name}) + if resp["errored_jobs"] or resp["cancelled_jobs"]: + l_inst_resp = cl.getPluginInstances({"feed_id": feed_id}) + l_error = [d_instance['plugin_name'] for d_instance in l_inst_resp['data'] if d_instance['status']=='finishedWithError'] + resp["errored_plugins"] = str(l_error) return resp except Exception as ex: return {"error": Error.cube.value + str(ex)} @@ -230,7 +234,7 @@ def get_analysis_status(response: dict) -> dict: analysis_details['progress'] = str(feed_progress) + "%" if errored > 0 or cancelled > 0: - analysis_details["error"] = str(errored + cancelled) + " job(s) failed" + analysis_details["error"] = f"{(errored + cancelled)} job(s) failed : {response['errored_plugins']}" if feed_progress == 100: analysis_details["state"] = State.COMPLETED else: diff --git a/app/controllers/subprocesses/wf_manager.py b/app/controllers/subprocesses/wf_manager.py index 31eea5a..0e16e12 100644 --- a/app/controllers/subprocesses/wf_manager.py +++ b/app/controllers/subprocesses/wf_manager.py @@ -92,7 +92,7 @@ def manage_workflow(db_key: str, test: bool): try: do_cube_start_analysis(pl_inst_id, request, cube_url) except Exception as ex: - logging.info(Error.analysis.value) + logging.info(Error.analysis.value + str(ex)) workflow.response.error = Error.analysis.value + str(ex) workflow.response.status = False update_workflow(key, workflow) From a91322052e65c5b5b42e8b683706283ece339301 Mon Sep 17 00:00:00 2001 From: Sandip Samal Date: Mon, 31 Jul 2023 13:49:17 -0400 Subject: [PATCH 2/4] updated pfdcm storage APIs --- app/controllers/auth.py | 2 +- app/controllers/pfdcm.py | 8 ++++---- app/main.py | 6 +++--- app/routes/pfdcm.py | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/controllers/auth.py b/app/controllers/auth.py index 4a366eb..c3f4dc4 100644 --- a/app/controllers/auth.py +++ b/app/controllers/auth.py @@ -7,7 +7,7 @@ from fastapi.security import OAuth2PasswordBearer from pydantic import ValidationError -ACCESS_TOKEN_EXPIRE_MINUTES = 600 # 600 minutes +ACCESS_TOKEN_EXPIRE_MINUTES = 3 # 600 minutes ALGORITHM = "HS256" JWT_SECRET_KEY = auth.JWT_SECRET_KEY diff --git a/app/controllers/pfdcm.py b/app/controllers/pfdcm.py index c938fc1..4b8bdd4 100644 --- a/app/controllers/pfdcm.py +++ b/app/controllers/pfdcm.py @@ -102,17 +102,17 @@ async def cube_list(service_name: str) -> list[str]: return d_results -# Get the list of `swift` servers available in a pfdcm instance -async def swift_list(service_name: str) -> list[str]: +# Get the list of `storage` services available in a pfdcm instance +async def storage_list(service_name: str) -> list[str]: d_results = [] pfdcm_server = retrieve_pfdcm(service_name) if not pfdcm_server: return d_results pfdcm_url = pfdcm_server['service_address'] - pfdcm_swift_list_api = f'{pfdcm_url}/SMDB/swift/list/' + pfdcm_storage_list_api = f'{pfdcm_url}/SMDB/storage/list/' async with httpx.AsyncClient() as client: try: - response = await client.get(pfdcm_swift_list_api) + response = await client.get(pfdcm_storage_list_api) d_results = json.loads(response.text) return d_results except: diff --git a/app/main.py b/app/main.py index 5de6ff3..71680c0 100644 --- a/app/main.py +++ b/app/main.py @@ -22,9 +22,9 @@ * **Get a `hello` response from a pfdcm instance.** * **Know `about` a pfdcm instance.** -* **Get the list of the names of all `cube` instances available in a pfdcm instance.** -* **Get the list of the names of all `swift` instances available in a pfdcm instance.** -* **Get the list of the names of all `PACS` instances available in a pfdcm instance.** +* **Get the list of the names of all `cube` services available in a pfdcm instance.** +* **Get the list of the names of all `storage` services available in a pfdcm instance.** +* **Get the list of the names of all `PACS` services available in a pfdcm instance.** ## Workflow diff --git a/app/routes/pfdcm.py b/app/routes/pfdcm.py index 86a38ad..85c6665 100644 --- a/app/routes/pfdcm.py +++ b/app/routes/pfdcm.py @@ -92,7 +92,7 @@ async def get_about_pfdcm(service_name: str) -> PfdcmQueryResponseSchema: ) async def cube_service_list(service_name: str) -> list[str]: """ - Get the list of PACS services registered to a `pfdcm` instance by providing its service name + Get the list of CUBE services registered to a `pfdcm` instance by providing its service name """ response = await pfdcm.cube_list(service_name) if not response: @@ -101,15 +101,15 @@ async def cube_service_list(service_name: str) -> list[str]: @router.get( - "/{service_name}/swift/list", + "/{service_name}/storage/list", response_description="About PFDCM", - summary="Get the list of swift services registered to a `pfdcm` instance" + summary="Get the list of storage services registered to a `pfdcm` instance" ) -async def swift_service_list(service_name: str) -> list[str]: +async def storage_service_list(service_name: str) -> list[str]: """ - Get the list of PACS services registered to a `pfdcm` instance by providing its service name + Get the list of storage services registered to a `pfdcm` instance by providing its service name """ - response = await pfdcm.swift_list(service_name) + response = await pfdcm.storage_list(service_name) if not response: raise HTTPException(status_code=404, detail=f"Unable to reach endpoints of {service_name}") return response @@ -136,4 +136,4 @@ async def delete_pfdcm(service_name: str): Delete a pfdcm record from the DB """ response = await pfdcm.delete_pfdcm(service_name) - return response \ No newline at end of file + return response From 8d917bb17af3f2f07bb8ec73f6d26595c4e7b7c2 Mon Sep 17 00:00:00 2001 From: "sandipsamal117@gmail.com" Date: Mon, 31 Jul 2023 18:02:45 -0400 Subject: [PATCH 3/4] Improved error logging --- app/controllers/auth.py | 2 +- app/controllers/subprocesses/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/auth.py b/app/controllers/auth.py index c3f4dc4..e86af2e 100644 --- a/app/controllers/auth.py +++ b/app/controllers/auth.py @@ -7,7 +7,7 @@ from fastapi.security import OAuth2PasswordBearer from pydantic import ValidationError -ACCESS_TOKEN_EXPIRE_MINUTES = 3 # 600 minutes +ACCESS_TOKEN_EXPIRE_MINUTES = 600 # minutes to expire ALGORITHM = "HS256" JWT_SECRET_KEY = auth.JWT_SECRET_KEY diff --git a/app/controllers/subprocesses/status.py b/app/controllers/subprocesses/status.py index bc9a6b1..6909a9f 100644 --- a/app/controllers/subprocesses/status.py +++ b/app/controllers/subprocesses/status.py @@ -204,7 +204,7 @@ def _get_feed_status(request: WorkflowRequestSchema, feed_id: str) -> dict: resp = cl.getFeed({"id": feed_id, "name_exact": feed_name}) if resp["errored_jobs"] or resp["cancelled_jobs"]: l_inst_resp = cl.getPluginInstances({"feed_id": feed_id}) - l_error = [d_instance['plugin_name'] for d_instance in l_inst_resp['data'] if d_instance['status']=='finishedWithError'] + l_error = [d_instance['plugin_name'] for d_instance in l_inst_resp['data'] if d_instance['status']=='finishedWithError' or d_instance['status'] == 'cancelled'] resp["errored_plugins"] = str(l_error) return resp except Exception as ex: From 59e04c1330e7e250cda0eaf6361cd00c3a45b2b0 Mon Sep 17 00:00:00 2001 From: "sandipsamal117@gmail.com" Date: Thu, 3 Aug 2023 11:52:44 -0400 Subject: [PATCH 4/4] Added API to show prod workflows --- app/config.py | 2 +- app/routes/workflow.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 8e394bf..eb9e7c0 100644 --- a/app/config.py +++ b/app/config.py @@ -3,7 +3,7 @@ class Settings(BaseSettings): pflink_mongodb: MongoDsn = 'mongodb://localhost:27017' - version: str = "3.5.4" + version: str = "3.6.0" class Auth(BaseSettings): diff --git a/app/routes/workflow.py b/app/routes/workflow.py index c7cb86d..f3af2df 100644 --- a/app/routes/workflow.py +++ b/app/routes/workflow.py @@ -22,6 +22,15 @@ async def create_workflow(data: WorkflowRequestSchema) -> WorkflowStatusResponse return response +@router.get("/list", response_description="All workflows retrieved") +async def get_workflows(): + """ + Fetch all workflows currently present in the database + """ + workflows = workflow.retrieve_workflows() + return workflows + + @router.delete("", response_description="All workflows deleted") async def delete_workflows(): """