From b77748d239df3a175cd521deb4bbe93de0f86198 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 31 Jul 2024 10:51:16 -0400 Subject: [PATCH 01/22] Adding additional metadata to typedef and modifying metadata value field schema --- weaver/typedefs.py | 8 ++++++++ weaver/wps_restapi/swagger_definitions.py | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/weaver/typedefs.py b/weaver/typedefs.py index 77a3dc463..b90aadd48 100644 --- a/weaver/typedefs.py +++ b/weaver/typedefs.py @@ -220,6 +220,14 @@ class CWL_SchemaName(Protocol): "$namespaces": NotRequired[CWL_Namespace], "$schemas": NotRequired[Dict[str, str]], "$graph": NotRequired[CWL_Graph], + "s:author": NotRequired[List[Dict[str, JSON]]], + "s:citation": NotRequired[List[Dict[str, str]]], + "s:codeRepository": NotRequired[Link], + "s:contributor": NotRequired[List[Dict[str, JSON]]], + "s:dateCreated": NotRequired[datetime], + "s:license": NotRequired[Optional[Union[Dict[str, JSON], Link]]], + "s:releaseNotes": NotRequired[Optional[Union[str, Link]]], + "s:version": NotRequired[Optional[Union[str, Number]]], }, total=False) CWL_WorkflowStepPackage = TypedDict("CWL_WorkflowStepPackage", { "id": str, # reference ID of the package diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index c71ce49e8..4e97120d5 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -804,6 +804,13 @@ class Link(LinkRelationship, LinkBase): _schema = f"{OGC_API_COMMON_PART1_SCHEMAS}/link.json" _schema_include_deserialize = False # only in OpenAPI otherwise too verbose +class MetadataValueField(OneOfKeywordSchema): + _one_of = [ + # pointer to a file or JSON schema relative item (as in OpenAPI definitions) + ExtendedSchemaNode(String(), ...), + # literal JSON schema, permissive since it can be anything + PermissiveMappingSchema(description="Flexible schema definition for the metadata value.") + ] class MetadataValue(NotKeywordSchema, ValueLanguage, MetadataBase): _not = [ @@ -812,7 +819,7 @@ class MetadataValue(NotKeywordSchema, ValueLanguage, MetadataBase): LinkRelationship(description="Field 'rel' must refer to a link reference with 'href'."), LinkLanguage(description="Field 'hreflang' must refer to a link reference with 'href'."), ] - value = ExtendedSchemaNode(String(), description="Plain text value of the information.") + value = MetadataValueField(description="Explicit schema definition of the metadata value.") class MetadataLink(Link): From 5a26ffeb519763e08102bd4247aca6744ca68381 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 31 Jul 2024 10:53:26 -0400 Subject: [PATCH 02/22] Adding list of supported metadata and begining of parsing it --- weaver/processes/wps_package.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 7e6ba5c21..7d44c0124 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -220,6 +220,16 @@ PACKAGE_SCHEMA_CACHE = {} # type: Dict[str, Tuple[str, str]] +SUPPORTED_METADATA_MAPPING = [ + "s:author" + "s:citation", + "s:codeRepository", + "s:contributor", + "s:dateCreated", + "s:license", + "s:releaseNotes", + "s:version" +] def get_status_location_log_path(status_location, out_dir=None): # type: (str, Optional[str]) -> str @@ -708,6 +718,30 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): set(wps_package_metadata.get("keywords", [])) | set(cwl_package_package.get("s:keywords", [])) ) + for metadata_mapping in SUPPORTED_METADATA_MAPPING: + # example s:author #TODO regroup similar parsing together + if metadata_mapping in cwl_package_package and isinstance(cwl_package_package["s:author"], list): + + metadata = wps_package_metadata.get("metadata", []) + for objects in cwl_package_package[metadata_mapping]: + class_name = objects["class"].strip("s:") + value = { + "$schema": f"https://schema.org/{class_name}" + } + for key, val in objects.items(): + if key.startswith("s:"): + value[key.strip("s:")] = val + metadata.append({ + "role": metadata_mapping.strip("s;"), + "value": value + }) + # specific use case with a different mapping + if metadata_mapping in cwl_package_package and isinstance(cwl_package_package["s:version"], list): + wps_package_metadata["s:version"] = list( + set(wps_package_metadata.get("version", [])) | set(cwl_package_package.get("s:version", [])) + ) + + def _patch_wps_process_description_url(reference, process_hint): # type: (str, Optional[JSON]) -> str From 7b758f4432158041e9b5cb41f530923be35254e6 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 21 Aug 2024 13:21:23 -0400 Subject: [PATCH 03/22] fixing schema warning --- weaver/wps_restapi/swagger_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 4e97120d5..846f9e5c0 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -807,7 +807,7 @@ class Link(LinkRelationship, LinkBase): class MetadataValueField(OneOfKeywordSchema): _one_of = [ # pointer to a file or JSON schema relative item (as in OpenAPI definitions) - ExtendedSchemaNode(String(), ...), + ExtendedSchemaNode(String()), # literal JSON schema, permissive since it can be anything PermissiveMappingSchema(description="Flexible schema definition for the metadata value.") ] From 52eddf498c45a987ab120b8af0f2d661f2985f01 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 21 Aug 2024 14:52:28 -0400 Subject: [PATCH 04/22] Fixing author metadata parsing and adding test --- tests/processes/test_wps_package.py | 40 ++++++++++++++++++++++++++++- weaver/processes/wps_package.py | 12 ++++----- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index bfc8ec270..9ee1fd637 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -48,7 +48,8 @@ _update_package_compatibility, format_extension_validator, get_application_requirement, - mask_process_inputs + mask_process_inputs, + _update_package_metadata ) from weaver.wps.service import WorkerRequest @@ -667,3 +668,40 @@ def test_mask_process_inputs(inputs, expect): def test_format_extension_validator_basic(data_input, mode, expect): # type: (Any, int, bool) -> None assert format_extension_validator(data_input, mode) == expect + +@pytest.mark.parametrize("original, expected", [ + ( + # Test author metadata with empty wps_package + { + "cwl_package_package": { + "s:author": [ + {"class": "s:Person", "s:name": "John Doe", "s:affiliation": "Example Inc."} + ], + + }, + "wps_package_metadata": {} + }, + { + "metadata": [ + { + "role": "author", + "value": { + "$schema": "https://schema.org/Person", + "name": "John Doe", + "affiliation": "Example Inc." + } + } + ] + } + ), + +]) +def test_process_metadata(original, expected): + # type: (CWL, CWL) -> None + cwl_package_package = original["cwl_package_package"] + wps_package_metadata = original["wps_package_metadata"] + _update_package_metadata(wps_package_metadata, cwl_package_package) + # Assertions + for key in expected: + assert key in wps_package_metadata + assert sorted(wps_package_metadata[key]) == sorted(expected[key]) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 7d44c0124..9a4fed4d0 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -221,7 +221,7 @@ PACKAGE_SCHEMA_CACHE = {} # type: Dict[str, Tuple[str, str]] SUPPORTED_METADATA_MAPPING = [ - "s:author" + "s:author", "s:citation", "s:codeRepository", "s:contributor", @@ -720,7 +720,7 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): for metadata_mapping in SUPPORTED_METADATA_MAPPING: # example s:author #TODO regroup similar parsing together - if metadata_mapping in cwl_package_package and isinstance(cwl_package_package["s:author"], list): + if metadata_mapping in cwl_package_package and metadata_mapping == "s:author": metadata = wps_package_metadata.get("metadata", []) for objects in cwl_package_package[metadata_mapping]: @@ -732,15 +732,15 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): if key.startswith("s:"): value[key.strip("s:")] = val metadata.append({ - "role": metadata_mapping.strip("s;"), + "role": metadata_mapping.strip("s:"), "value": value }) # specific use case with a different mapping - if metadata_mapping in cwl_package_package and isinstance(cwl_package_package["s:version"], list): - wps_package_metadata["s:version"] = list( + if metadata_mapping in cwl_package_package and metadata_mapping == "s:version": + wps_package_metadata["version"] = ( set(wps_package_metadata.get("version", [])) | set(cwl_package_package.get("s:version", [])) ) - + wps_package_metadata["metadata"] = metadata def _patch_wps_process_description_url(reference, process_hint): From 493edbd9384ed6d1a4cea97974bd5de3bb89b113 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 21 Aug 2024 15:10:30 -0400 Subject: [PATCH 05/22] Changing docker-compose commands to docker compose --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 9aa4fe7c2..436a83d64 100644 --- a/Makefile +++ b/Makefile @@ -802,22 +802,22 @@ DOCKER_TEST_EXEC_ARGS ?= .PHONY: docker-test docker-test: docker-build stop ## execute smoke test of the built images (validate that they boots and reply) @echo "Smoke test of built application docker images" - docker-compose $(DOCKER_TEST_COMPOSES) up -d + docker compose $(DOCKER_TEST_COMPOSES) up -d sleep 10 ## leave some time to boot @echo "Pinging Weaver API entrypoint to validate response..." @curl localhost:4001 | grep "Weaver Information" || \ - ( docker-compose $(DOCKER_TEST_COMPOSES) logs weaver worker || true && \ - docker-compose $(DOCKER_TEST_COMPOSES) stop; exit 1 ) - docker-compose $(DOCKER_TEST_COMPOSES) exec $(DOCKER_TEST_EXEC_ARGS) weaver bash /tests/run_tests.sh - docker-compose $(DOCKER_TEST_COMPOSES) stop + ( docker compose $(DOCKER_TEST_COMPOSES) logs weaver worker || true && \ + docker compose $(DOCKER_TEST_COMPOSES) stop; exit 1 ) + docker compose $(DOCKER_TEST_COMPOSES) exec $(DOCKER_TEST_EXEC_ARGS) weaver bash /tests/run_tests.sh + docker compose $(DOCKER_TEST_COMPOSES) stop .PHONY: docker-stat -docker-stat: ## query docker-compose images status (from 'docker-test') - docker-compose $(DOCKER_TEST_COMPOSES) ps +docker-stat: ## query docker compose images status (from 'docker-test') + docker compose $(DOCKER_TEST_COMPOSES) ps .PHONY: docker-clean docker-clean: ## remove all built docker images (only matching current/latest versions) - docker-compose $(DOCKER_TEST_COMPOSES) down || true + docker compose $(DOCKER_TEST_COMPOSES) down || true docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)-manager" || true docker rmi -f "$(DOCKER_REPO):latest-manager" || true docker rmi -f "$(APP_NAME):$(APP_VERSION)-manager" || true From de023cd0c6849023421401f1b5a7c74fe9b4fa63 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 21 Aug 2024 15:17:52 -0400 Subject: [PATCH 06/22] Removing version of yaml for smoke test, obsolete in v2 --- tests/smoke/docker-compose.smoke-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/smoke/docker-compose.smoke-test.yml b/tests/smoke/docker-compose.smoke-test.yml index 36e6e541f..f01602065 100644 --- a/tests/smoke/docker-compose.smoke-test.yml +++ b/tests/smoke/docker-compose.smoke-test.yml @@ -1,4 +1,3 @@ -version: "3.4" services: mongodb: image: mongo:5.0 From b7b9940339729819d9fedc2123bfaaa1e4bde77e Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 21 Aug 2024 15:24:29 -0400 Subject: [PATCH 07/22] deplacing set of metadata after loop --- weaver/processes/wps_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 9a4fed4d0..5d63c07d8 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -735,12 +735,12 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): "role": metadata_mapping.strip("s:"), "value": value }) + wps_package_metadata["metadata"] = metadata # specific use case with a different mapping if metadata_mapping in cwl_package_package and metadata_mapping == "s:version": wps_package_metadata["version"] = ( set(wps_package_metadata.get("version", [])) | set(cwl_package_package.get("s:version", [])) ) - wps_package_metadata["metadata"] = metadata def _patch_wps_process_description_url(reference, process_hint): From 9e692e49f84ce7bbec0d198ccfbfc11f027eca01 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 10:07:58 -0400 Subject: [PATCH 08/22] Adding test and href url parsing for metadata --- tests/processes/test_wps_package.py | 48 ++++++++++++++++++++++- tests/smoke/docker-compose.smoke-test.yml | 1 + weaver/processes/wps_package.py | 45 ++++++++++++--------- 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index 9ee1fd637..5b5a993dd 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -677,7 +677,7 @@ def test_format_extension_validator_basic(data_input, mode, expect): "s:author": [ {"class": "s:Person", "s:name": "John Doe", "s:affiliation": "Example Inc."} ], - + }, "wps_package_metadata": {} }, @@ -694,7 +694,51 @@ def test_format_extension_validator_basic(data_input, mode, expect): ] } ), - + ( + # Test codeRepository + { + "cwl_package_package": { + "s:codeRepository": "https://gitlab.com/", + + }, + "wps_package_metadata": {} + }, + { + "metadata": [ + { + "rel": "codeRepository", + "href": "https://gitlab.com/" + } + ] + } + ), + ( + # Test Version with existing metadata + { + "cwl_package_package": { + "s:version": "1.0", + + }, + "wps_package_metadata": { + "metadata": [ + { + "rel": "codeRepository", + "href": "https://gitlab.com/" + } + ] + } + }, + { + "metadata": [ + { + "rel": "codeRepository", + "href": "https://gitlab.com/" + }, + + ], + "version":"1.0" + } + ), ]) def test_process_metadata(original, expected): # type: (CWL, CWL) -> None diff --git a/tests/smoke/docker-compose.smoke-test.yml b/tests/smoke/docker-compose.smoke-test.yml index f01602065..cb770cc0a 100644 --- a/tests/smoke/docker-compose.smoke-test.yml +++ b/tests/smoke/docker-compose.smoke-test.yml @@ -1,3 +1,4 @@ +version: "3.8" services: mongodb: image: mongo:5.0 diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 5d63c07d8..56f8d2af9 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -228,7 +228,6 @@ "s:dateCreated", "s:license", "s:releaseNotes", - "s:version" ] def get_status_location_log_path(status_location, out_dir=None): @@ -718,29 +717,37 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): set(wps_package_metadata.get("keywords", [])) | set(cwl_package_package.get("s:keywords", [])) ) - for metadata_mapping in SUPPORTED_METADATA_MAPPING: - # example s:author #TODO regroup similar parsing together - if metadata_mapping in cwl_package_package and metadata_mapping == "s:author": + # specific use case with a different mapping + if "s:version" in cwl_package_package: + wps_package_metadata["version"] = str( + str(cwl_package_package.get("s:version", "")) or str(wps_package_metadata.get("version")) + ) + for metadata_mapping in SUPPORTED_METADATA_MAPPING: + if metadata_mapping in cwl_package_package: metadata = wps_package_metadata.get("metadata", []) - for objects in cwl_package_package[metadata_mapping]: - class_name = objects["class"].strip("s:") - value = { - "$schema": f"https://schema.org/{class_name}" - } - for key, val in objects.items(): - if key.startswith("s:"): - value[key.strip("s:")] = val + if (isinstance((cwl_package_package[metadata_mapping]),str) + and urlparse(cwl_package_package[metadata_mapping]) != "" + ): metadata.append({ - "role": metadata_mapping.strip("s:"), - "value": value + "rel": metadata_mapping.strip("s:"), + "href": cwl_package_package[metadata_mapping] }) + else: + for objects in cwl_package_package[metadata_mapping]: + class_name = objects["class"].strip("s:") + value = { + "$schema": f"https://schema.org/{class_name}" + } + for key, val in objects.items(): + if key.startswith("s:"): + value[key.strip("s:")] = val + metadata.append({ + "role": metadata_mapping.strip("s:"), + "value": value + }) + wps_package_metadata["metadata"] = metadata - # specific use case with a different mapping - if metadata_mapping in cwl_package_package and metadata_mapping == "s:version": - wps_package_metadata["version"] = ( - set(wps_package_metadata.get("version", [])) | set(cwl_package_package.get("s:version", [])) - ) def _patch_wps_process_description_url(reference, process_hint): From c7f4626c503da86a3f3b8b32484397132eb6971d Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 10:21:13 -0400 Subject: [PATCH 09/22] Fixing pep8 --- tests/processes/test_wps_package.py | 6 +++--- weaver/processes/wps_package.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index 5b5a993dd..ea81a151f 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -669,6 +669,7 @@ def test_format_extension_validator_basic(data_input, mode, expect): # type: (Any, int, bool) -> None assert format_extension_validator(data_input, mode) == expect + @pytest.mark.parametrize("original, expected", [ ( # Test author metadata with empty wps_package @@ -699,7 +700,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "cwl_package_package": { "s:codeRepository": "https://gitlab.com/", - }, "wps_package_metadata": {} }, @@ -716,10 +716,10 @@ def test_format_extension_validator_basic(data_input, mode, expect): # Test Version with existing metadata { "cwl_package_package": { - "s:version": "1.0", + "s:version": "1.0" }, - "wps_package_metadata": { + "wps_package_metadata": { "metadata": [ { "rel": "codeRepository", diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 56f8d2af9..333d4c3aa 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -726,9 +726,9 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): for metadata_mapping in SUPPORTED_METADATA_MAPPING: if metadata_mapping in cwl_package_package: metadata = wps_package_metadata.get("metadata", []) - if (isinstance((cwl_package_package[metadata_mapping]),str) + if (isinstance((cwl_package_package[metadata_mapping]), str) and urlparse(cwl_package_package[metadata_mapping]) != "" - ): + ): metadata.append({ "rel": metadata_mapping.strip("s:"), "href": cwl_package_package[metadata_mapping] From 4d6adf9552c621517e69ddaae7cde8b58417e2c4 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 10:37:22 -0400 Subject: [PATCH 10/22] Fixing pep8 indent --- tests/processes/test_wps_package.py | 12 ++++++------ weaver/processes/wps_package.py | 4 ++-- weaver/wps_restapi/swagger_definitions.py | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index ea81a151f..debd8f3e3 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -721,11 +721,11 @@ def test_format_extension_validator_basic(data_input, mode, expect): }, "wps_package_metadata": { "metadata": [ - { - "rel": "codeRepository", - "href": "https://gitlab.com/" - } - ] + { + "rel": "codeRepository", + "href": "https://gitlab.com/" + } + ] } }, { @@ -736,7 +736,7 @@ def test_format_extension_validator_basic(data_input, mode, expect): }, ], - "version":"1.0" + "version": "1.s0" } ), ]) diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 333d4c3aa..112840d04 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -230,6 +230,7 @@ "s:releaseNotes", ] + def get_status_location_log_path(status_location, out_dir=None): # type: (str, Optional[str]) -> str log_path = f"{os.path.splitext(status_location)[0]}.log" @@ -727,8 +728,7 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): if metadata_mapping in cwl_package_package: metadata = wps_package_metadata.get("metadata", []) if (isinstance((cwl_package_package[metadata_mapping]), str) - and urlparse(cwl_package_package[metadata_mapping]) != "" - ): + and urlparse(cwl_package_package[metadata_mapping]) != ""): metadata.append({ "rel": metadata_mapping.strip("s:"), "href": cwl_package_package[metadata_mapping] diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 846f9e5c0..ffd1abadb 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -804,6 +804,7 @@ class Link(LinkRelationship, LinkBase): _schema = f"{OGC_API_COMMON_PART1_SCHEMAS}/link.json" _schema_include_deserialize = False # only in OpenAPI otherwise too verbose + class MetadataValueField(OneOfKeywordSchema): _one_of = [ # pointer to a file or JSON schema relative item (as in OpenAPI definitions) @@ -812,6 +813,7 @@ class MetadataValueField(OneOfKeywordSchema): PermissiveMappingSchema(description="Flexible schema definition for the metadata value.") ] + class MetadataValue(NotKeywordSchema, ValueLanguage, MetadataBase): _not = [ # make sure value metadata does not allow 'rel' and 'hreflang' reserved for link reference From 3ecceb673df15f6e11cb29cd924e696c8d1ec0ee Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 10:47:31 -0400 Subject: [PATCH 11/22] Fix test typo --- tests/processes/test_wps_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index debd8f3e3..49d4d0d2c 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -736,7 +736,7 @@ def test_format_extension_validator_basic(data_input, mode, expect): }, ], - "version": "1.s0" + "version": "1.0" } ), ]) From 07afac06ac9b1a979d37ec9c3284bca75414dc82 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 10:57:45 -0400 Subject: [PATCH 12/22] Fixing import sorting --- tests/processes/test_wps_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index 49d4d0d2c..df8392eb9 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -46,10 +46,10 @@ WpsPackage, _load_package_content, _update_package_compatibility, + _update_package_metadata, format_extension_validator, get_application_requirement, mask_process_inputs, - _update_package_metadata ) from weaver.wps.service import WorkerRequest From b74f3ec1db144796a98ce5980ee718455887da7c Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 29 Aug 2024 11:07:10 -0400 Subject: [PATCH 13/22] Removing comma --- tests/processes/test_wps_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index df8392eb9..d09d5b665 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -49,7 +49,7 @@ _update_package_metadata, format_extension_validator, get_application_requirement, - mask_process_inputs, + mask_process_inputs ) from weaver.wps.service import WorkerRequest From e635e9cec39094c5c67d15f6ad423994d3fc8f6b Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Tue, 10 Sep 2024 13:33:47 -0400 Subject: [PATCH 14/22] Adding description to extended schema --- weaver/wps_restapi/swagger_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index ffd1abadb..34c3e07ea 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -808,7 +808,7 @@ class Link(LinkRelationship, LinkBase): class MetadataValueField(OneOfKeywordSchema): _one_of = [ # pointer to a file or JSON schema relative item (as in OpenAPI definitions) - ExtendedSchemaNode(String()), + ExtendedSchemaNode(String(), description="Plain text value of the information."), # literal JSON schema, permissive since it can be anything PermissiveMappingSchema(description="Flexible schema definition for the metadata value.") ] From ea2d487abb7ddbce24bd09640ac370d8cd2ec9cb Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Tue, 10 Sep 2024 14:11:09 -0400 Subject: [PATCH 15/22] Fixing typedef schema, adding test for softwareVersion and adding content type parsing for codeRepository --- tests/processes/test_wps_package.py | 32 +++++++++++++++++++++++------ weaver/processes/wps_package.py | 21 +++++++++++++++---- weaver/typedefs.py | 3 ++- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index d09d5b665..879bc4c9e 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -678,11 +678,13 @@ def test_format_extension_validator_basic(data_input, mode, expect): "s:author": [ {"class": "s:Person", "s:name": "John Doe", "s:affiliation": "Example Inc."} ], - }, "wps_package_metadata": {} }, { + "abstract": "", + "title": "", + "version": "", "metadata": [ { "role": "author", @@ -704,8 +706,12 @@ def test_format_extension_validator_basic(data_input, mode, expect): "wps_package_metadata": {} }, { + "abstract": "", + "title": "", + "version": "", "metadata": [ { + "type": "text/html", "rel": "codeRepository", "href": "https://gitlab.com/" } @@ -717,11 +723,11 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "cwl_package_package": { "s:version": "1.0" - }, "wps_package_metadata": { "metadata": [ { + "type": "text/html", "rel": "codeRepository", "href": "https://gitlab.com/" } @@ -729,16 +735,32 @@ def test_format_extension_validator_basic(data_input, mode, expect): } }, { + "abstract": "", + "title": "", "metadata": [ { + "type": "text/html", "rel": "codeRepository", "href": "https://gitlab.com/" }, - ], "version": "1.0" } ), + ( + # Test softwareVersion + { + "cwl_package_package": { + "s:softwareVersion": "1.0.0" + }, + "wps_package_metadata": {} + }, + { + "abstract": "", + "title": "", + "version": "1.0.0" + } + ), ]) def test_process_metadata(original, expected): # type: (CWL, CWL) -> None @@ -746,6 +768,4 @@ def test_process_metadata(original, expected): wps_package_metadata = original["wps_package_metadata"] _update_package_metadata(wps_package_metadata, cwl_package_package) # Assertions - for key in expected: - assert key in wps_package_metadata - assert sorted(wps_package_metadata[key]) == sorted(expected[key]) + assert wps_package_metadata == expected diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 112840d04..8e4598def 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -719,17 +719,30 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): ) # specific use case with a different mapping - if "s:version" in cwl_package_package: + # https://docs.ogc.org/bp/20-089r1.html#toc31 + if "s:version" in cwl_package_package or "s:softwareVersion" in cwl_package_package: wps_package_metadata["version"] = str( - str(cwl_package_package.get("s:version", "")) or str(wps_package_metadata.get("version")) + str(wps_package_metadata.get("version", "")) + or cwl_package_package.get("s:version") + or cwl_package_package.get("s:softwareVersion") ) + else: + wps_package_metadata["version"] = str(wps_package_metadata.get("version", "")) for metadata_mapping in SUPPORTED_METADATA_MAPPING: if metadata_mapping in cwl_package_package: metadata = wps_package_metadata.get("metadata", []) - if (isinstance((cwl_package_package[metadata_mapping]), str) - and urlparse(cwl_package_package[metadata_mapping]) != ""): + if ( + isinstance((cwl_package_package[metadata_mapping]), str) + and urlparse(cwl_package_package[metadata_mapping]).scheme != "" + ): + url = cwl_package_package[metadata_mapping] + if metadata_mapping == "s:codeRepository": + type = "text/html" + else: + type = get_content_type(os.path.splitext(url)[-1], default=ContentType.TEXT_PLAIN) metadata.append({ + "type": type, "rel": metadata_mapping.strip("s:"), "href": cwl_package_package[metadata_mapping] }) diff --git a/weaver/typedefs.py b/weaver/typedefs.py index b90aadd48..00203d778 100644 --- a/weaver/typedefs.py +++ b/weaver/typedefs.py @@ -221,13 +221,14 @@ class CWL_SchemaName(Protocol): "$schemas": NotRequired[Dict[str, str]], "$graph": NotRequired[CWL_Graph], "s:author": NotRequired[List[Dict[str, JSON]]], - "s:citation": NotRequired[List[Dict[str, str]]], + "s:citation": NotRequired[List[Union[str, JSON]]], "s:codeRepository": NotRequired[Link], "s:contributor": NotRequired[List[Dict[str, JSON]]], "s:dateCreated": NotRequired[datetime], "s:license": NotRequired[Optional[Union[Dict[str, JSON], Link]]], "s:releaseNotes": NotRequired[Optional[Union[str, Link]]], "s:version": NotRequired[Optional[Union[str, Number]]], + "s:softwareVersion": NotRequired[Optional[Union[str, Number]]], }, total=False) CWL_WorkflowStepPackage = TypedDict("CWL_WorkflowStepPackage", { "id": str, # reference ID of the package From 555aeb2b2cf9568a7926eba9008be2eaa535f044 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Tue, 10 Sep 2024 14:32:38 -0400 Subject: [PATCH 16/22] Adding changes to changelog --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5e4c3c321..19d19116f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,10 @@ Changes ******* -.. **REPLACE AND/OR ADD SECTION ENTRIES ACCORDINGLY WITH APPLIED CHANGES** +Changes: +-------- +- Add parsing of additional metadata from schema.org in CWL document to convert into process fields (fixes `#463 `_) +- Add more metadata mapping details in documentation (fixes `#613 `_) .. _changes_latest: From 4ad02703988f883a3dbbfdd6524641a3453cdb70 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Wed, 11 Sep 2024 15:22:03 -0400 Subject: [PATCH 17/22] Adding test for metadata --- tests/processes/test_wps_package.py | 90 ++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index ba4bc4021..156bf7cbb 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -911,6 +911,7 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", + "version": "1.0", "metadata": [ { "type": "text/html", @@ -918,7 +919,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): "href": "https://gitlab.com/" }, ], - "version": "1.0" } ), ( @@ -935,6 +935,94 @@ def test_format_extension_validator_basic(data_input, mode, expect): "version": "1.0.0" } ), + ( + # Test contributor + { + "cwl_package_package": { + "s:contributor": [ + {"class": "s:Person", "s:name": "John Doe", "s:affiliation": "Example Inc."} + ], + }, + "wps_package_metadata": {} + }, + { + "abstract": "", + "title": "", + "version": "", + "metadata": [ + { + "role": "contributor", + "value": { + "$schema": "https://schema.org/Person", + "name": "John Doe", + "affiliation": "Example Inc." + } + } + ] + } + ), + ( + # Test citation + { + "cwl_package_package": { + "s:citation": "https://dx.doi.org/10.6084/m9.figshare.3115156.v2" + }, + "wps_package_metadata": {} + }, + { + "abstract": "", + "title": "", + "version": "", + "metadata": [ + { + "type": "text/plain", + "rel": "citation", + "href": "https://dx.doi.org/10.6084/m9.figshare.3115156.v2" + }, + ], + } + ), + ( + # Test dateCreated with existing metadata + { + "cwl_package_package": { + "s:dateCreated": [ + {"class": "s:DateTime", "s:dateCreated": "2016-12-13"} + ], + }, + "wps_package_metadata": { + "abstract": "", + "title": "", + "version": "", + "metadata": [ + { + "type": "text/plain", + "rel": "citation", + "href": "https://dx.doi.org/10.6084/m9.figshare.3115156.v2" + }, + ], + } + }, + { + "abstract": "", + "title": "", + "version": "", + "metadata": [ + { + "type": "text/plain", + "rel": "citation", + "href": "https://dx.doi.org/10.6084/m9.figshare.3115156.v2" + }, + { + "role": "dateCreated", + "value": { + "$schema": "https://schema.org/DateTime", + "dateCreated": "2016-12-13", + } + } + ] + } + ), ]) def test_process_metadata(original, expected): # type: (CWL, CWL) -> None From a31f211894450b771c59180efaa43c328ddc3268 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 12 Sep 2024 10:08:23 -0400 Subject: [PATCH 18/22] Fixing version metadata to remove empty value --- tests/processes/test_wps_package.py | 6 ------ weaver/processes/wps_package.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index 156bf7cbb..cba546e41 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -858,7 +858,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", - "version": "", "metadata": [ { "role": "author", @@ -882,7 +881,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", - "version": "", "metadata": [ { "type": "text/html", @@ -948,7 +946,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", - "version": "", "metadata": [ { "role": "contributor", @@ -972,7 +969,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", - "version": "", "metadata": [ { "type": "text/plain", @@ -993,7 +989,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): "wps_package_metadata": { "abstract": "", "title": "", - "version": "", "metadata": [ { "type": "text/plain", @@ -1006,7 +1001,6 @@ def test_format_extension_validator_basic(data_input, mode, expect): { "abstract": "", "title": "", - "version": "", "metadata": [ { "type": "text/plain", diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 10eebcea9..4d52acc38 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -802,13 +802,18 @@ def _update_package_metadata(wps_package_metadata, cwl_package_package): # specific use case with a different mapping # https://docs.ogc.org/bp/20-089r1.html#toc31 if "s:version" in cwl_package_package or "s:softwareVersion" in cwl_package_package: - wps_package_metadata["version"] = str( - str(wps_package_metadata.get("version", "")) + version_value = ( + wps_package_metadata.get("version") or cwl_package_package.get("s:version") or cwl_package_package.get("s:softwareVersion") ) + # Only set the key if version_value is not empty or null + if version_value: + wps_package_metadata["version"] = str(version_value) else: - wps_package_metadata["version"] = str(wps_package_metadata.get("version", "")) + version_value = wps_package_metadata.get("version") + if version_value: + wps_package_metadata["version"] = str(version_value) for metadata_mapping in SUPPORTED_METADATA_MAPPING: if metadata_mapping in cwl_package_package: From 298fe1c7e4c38b1420698d78c1f71673e082274f Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 12 Sep 2024 10:22:27 -0400 Subject: [PATCH 19/22] Adding metadata doc --- docs/source/package.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/source/package.rst b/docs/source/package.rst index 6bc46b9c6..d04e0228e 100644 --- a/docs/source/package.rst +++ b/docs/source/package.rst @@ -1312,6 +1312,20 @@ Below is a list of compatible elements. +-----------------------------------------+----------------------------------------------------------+ | ``abstract``/``description`` | ``doc`` | +-----------------------------------------+----------------------------------------------------------+ +| ``version`` | ``s:version``/``s:softwareVersion`` | ++-----------------------------------------+----------------------------------------------------------+ + +Additional compatible elements found under ``metadata`` in :term:`WPS` Context + +Parameters in :term:`CWL` Context: + +- ``s:author`` +- ``s:citation`` +- ``s:codeRepository`` +- ``s:contributor`` +- ``s:dateCreated`` +- ``s:license`` +- ``s:releaseNotes`` .. rubric:: Footnotes From 39bd3e50631f2eb9dbd4f62503bd1659613fd894 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 12 Sep 2024 10:31:02 -0400 Subject: [PATCH 20/22] Fix changelog --- CHANGES.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 938352439..74281262e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,10 +3,7 @@ Changes ******* -Changes: --------- -- Add parsing of additional metadata from schema.org in CWL document to convert into process fields (fixes `#463 `_) -- Add more metadata mapping details in documentation (fixes `#613 `_) +.. **REPLACE AND/OR ADD SECTION ENTRIES ACCORDINGLY WITH APPLIED CHANGES** .. _changes_latest: @@ -33,6 +30,9 @@ Changes: and a sample `crim-ca/ncml2stac `_ repository making use of it with the `Weaver` `CLI` to generate a deployed `OGC API - Processes` definition (fixes `#63 `_). +- Add parsing of additional metadata from schema.org in CWL document to convert into process fields + (fixes `#463 `_) +- Add more metadata mapping details in documentation (fixes `#613 `_) Fixes: ------ From 667702e99b04cf8331b16d16f3faf1083ab10560 Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 12 Sep 2024 10:41:38 -0400 Subject: [PATCH 21/22] removing whitespace --- docs/source/package.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/package.rst b/docs/source/package.rst index d04e0228e..4c83d9b27 100644 --- a/docs/source/package.rst +++ b/docs/source/package.rst @@ -1315,7 +1315,7 @@ Below is a list of compatible elements. | ``version`` | ``s:version``/``s:softwareVersion`` | +-----------------------------------------+----------------------------------------------------------+ -Additional compatible elements found under ``metadata`` in :term:`WPS` Context +Additional compatible elements found under ``metadata`` in :term:`WPS` Context Parameters in :term:`CWL` Context: From b76cfd1397677a6c392e08f4b8dace0eddf54f9a Mon Sep 17 00:00:00 2001 From: Nazim Azeli Date: Thu, 12 Sep 2024 13:15:47 -0400 Subject: [PATCH 22/22] Fixing doc --- CHANGES.rst | 4 ++-- docs/source/package.rst | 21 ++++++++------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 74281262e..5b7ce4aa7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -31,8 +31,8 @@ Changes: making use of it with the `Weaver` `CLI` to generate a deployed `OGC API - Processes` definition (fixes `#63 `_). - Add parsing of additional metadata from schema.org in CWL document to convert into process fields - (fixes `#463 `_) -- Add more metadata mapping details in documentation (fixes `#613 `_) + (fixes `#463 `_). +- Add more metadata mapping details in documentation (fixes `#613 `_). Fixes: ------ diff --git a/docs/source/package.rst b/docs/source/package.rst index 4c83d9b27..d712c3210 100644 --- a/docs/source/package.rst +++ b/docs/source/package.rst @@ -1307,26 +1307,21 @@ Below is a list of compatible elements. +-----------------------------------------+----------------------------------------------------------+ | ``metadata`` | ``$schemas``/``$namespace`` | | (using ``title`` and ``href`` fields) | (using namespace name and HTTP references) | +| | ``s:author`` | +| | ``s:citation`` | +| | ``s:codeRepository`` | +| | ``s:contributor`` | +| | ``s:dateCreated`` | +| | ``s:license`` | +| | ``s:releaseNotes`` [#cwl_schemaorg]_ | +-----------------------------------------+----------------------------------------------------------+ | ``title`` | ``label`` | +-----------------------------------------+----------------------------------------------------------+ | ``abstract``/``description`` | ``doc`` | +-----------------------------------------+----------------------------------------------------------+ -| ``version`` | ``s:version``/``s:softwareVersion`` | +| ``version`` | ``s:version``/``s:softwareVersion`` [#cwl_schemaorg]_ | +-----------------------------------------+----------------------------------------------------------+ -Additional compatible elements found under ``metadata`` in :term:`WPS` Context - -Parameters in :term:`CWL` Context: - -- ``s:author`` -- ``s:citation`` -- ``s:codeRepository`` -- ``s:contributor`` -- ``s:dateCreated`` -- ``s:license`` -- ``s:releaseNotes`` - .. rubric:: Footnotes .. [#cwl_schemaorg]