v1.0.0-rc5

.github/workflows/unit-tests.yml

@@ -36,6 +36,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_SCANNER_OPTS: -Dsonar.working.directory=/tmp/sonar -X
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1

Makefile

@@ -109,7 +109,7 @@ prod-dotenv-file:
 	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> ${ENV_FILE}
 	@ echo BACKEND_INFRA_FOLDER=${BACKEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo FRONTEND_INFRA_FOLDER=${FRONTEND_INFRA_FOLDER} >> ${ENV_FILE}
-	@ echo NODE_ENV=development >> ${ENV_FILE}
+	@ echo NODE_ENV=production >> ${ENV_FILE}
 	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> ${ENV_FILE}
 	@ vault kv get -format="json" mapping-workbench-prod/app | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> ${ENV_FILE}
 

mapping_workbench/backend/core/entrypoints/api/main.py

@@ -35,7 +35,8 @@
     routes_for_specific as specific_triple_map_fragment_routes
 from mapping_workbench.backend.triple_map_registry.entrypoints.api import routes as triple_map_registry_routes
 from mapping_workbench.backend.user.entrypoints.api import routes as user_routes
-from mapping_workbench.backend.xsd_schema.entrypoints.api import routes as xsd_schema_router
+from mapping_workbench.backend.xsd_schema.entrypoints.api import routes as xsd_schema_routes
+from mapping_workbench.backend.demo.entrypoints.api import routes as demo_routes
 
 ROOT_API_PATH = "/api/v1"
 
@@ -95,8 +96,9 @@ async def on_startup():
     task_manager_routes.router,
     tasks_routes.router,  # Deprecated
     fields_registry.router,
-    xsd_schema_router.router,
-    cm_groups_routes.router
+    xsd_schema_routes.router,
+    cm_groups_routes.router,
+    demo_routes.router
 ]
 
 for secured_router in secured_routers:

mapping_workbench/backend/core/services/project_initilisers.py

@@ -18,7 +18,8 @@
 from mapping_workbench.backend.security.models.security import AccessToken
 from mapping_workbench.backend.shacl_test_suite.models.entity import SHACLTestSuite, SHACLTestFileResource
 from mapping_workbench.backend.sparql_test_suite.models.entity import SPARQLTestSuite, SPARQLTestFileResource
-from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource
+from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \
+    TestDataManifestationHistory
 from mapping_workbench.backend.triple_map_fragment.models.entity import SpecificTripleMapFragment, \
     GenericTripleMapFragment
 from mapping_workbench.backend.triple_map_registry.models.entity import TripleMapRegistry
@@ -58,6 +59,7 @@ async def init_project_models(mongodb_database: AsyncIOMotorDatabase):
             ResourceFile,
             TestDataSuite,
             TestDataFileResource,
+            TestDataManifestationHistory,
             MappingPackage,
             MappingPackageStateGate,
             MappingRuleRegistry,

mapping_workbench/backend/demo/entrypoints/api/routes.py

@@ -0,0 +1,22 @@
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from mapping_workbench.backend.config import settings
+
+ROUTE_PREFIX = "/demo"
+TAG = "demo"
+
+sub_router = APIRouter()
+
+
+@sub_router.post(
+    "/reset",
+    name="demo:reset"
+)
+async def metadata() -> JSONResponse:
+    return JSONResponse(content={
+        "message": "Resetting the demo data"
+    })
+
+
+router = APIRouter()
+router.include_router(sub_router, prefix=ROUTE_PREFIX, tags=[TAG])

mapping_workbench/backend/fields_registry/adapters/github_download.py

@@ -34,6 +34,7 @@ def download(self, result_dir_path: pathlib.Path, download_resources_filter: Lis
             subprocess.run(bash_script, shell=True,
                            stdout=subprocess.DEVNULL,
                            stderr=subprocess.STDOUT)
+
             dir_contents = list(temp_dir_path.iterdir())
             mwb_logger.log_all_info(f"Downloaded path {dir_contents}")
 

mapping_workbench/backend/fields_registry/entrypoints/api/routes.py

@@ -194,7 +194,7 @@ async def route_search_structural_elements_versioned_view_by_eforms_version(
     status_code=status.HTTP_201_CREATED
 )
 async def route_task_import_eforms_xsd(
-        github_repository_url: str = Form(...),
+        github_repository_url: str = Form(default=None),
         branch_or_tag_name: str = Form(...),
         project_id: PydanticObjectId = Form(...),
         user: User = Depends(current_active_user)

mapping_workbench/backend/fields_registry/services/data.py

@@ -7,14 +7,14 @@
 
 
 async def get_structural_element_by_unique_fields(
-        sdk_element_id, bt_id, absolute_xpath, project_id: PydanticObjectId, name: str = None
+        sdk_element_id, absolute_xpath, project_id: PydanticObjectId, bt_id: str = None, name: str = None
 ) -> StructuralElement:
     project_link = Project.link_from_id(project_id)
     return await StructuralElement.find_one(
         StructuralElement.project == project_link,
         StructuralElement.sdk_element_id == sdk_element_id,
-        #StructuralElement.name == name,
-        StructuralElement.bt_id == bt_id,
+        # StructuralElement.name == name,
+        # StructuralElement.bt_id == bt_id,
         StructuralElement.absolute_xpath == absolute_xpath
     )
 

mapping_workbench/backend/mapping_package/models/entity.py

@@ -76,6 +76,7 @@ class MappingPackageOut(BaseProjectResourceEntityOutSchema):
     shacl_test_suites: Optional[List[Link[SHACLTestSuite]]] = None
     sparql_test_suites: Optional[List[Link[SPARQLTestSuite]]] = None
     resource_collections: Optional[List[Link[ResourceCollection]]] = None
+    process_status: Optional[str] = None
 
 
 class MappingPackageListFilters(BaseTitledEntityListFiltersSchema):

mapping_workbench/backend/mapping_package/services/api.py

@@ -12,7 +12,11 @@
     api_entity_is_found, prepare_search_param, pagination_params
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage, MappingPackageCreateIn, \
     MappingPackageUpdateIn, MappingPackageOut, MappingPackageStateGate
+from mapping_workbench.backend.mapping_package.services.data import mapping_package_process_status
+from mapping_workbench.backend.package_processor.services import TASK_ENTITY_TYPE, TASK_ENTITY_ACTION
 from mapping_workbench.backend.state_manager.services.object_state_manager import delete_object_state
+from mapping_workbench.backend.task_manager.entrypoints import AppTaskManager
+from mapping_workbench.backend.tasks.models.task_result import TaskMetadataMeta
 from mapping_workbench.backend.triple_map_fragment.models.entity import SpecificTripleMapFragment, \
     GenericTripleMapFragment
 from mapping_workbench.backend.user.models.user import User
@@ -33,6 +37,25 @@ async def list_mapping_packages(filters: dict = None, page: int = None, limit: i
         limit=limit
     ).to_list()
 
+    tasks_metadata = AppTaskManager.get_task_statuses()
+
+    tasks_statuses = {
+        task_item.meta.entity.id: task_item.task_status.value
+        for task_item in tasks_metadata
+        if isinstance(task_item.meta, TaskMetadataMeta)
+           and task_item.meta.entity
+           and task_item.meta.entity.type == TASK_ENTITY_TYPE
+           and task_item.meta.entity.action == TASK_ENTITY_ACTION
+    }
+
+    for item in items:
+        item_id = str(item.id)
+        tasks_status = None
+        if item_id in tasks_statuses:
+            tasks_status = tasks_statuses[item_id]
+
+        item.process_status = mapping_package_process_status(tasks_status)
+
     total_count: int = await MappingPackage.find(query_filters).count()
     return items, total_count
 

mapping_workbench/backend/mapping_package/services/data.py

@@ -6,6 +6,7 @@
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage, MappingPackageState, \
     MappingPackageStateGate
 from mapping_workbench.backend.state_manager.services.object_state_manager import load_object_state
+from mapping_workbench.backend.tasks.models.task_response import TaskProgressStatus
 
 DEFAULT_PACKAGE_NAME = "DEFAULT"
 DEFAULT_PACKAGE_IDENTIFIER = "default"
@@ -55,3 +56,14 @@ def get_mapping_package_state_ns_definitions(mapping_package_state: MappingPacka
         )
     }
     return ns_definitions
+
+
+def mapping_package_process_status(task_status):
+    if task_status == TaskProgressStatus.RUNNING:
+        return "PROCESSING"
+    elif task_status == TaskProgressStatus.FINISHED:
+        return "PROCESSED_SUCCESS"
+    elif task_status in [TaskProgressStatus.TIMEOUT, TaskProgressStatus.FAILED, TaskProgressStatus.CANCELED]:
+        return "PROCESSED_ERROR"
+    else:
+        return "UNPROCESSED"

mapping_workbench/backend/package_importer/adapters/eforms/importer.py

@@ -8,7 +8,7 @@
 from mapping_workbench.backend.package_importer.adapters.importer_abc import PackageImporterABC
 from mapping_workbench.backend.package_importer.models.imported_mapping_suite import ImportedMappingSuite
 from mapping_workbench.backend.project.models.entity import Project
-from mapping_workbench.backend.tasks.models.task_response import TaskResponse
+from mapping_workbench.backend.tasks.models.task_response import TaskResponse, TaskResultWarning
 from mapping_workbench.backend.triple_map_fragment.models.entity import GenericTripleMapFragment
 from mapping_workbench.backend.user.models.user import User
 
@@ -71,19 +71,37 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
         for mono_rule in mono_package.conceptual_rules:
             source_structural_element: StructuralElement = await get_structural_element_by_unique_fields(
                 sdk_element_id=mono_rule.eforms_sdk_id,
-                bt_id=mono_rule.bt_id,
                 absolute_xpath=mono_rule.absolute_xpath,
                 project_id=self.project.id,
+                # bt_id=mono_rule.bt_id,
                 # name=mono_rule.field_name
             )
 
             if not source_structural_element:
+                m = f"{mono_rule.eforms_sdk_id}"
+                mwb_logger.log_all_warning(m)
+                self.warnings.append(TaskResultWarning(message=m, type="Not Found SDK Elements"))
                 continue
 
+            if source_structural_element.bt_id != mono_rule.bt_id:
+                m = f"{mono_rule.eforms_sdk_id}, {source_structural_element.bt_id} <> {mono_rule.bt_id}, {mono_rule.absolute_xpath}"
+                mwb_logger.log_all_warning(m)
+                self.warnings.append(TaskResultWarning(message=m, type="CM(sdk_id, sdk_bt_id <> bt_id, xpath) BT ID Mismatch"))
+
             if source_structural_element.name != mono_rule.field_name:
                 m = f"Field[{source_structural_element.sdk_element_id}] has Imported Name ({mono_rule.field_name}) <> Current Name ({source_structural_element.name})"
                 mwb_logger.log_all_warning(m)
-                self.warnings.append(m)
+                self.warnings.append(TaskResultWarning(message=m, type="Field Name Mismatch"))
+
+            if not self.is_cm_rule_path_valid(mono_rule.class_path):
+                m = f"{mono_rule.class_path}"
+                mwb_logger.log_all_warning(m)
+                self.warnings.append(TaskResultWarning(message=m, type="Class Path Mismatch"))
+
+            if not self.is_cm_rule_path_valid(mono_rule.property_path):
+                m = f"{mono_rule.property_path}"
+                mwb_logger.log_all_warning(m)
+                self.warnings.append(TaskResultWarning(message=m, type="Property Path Mismatch"))
 
             # A conceptual mapping rule may have same structural element but different Ontology Fragment
             rule: ConceptualMappingRule = await ConceptualMappingRule.find_one(
@@ -93,19 +111,12 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
                 ConceptualMappingRule.target_class_path == mono_rule.class_path,
                 ConceptualMappingRule.target_property_path == mono_rule.property_path
             )
-            if rule:
-                mwb_logger.log_all_info(
-                    f"CM Rule with Ontology Fragment: {rule.target_class_path} | {rule.target_property_path} already exist")
-                continue
-
-            rule = ConceptualMappingRule(
-                source_structural_element=StructuralElement.link_from_id(source_structural_element.id)
-            )
+            if not rule:
+                rule = ConceptualMappingRule(
+                    source_structural_element=StructuralElement.link_from_id(source_structural_element.id)
+                )
 
-            # rule: ConceptualMappingRule = ConceptualMappingRule()
             rule.project = self.project_link
-            if source_structural_element:
-                rule.source_structural_element = source_structural_element
 
             if not rule.refers_to_mapping_package_ids:
                 rule.refers_to_mapping_package_ids = []
@@ -140,8 +151,7 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
             if mono_rule.feedback_notes:
                 rule.feedback_notes = [ConceptualMappingRuleComment(comment=mono_rule.feedback_notes)]
 
-            # await rule.on_update(self.user).save() if rule.id else \
-            await rule.on_create(self.user).create()
+            await rule.on_update(self.user).save() if rule.id else await rule.on_create(self.user).create()
 
             sort_order += 1
 

mapping_workbench/backend/package_importer/adapters/importer_abc.py

@@ -1,3 +1,4 @@
+import re
 from abc import ABC, abstractmethod
 from itertools import takewhile
 from pathlib import Path
@@ -22,7 +23,7 @@
 from mapping_workbench.backend.sparql_test_suite.services.data import SPARQL_CM_ASSERTIONS_SUITE_TITLE, \
     SPARQL_INTEGRATION_TESTS_SUITE_TITLE
 from mapping_workbench.backend.task_manager.adapters.task_progress import TaskProgress
-from mapping_workbench.backend.tasks.models.task_response import TaskResponse
+from mapping_workbench.backend.tasks.models.task_response import TaskResponse, TaskResultWarning
 from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \
     TestDataFileResourceFormat
 from mapping_workbench.backend.triple_map_fragment.models.entity import TripleMapFragmentFormat, \
@@ -32,7 +33,7 @@
 
 class PackageImporterABC(ABC):
     package: MappingPackage
-    warnings: List[str] = []
+    warnings: List[TaskResultWarning] = []
     task_progress: TaskProgress
 
     def __init__(self, project: Project, user: User, task_response: TaskResponse = None):
@@ -43,7 +44,6 @@ def __init__(self, project: Project, user: User, task_response: TaskResponse = N
         self.task_progress = TaskProgress(self.task_response)
         self.package = None
 
-
     @abstractmethod
     async def import_from_mono_mapping_suite(self, mono_package: ImportedMappingSuite):
         """
@@ -366,6 +366,12 @@ async def add_mapping_package_from_mono(self, mono_package: ImportedMappingSuite
 
         self.task_progress.finish_current_action_step()
 
+    @classmethod
+    def is_cm_rule_path_valid(cls, cm_rule_path: str) -> bool:
+        if not cm_rule_path:
+            return True
+        return len(cm_rule_path.split('/')) == len(cm_rule_path.split(" / "))
+
     @classmethod
     async def clear_project_data(cls, project: Project):
         project_link = Project.link_from_id(project.id)

mapping_workbench/backend/package_importer/adapters/standard/importer.py

@@ -50,9 +50,9 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
         for mono_rule in mono_package.conceptual_rules:
             source_structural_element: StructuralElement = await get_structural_element_by_unique_fields(
                 sdk_element_id=mono_rule.field_name,
-                bt_id=mono_rule.bt_id,
                 absolute_xpath=mono_rule.absolute_xpath,
                 project_id=self.project.id,
+                # bt_id=mono_rule.bt_id,
                 # name=mono_rule.field_name
             )
 
@@ -78,8 +78,6 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
                 )
 
             rule.project = self.project_link
-            if source_structural_element:
-                rule.source_structural_element = source_structural_element
 
             if not rule.refers_to_mapping_package_ids:
                 rule.refers_to_mapping_package_ids = []

mapping_workbench/backend/package_importer/entrypoints/api/routes.py

@@ -48,7 +48,7 @@ async def route_import_package_archive(
         user: User = Depends(current_active_user)
 ):
     imported_mapping_package: ImportedMappingSuiteResponse = await import_mapping_package_from_archive(
-        file.file.read(), await get_project(project), package_type, user
+        file.file.read(), await get_project(project), package_type, False, user
     )
 
     return imported_mapping_package
@@ -77,7 +77,8 @@ async def route_task_import_package(
         None,
         user.email,
         True,
-        file.file.read(), await get_project(project), package_type, trigger_package_processing, cleanup_project, user
+        file.file.read(), await get_project(project), package_type, trigger_package_processing, cleanup_project,
+        user
     )
 
     return task.task_metadata