diff --git a/mapping_workbench/backend/core/services/project_initilisers.py b/mapping_workbench/backend/core/services/project_initilisers.py index 58265a4a..f075fd22 100644 --- a/mapping_workbench/backend/core/services/project_initilisers.py +++ b/mapping_workbench/backend/core/services/project_initilisers.py @@ -18,7 +18,8 @@ from mapping_workbench.backend.security.models.security import AccessToken from mapping_workbench.backend.shacl_test_suite.models.entity import SHACLTestSuite, SHACLTestFileResource from mapping_workbench.backend.sparql_test_suite.models.entity import SPARQLTestSuite, SPARQLTestFileResource -from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource +from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \ + TestDataManifestationHistory from mapping_workbench.backend.triple_map_fragment.models.entity import SpecificTripleMapFragment, \ GenericTripleMapFragment from mapping_workbench.backend.triple_map_registry.models.entity import TripleMapRegistry @@ -58,6 +59,7 @@ async def init_project_models(mongodb_database: AsyncIOMotorDatabase): ResourceFile, TestDataSuite, TestDataFileResource, + TestDataManifestationHistory, MappingPackage, MappingPackageStateGate, MappingRuleRegistry, diff --git a/mapping_workbench/backend/test_data_suite/entrypoints/api/routes.py b/mapping_workbench/backend/test_data_suite/entrypoints/api/routes.py index 3e0639dd..b758f29c 100644 --- a/mapping_workbench/backend/test_data_suite/entrypoints/api/routes.py +++ b/mapping_workbench/backend/test_data_suite/entrypoints/api/routes.py @@ -14,7 +14,8 @@ from mapping_workbench.backend.task_manager.services.task_wrapper import add_task from mapping_workbench.backend.test_data_suite.adapters.rml_mapper import RMLMapperException from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \ - TestDataFileResourceUpdateIn, TestDataFileResourceCreateIn + TestDataFileResourceUpdateIn, TestDataFileResourceCreateIn, TestDataManifestationHistory, \ + TestDataManifestationHistoryOut from mapping_workbench.backend.test_data_suite.models.entity_api_response import \ APIListTestDataSuitesPaginatedResponse, APIListTestDataFileResourcesPaginatedResponse from mapping_workbench.backend.test_data_suite.services import tasks @@ -27,7 +28,7 @@ list_test_data_suite_file_resources, create_test_data_suite_file_resource, get_test_data_file_resource, - delete_test_data_file_resource, update_test_data_file_resource + delete_test_data_file_resource, update_test_data_file_resource, get_test_data_transform_history ) from mapping_workbench.backend.test_data_suite.services.import_test_data_suite import \ import_test_data_suites_from_archive @@ -416,3 +417,15 @@ async def route_task_import_test_data_suites( ): await import_test_data_suites_from_archive(project, file, user) return APIEmptyContentResponse() + + +@router.get( + "/file_resources/{id}/transform/history", + description=f"Get Test Data Transform History", + name=f"{FILE_RESOURCE_NAME_FOR_ONE}:get_transform_history", +) +async def route_get_test_data_file_resource_transform_history( + project: PydanticObjectId, + test_data_file_resource: TestDataFileResource = Depends(get_test_data_file_resource) +) -> List[TestDataManifestationHistoryOut]: + return await get_test_data_transform_history(test_data_file_resource, project) diff --git a/mapping_workbench/backend/test_data_suite/models/entity.py b/mapping_workbench/backend/test_data_suite/models/entity.py index 8c322c04..d712ddf0 100644 --- a/mapping_workbench/backend/test_data_suite/models/entity.py +++ b/mapping_workbench/backend/test_data_suite/models/entity.py @@ -1,14 +1,18 @@ +from datetime import datetime from enum import Enum from typing import Optional, List import pymongo from beanie import Link, PydanticObjectId -from pydantic import BaseModel +from dateutil.tz import tzlocal +from pydantic import BaseModel, Field from pymongo import IndexModel +from mapping_workbench.backend.core.models.base_entity import BaseEntityOutSchema from mapping_workbench.backend.core.models.base_mapping_package_resource_entity import \ BaseMappingPackagesResourceSchemaTrait -from mapping_workbench.backend.core.models.base_project_resource_entity import BaseProjectResourceEntity +from mapping_workbench.backend.core.models.base_project_resource_entity import BaseProjectResourceEntity, \ + BaseProjectAbleResourceEntity from mapping_workbench.backend.file_resource.models.file_resource import FileResource, FileResourceCollection, \ FileResourceIn, FileResourceFormat, FileResourceState from mapping_workbench.backend.package_validator.models.shacl_validation import SHACLTestDataValidationResult @@ -61,6 +65,23 @@ class TestDataState(TestDataValidation, ObjectState): rdf_manifestation: Optional[FileResourceState] = None +class TestDataManifestationHistory(BaseProjectAbleResourceEntity): + test_data_id: PydanticObjectId + in_manifestation: Optional[str] = None + out_manifestation: Optional[str] = None + created_at: Optional[datetime] = Field(default_factory=lambda: datetime.now(tzlocal())) + + class Settings(FileResource.Settings): + name = "test_data_manifestation_history" + + +class TestDataManifestationHistoryOut(BaseEntityOutSchema): + test_data_id: PydanticObjectId + in_manifestation: Optional[str] = None + out_manifestation: Optional[str] = None + created_at: Optional[datetime] = Field(default_factory=lambda: datetime.now(tzlocal())) + + class TestDataFileResource(FileResource, StatefulObjectABC): identifier: Optional[str] = None format: Optional[TestDataFileResourceFormat] = None diff --git a/mapping_workbench/backend/test_data_suite/services/api.py b/mapping_workbench/backend/test_data_suite/services/api.py index 67796a72..117758e3 100644 --- a/mapping_workbench/backend/test_data_suite/services/api.py +++ b/mapping_workbench/backend/test_data_suite/services/api.py @@ -1,5 +1,6 @@ from typing import List +import pymongo from beanie import PydanticObjectId from mapping_workbench.backend.core.models.base_entity import BaseEntityFiltersSchema @@ -9,8 +10,10 @@ from mapping_workbench.backend.mapping_package.models.entity import MappingPackage from mapping_workbench.backend.mapping_package.services.link import unassign_resources_from_mapping_packages, \ ResourceField +from mapping_workbench.backend.project.models.entity import Project from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \ - TestDataFileResourceUpdateIn, TestDataFileResourceCreateIn + TestDataFileResourceUpdateIn, TestDataFileResourceCreateIn, TestDataManifestationHistory, \ + TestDataManifestationHistoryOut from mapping_workbench.backend.test_data_suite.services.transform_test_data import transform_test_data_file_resource from mapping_workbench.backend.user.models.user import User @@ -136,3 +139,19 @@ async def get_test_data_file_resource(id: PydanticObjectId) -> TestDataFileResou async def delete_test_data_file_resource(test_data_file_resource: TestDataFileResource): return await test_data_file_resource.delete() + + +async def get_test_data_transform_history( + test_data_file_resource: TestDataFileResource, + project_id: PydanticObjectId +) -> List[TestDataManifestationHistoryOut]: + Project.link_from_id(project_id) + items: List[TestDataManifestationHistoryOut] = await TestDataManifestationHistory.find( + TestDataManifestationHistory.project == Project.link_from_id(project_id), + TestDataManifestationHistory.test_data_id == test_data_file_resource.id, + projection_model=TestDataManifestationHistoryOut, + fetch_links=False, + sort=[(str(TestDataManifestationHistory.created_at), pymongo.DESCENDING)] + ).to_list() + + return items diff --git a/mapping_workbench/backend/test_data_suite/services/test_data_transform_history.py b/mapping_workbench/backend/test_data_suite/services/test_data_transform_history.py new file mode 100644 index 00000000..a4cedc4b --- /dev/null +++ b/mapping_workbench/backend/test_data_suite/services/test_data_transform_history.py @@ -0,0 +1,17 @@ +from beanie import PydanticObjectId + +from mapping_workbench.backend.project.models.entity import Project +from mapping_workbench.backend.test_data_suite.models.entity import TestDataFileResource, TestDataManifestationHistory + + +async def add_test_data_transform_to_history( + test_data_file_resource: TestDataFileResource, + project_id: PydanticObjectId +): + history_item = TestDataManifestationHistory( + project=Project.link_from_id(project_id), + test_data_id=test_data_file_resource.id, + in_manifestation=test_data_file_resource.content, + out_manifestation=test_data_file_resource.rdf_manifestation + ) + await history_item.create() diff --git a/mapping_workbench/backend/test_data_suite/services/transform_test_data.py b/mapping_workbench/backend/test_data_suite/services/transform_test_data.py index 43cb65ab..0709e356 100644 --- a/mapping_workbench/backend/test_data_suite/services/transform_test_data.py +++ b/mapping_workbench/backend/test_data_suite/services/transform_test_data.py @@ -15,6 +15,8 @@ TestDataSuiteState from mapping_workbench.backend.test_data_suite.services import DATA_SOURCE_PATH_NAME, \ TRANSFORMATION_PATH_NAME, MAPPINGS_PATH_NAME, RESOURCES_PATH_NAME +from mapping_workbench.backend.test_data_suite.services.test_data_transform_history import \ + add_test_data_transform_to_history from mapping_workbench.backend.test_data_suite.services.data import get_test_data_file_resources_for_project, \ get_test_data_file_resources_for_package from mapping_workbench.backend.triple_map_fragment.models.entity import TripleMapFragment, TripleMapFragmentState, \ @@ -119,7 +121,6 @@ async def transform_test_data_file_resource( if not isinstance(rml_mapper, RMLMapperABC): rml_mapper: RMLMapper = RMLMapper(rml_mapper_path=Path(settings.RML_MAPPER_PATH)) - test_data_file_resource.rdf_manifestation = await transform_test_data_file_resource_content( content=test_data_file_resource.content, mappings=mappings, @@ -137,6 +138,11 @@ async def transform_test_data_file_resource( if save: await test_data_file_resource.save() + await add_test_data_transform_to_history( + test_data_file_resource=test_data_file_resource, + project_id=project_id + ) + return test_data_file_resource