Merge pull request #47 from meaningfy-ws/feature/MWB-224

Feature/mwb 224

Makefile

@@ -2,18 +2,18 @@ PROJECT_PATH = $(shell pwd)
 
 ENV_FILE := .env
 
+# include .env files if they exist
+-include ${ENV_FILE}
+
 NAME := mapping_workbench
+DOCKER_PROJECT := ${NAME}_${ENVIRONMENT}
 BACKEND_INFRA_FOLDER := ${PROJECT_PATH}/${NAME}/backend
 FRONTEND_HOME := ${NAME}/frontend
 FRONTEND_INFRA_FOLDER := ${PROJECT_PATH}/${FRONTEND_HOME}
 
 RML_MAPPER_PATH = ${PROJECT_PATH}/.rmlmapper/rmlmapper.jar
 
 
-# include .env files if they exist
--include ${ENV_FILE}
-
-
 #-----------------------------------------------------------------------------
 # INSTALLING
 #----------------------------------------------------------------------
@@ -107,37 +107,37 @@ prod-dotenv-file:
 
 build-backend:
 	@ echo "Building the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
 
 start-backend:
 	@ echo "Starting the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-backend:
 	@ echo "Stopping the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} down
 
 build-frontend:
 	@ echo "Building the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
 
 start-frontend:
 	@ echo "Starting the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-frontend:
 	@ echo "Stopping the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.yml --env-file ${ENV_FILE} down
 
 start-mongo: build-externals
 	@ echo "Starting the Mongo services"
-	@ docker-compose -p ${NAME} --file ./infra/mongodb/docker-compose.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/mongodb/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-mongo:
 	@ echo "Stopping the Mongo services"
-	@ docker-compose -p ${NAME} --file ./infra/mongodb/docker-compose.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/mongodb/docker-compose.yml --env-file ${ENV_FILE} down
 
 
 #-----------------------------------------------------------------------------
@@ -146,29 +146,29 @@ stop-mongo:
 
 build-backend-dev:
 	@ echo "Building the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d --force-recreate
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d --force-recreate
 
 start-backend-dev:
 	@ echo "Starting the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
 
 stop-backend-dev:
 	@ echo "Stopping the BACKEND"
-	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/backend/docker-compose.dev.yml --env-file ${ENV_FILE} down
 
 build-frontend-dev:
 	@ echo "Building the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d --force-recreate
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d --force-recreate
 
 start-frontend-dev:
 	@ echo "Starting the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
 
 stop-frontend-dev:
 	@ echo "Stopping the FRONTEND"
-	@ docker-compose -p ${NAME} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/frontend/docker-compose.dev.yml --env-file ${ENV_FILE} down
 
 clear-frontend:
 	@ cd ${FRONTEND_HOME} && rm -rf build && rm -rf node_modules && rm -f .env* &&  rm -f package-lock.json
@@ -185,11 +185,11 @@ start-mongo-console-mode:
 
 start-mongo-dev: build-externals
 	@ echo "Starting the Mongo services"
-	@ docker-compose -p ${NAME} --file ./infra/mongodb/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/mongodb/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
 
 stop-mongo-dev:
 	@ echo "Stopping the Mongo services"
-	@ docker-compose -p ${NAME} --file ./infra/mongodb/docker-compose.dev.yml --env-file ${ENV_FILE} down
+	@ docker-compose -p ${DOCKER_PROJECT} --file ./infra/mongodb/docker-compose.dev.yml --env-file ${ENV_FILE} down
 
 #-----------------------------------------------------------------------------
 # SERVER SERVICES

mapping_workbench/backend/core/entrypoints/api/main.py

@@ -2,6 +2,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from httpx_oauth.clients.google import GoogleOAuth2
 
+from mapping_workbench.backend.fields_registry.entrypoints.api import routes as fields_registry
 from mapping_workbench.backend.conceptual_mapping_rule.entrypoints.api import routes as conceptual_mapping_rule_routes
 from mapping_workbench.backend.config import settings
 from mapping_workbench.backend.config.entrypoints.api import routes as config_routes
@@ -77,7 +78,8 @@ async def on_startup():
     generic_triple_map_fragment_routes.router,
     config_routes.router,
     ontology_routes.router,
-    tasks_routes.router
+    tasks_routes.router,
+    fields_registry.router
 ]
 
 for secured_router in secured_routers:

mapping_workbench/backend/mapping_package/models/entity.py

@@ -13,7 +13,7 @@
     BaseProjectResourceEntityInSchema, BaseProjectResourceEntityOutSchema
 from mapping_workbench.backend.shacl_test_suite.models.entity import SHACLTestSuite
 from mapping_workbench.backend.state_manager.models.state_object import ObjectState, StatefulObjectABC
-from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataSuiteState
+#from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataSuiteState
 
 
 class MappingPackageException(Exception):
@@ -30,7 +30,7 @@ class MappingPackageIn(BaseProjectResourceEntityInSchema):
     end_date: Optional[datetime] = None
     min_xsd_version: Optional[str] = None
     max_xsd_version: Optional[str] = None
-    test_data_suites: Optional[List[Optional[Link[TestDataSuite]]]] = None
+    #test_data_suites: Optional[List[Optional[Link[TestDataSuite]]]] = None
     shacl_test_suites: Optional[List[Optional[Link[SHACLTestSuite]]]] = None
 
 
@@ -56,7 +56,7 @@ class MappingPackageOut(BaseProjectResourceEntityOutSchema):
     end_date: Optional[datetime] = None
     min_xsd_version: Optional[str] = None
     max_xsd_version: Optional[str] = None
-    test_data_suites: Optional[List[Link[TestDataSuite]]] = None
+    #test_data_suites: Optional[List[Link[TestDataSuite]]] = None
     shacl_test_suites: Optional[List[Link[SHACLTestSuite]]] = None
 
 
@@ -74,7 +74,7 @@ class MappingPackageState(ObjectState):
     end_date: Optional[datetime] = None
     min_xsd_version: Optional[str] = None
     max_xsd_version: Optional[str] = None
-    test_data_suites: List[TestDataSuiteState] = []
+    #test_data_suites: List[TestDataSuiteState] = []
     shacl_test_suites: List[SHACLTestSuite] = []
     conceptual_mapping_rule_states: List[ConceptualMappingRuleState] = []
 
@@ -89,7 +89,7 @@ class MappingPackage(BaseProjectResourceEntity, StatefulObjectABC):
     end_date: Optional[datetime] = None
     min_xsd_version: Optional[str] = None
     max_xsd_version: Optional[str] = None
-    test_data_suites: Optional[List[Link[TestDataSuite]]] = None
+    #test_data_suites: Optional[List[Link[TestDataSuite]]] = None
     shacl_test_suites: Optional[List[Link[SHACLTestSuite]]] = None
 
     async def get_conceptual_mapping_rules(self) -> List[ConceptualMappingRuleState]:

mapping_workbench/backend/ontology_file_collection/entrypoints/api/routes.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from beanie import PydanticObjectId
 from fastapi import APIRouter, status, Depends
 from starlette.requests import Request

mapping_workbench/backend/resource_collection/entrypoints/api/routes.py

@@ -47,13 +47,22 @@
     response_model=APIListResourceCollectionsPaginatedResponse
 )
 async def route_list_resource_collections(
-        project: PydanticObjectId = None
+        project: PydanticObjectId = None,
+        page: int = None,
+        limit: int = None,
+        q: str = None
 ):
     filters: dict = {}
     if project:
         filters['project'] = Project.link_from_id(project)
-    items: List[ResourceCollection] = await list_resource_collections(filters)
-    return APIListResourceCollectionsPaginatedResponse(items=items, count=len(items))
+    if q is not None:
+        filters['q'] = q
+
+    items, total_count = await list_resource_collections(filters, page, limit)
+    return APIListResourceCollectionsPaginatedResponse(
+        items=items,
+        count=total_count
+    )
 
 
 @router.post(
@@ -112,10 +121,24 @@ async def route_delete_resource_collection(resource_collection: ResourceCollecti
     response_model=APIListResourceFilesPaginatedResponse
 )
 async def route_list_resource_collection_file_resources(
-        id: PydanticObjectId = None
+        resource_collection: ResourceCollection = Depends(get_resource_collection),
+        project: PydanticObjectId = None,
+        page: int = None,
+        limit: int = None,
+        q: str = None
 ):
-    items: List[ResourceFile] = await list_resource_collection_file_resources(id)
-    return APIListResourceFilesPaginatedResponse(items=items, count=len(items))
+    filters: dict = {}
+    if project:
+        filters['project'] = Project.link_from_id(project)
+    if q is not None:
+        filters['q'] = q
+
+    items, total_count = \
+        await list_resource_collection_file_resources(resource_collection, filters, page, limit)
+    return APIListResourceFilesPaginatedResponse(
+        items=items,
+        count=total_count
+    )
 
 
 @router.post(

mapping_workbench/backend/resource_collection/services/api.py

@@ -1,25 +1,34 @@
 from typing import List
 
 from beanie import PydanticObjectId
-from bson import DBRef
 
 from mapping_workbench.backend.core.models.base_entity import BaseEntityFiltersSchema
 from mapping_workbench.backend.core.services.exceptions import ResourceNotFoundException
 from mapping_workbench.backend.core.services.request import request_update_data, api_entity_is_found, \
-    request_create_data
+    request_create_data, prepare_search_param, pagination_params
 from mapping_workbench.backend.resource_collection.models.entity import ResourceCollection, ResourceFile, \
     ResourceFileUpdateIn, ResourceFileCreateIn
 from mapping_workbench.backend.user.models.user import User
 
 
-async def list_resource_collections(filters=None) -> List[ResourceCollection]:
+async def list_resource_collections(filters: dict = None, page: int = None, limit: int = None) -> \
+        (List[ResourceCollection], int):
     query_filters: dict = dict(filters or {}) | dict(BaseEntityFiltersSchema())
-    return await ResourceCollection.find(
+
+    prepare_search_param(query_filters)
+    skip, limit = pagination_params(page, limit)
+
+    items: List[ResourceCollection] = await ResourceCollection.find(
         query_filters,
         projection_model=ResourceCollection,
-        fetch_links=False
+        fetch_links=False,
+        skip=skip,
+        limit=limit
     ).to_list()
 
+    total_count: int = await ResourceCollection.find(query_filters).count()
+    return items, total_count
+
 
 async def create_resource_collection(resource_collection: ResourceCollection, user: User) -> ResourceCollection:
     resource_collection.on_create(user=user)
@@ -48,15 +57,23 @@ async def delete_resource_collection(resource_collection: ResourceCollection):
 
 
 async def list_resource_collection_file_resources(
-        id: PydanticObjectId = None,
-        filters=None
-) -> List[ResourceFile]:
+        resource_collection: ResourceCollection,
+        filters=None, page: int = None, limit: int = None
+):
     query_filters: dict = dict(filters or {}) | dict(BaseEntityFiltersSchema())
-    return await ResourceFile.find(
-        ResourceFile.resource_collection == ResourceCollection.link_from_id(id),
+    query_filters['resource_collection'] = ResourceCollection.link_from_id(resource_collection.id)
+
+    prepare_search_param(query_filters)
+    skip, limit = pagination_params(page, limit)
+
+    items: List[ResourceFile] = await ResourceFile.find(
         query_filters,
-        fetch_links=False
+        fetch_links=False,
+        skip=skip,
+        limit=limit
     ).to_list()
+    total_count: int = await ResourceFile.find(query_filters).count()
+    return items, total_count
 
 
 async def create_resource_collection_file_resource(

mapping_workbench/backend/shacl_test_suite/entrypoints/api/routes.py

@@ -48,15 +48,24 @@
 )
 async def route_list_shacl_test_suites(
         project: PydanticObjectId = None,
-        ids: Annotated[List[PydanticObjectId | str] | None, Query()] = None
+        ids: Annotated[List[PydanticObjectId | str] | None, Query()] = None,
+        page: int = None,
+        limit: int = None,
+        q: str = None
 ):
     filters: dict = {}
     if project:
         filters['project'] = Project.link_from_id(project)
     if ids is not None:
         filters['_id'] = {"$in": ids}
-    items: List[SHACLTestSuite] = await list_shacl_test_suites(filters)
-    return APIListSHACLTestSuitesPaginatedResponse(items=items, count=len(items))
+    if q is not None:
+        filters['q'] = q
+
+    items, total_count = await list_shacl_test_suites(filters, page, limit)
+    return APIListSHACLTestSuitesPaginatedResponse(
+        items=items,
+        count=total_count
+    )
 
 
 @router.post(
@@ -115,10 +124,25 @@ async def route_delete_shacl_test_suite(shacl_test_suite: SHACLTestSuite = Depen
     response_model=APIListSHACLTestFileResourcesPaginatedResponse
 )
 async def route_list_shacl_test_suite_file_resources(
-        id: PydanticObjectId = None
+        shacl_test_suite: SHACLTestSuite = Depends(get_shacl_test_suite),
+        project: PydanticObjectId = None,
+        page: int = None,
+        limit: int = None,
+        q: str = None
 ):
-    items: List[SHACLTestFileResource] = await list_shacl_test_suite_file_resources(id)
-    return APIListSHACLTestFileResourcesPaginatedResponse(items=items, count=len(items))
+
+    filters: dict = {}
+    if project:
+        filters['project'] = Project.link_from_id(project)
+    if q is not None:
+        filters['q'] = q
+
+    items, total_count = \
+        await list_shacl_test_suite_file_resources(shacl_test_suite, filters, page, limit)
+    return APIListSHACLTestFileResourcesPaginatedResponse(
+        items=items,
+        count=total_count
+    )
 
 
 @router.post(