From 496bba5a032cf617e02f5a223f785fd4284afdc1 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 27 Nov 2024 14:20:39 +0100 Subject: [PATCH 01/29] api and handlers --- .../folders/_workspaces_api.py | 85 +++++++++++++++++++ .../folders/_workspaces_handlers.py | 76 +++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py create mode 100644 services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py new file mode 100644 index 00000000000..b79939022e8 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -0,0 +1,85 @@ +import logging + +from aiohttp import web +from models_library.access_rights import AccessRights +from models_library.folders import FolderID +from models_library.products import ProductName +from models_library.users import UserID +from models_library.workspaces import WorkspaceID + +from ..users.api import get_user +from ..workspaces.api import check_user_workspace_access +from . import _folders_db + +_logger = logging.getLogger(__name__) + + +async def move_folder_into_workspace( + app: web.Application, + *, + user_id: UserID, + folder_id: FolderID, + workspace_id: WorkspaceID | None, + product_name: ProductName, +) -> None: + # 1. User needs to have delete permission on source folder + folder_db = await _folders_db.get( + app, folder_id=folder_id, product_name=product_name + ) + workspace_is_private = True + user_folder_access_rights = AccessRights(read=True, write=True, delete=True) + if folder_db.workspace_id: + user_workspace_access_rights = await check_user_workspace_access( + app, + user_id=user_id, + workspace_id=folder_db.workspace_id, + product_name=product_name, + permission="delete", + ) + workspace_is_private = False + user_folder_access_rights = user_workspace_access_rights.my_access_rights + + # Here we have checked user has delete access rights on the folder he is moving + + # 2. User needs to have write permission on destination workspace + if workspace_id is not None: + user_workspace_access_rights = await check_user_workspace_access( + app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission="write", + ) + + # Here we have already guaranties that user has all the right permissions to do this operation + + # Get all project children + # await _folders_db. + # Get all folder children + children_folders_list = await _folders_db.get_folders_recursively( + app, connection=None, folder_id=folder_id, product_name=product_name + ) + + # 3. Delete project to folders (for everybody) + await project_to_folders_db.delete_all_project_to_folder_by_project_id( + app, + project_id=project_id, + ) + + # 4. Update workspace ID on the project resource + await project_api.patch_project( + project_uuid=project_id, + new_partial_project_data={"workspace_id": workspace_id}, + ) + + # 5. Remove all project permissions, leave only the user who moved the project + user = await get_user(app, user_id=user_id) + await project_groups_db.delete_all_project_groups(app, project_id=project_id) + await project_groups_db.update_or_insert_project_group( + app, + project_id=project_id, + group_id=user["primary_gid"], + read=True, + write=True, + delete=True, + ) diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py new file mode 100644 index 00000000000..0905974f863 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py @@ -0,0 +1,76 @@ +import functools +import logging +from typing import Annotated + +from aiohttp import web +from models_library.folders import FolderID +from models_library.utils.common_validators import null_or_none_str_to_none_validator +from models_library.workspaces import WorkspaceID +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field +from servicelib.aiohttp import status +from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as +from servicelib.aiohttp.typing_extension import Handler + +from .._meta import api_version_prefix as VTAG +from ..folders.errors import FolderAccessForbiddenError, FolderNotFoundError +from ..login.decorators import login_required +from ..security.decorators import permission_required +from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError +from . import _workspaces_api +from ._models import FoldersRequestContext + +_logger = logging.getLogger(__name__) + + +def _handle_folders_workspaces_exceptions(handler: Handler): + @functools.wraps(handler) + async def wrapper(request: web.Request) -> web.StreamResponse: + try: + return await handler(request) + + except ( + FolderNotFoundError, + WorkspaceNotFoundError, + ) as exc: + raise web.HTTPNotFound(reason=f"{exc}") from exc + + except ( + FolderAccessForbiddenError, + WorkspaceAccessForbiddenError, + ) as exc: + raise web.HTTPForbidden(reason=f"{exc}") from exc + + return wrapper + + +routes = web.RouteTableDef() + + +class _FolderWorkspacesPathParams(BaseModel): + folder_id: FolderID + workspace_id: Annotated[ + WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) + ] = Field(default=None) + + model_config = ConfigDict(extra="forbid") + + +@routes.put( + f"/{VTAG}/folders/{{folder_id}}/workspaces/{{workspace_id}}", + name="replace_folder_workspace", +) +@login_required +@permission_required("folder.update") +@_handle_folders_workspaces_exceptions +async def replace_project_workspace(request: web.Request): + req_ctx = FoldersRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(_FolderWorkspacesPathParams, request) + + await _workspaces_api.move_folder_into_workspace( + app=request.app, + user_id=req_ctx.user_id, + folder_id=path_params.folder_id, + workspace_id=path_params.workspace_id, + product_name=req_ctx.product_name, + ) + return web.json_response(status=status.HTTP_204_NO_CONTENT) From fcd07013f1b303d17f1ef785b28ead4a06f8953e Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 27 Nov 2024 17:19:43 +0100 Subject: [PATCH 02/29] db --- .../folders/_folders_db.py | 68 +++++++++-- .../folders/_workspaces_api.py | 106 +++++++++++++----- .../folders/_workspaces_handlers.py | 3 + .../projects/_folders_db.py | 78 ++++++++++++- .../projects/_workspaces_api.py | 1 + .../src/simcore_service_webserver/utils.py | 14 +++ 6 files changed, 227 insertions(+), 43 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py index 7e3a54d0bb5..c2fd49895f7 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py @@ -6,7 +6,7 @@ import logging from datetime import datetime -from typing import Any, Final, cast +from typing import Final, cast import sqlalchemy as sa from aiohttp import web @@ -33,6 +33,7 @@ from simcore_postgres_database.utils_workspaces_sql import ( create_my_workspace_access_rights_subquery, ) +from simcore_service_webserver.utils import UnSet, as_dict_exclude_unset from sqlalchemy import func from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.orm import aliased @@ -43,18 +44,9 @@ _logger = logging.getLogger(__name__) - -class UnSet: - ... - - _unset: Final = UnSet() -def as_dict_exclude_unset(**params) -> dict[str, Any]: - return {k: v for k, v in params.items() if not isinstance(v, UnSet)} - - _SELECTION_ARGS = ( folders_v2.c.folder_id, folders_v2.c.name, @@ -324,6 +316,7 @@ async def update( parent_folder_id: FolderID | None | UnSet = _unset, trashed_at: datetime | None | UnSet = _unset, trashed_explicitly: bool | UnSet = _unset, + workspace_id: WorkspaceID | None | UnSet = _unset, ) -> FolderDB: """ Batch/single patch of folder/s @@ -334,6 +327,7 @@ async def update( parent_folder_id=parent_folder_id, trashed_at=trashed_at, trashed_explicitly=trashed_explicitly, + workspace_id=workspace_id, ) query = ( @@ -467,6 +461,60 @@ async def get_projects_recursively_only_if_user_is_owner( return [ProjectID(row[0]) async for row in result] +async def get_all_folders_and_projects_recursively( + app: web.Application, + connection: AsyncConnection | None = None, + *, + folder_id: FolderID, + private_workspace_user_id_or_none: UserID | None, + product_name: ProductName, +) -> tuple[list[FolderID], list[ProjectID]]: + """ + The purpose of this function is to retrieve all projects within the provided folder ID. + """ + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + + # Step 1: Define the base case for the recursive CTE + base_query = select( + folders_v2.c.folder_id, folders_v2.c.parent_folder_id + ).where( + (folders_v2.c.folder_id == folder_id) # <-- specified folder id + & (folders_v2.c.product_name == product_name) + ) + folder_hierarchy_cte = base_query.cte(name="folder_hierarchy", recursive=True) + + # Step 2: Define the recursive case + folder_alias = aliased(folders_v2) + recursive_query = select( + folder_alias.c.folder_id, folder_alias.c.parent_folder_id + ).select_from( + folder_alias.join( + folder_hierarchy_cte, + folder_alias.c.parent_folder_id == folder_hierarchy_cte.c.folder_id, + ) + ) + + # Step 3: Combine base and recursive cases into a CTE + folder_hierarchy_cte = folder_hierarchy_cte.union_all(recursive_query) + + # Step 4: Execute the query to get all descendants + final_query = select(folder_hierarchy_cte) + result = await conn.stream(final_query) + # list of tuples [(folder_id, parent_folder_id), ...] ex. [(1, None), (2, 1)] + folder_ids = [item[0] async for item in result] + + query = select(projects_to_folders.c.project_uuid).where( + (projects_to_folders.c.folder_id.in_(folder_ids)) + & (projects_to_folders.c.user_id == private_workspace_user_id_or_none) + ) + + result = await conn.stream(query) + project_ids = [ProjectID(row[0]) async for row in result] + + return folder_ids, project_ids + + async def get_folders_recursively( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py index b79939022e8..ef2eb872e31 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -1,12 +1,15 @@ import logging from aiohttp import web -from models_library.access_rights import AccessRights from models_library.folders import FolderID from models_library.products import ProductName from models_library.users import UserID from models_library.workspaces import WorkspaceID +from simcore_service_webserver.projects.db import ProjectDBAPI +from ..projects import _folders_db as project_to_folders_db +from ..projects import _groups_db as project_groups_db +from ..projects._access_rights_api import check_user_project_permission from ..users.api import get_user from ..workspaces.api import check_user_workspace_access from . import _folders_db @@ -22,14 +25,15 @@ async def move_folder_into_workspace( workspace_id: WorkspaceID | None, product_name: ProductName, ) -> None: + projects_db = ProjectDBAPI.get_from_app_context(app) + # 1. User needs to have delete permission on source folder folder_db = await _folders_db.get( app, folder_id=folder_id, product_name=product_name ) workspace_is_private = True - user_folder_access_rights = AccessRights(read=True, write=True, delete=True) if folder_db.workspace_id: - user_workspace_access_rights = await check_user_workspace_access( + await check_user_workspace_access( app, user_id=user_id, workspace_id=folder_db.workspace_id, @@ -37,13 +41,10 @@ async def move_folder_into_workspace( permission="delete", ) workspace_is_private = False - user_folder_access_rights = user_workspace_access_rights.my_access_rights - - # Here we have checked user has delete access rights on the folder he is moving - # 2. User needs to have write permission on destination workspace + # 2. User needs to have write permission on destination workspace if workspace_id is not None: - user_workspace_access_rights = await check_user_workspace_access( + await check_user_workspace_access( app, user_id=user_id, workspace_id=workspace_id, @@ -51,35 +52,80 @@ async def move_folder_into_workspace( permission="write", ) - # Here we have already guaranties that user has all the right permissions to do this operation + # 3. User needs to have delete permission on all the projects inside source folder + ( + folder_ids, + project_ids, + ) = await _folders_db.get_all_folders_and_projects_recursively( + app, + connection=None, + folder_id=folder_id, + private_workspace_user_id_or_none=user_id if workspace_is_private else None, + product_name=product_name, + ) + # NOTE: Not the most effective, can be improved + for project_id in project_ids: + await check_user_project_permission( + app, + project_id=project_id, + user_id=user_id, + product_name=product_name, + permission="delete", + ) - # Get all project children - # await _folders_db. - # Get all folder children - children_folders_list = await _folders_db.get_folders_recursively( - app, connection=None, folder_id=folder_id, product_name=product_name + # ⬆️ Here we have already guaranties that user has all the right permissions to do this operation ⬆️ + + # 4. Update workspace ID on the project resource + for project_id in project_ids: + await projects_db.patch_project( + project_uuid=project_id, + new_partial_project_data={"workspace_id": workspace_id}, + ) + + # 5. BATCH update of folders with workspace_id + await _folders_db.update( + app, + connection=None, + folders_id_or_ids=set(folder_ids), + product_name=product_name, + workspace_id=workspace_id, # <-- Updating workspace_id ) - # 3. Delete project to folders (for everybody) - await project_to_folders_db.delete_all_project_to_folder_by_project_id( + # 6. Update source folder parent folder ID with NULL (it will appear in the root directory) + await _folders_db.update( app, - project_id=project_id, + connection=None, + folders_id_or_ids=folder_id, + product_name=product_name, + parent_folder_id=None, # <-- Updating parent folder ID ) - # 4. Update workspace ID on the project resource - await project_api.patch_project( - project_uuid=project_id, - new_partial_project_data={"workspace_id": workspace_id}, + # 7. Remove all records of project to folders that are not in the folders that we are moving + # (ex. If we are moving from private workspace, the same project can be in different folders for different users) + await project_to_folders_db.delete_all_project_to_folder_by_project_ids_not_in_folder_ids( + app, + connection=None, + project_id_or_ids=set(project_ids), + not_in_folder_ids=set(folder_ids), ) - # 5. Remove all project permissions, leave only the user who moved the project - user = await get_user(app, user_id=user_id) - await project_groups_db.delete_all_project_groups(app, project_id=project_id) - await project_groups_db.update_or_insert_project_group( + # 8. Update the user id field for the remaining folders + await project_to_folders_db.update_project_to_folder( app, - project_id=project_id, - group_id=user["primary_gid"], - read=True, - write=True, - delete=True, + connection=None, + folders_id_or_ids=set(folder_ids), + user_id=user_id if workspace_id is None else None, ) + + # 9. Remove all project permissions, leave only the user who moved the project + user = await get_user(app, user_id=user_id) + for project_id in project_ids: + await project_groups_db.delete_all_project_groups(app, project_id=project_id) + await project_groups_db.update_or_insert_project_group( + app, + project_id=project_id, + group_id=user["primary_gid"], + read=True, + write=True, + delete=True, + ) diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py index 0905974f863..421643c5114 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py @@ -14,6 +14,7 @@ from .._meta import api_version_prefix as VTAG from ..folders.errors import FolderAccessForbiddenError, FolderNotFoundError from ..login.decorators import login_required +from ..projects.exceptions import ProjectInvalidRightsError, ProjectNotFoundError from ..security.decorators import permission_required from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _workspaces_api @@ -29,12 +30,14 @@ async def wrapper(request: web.Request) -> web.StreamResponse: return await handler(request) except ( + ProjectInvalidRightsError, FolderNotFoundError, WorkspaceNotFoundError, ) as exc: raise web.HTTPNotFound(reason=f"{exc}") from exc except ( + ProjectNotFoundError, FolderAccessForbiddenError, WorkspaceAccessForbiddenError, ) as exc: diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index 59ea8ebe282..cb40bdffbcd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -6,6 +6,7 @@ import logging from datetime import datetime +from typing import Final from aiohttp import web from models_library.folders import FolderID @@ -13,15 +14,17 @@ from models_library.users import UserID from pydantic import BaseModel from simcore_postgres_database.models.projects_to_folders import projects_to_folders +from simcore_postgres_database.utils_repos import transaction_context +from simcore_service_webserver.utils import UnSet, as_dict_exclude_unset from sqlalchemy import func, literal_column +from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select -from ..db.plugin import get_database_engine +from ..db.plugin import get_asyncpg_engine, get_database_engine _logger = logging.getLogger(__name__) - -_logger = logging.getLogger(__name__) +_unset: Final = UnSet() ### Models @@ -110,3 +113,72 @@ async def delete_all_project_to_folder_by_project_id( projects_to_folders.c.project_uuid == f"{project_id}" ) ) + + +### AsyncPg + + +async def update_project_to_folder( + app: web.Application, + connection: AsyncConnection | None = None, + *, + folders_id_or_ids: FolderID | set[FolderID], + filter_by_user_id: UserID | None | UnSet = _unset, + # updatable columns + user_id: UserID | None | UnSet = _unset, +) -> None: + """ + Batch/single patch of project to folders + """ + # NOTE: exclude unset can also be done using a pydantic model and dict(exclude_unset=True) + updated = as_dict_exclude_unset( + user_id=user_id, + ) + + query = projects_to_folders.update().values(modified=func.now(), **updated) + + if isinstance(folders_id_or_ids, set): + # batch-update + query = query.where( + projects_to_folders.c.folder_id.in_(list(folders_id_or_ids)) + ) + else: + # single-update + query = query.where(projects_to_folders.c.folder_id == folders_id_or_ids) + + if not isinstance(filter_by_user_id, UnSet): + query = query.where(projects_to_folders.c.user_id == filter_by_user_id) + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream(query) + + +async def delete_all_project_to_folder_by_project_ids_not_in_folder_ids( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id_or_ids: ProjectID | set[ProjectID], + # Optional filter + not_in_folder_ids: set[FolderID], +) -> None: + query = projects_to_folders.delete() + + if isinstance(project_id_or_ids, set): + # batch-delete + query = query.where( + projects_to_folders.c.project_uuid.in_(list(project_id_or_ids)) + ) + else: + # single-delete + query = query.where( + projects_to_folders.c.project_uuid == f"{project_id_or_ids}" + ) + + query = query.where( + projects_to_folders.c.folder_id.not_in( # <-- NOT IN! + [f"{folder_id}" for folder_id in not_in_folder_ids] + ) + ) + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream(query) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py index 105decdd3ac..39441f42716 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py @@ -55,6 +55,7 @@ async def move_project_into_workspace( project_uuid=project_id, new_partial_project_data={"workspace_id": workspace_id}, ) + # NOTE: MD: should I also patch the project owner? -> probably yes, or if it is more like "original owner" then probably no # 5. Remove all project permissions, leave only the user who moved the project user = await get_user(app, user_id=user_id) diff --git a/services/web/server/src/simcore_service_webserver/utils.py b/services/web/server/src/simcore_service_webserver/utils.py index c6eade6345d..1f73ac06e0a 100644 --- a/services/web/server/src/simcore_service_webserver/utils.py +++ b/services/web/server/src/simcore_service_webserver/utils.py @@ -194,3 +194,17 @@ def compute_sha1_on_small_dataset(d: Any) -> SHA1Str: # SEE options in https://github.com/ijl/orjson#option data_bytes = orjson.dumps(d, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS) return SHA1Str(hashlib.sha1(data_bytes).hexdigest()) # nosec # NOSONAR + + +# ----------------------------------------------- +# +# UNSET +# + + +class UnSet: + ... + + +def as_dict_exclude_unset(**params) -> dict[str, Any]: + return {k: v for k, v in params.items() if not isinstance(v, UnSet)} From 33b48e6b2c6c4d05d30cdafe4df7485c5e9e3d6f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 27 Nov 2024 17:37:18 +0100 Subject: [PATCH 03/29] open api specs --- api/specs/web-server/_folders.py | 23 ++ .../api/v0/openapi.yaml | 331 +----------------- 2 files changed, 35 insertions(+), 319 deletions(-) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index f98b5e98308..dda18a64372 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -23,6 +23,9 @@ FoldersListQueryParams, FoldersPathParams, ) +from simcore_service_webserver.folders._workspaces_handlers import ( + _FolderWorkspacesPathParams, +) router = APIRouter( prefix=f"/{API_VTAG}", @@ -92,3 +95,23 @@ async def delete_folder( _path: Annotated[FoldersPathParams, Depends()], ): ... + + +### Move Folder to Workspace + + +router = APIRouter( + prefix=f"/{API_VTAG}", + tags=["folders", "workspaces"], +) + + +@router.put( + "/folders/{folder_id}/workspaces/{workspace_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Move folder to the workspace", +) +async def replace_folder_workspace( + _path: Annotated[_FolderWorkspacesPathParams, Depends()], +): + ... diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 688d2187cb3..878138f87a7 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2594,173 +2594,13 @@ paths: responses: '200': description: Successful Response - /v0/folders: - post: - tags: - - folders - summary: Create Folder - operationId: create_folder - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/FolderCreateBodyParams' - responses: - '201': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FolderGet_' - get: - tags: - - folders - summary: List Folders - operationId: list_folders - parameters: - - name: filters - in: query - required: false - schema: - anyOf: - - type: string - contentMediaType: application/json - contentSchema: {} - - type: 'null' - title: Filters - - name: order_by - in: query - required: false - schema: - type: string - contentMediaType: application/json - contentSchema: {} - default: '{"field":"modified","direction":"desc"}' - title: Order By - - name: limit - in: query - required: false - schema: - type: integer - default: 20 - title: Limit - - name: offset - in: query - required: false - schema: - type: integer - default: 0 - title: Offset - - name: folder_id - in: query - required: false - schema: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Folder Id - - name: workspace_id - in: query - required: false - schema: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Workspace Id - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_FolderGet__' - /v0/folders:search: - get: - tags: - - folders - summary: List Folders Full Search - operationId: list_folders_full_search - parameters: - - name: filters - in: query - required: false - schema: - anyOf: - - type: string - contentMediaType: application/json - contentSchema: {} - - type: 'null' - title: Filters - - name: order_by - in: query - required: false - schema: - type: string - contentMediaType: application/json - contentSchema: {} - default: '{"field":"modified","direction":"desc"}' - title: Order By - - name: limit - in: query - required: false - schema: - type: integer - default: 20 - title: Limit - - name: offset - in: query - required: false - schema: - type: integer - default: 0 - title: Offset - - name: text - in: query - required: false - schema: - anyOf: - - type: string - - type: 'null' - title: Text - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_FolderGet__' - /v0/folders/{folder_id}: - get: - tags: - - folders - summary: Get Folder - operationId: get_folder - parameters: - - name: folder_id - in: path - required: true - schema: - type: integer - exclusiveMinimum: true - title: Folder Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FolderGet_' + /v0/folders/{folder_id}/workspaces/{workspace_id}: put: tags: - folders - summary: Replace Folder - operationId: replace_folder + - workspaces + summary: Move folder to the workspace + operationId: replace_folder_workspace parameters: - name: folder_id in: path @@ -2770,33 +2610,16 @@ paths: exclusiveMinimum: true title: Folder Id minimum: 0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/FolderReplaceBodyParams' - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FolderGet_' - delete: - tags: - - folders - summary: Delete Folder - operationId: delete_folder - parameters: - - name: folder_id + - name: workspace_id in: path required: true schema: - type: integer - exclusiveMinimum: true - title: Folder Id - minimum: 0 + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Workspace Id responses: '204': description: Successful Response @@ -4396,7 +4219,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: ProjectNotFoundError, UserDefaultWalletNotFoundError + description: UserDefaultWalletNotFoundError, ProjectNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -7564,19 +7387,6 @@ components: title: Error type: object title: Envelope[FileUploadSchema] - Envelope_FolderGet_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FolderGet' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FolderGet] Envelope_GetCreditPrice_: properties: data: @@ -8451,22 +8261,6 @@ components: title: Error type: object title: Envelope[list[FileMetaDataGet]] - Envelope_list_FolderGet__: - properties: - data: - anyOf: - - items: - $ref: '#/components/schemas/FolderGet' - type: array - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[list[FolderGet]] Envelope_list_GroupUserGet__: properties: data: @@ -9240,107 +9034,6 @@ components: - urls - links title: FileUploadSchema - FolderCreateBodyParams: - properties: - name: - type: string - maxLength: 100 - minLength: 1 - title: Name - parentFolderId: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Parentfolderid - workspaceId: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Workspaceid - additionalProperties: false - type: object - required: - - name - title: FolderCreateBodyParams - FolderGet: - properties: - folderId: - type: integer - exclusiveMinimum: true - title: Folderid - minimum: 0 - parentFolderId: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Parentfolderid - name: - type: string - title: Name - createdAt: - type: string - format: date-time - title: Createdat - modifiedAt: - type: string - format: date-time - title: Modifiedat - trashedAt: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Trashedat - owner: - type: integer - exclusiveMinimum: true - title: Owner - minimum: 0 - workspaceId: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Workspaceid - myAccessRights: - $ref: '#/components/schemas/AccessRights' - type: object - required: - - folderId - - name - - createdAt - - modifiedAt - - trashedAt - - owner - - workspaceId - - myAccessRights - title: FolderGet - FolderReplaceBodyParams: - properties: - name: - type: string - maxLength: 100 - minLength: 1 - title: Name - parentFolderId: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Parentfolderid - additionalProperties: false - type: object - required: - - name - title: FolderReplaceBodyParams GenerateInvitation: properties: guest: From dc092c21e3038017c5348cff16ff46162c32e4ce Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 13:26:40 +0100 Subject: [PATCH 04/29] adding unit tests --- .../folders/_folders_db.py | 2 + .../folders/_workspaces_api.py | 3 +- .../folders/plugin.py | 3 +- .../projects/_folders_db.py | 14 +- .../unit/with_dbs/04/workspaces/conftest.py | 2 + ...aces__moving_folders_between_workspaces.py | 295 ++++++++++++++++++ 6 files changed, 310 insertions(+), 9 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py index c2fd49895f7..ce6633f8c01 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py @@ -317,6 +317,7 @@ async def update( trashed_at: datetime | None | UnSet = _unset, trashed_explicitly: bool | UnSet = _unset, workspace_id: WorkspaceID | None | UnSet = _unset, + user_id: UserID | None | UnSet = _unset, ) -> FolderDB: """ Batch/single patch of folder/s @@ -328,6 +329,7 @@ async def update( trashed_at=trashed_at, trashed_explicitly=trashed_explicitly, workspace_id=workspace_id, + user_id=user_id, ) query = ( diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py index ef2eb872e31..da4d689137d 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -89,6 +89,7 @@ async def move_folder_into_workspace( folders_id_or_ids=set(folder_ids), product_name=product_name, workspace_id=workspace_id, # <-- Updating workspace_id + user_id=user_id if workspace_id is None else None, # <-- Updating user_id ) # 6. Update source folder parent folder ID with NULL (it will appear in the root directory) @@ -114,7 +115,7 @@ async def move_folder_into_workspace( app, connection=None, folders_id_or_ids=set(folder_ids), - user_id=user_id if workspace_id is None else None, + user_id=user_id if workspace_id is None else None, # <-- Updating user_id ) # 9. Remove all project permissions, leave only the user who moved the project diff --git a/services/web/server/src/simcore_service_webserver/folders/plugin.py b/services/web/server/src/simcore_service_webserver/folders/plugin.py index 8ddef03ec1f..2601962e52f 100644 --- a/services/web/server/src/simcore_service_webserver/folders/plugin.py +++ b/services/web/server/src/simcore_service_webserver/folders/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _folders_handlers, _trash_handlers +from . import _folders_handlers, _trash_handlers, _workspaces_handlers _logger = logging.getLogger(__name__) @@ -25,3 +25,4 @@ def setup_folders(app: web.Application): # routes app.router.add_routes(_folders_handlers.routes) app.router.add_routes(_trash_handlers.routes) + app.router.add_routes(_workspaces_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index cb40bdffbcd..e31d2f56f0d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -123,7 +123,7 @@ async def update_project_to_folder( connection: AsyncConnection | None = None, *, folders_id_or_ids: FolderID | set[FolderID], - filter_by_user_id: UserID | None | UnSet = _unset, + # filter_by_user_id: UserID | None | UnSet = _unset, # updatable columns user_id: UserID | None | UnSet = _unset, ) -> None: @@ -146,8 +146,8 @@ async def update_project_to_folder( # single-update query = query.where(projects_to_folders.c.folder_id == folders_id_or_ids) - if not isinstance(filter_by_user_id, UnSet): - query = query.where(projects_to_folders.c.user_id == filter_by_user_id) + # if not isinstance(filter_by_user_id, UnSet): + # query = query.where(projects_to_folders.c.user_id == filter_by_user_id) async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream(query) @@ -166,7 +166,9 @@ async def delete_all_project_to_folder_by_project_ids_not_in_folder_ids( if isinstance(project_id_or_ids, set): # batch-delete query = query.where( - projects_to_folders.c.project_uuid.in_(list(project_id_or_ids)) + projects_to_folders.c.project_uuid.in_( + [f"{project_id}" for project_id in project_id_or_ids] + ) ) else: # single-delete @@ -175,9 +177,7 @@ async def delete_all_project_to_folder_by_project_ids_not_in_folder_ids( ) query = query.where( - projects_to_folders.c.folder_id.not_in( # <-- NOT IN! - [f"{folder_id}" for folder_id in not_in_folder_ids] - ) + projects_to_folders.c.folder_id.not_in(not_in_folder_ids) # <-- NOT IN! ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py b/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py index 744b30da23b..fa008269aaf 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py @@ -5,6 +5,7 @@ import pytest import sqlalchemy as sa +from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.workspaces import workspaces @@ -13,3 +14,4 @@ def workspaces_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: with postgres_db.connect() as con: yield con.execute(workspaces.delete()) + con.execute(projects.delete()) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py new file mode 100644 index 00000000000..22176798927 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -0,0 +1,295 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + + +from copy import deepcopy +from http import HTTPStatus +from http.client import NO_CONTENT + +import pytest +from aiohttp.test_utils import TestClient +from pytest_mock import MockerFixture +from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.webserver_login import UserInfoDict +from pytest_simcore.helpers.webserver_projects import create_project +from servicelib.aiohttp import status +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.fixture +def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): + mocker.patch( + "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", + spec=True, + return_value=[], + ) + mocker.patch( + "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + spec=True, + return_value=[], + ) + mocker.patch( + "simcore_service_webserver.projects._crud_handlers.project_uses_available_services", + spec=True, + return_value=True, + ) + + +# @pytest.mark.parametrize(*standard_role_response(), ids=str) +# async def test_moving_between_workspaces_user_role_permissions( +# client: TestClient, +# logged_user: UserInfoDict, +# user_project: ProjectDict, +# expected: ExpectedResponse, +# mock_catalog_api_get_services_for_user_in_product: MockerFixture, +# fake_project: ProjectDict, +# workspaces_clean_db: None, +# ): +# # Move project from workspace to your private workspace +# base_url = client.app.router["replace_folder_workspace"].url_for( +# folder_id="1", workspace_id="null" +# ) +# resp = await client.put(f"{base_url}") +# await assert_status(resp, expected.no_content) + + +## Usecases to test: +# 1. Private workspace -> Shared workspace +# 2. Shared workspace -> Shared workspace +# 3. Shared workspace -> Private workspace + + +async def _setup_test( + client: TestClient, + logged_user: UserInfoDict, + fake_project: ProjectDict, +) -> str: + assert client.app + + ### Project creation + + # Create 2 projects + project_data = deepcopy(fake_project) + first_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + second_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + + ### Folder creation + + # Create folder + url = client.app.router["create_folder"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "Original user folder", + }, + ) + first_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # Create sub folder of previous folder + url = client.app.router["create_folder"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "Second user folder", + "parentFolderId": f"{first_folder['folderId']}", + }, + ) + second_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # Create sub sub folder of previous sub folder + url = client.app.router["create_folder"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "Third user folder", + "parentFolderId": f"{second_folder['folderId']}", + }, + ) + third_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) + + ### Move projects to subfolder + # add first project to the folder + url = client.app.router["replace_project_folder"].url_for( + folder_id=f"{second_folder['folderId']}", project_id=f"{first_project['uuid']}" + ) + resp = await client.put(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + # add second project to the folder + url = client.app.router["replace_project_folder"].url_for( + folder_id=f"{second_folder['folderId']}", project_id=f"{second_project['uuid']}" + ) + resp = await client.put(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + ## Double check whether everything is setup OK + url = ( + client.app.router["list_projects"] + .url_for() + .with_query({"folder_id": f"{second_folder['folderId']}"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 2 + + url = ( + client.app.router["list_projects"] + .url_for() + .with_query({"folder_id": f"{first_folder['folderId']}"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 0 + + url = client.app.router["list_projects"].url_for().with_query({"folder_id": "null"}) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 0 + + url = client.app.router["list_folders"].url_for().with_query({"folder_id": "null"}) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + url = ( + client.app.router["list_folders"] + .url_for() + .with_query({"folder_id": f"{first_folder['folderId']}"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + return f"{second_folder['folderId']}" + + +async def _move_folder_to_workspace_and_assert( + client: TestClient, folder_id: str, workspace_id: str +): + assert client.app + + # MOVE + base_url = client.app.router["replace_folder_workspace"].url_for( + folder_id=folder_id, + workspace_id=workspace_id, + ) + resp = await client.put(f"{base_url}") + await assert_status(resp, NO_CONTENT) + + # ASSERT + url = ( + client.app.router["list_projects"] + .url_for() + .with_query( + { + "folder_id": folder_id, + "workspace_id": workspace_id, + } + ) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 2 + + url = ( + client.app.router["list_folders"] + .url_for() + .with_query( + { + "folder_id": folder_id, + "workspace_id": workspace_id, + } + ) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_moving_between_private_and_shared_workspaces( + client: TestClient, + logged_user: UserInfoDict, + # user_project: ProjectDict, + expected: HTTPStatus, + mock_catalog_api_get_services_for_user_in_product: MockerFixture, + fake_project: ProjectDict, + workspaces_clean_db: None, +): + assert client.app + + moving_folder_id = await _setup_test(client, logged_user, fake_project) + + # We will test these scenarios of moving folders: + # 1. Private workspace -> Shared workspace + # 2. Shared workspace A -> Shared workspace B + # 3. Shared workspace A -> Shared workspace A (Corner case - This endpoint is not used like this) + # 4. Shared workspace -> Private workspace + # 5. Private workspace -> Private workspace (Corner case - This endpoint is not used like this) + + # create a new workspace + url = client.app.router["create_workspace"].url_for() + resp = await client.post( + url.path, + json={ + "name": "A", + "description": "A", + "thumbnail": None, + }, + ) + added_workspace, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # 1. Private workspace -> Shared workspace A + await _move_folder_to_workspace_and_assert( + client, + folder_id=moving_folder_id, + workspace_id=f"{added_workspace['workspaceId']}", + ) + + # create a new workspace + url = client.app.router["create_workspace"].url_for() + resp = await client.post( + url.path, + json={ + "name": "B", + "description": "B", + "thumbnail": None, + }, + ) + second_workspace, _ = await assert_status(resp, status.HTTP_201_CREATED) + # 2. Shared workspace A -> Shared workspace B + await _move_folder_to_workspace_and_assert( + client, + folder_id=moving_folder_id, + workspace_id=f"{second_workspace['workspaceId']}", + ) + + # 3. (Corner case) Shared workspace A -> Shared workspace A + await _move_folder_to_workspace_and_assert( + client, + folder_id=moving_folder_id, + workspace_id=f"{second_workspace['workspaceId']}", + ) + + # 4. Shared workspace -> Private workspace + await _move_folder_to_workspace_and_assert( + client, folder_id=moving_folder_id, workspace_id="null" + ) + + # 5. (Corner case) Private workspace -> Private workspace + await _move_folder_to_workspace_and_assert( + client, folder_id=moving_folder_id, workspace_id="null" + ) From 3f8e8b1db933f4f95e622d404fc5a93b32d67f3a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 13:28:53 +0100 Subject: [PATCH 05/29] adding unit tests --- .../test_workspaces__moving_folders_between_workspaces.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 22176798927..1aa772367e3 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -223,7 +223,6 @@ async def _move_folder_to_workspace_and_assert( async def test_moving_between_private_and_shared_workspaces( client: TestClient, logged_user: UserInfoDict, - # user_project: ProjectDict, expected: HTTPStatus, mock_catalog_api_get_services_for_user_in_product: MockerFixture, fake_project: ProjectDict, From e6908f0e774560bfd586f6008da1c42617fb6001 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:39:48 +0100 Subject: [PATCH 06/29] refactor project DB --- .../helpers/webserver_projects.py | 2 +- .../folders/_workspaces_api.py | 11 +- .../projects/_db_v2.py | 59 ++++++++++ .../projects/_groups_db.py | 101 +++++++++++------- .../projects/_workspaces_api.py | 4 +- .../simcore_service_webserver/projects/db.py | 45 +------- .../projects/projects_api.py | 7 +- services/web/server/tests/conftest.py | 2 +- .../integration/01/test_garbage_collection.py | 2 +- .../tests/unit/with_dbs/03/test_project_db.py | 2 +- ...aces__moving_folders_between_workspaces.py | 2 + 11 files changed, 143 insertions(+), 94 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_db_v2.py diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 55065daaf76..092ab82d655 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -95,7 +95,7 @@ async def create_project( for group_id, permissions in _access_rights.items(): await update_or_insert_project_group( app, - new_project["uuid"], + project_id=new_project["uuid"], group_id=int(group_id), read=permissions["read"], write=permissions["write"], diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py index da4d689137d..3a153d6237e 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -5,8 +5,8 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.workspaces import WorkspaceID -from simcore_service_webserver.projects.db import ProjectDBAPI +from ..projects import _db_v2 as projects_db from ..projects import _folders_db as project_to_folders_db from ..projects import _groups_db as project_groups_db from ..projects._access_rights_api import check_user_project_permission @@ -25,8 +25,6 @@ async def move_folder_into_workspace( workspace_id: WorkspaceID | None, product_name: ProductName, ) -> None: - projects_db = ProjectDBAPI.get_from_app_context(app) - # 1. User needs to have delete permission on source folder folder_db = await _folders_db.get( app, folder_id=folder_id, product_name=product_name @@ -78,6 +76,8 @@ async def move_folder_into_workspace( # 4. Update workspace ID on the project resource for project_id in project_ids: await projects_db.patch_project( + app=app, + connection=None, project_uuid=project_id, new_partial_project_data={"workspace_id": workspace_id}, ) @@ -121,9 +121,12 @@ async def move_folder_into_workspace( # 9. Remove all project permissions, leave only the user who moved the project user = await get_user(app, user_id=user_id) for project_id in project_ids: - await project_groups_db.delete_all_project_groups(app, project_id=project_id) + await project_groups_db.delete_all_project_groups( + app, connection=None, project_id=project_id + ) await project_groups_db.update_or_insert_project_group( app, + connection=None, project_id=project_id, group_id=user["primary_gid"], read=True, diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_v2.py b/services/web/server/src/simcore_service_webserver/projects/_db_v2.py new file mode 100644 index 00000000000..3c94e9e7cdc --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_db_v2.py @@ -0,0 +1,59 @@ +import logging + +import sqlalchemy as sa +from aiohttp import web +from models_library.projects import ProjectID +from simcore_postgres_database.utils_repos import transaction_context +from simcore_postgres_database.webserver_models import projects +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.plugin import get_asyncpg_engine +from .exceptions import ProjectNotFoundError +from .models import ProjectDB + +_logger = logging.getLogger(__name__) + + +# NOTE: MD: I intentionally didn't include the workbench. There is a special interface +# for the workbench, and at some point, this column should be removed from the table. +# The same holds true for access_rights/ui/classifiers/quality, but we have decided to proceed step by step. +_SELECTION_PROJECT_DB_ARGS = [ # noqa: RUF012 + projects.c.id, + projects.c.type, + projects.c.uuid, + projects.c.name, + projects.c.description, + projects.c.thumbnail, + projects.c.prj_owner, + projects.c.creation_date, + projects.c.last_change_date, + projects.c.ui, + projects.c.classifiers, + projects.c.dev, + projects.c.quality, + projects.c.published, + projects.c.hidden, + projects.c.workspace_id, + projects.c.trashed_at, +] + + +async def patch_project( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_uuid: ProjectID, + new_partial_project_data: dict, +) -> ProjectDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + projects.update() + .values(last_change_date=sa.func.now(), **new_partial_project_data) + .where(projects.c.uuid == f"{project_uuid}") + .returning(*_SELECTION_PROJECT_DB_ARGS) + ) + row = await result.first() + if row is None: + raise ProjectNotFoundError(project_uuid=project_uuid) + return ProjectDB.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py index 5b963b90cdb..0f9169fabea 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py @@ -3,6 +3,7 @@ - Adds a layer to the postgres API with a focus on the projects comments """ + import logging from datetime import datetime @@ -11,11 +12,13 @@ from models_library.users import GroupID from pydantic import BaseModel, TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups +from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy import func, literal_column from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select -from ..db.plugin import get_database_engine +from ..db.plugin import get_asyncpg_engine from .exceptions import ProjectGroupNotFoundError _logger = logging.getLogger(__name__) @@ -37,33 +40,38 @@ class ProjectGroupGetDB(BaseModel): async def create_project_group( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, group_id: GroupID, - *, read: bool, write: bool, delete: bool, ) -> ProjectGroupGetDB: - async with get_database_engine(app).acquire() as conn: - result = await conn.execute( - project_to_groups.insert() - .values( - project_uuid=f"{project_id}", - gid=group_id, - read=read, - write=write, - delete=delete, - created=func.now(), - modified=func.now(), - ) - .returning(literal_column("*")) + query = ( + project_to_groups.insert() + .values( + project_uuid=f"{project_id}", + gid=group_id, + read=read, + write=write, + delete=delete, + created=func.now(), + modified=func.now(), ) + .returning(literal_column("*")) + ) + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(query) row = await result.first() return ProjectGroupGetDB.model_validate(row) async def list_project_groups( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, ) -> list[ProjectGroupGetDB]: stmt = ( @@ -79,14 +87,16 @@ async def list_project_groups( .where(project_to_groups.c.project_uuid == f"{project_id}") ) - async with get_database_engine(app).acquire() as conn: - result = await conn.execute(stmt) - rows = await result.fetchall() or [] + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(stmt) + rows = await result.first() or [] return TypeAdapter(list[ProjectGroupGetDB]).validate_python(rows) async def get_project_group( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, group_id: GroupID, ) -> ProjectGroupGetDB: @@ -106,8 +116,8 @@ async def get_project_group( ) ) - async with get_database_engine(app).acquire() as conn: - result = await conn.execute(stmt) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(stmt) row = await result.first() if row is None: raise ProjectGroupNotFoundError( @@ -118,27 +128,31 @@ async def get_project_group( async def replace_project_group( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, group_id: GroupID, - *, read: bool, write: bool, delete: bool, ) -> ProjectGroupGetDB: - async with get_database_engine(app).acquire() as conn: - result = await conn.execute( - project_to_groups.update() - .values( - read=read, - write=write, - delete=delete, - ) - .where( - (project_to_groups.c.project_uuid == f"{project_id}") - & (project_to_groups.c.gid == group_id) - ) - .returning(literal_column("*")) + + query = ( + project_to_groups.update() + .values( + read=read, + write=write, + delete=delete, ) + .where( + (project_to_groups.c.project_uuid == f"{project_id}") + & (project_to_groups.c.gid == group_id) + ) + .returning(literal_column("*")) + ) + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(query) row = await result.first() if row is None: raise ProjectGroupNotFoundError( @@ -149,14 +163,15 @@ async def replace_project_group( async def update_or_insert_project_group( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, group_id: GroupID, - *, read: bool, write: bool, delete: bool, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: insert_stmt = pg_insert(project_to_groups).values( project_uuid=f"{project_id}", gid=group_id, @@ -175,16 +190,18 @@ async def update_or_insert_project_group( "modified": func.now(), }, ) - await conn.execute(on_update_stmt) + await conn.stream(on_update_stmt) async def delete_project_group( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, group_id: GroupID, ) -> None: - async with get_database_engine(app).acquire() as conn: - await conn.execute( + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream( project_to_groups.delete().where( (project_to_groups.c.project_uuid == f"{project_id}") & (project_to_groups.c.gid == group_id) @@ -194,10 +211,12 @@ async def delete_project_group( async def delete_all_project_groups( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, ) -> None: - async with get_database_engine(app).acquire() as conn: - await conn.execute( + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream( project_to_groups.delete().where( project_to_groups.c.project_uuid == f"{project_id}" ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py index 39441f42716..0810640f34d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py @@ -9,6 +9,7 @@ from ..projects._access_rights_api import get_user_project_access_rights from ..users.api import get_user from ..workspaces.api import check_user_workspace_access +from . import _db_v2 as project_db_v2 from . import _folders_db as project_to_folders_db from . import _groups_db as project_groups_db from .db import APP_PROJECT_DBAPI, ProjectDBAPI @@ -51,7 +52,8 @@ async def move_project_into_workspace( ) # 4. Update workspace ID on the project resource - await project_api.patch_project( + await project_db_v2.patch_project( + app=app, project_uuid=project_id, new_partial_project_data={"workspace_id": workspace_id}, ) diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index cdaed691e71..3303e7d01c0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -85,6 +85,7 @@ patch_workbench, update_workbench, ) +from ._db_v2 import _SELECTION_PROJECT_DB_ARGS from .exceptions import ( ProjectDeleteError, ProjectInvalidRightsError, @@ -676,33 +677,10 @@ async def get_project( project_type, ) - # NOTE: MD: I intentionally didn't include the workbench. There is a special interface - # for the workbench, and at some point, this column should be removed from the table. - # The same holds true for access_rights/ui/classifiers/quality, but we have decided to proceed step by step. - _SELECTION_PROJECT_DB_ARGS = [ # noqa: RUF012 - projects.c.id, - projects.c.type, - projects.c.uuid, - projects.c.name, - projects.c.description, - projects.c.thumbnail, - projects.c.prj_owner, - projects.c.creation_date, - projects.c.last_change_date, - projects.c.ui, - projects.c.classifiers, - projects.c.dev, - projects.c.quality, - projects.c.published, - projects.c.hidden, - projects.c.workspace_id, - projects.c.trashed_at, - ] - async def get_project_db(self, project_uuid: ProjectID) -> ProjectDB: async with self.engine.acquire() as conn: result = await conn.execute( - sa.select(*self._SELECTION_PROJECT_DB_ARGS).where( + sa.select(*_SELECTION_PROJECT_DB_ARGS).where( projects.c.uuid == f"{project_uuid}" ) ) @@ -716,9 +694,7 @@ async def get_user_specific_project_data_db( ) -> UserSpecificProjectDataDB: async with self.engine.acquire() as conn: result = await conn.execute( - sa.select( - *self._SELECTION_PROJECT_DB_ARGS, projects_to_folders.c.folder_id - ) + sa.select(*_SELECTION_PROJECT_DB_ARGS, projects_to_folders.c.folder_id) .select_from( projects.join( projects_to_folders, @@ -865,21 +841,6 @@ async def replace_project( msg = "linter unhappy without this" raise RuntimeError(msg) - async def patch_project( - self, project_uuid: ProjectID, new_partial_project_data: dict - ) -> ProjectDB: - async with self.engine.acquire() as conn: - result = await conn.execute( - projects.update() - .values(last_change_date=sa.func.now(), **new_partial_project_data) - .where(projects.c.uuid == f"{project_uuid}") - .returning(*self._SELECTION_PROJECT_DB_ARGS) - ) - row = await result.fetchone() - if row is None: - raise ProjectNotFoundError(project_uuid=project_uuid) - return ProjectDB.model_validate(row) - async def get_project_product(self, project_uuid: ProjectID) -> ProductName: async with self.engine.acquire() as conn: result = await conn.execute( diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 6876c63718d..8830ad829b1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -120,7 +120,9 @@ from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError from ..workspaces import _workspaces_db as workspaces_db -from . import _crud_api_delete, _nodes_api +from . import _crud_api_delete +from . import _db_v2 as project_db_v2 +from . import _nodes_api from ._access_rights_api import ( check_user_project_permission, has_user_project_access_rights, @@ -289,7 +291,8 @@ async def patch_project( raise ProjectOwnerNotFoundInTheProjectAccessRightsError # 4. Patch the project - await db.patch_project( + await project_db_v2.patch_project( + app=app, project_uuid=project_uuid, new_partial_project_data=_project_patch_exclude_unset, ) diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 7085050f331..f215368ad1d 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -358,7 +358,7 @@ async def _creator( for group_id, permissions in _access_rights.items(): await update_or_insert_project_group( client.app, - data["uuid"], + project_id=data["uuid"], group_id=int(group_id), read=permissions["read"], write=permissions["write"], diff --git a/services/web/server/tests/integration/01/test_garbage_collection.py b/services/web/server/tests/integration/01/test_garbage_collection.py index c52977d7115..d3aee60764d 100644 --- a/services/web/server/tests/integration/01/test_garbage_collection.py +++ b/services/web/server/tests/integration/01/test_garbage_collection.py @@ -237,7 +237,7 @@ async def new_project( for group_id, permissions in access_rights.items(): await update_or_insert_project_group( client.app, - project["uuid"], + project_id=project["uuid"], group_id=int(group_id), read=permissions["read"], write=permissions["write"], diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index fadfe561267..1d73a0e88c4 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -201,7 +201,7 @@ async def _inserter(prj: dict[str, Any], **overrides) -> dict[str, Any]: for group_id, permissions in _access_rights.items(): await update_or_insert_project_group( client.app, - new_project["uuid"], + project_id=new_project["uuid"], group_id=int(group_id), read=permissions["read"], write=permissions["write"], diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 1aa772367e3..5314bd51dc8 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -17,6 +17,7 @@ from pytest_simcore.helpers.webserver_projects import create_project from servicelib.aiohttp import status from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.projects.models import ProjectDict @@ -229,6 +230,7 @@ async def test_moving_between_private_and_shared_workspaces( workspaces_clean_db: None, ): assert client.app + setup_db(client.app) moving_folder_id = await _setup_test(client, logged_user, fake_project) From 57b63b1b59e77395a880143bf39530caf0d0366d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:43:51 +0100 Subject: [PATCH 07/29] adding transaction --- .../folders/_workspaces_api.py | 111 +++++++++--------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py index 3a153d6237e..3ede58be60c 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -5,7 +5,9 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.workspaces import WorkspaceID +from simcore_postgres_database.utils_repos import transaction_context +from ..db.plugin import get_asyncpg_engine from ..projects import _db_v2 as projects_db from ..projects import _folders_db as project_to_folders_db from ..projects import _groups_db as project_groups_db @@ -73,63 +75,64 @@ async def move_folder_into_workspace( # ⬆️ Here we have already guaranties that user has all the right permissions to do this operation ⬆️ - # 4. Update workspace ID on the project resource - for project_id in project_ids: - await projects_db.patch_project( - app=app, - connection=None, - project_uuid=project_id, - new_partial_project_data={"workspace_id": workspace_id}, - ) - - # 5. BATCH update of folders with workspace_id - await _folders_db.update( - app, - connection=None, - folders_id_or_ids=set(folder_ids), - product_name=product_name, - workspace_id=workspace_id, # <-- Updating workspace_id - user_id=user_id if workspace_id is None else None, # <-- Updating user_id - ) + async with transaction_context(get_asyncpg_engine(app)) as conn: + # 4. Update workspace ID on the project resource + for project_id in project_ids: + await projects_db.patch_project( + app=app, + connection=conn, + project_uuid=project_id, + new_partial_project_data={"workspace_id": workspace_id}, + ) - # 6. Update source folder parent folder ID with NULL (it will appear in the root directory) - await _folders_db.update( - app, - connection=None, - folders_id_or_ids=folder_id, - product_name=product_name, - parent_folder_id=None, # <-- Updating parent folder ID - ) - - # 7. Remove all records of project to folders that are not in the folders that we are moving - # (ex. If we are moving from private workspace, the same project can be in different folders for different users) - await project_to_folders_db.delete_all_project_to_folder_by_project_ids_not_in_folder_ids( - app, - connection=None, - project_id_or_ids=set(project_ids), - not_in_folder_ids=set(folder_ids), - ) + # 5. BATCH update of folders with workspace_id + await _folders_db.update( + app, + connection=conn, + folders_id_or_ids=set(folder_ids), + product_name=product_name, + workspace_id=workspace_id, # <-- Updating workspace_id + user_id=user_id if workspace_id is None else None, # <-- Updating user_id + ) - # 8. Update the user id field for the remaining folders - await project_to_folders_db.update_project_to_folder( - app, - connection=None, - folders_id_or_ids=set(folder_ids), - user_id=user_id if workspace_id is None else None, # <-- Updating user_id - ) + # 6. Update source folder parent folder ID with NULL (it will appear in the root directory) + await _folders_db.update( + app, + connection=conn, + folders_id_or_ids=folder_id, + product_name=product_name, + parent_folder_id=None, # <-- Updating parent folder ID + ) - # 9. Remove all project permissions, leave only the user who moved the project - user = await get_user(app, user_id=user_id) - for project_id in project_ids: - await project_groups_db.delete_all_project_groups( - app, connection=None, project_id=project_id + # 7. Remove all records of project to folders that are not in the folders that we are moving + # (ex. If we are moving from private workspace, the same project can be in different folders for different users) + await project_to_folders_db.delete_all_project_to_folder_by_project_ids_not_in_folder_ids( + app, + connection=conn, + project_id_or_ids=set(project_ids), + not_in_folder_ids=set(folder_ids), ) - await project_groups_db.update_or_insert_project_group( + + # 8. Update the user id field for the remaining folders + await project_to_folders_db.update_project_to_folder( app, - connection=None, - project_id=project_id, - group_id=user["primary_gid"], - read=True, - write=True, - delete=True, + connection=conn, + folders_id_or_ids=set(folder_ids), + user_id=user_id if workspace_id is None else None, # <-- Updating user_id ) + + # 9. Remove all project permissions, leave only the user who moved the project + user = await get_user(app, user_id=user_id) + for project_id in project_ids: + await project_groups_db.delete_all_project_groups( + app, connection=conn, project_id=project_id + ) + await project_groups_db.update_or_insert_project_group( + app, + connection=conn, + project_id=project_id, + group_id=user["primary_gid"], + read=True, + write=True, + delete=True, + ) From 5e8e5e3622791ab39ad577723810d071e13e0dd1 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:45:59 +0100 Subject: [PATCH 08/29] open api specs --- api/specs/web-server/_folders.py | 7 +- .../api/v0/openapi.yaml | 337 ++++++++++++++++++ 2 files changed, 338 insertions(+), 6 deletions(-) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index dda18a64372..feeda29fee6 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -100,16 +100,11 @@ async def delete_folder( ### Move Folder to Workspace -router = APIRouter( - prefix=f"/{API_VTAG}", - tags=["folders", "workspaces"], -) - - @router.put( "/folders/{folder_id}/workspaces/{workspace_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Move folder to the workspace", + tags=["folders", "workspaces"], ) async def replace_folder_workspace( _path: Annotated[_FolderWorkspacesPathParams, Depends()], diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 878138f87a7..bec2fea00d9 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2594,10 +2594,217 @@ paths: responses: '200': description: Successful Response + /v0/folders: + post: + tags: + - folders + summary: Create Folder + operationId: create_folder + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FolderCreateBodyParams' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_FolderGet_' + get: + tags: + - folders + summary: List Folders + operationId: list_folders + parameters: + - name: filters + in: query + required: false + schema: + anyOf: + - type: string + contentMediaType: application/json + contentSchema: {} + - type: 'null' + title: Filters + - name: order_by + in: query + required: false + schema: + type: string + contentMediaType: application/json + contentSchema: {} + default: '{"field":"modified","direction":"desc"}' + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + - name: folder_id + in: query + required: false + schema: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Folder Id + - name: workspace_id + in: query + required: false + schema: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Workspace Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_list_FolderGet__' + /v0/folders:search: + get: + tags: + - folders + summary: List Folders Full Search + operationId: list_folders_full_search + parameters: + - name: filters + in: query + required: false + schema: + anyOf: + - type: string + contentMediaType: application/json + contentSchema: {} + - type: 'null' + title: Filters + - name: order_by + in: query + required: false + schema: + type: string + contentMediaType: application/json + contentSchema: {} + default: '{"field":"modified","direction":"desc"}' + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + - name: text + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: Text + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_list_FolderGet__' + /v0/folders/{folder_id}: + get: + tags: + - folders + summary: Get Folder + operationId: get_folder + parameters: + - name: folder_id + in: path + required: true + schema: + type: integer + exclusiveMinimum: true + title: Folder Id + minimum: 0 + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_FolderGet_' + put: + tags: + - folders + summary: Replace Folder + operationId: replace_folder + parameters: + - name: folder_id + in: path + required: true + schema: + type: integer + exclusiveMinimum: true + title: Folder Id + minimum: 0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FolderReplaceBodyParams' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_FolderGet_' + delete: + tags: + - folders + summary: Delete Folder + operationId: delete_folder + parameters: + - name: folder_id + in: path + required: true + schema: + type: integer + exclusiveMinimum: true + title: Folder Id + minimum: 0 + responses: + '204': + description: Successful Response /v0/folders/{folder_id}/workspaces/{workspace_id}: put: tags: - folders + - folders - workspaces summary: Move folder to the workspace operationId: replace_folder_workspace @@ -7387,6 +7594,19 @@ components: title: Error type: object title: Envelope[FileUploadSchema] + Envelope_FolderGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/FolderGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[FolderGet] Envelope_GetCreditPrice_: properties: data: @@ -8261,6 +8481,22 @@ components: title: Error type: object title: Envelope[list[FileMetaDataGet]] + Envelope_list_FolderGet__: + properties: + data: + anyOf: + - items: + $ref: '#/components/schemas/FolderGet' + type: array + - type: 'null' + title: Data + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[list[FolderGet]] Envelope_list_GroupUserGet__: properties: data: @@ -9034,6 +9270,107 @@ components: - urls - links title: FileUploadSchema + FolderCreateBodyParams: + properties: + name: + type: string + maxLength: 100 + minLength: 1 + title: Name + parentFolderId: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Parentfolderid + workspaceId: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Workspaceid + additionalProperties: false + type: object + required: + - name + title: FolderCreateBodyParams + FolderGet: + properties: + folderId: + type: integer + exclusiveMinimum: true + title: Folderid + minimum: 0 + parentFolderId: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Parentfolderid + name: + type: string + title: Name + createdAt: + type: string + format: date-time + title: Createdat + modifiedAt: + type: string + format: date-time + title: Modifiedat + trashedAt: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Trashedat + owner: + type: integer + exclusiveMinimum: true + title: Owner + minimum: 0 + workspaceId: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Workspaceid + myAccessRights: + $ref: '#/components/schemas/AccessRights' + type: object + required: + - folderId + - name + - createdAt + - modifiedAt + - trashedAt + - owner + - workspaceId + - myAccessRights + title: FolderGet + FolderReplaceBodyParams: + properties: + name: + type: string + maxLength: 100 + minLength: 1 + title: Name + parentFolderId: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Parentfolderid + additionalProperties: false + type: object + required: + - name + title: FolderReplaceBodyParams GenerateInvitation: properties: guest: From f45879c19bc0ba65c3f195c86e1dc1a5b51c4146 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:48:20 +0100 Subject: [PATCH 09/29] final cleanup --- .../src/simcore_service_webserver/projects/_folders_db.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index e31d2f56f0d..1eecaa76aa7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -123,7 +123,6 @@ async def update_project_to_folder( connection: AsyncConnection | None = None, *, folders_id_or_ids: FolderID | set[FolderID], - # filter_by_user_id: UserID | None | UnSet = _unset, # updatable columns user_id: UserID | None | UnSet = _unset, ) -> None: @@ -146,9 +145,6 @@ async def update_project_to_folder( # single-update query = query.where(projects_to_folders.c.folder_id == folders_id_or_ids) - # if not isinstance(filter_by_user_id, UnSet): - # query = query.where(projects_to_folders.c.user_id == filter_by_user_id) - async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream(query) From 1ce4f4e124544325f5112d655cdf22590819b398 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:52:36 +0100 Subject: [PATCH 10/29] final cleanup --- .../projects/_folders_db.py | 12 ++-- .../projects/_workspaces_api.py | 56 ++++++++++--------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index 1eecaa76aa7..db41c2a7265 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -103,21 +103,23 @@ async def delete_project_to_folder( ) +### AsyncPg + + async def delete_all_project_to_folder_by_project_id( app: web.Application, + connection: AsyncConnection | None = None, + *, project_id: ProjectID, ) -> None: - async with get_database_engine(app).acquire() as conn: - await conn.execute( + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream( projects_to_folders.delete().where( projects_to_folders.c.project_uuid == f"{project_id}" ) ) -### AsyncPg - - async def update_project_to_folder( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py index 0810640f34d..ee5cadb1f34 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py @@ -5,14 +5,15 @@ from models_library.projects import ProjectID from models_library.users import UserID from models_library.workspaces import WorkspaceID +from simcore_postgres_database.utils_repos import transaction_context +from ..db.plugin import get_asyncpg_engine from ..projects._access_rights_api import get_user_project_access_rights from ..users.api import get_user from ..workspaces.api import check_user_workspace_access from . import _db_v2 as project_db_v2 from . import _folders_db as project_to_folders_db from . import _groups_db as project_groups_db -from .db import APP_PROJECT_DBAPI, ProjectDBAPI from .exceptions import ProjectInvalidRightsError _logger = logging.getLogger(__name__) @@ -26,8 +27,6 @@ async def move_project_into_workspace( workspace_id: WorkspaceID | None, product_name: ProductName, ) -> None: - project_api: ProjectDBAPI = app[APP_PROJECT_DBAPI] - # 1. User needs to have delete permission on project project_access_rights = await get_user_project_access_rights( app, project_id=project_id, user_id=user_id, product_name=product_name @@ -45,28 +44,33 @@ async def move_project_into_workspace( permission="write", ) - # 3. Delete project to folders (for everybody) - await project_to_folders_db.delete_all_project_to_folder_by_project_id( - app, - project_id=project_id, - ) + async with transaction_context(get_asyncpg_engine(app)) as conn: + # 3. Delete project to folders (for everybody) + await project_to_folders_db.delete_all_project_to_folder_by_project_id( + app, + connection=conn, + project_id=project_id, + ) - # 4. Update workspace ID on the project resource - await project_db_v2.patch_project( - app=app, - project_uuid=project_id, - new_partial_project_data={"workspace_id": workspace_id}, - ) - # NOTE: MD: should I also patch the project owner? -> probably yes, or if it is more like "original owner" then probably no + # 4. Update workspace ID on the project resource + await project_db_v2.patch_project( + app=app, + connection=conn, + project_uuid=project_id, + new_partial_project_data={"workspace_id": workspace_id}, + ) - # 5. Remove all project permissions, leave only the user who moved the project - user = await get_user(app, user_id=user_id) - await project_groups_db.delete_all_project_groups(app, project_id=project_id) - await project_groups_db.update_or_insert_project_group( - app, - project_id=project_id, - group_id=user["primary_gid"], - read=True, - write=True, - delete=True, - ) + # 5. Remove all project permissions, leave only the user who moved the project + user = await get_user(app, user_id=user_id) + await project_groups_db.delete_all_project_groups( + app, connection=conn, project_id=project_id + ) + await project_groups_db.update_or_insert_project_group( + app, + connection=conn, + project_id=project_id, + group_id=user["primary_gid"], + read=True, + write=True, + delete=True, + ) From b7ee5cfeb6fff4f8aba22b39853f343a19a2305f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:54:15 +0100 Subject: [PATCH 11/29] final cleanup --- ...aces__moving_folders_between_workspaces.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 5314bd51dc8..56fd1f03c41 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -40,30 +40,6 @@ def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): ) -# @pytest.mark.parametrize(*standard_role_response(), ids=str) -# async def test_moving_between_workspaces_user_role_permissions( -# client: TestClient, -# logged_user: UserInfoDict, -# user_project: ProjectDict, -# expected: ExpectedResponse, -# mock_catalog_api_get_services_for_user_in_product: MockerFixture, -# fake_project: ProjectDict, -# workspaces_clean_db: None, -# ): -# # Move project from workspace to your private workspace -# base_url = client.app.router["replace_folder_workspace"].url_for( -# folder_id="1", workspace_id="null" -# ) -# resp = await client.put(f"{base_url}") -# await assert_status(resp, expected.no_content) - - -## Usecases to test: -# 1. Private workspace -> Shared workspace -# 2. Shared workspace -> Shared workspace -# 3. Shared workspace -> Private workspace - - async def _setup_test( client: TestClient, logged_user: UserInfoDict, From e2d205ec9b79e35e675f598fd53b30f0f3654711 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 28 Nov 2024 17:59:40 +0100 Subject: [PATCH 12/29] open api specs --- api/specs/web-server/_folders.py | 2 +- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index 21497e735c7..fcd1309b618 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -109,7 +109,7 @@ async def delete_folder( "/folders/{folder_id}/workspaces/{workspace_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Move folder to the workspace", - tags=["folders", "workspaces"], + tags=["workspaces"], ) async def replace_folder_workspace( _path: Annotated[_FolderWorkspacesPathParams, Depends()], diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index c4ea4ae04ea..af862ed18ea 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2948,7 +2948,6 @@ paths: put: tags: - folders - - folders - workspaces summary: Move folder to the workspace operationId: replace_folder_workspace From 4b87ce3b966d832a21c33afbc9b95bb1744404c5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 11:37:42 +0100 Subject: [PATCH 13/29] review @pcrespov --- api/specs/web-server/_folders.py | 3 -- .../folders/_folders_db.py | 6 +-- .../folders/_workspaces_api.py | 4 +- .../projects/_folders_db.py | 1 - .../projects/{_db_v2.py => _projects_db.py} | 0 .../projects/_workspaces_api.py | 2 +- .../simcore_service_webserver/projects/db.py | 2 +- .../projects/projects_api.py | 5 +-- ...aces__moving_folders_between_workspaces.py | 42 ++++++++++--------- 9 files changed, 32 insertions(+), 33 deletions(-) rename services/web/server/src/simcore_service_webserver/projects/{_db_v2.py => _projects_db.py} (100%) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index fcd1309b618..b08dcfde7e4 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -102,9 +102,6 @@ async def delete_folder( ... -### Move Folder to Workspace - - @router.put( "/folders/{folder_id}/workspaces/{workspace_id}", status_code=status.HTTP_204_NO_CONTENT, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py index ce6633f8c01..88bb3987de4 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py @@ -463,7 +463,7 @@ async def get_projects_recursively_only_if_user_is_owner( return [ProjectID(row[0]) async for row in result] -async def get_all_folders_and_projects_recursively( +async def get_all_folders_and_projects_ids_recursively( app: web.Application, connection: AsyncConnection | None = None, *, @@ -504,7 +504,7 @@ async def get_all_folders_and_projects_recursively( final_query = select(folder_hierarchy_cte) result = await conn.stream(final_query) # list of tuples [(folder_id, parent_folder_id), ...] ex. [(1, None), (2, 1)] - folder_ids = [item[0] async for item in result] + folder_ids = [item.folder_id async for item in result] query = select(projects_to_folders.c.project_uuid).where( (projects_to_folders.c.folder_id.in_(folder_ids)) @@ -512,7 +512,7 @@ async def get_all_folders_and_projects_recursively( ) result = await conn.stream(query) - project_ids = [ProjectID(row[0]) async for row in result] + project_ids = [ProjectID(row.project_uuid) async for row in result] return folder_ids, project_ids diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py index 3ede58be60c..115ff2c8d8e 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py @@ -8,9 +8,9 @@ from simcore_postgres_database.utils_repos import transaction_context from ..db.plugin import get_asyncpg_engine -from ..projects import _db_v2 as projects_db from ..projects import _folders_db as project_to_folders_db from ..projects import _groups_db as project_groups_db +from ..projects import _projects_db as projects_db from ..projects._access_rights_api import check_user_project_permission from ..users.api import get_user from ..workspaces.api import check_user_workspace_access @@ -56,7 +56,7 @@ async def move_folder_into_workspace( ( folder_ids, project_ids, - ) = await _folders_db.get_all_folders_and_projects_recursively( + ) = await _folders_db.get_all_folders_and_projects_ids_recursively( app, connection=None, folder_id=folder_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index db41c2a7265..e655cc17bf5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -156,7 +156,6 @@ async def delete_all_project_to_folder_by_project_ids_not_in_folder_ids( connection: AsyncConnection | None = None, *, project_id_or_ids: ProjectID | set[ProjectID], - # Optional filter not_in_folder_ids: set[FolderID], ) -> None: query = projects_to_folders.delete() diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_v2.py b/services/web/server/src/simcore_service_webserver/projects/_projects_db.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/projects/_db_v2.py rename to services/web/server/src/simcore_service_webserver/projects/_projects_db.py diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py index ee5cadb1f34..4862c1d047d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py @@ -11,9 +11,9 @@ from ..projects._access_rights_api import get_user_project_access_rights from ..users.api import get_user from ..workspaces.api import check_user_workspace_access -from . import _db_v2 as project_db_v2 from . import _folders_db as project_to_folders_db from . import _groups_db as project_groups_db +from . import _projects_db as project_db_v2 from .exceptions import ProjectInvalidRightsError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index 3303e7d01c0..b0fc7c5551a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -85,7 +85,7 @@ patch_workbench, update_workbench, ) -from ._db_v2 import _SELECTION_PROJECT_DB_ARGS +from ._projects_db import _SELECTION_PROJECT_DB_ARGS from .exceptions import ( ProjectDeleteError, ProjectInvalidRightsError, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 8830ad829b1..49f6706e342 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -120,9 +120,8 @@ from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError from ..workspaces import _workspaces_db as workspaces_db -from . import _crud_api_delete -from . import _db_v2 as project_db_v2 -from . import _nodes_api +from . import _crud_api_delete, _nodes_api +from . import _projects_db as project_db_v2 from ._access_rights_api import ( check_user_project_permission, has_user_project_access_rights, diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 56fd1f03c41..9bc1b820fd7 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -6,7 +6,6 @@ from copy import deepcopy -from http import HTTPStatus from http.client import NO_CONTENT import pytest @@ -21,6 +20,11 @@ from simcore_service_webserver.projects.models import ProjectDict +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( @@ -40,12 +44,14 @@ def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): ) -async def _setup_test( +@pytest.fixture +async def moving_folder_id( client: TestClient, logged_user: UserInfoDict, fake_project: ProjectDict, ) -> str: assert client.app + setup_db(client.app) ### Project creation @@ -53,13 +59,13 @@ async def _setup_test( project_data = deepcopy(fake_project) first_project = await create_project( client.app, - project_data, + params_override=project_data, user_id=logged_user["id"], product_name="osparc", ) second_project = await create_project( client.app, - project_data, + params_override=project_data, user_id=logged_user["id"], product_name="osparc", ) @@ -159,11 +165,11 @@ async def _move_folder_to_workspace_and_assert( assert client.app # MOVE - base_url = client.app.router["replace_folder_workspace"].url_for( + url = client.app.router["replace_folder_workspace"].url_for( folder_id=folder_id, workspace_id=workspace_id, ) - resp = await client.put(f"{base_url}") + resp = await client.put(f"{url}") await assert_status(resp, NO_CONTENT) # ASSERT @@ -196,19 +202,17 @@ async def _move_folder_to_workspace_and_assert( assert len(data) == 1 -@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +# @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) async def test_moving_between_private_and_shared_workspaces( client: TestClient, logged_user: UserInfoDict, - expected: HTTPStatus, + expected, mock_catalog_api_get_services_for_user_in_product: MockerFixture, fake_project: ProjectDict, + moving_folder_id: str, workspaces_clean_db: None, ): assert client.app - setup_db(client.app) - - moving_folder_id = await _setup_test(client, logged_user, fake_project) # We will test these scenarios of moving folders: # 1. Private workspace -> Shared workspace @@ -220,45 +224,45 @@ async def test_moving_between_private_and_shared_workspaces( # create a new workspace url = client.app.router["create_workspace"].url_for() resp = await client.post( - url.path, + f"{url}", json={ "name": "A", "description": "A", "thumbnail": None, }, ) - added_workspace, _ = await assert_status(resp, status.HTTP_201_CREATED) + shared_workspace_A, _ = await assert_status(resp, status.HTTP_201_CREATED) # 1. Private workspace -> Shared workspace A await _move_folder_to_workspace_and_assert( client, folder_id=moving_folder_id, - workspace_id=f"{added_workspace['workspaceId']}", + workspace_id=f"{shared_workspace_A['workspaceId']}", ) # create a new workspace url = client.app.router["create_workspace"].url_for() resp = await client.post( - url.path, + f"{url}", json={ "name": "B", "description": "B", "thumbnail": None, }, ) - second_workspace, _ = await assert_status(resp, status.HTTP_201_CREATED) + shared_workspace_B, _ = await assert_status(resp, status.HTTP_201_CREATED) # 2. Shared workspace A -> Shared workspace B await _move_folder_to_workspace_and_assert( client, folder_id=moving_folder_id, - workspace_id=f"{second_workspace['workspaceId']}", + workspace_id=f"{shared_workspace_B['workspaceId']}", ) - # 3. (Corner case) Shared workspace A -> Shared workspace A + # 3. (Corner case) Shared workspace B -> Shared workspace B await _move_folder_to_workspace_and_assert( client, folder_id=moving_folder_id, - workspace_id=f"{second_workspace['workspaceId']}", + workspace_id=f"{shared_workspace_B['workspaceId']}", ) # 4. Shared workspace -> Private workspace From 0d71046d50a34fe45bc736bab98a02b64e01123a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 13:00:49 +0100 Subject: [PATCH 14/29] review @pcrespov --- .../test_workspaces__moving_folders_between_workspaces.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 9bc1b820fd7..aa5aef862e1 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -202,11 +202,9 @@ async def _move_folder_to_workspace_and_assert( assert len(data) == 1 -# @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) async def test_moving_between_private_and_shared_workspaces( client: TestClient, logged_user: UserInfoDict, - expected, mock_catalog_api_get_services_for_user_in_product: MockerFixture, fake_project: ProjectDict, moving_folder_id: str, From 7061af79250cd3a25d6502d3a73f24d320926473 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 13:20:41 +0100 Subject: [PATCH 15/29] review @pcrespov --- .../folders/_models.py | 14 ++++-- .../folders/_workspaces_handlers.py | 47 ++----------------- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_models.py b/services/web/server/src/simcore_service_webserver/folders/_models.py index 9cac8a2f1a1..553d43bd64c 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_models.py +++ b/services/web/server/src/simcore_service_webserver/folders/_models.py @@ -18,10 +18,9 @@ null_or_none_str_to_none_validator, ) from models_library.workspaces import WorkspaceID -from pydantic import BeforeValidator, ConfigDict, Field -from servicelib.request_keys import RQT_USERID_KEY +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field -from .._constants import RQ_PRODUCT_KEY +from .._constants import RQ_PRODUCT_KEY, RQT_USERID_KEY _logger = logging.getLogger(__name__) @@ -88,3 +87,12 @@ class FolderSearchQueryParams( class FolderTrashQueryParams(RemoveQueryParams): ... + + +class _FolderWorkspacesPathParams(BaseModel): + folder_id: FolderID + workspace_id: Annotated[ + WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) + ] = Field(default=None) + + model_config = ConfigDict(extra="forbid") diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py index 421643c5114..ac00317c461 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py @@ -1,70 +1,29 @@ -import functools import logging -from typing import Annotated from aiohttp import web -from models_library.folders import FolderID -from models_library.utils.common_validators import null_or_none_str_to_none_validator -from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, BeforeValidator, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as -from servicelib.aiohttp.typing_extension import Handler from .._meta import api_version_prefix as VTAG -from ..folders.errors import FolderAccessForbiddenError, FolderNotFoundError from ..login.decorators import login_required -from ..projects.exceptions import ProjectInvalidRightsError, ProjectNotFoundError from ..security.decorators import permission_required -from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _workspaces_api -from ._models import FoldersRequestContext +from ._exceptions_handlers import handle_plugin_requests_exceptions +from ._models import FoldersRequestContext, _FolderWorkspacesPathParams _logger = logging.getLogger(__name__) -def _handle_folders_workspaces_exceptions(handler: Handler): - @functools.wraps(handler) - async def wrapper(request: web.Request) -> web.StreamResponse: - try: - return await handler(request) - - except ( - ProjectInvalidRightsError, - FolderNotFoundError, - WorkspaceNotFoundError, - ) as exc: - raise web.HTTPNotFound(reason=f"{exc}") from exc - - except ( - ProjectNotFoundError, - FolderAccessForbiddenError, - WorkspaceAccessForbiddenError, - ) as exc: - raise web.HTTPForbidden(reason=f"{exc}") from exc - - return wrapper - - routes = web.RouteTableDef() -class _FolderWorkspacesPathParams(BaseModel): - folder_id: FolderID - workspace_id: Annotated[ - WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) - ] = Field(default=None) - - model_config = ConfigDict(extra="forbid") - - @routes.put( f"/{VTAG}/folders/{{folder_id}}/workspaces/{{workspace_id}}", name="replace_folder_workspace", ) @login_required @permission_required("folder.update") -@_handle_folders_workspaces_exceptions +@handle_plugin_requests_exceptions async def replace_project_workspace(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_FolderWorkspacesPathParams, request) From b77602cb7872e505aa4a368814d31d183bcdd36e Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 13:27:52 +0100 Subject: [PATCH 16/29] open api specs --- api/specs/web-server/_folders.py | 6 +++--- api/specs/web-server/_projects_workspaces.py | 6 +++--- .../simcore_service_webserver/api/v0/openapi.yaml | 12 ++++++------ .../folders/_workspaces_handlers.py | 8 ++++---- .../projects/_workspaces_handlers.py | 12 +++++++----- ..._workspaces__moving_folders_between_workspaces.py | 2 +- ...workspaces__moving_projects_between_workspaces.py | 10 +++++----- 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index b08dcfde7e4..2aa77e485d4 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -102,13 +102,13 @@ async def delete_folder( ... -@router.put( - "/folders/{folder_id}/workspaces/{workspace_id}", +@router.post( + "/folders/{folder_id}/workspaces/{workspace_id}:move", status_code=status.HTTP_204_NO_CONTENT, summary="Move folder to the workspace", tags=["workspaces"], ) -async def replace_folder_workspace( +async def move_folder_to_workspace( _path: Annotated[_FolderWorkspacesPathParams, Depends()], ): ... diff --git a/api/specs/web-server/_projects_workspaces.py b/api/specs/web-server/_projects_workspaces.py index 533d3c72a9b..caaccfca05c 100644 --- a/api/specs/web-server/_projects_workspaces.py +++ b/api/specs/web-server/_projects_workspaces.py @@ -23,12 +23,12 @@ ) -@router.put( - "/projects/{project_id}/workspaces/{workspace_id}", +@router.post( + "/projects/{project_id}/workspaces/{workspace_id}:move", status_code=status.HTTP_204_NO_CONTENT, summary="Move project to the workspace", ) -async def replace_project_workspace( +async def move_project_to_workspace( _path: Annotated[_ProjectWorkspacesPathParams, Depends()], ): ... diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index af862ed18ea..84951101670 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2944,13 +2944,13 @@ paths: schema: $ref: '#/components/schemas/EnvelopedError' description: Service Unavailable - /v0/folders/{folder_id}/workspaces/{workspace_id}: - put: + /v0/folders/{folder_id}/workspaces/{workspace_id}:move: + post: tags: - folders - workspaces summary: Move folder to the workspace - operationId: replace_folder_workspace + operationId: move_folder_to_workspace parameters: - name: folder_id in: path @@ -4759,13 +4759,13 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_WalletGet_' - /v0/projects/{project_id}/workspaces/{workspace_id}: - put: + /v0/projects/{project_id}/workspaces/{workspace_id}:move: + post: tags: - projects - workspaces summary: Move project to the workspace - operationId: replace_project_workspace + operationId: move_project_to_workspace parameters: - name: project_id in: path diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py index ac00317c461..faa505ecd31 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py @@ -17,14 +17,14 @@ routes = web.RouteTableDef() -@routes.put( - f"/{VTAG}/folders/{{folder_id}}/workspaces/{{workspace_id}}", - name="replace_folder_workspace", +@routes.post( + f"/{VTAG}/folders/{{folder_id}}/workspaces/{{workspace_id}}:move", + name="move_folder_to_workspace", ) @login_required @permission_required("folder.update") @handle_plugin_requests_exceptions -async def replace_project_workspace(request: web.Request): +async def move_folder_to_workspace(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_FolderWorkspacesPathParams, request) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py index ff881b418af..ef3d20b3c5a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py @@ -51,19 +51,21 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _ProjectWorkspacesPathParams(BaseModel): project_id: ProjectID - workspace_id: Annotated[WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator)] = Field(default=None) + workspace_id: Annotated[ + WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) + ] = Field(default=None) model_config = ConfigDict(extra="forbid") -@routes.put( - f"/{VTAG}/projects/{{project_id}}/workspaces/{{workspace_id}}", - name="replace_project_workspace", +@routes.post( + f"/{VTAG}/projects/{{project_id}}/workspaces/{{workspace_id}}:move", + name="move_project_to_workspace", ) @login_required @permission_required("project.workspaces.*") @_handle_projects_workspaces_exceptions -async def replace_project_workspace(request: web.Request): +async def move_project_to_workspace(request: web.Request): req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectWorkspacesPathParams, request diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index aa5aef862e1..91d7dfc331a 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -165,7 +165,7 @@ async def _move_folder_to_workspace_and_assert( assert client.app # MOVE - url = client.app.router["replace_folder_workspace"].url_for( + url = client.app.router["move_folder_to_workspace"].url_for( folder_id=folder_id, workspace_id=workspace_id, ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index 21b16ea9738..db186f5fbcf 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -55,7 +55,7 @@ async def test_moving_between_workspaces_user_role_permissions( workspaces_clean_db: None, ): # Move project from workspace to your private workspace - base_url = client.app.router["replace_project_workspace"].url_for( + base_url = client.app.router["move_project_to_workspace"].url_for( project_id=fake_project["uuid"], workspace_id="null" ) resp = await client.put(f"{base_url}") @@ -103,7 +103,7 @@ async def test_moving_between_private_and_shared_workspaces( assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID # Move project from workspace to your private workspace - base_url = client.app.router["replace_project_workspace"].url_for( + base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id="null" ) resp = await client.put(f"{base_url}") @@ -116,7 +116,7 @@ async def test_moving_between_private_and_shared_workspaces( assert data["workspaceId"] is None # <-- Workspace ID is None # Move project from your private workspace to shared workspace - base_url = client.app.router["replace_project_workspace"].url_for( + base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{added_workspace['workspaceId']}" ) resp = await client.put(f"{base_url}") @@ -182,7 +182,7 @@ async def test_moving_between_shared_and_shared_workspaces( assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID # Move project from workspace to your private workspace - base_url = client.app.router["replace_project_workspace"].url_for( + base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{second_workspace['workspaceId']}" ) resp = await client.put(f"{base_url}") @@ -262,7 +262,7 @@ async def test_moving_between_workspaces_check_removed_from_folder( assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID # Move project from workspace to your private workspace - base_url = client.app.router["replace_project_workspace"].url_for( + base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id="none" ) resp = await client.put(f"{base_url}") From 4e4675fbd2d8d77618bd06746483edde79eb6ca2 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 13:30:04 +0100 Subject: [PATCH 17/29] frontend changes --- .../client/source/class/osparc/data/Resources.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/data/Resources.js b/services/static-webserver/client/source/class/osparc/data/Resources.js index 8ff5eb822ba..93ab14d086f 100644 --- a/services/static-webserver/client/source/class/osparc/data/Resources.js +++ b/services/static-webserver/client/source/class/osparc/data/Resources.js @@ -274,8 +274,8 @@ qx.Class.define("osparc.data.Resources", { url: statics.API + "/projects/{studyId}/folders/{folderId}" }, moveToWorkspace: { - method: "PUT", - url: statics.API + "/projects/{studyId}/workspaces/{workspaceId}" + method: "POST", + url: statics.API + "/projects/{studyId}/workspaces/{workspaceId}:move" }, } }, From c37f3dc2c57895a1799105991c841d2e9a13bb7b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 13:32:36 +0100 Subject: [PATCH 18/29] clieaning --- ...st_workspaces__moving_folders_between_workspaces.py | 2 +- ...t_workspaces__moving_projects_between_workspaces.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index 91d7dfc331a..ea7105a3338 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -169,7 +169,7 @@ async def _move_folder_to_workspace_and_assert( folder_id=folder_id, workspace_id=workspace_id, ) - resp = await client.put(f"{url}") + resp = await client.post(f"{url}") await assert_status(resp, NO_CONTENT) # ASSERT diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index db186f5fbcf..a81c76012a0 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -58,7 +58,7 @@ async def test_moving_between_workspaces_user_role_permissions( base_url = client.app.router["move_project_to_workspace"].url_for( project_id=fake_project["uuid"], workspace_id="null" ) - resp = await client.put(f"{base_url}") + resp = await client.post(f"{base_url}") await assert_status(resp, expected.no_content) @@ -106,7 +106,7 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id="null" ) - resp = await client.put(f"{base_url}") + resp = await client.post(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace @@ -119,7 +119,7 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{added_workspace['workspaceId']}" ) - resp = await client.put(f"{base_url}") + resp = await client.post(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace @@ -185,7 +185,7 @@ async def test_moving_between_shared_and_shared_workspaces( base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{second_workspace['workspaceId']}" ) - resp = await client.put(f"{base_url}") + resp = await client.post(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace @@ -265,7 +265,7 @@ async def test_moving_between_workspaces_check_removed_from_folder( base_url = client.app.router["move_project_to_workspace"].url_for( project_id=project["uuid"], workspace_id="none" ) - resp = await client.put(f"{base_url}") + resp = await client.post(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace From 0fb74d4cb34d000c2c9ed3f030678dd323cfa906 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 14:01:09 +0100 Subject: [PATCH 19/29] frontend changes --- .../client/source/class/osparc/dashboard/StudyBrowser.js | 5 ----- .../client/source/class/osparc/data/Resources.js | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js index 882848dd295..1c5a82f30fd 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js @@ -485,11 +485,6 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { const data = e.getData(); const destWorkspaceId = data["workspaceId"]; const destFolderId = data["folderId"]; - if (destWorkspaceId !== currentWorkspaceId) { - const msg = this.tr("Moving folders to Shared Workspaces are coming soon"); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); - return; - } const moveFolder = () => { Promise.all([ this.__moveFolderToWorkspace(folderId, destWorkspaceId), diff --git a/services/static-webserver/client/source/class/osparc/data/Resources.js b/services/static-webserver/client/source/class/osparc/data/Resources.js index 93ab14d086f..e75a9127712 100644 --- a/services/static-webserver/client/source/class/osparc/data/Resources.js +++ b/services/static-webserver/client/source/class/osparc/data/Resources.js @@ -323,8 +323,8 @@ qx.Class.define("osparc.data.Resources", { url: statics.API + "/folders/{folderId}" }, moveToWorkspace: { - method: "PUT", - url: statics.API + "/folders/{folderId}/folders/{workspaceId}" + method: "POST", + url: statics.API + "/folders/{folderId}/folders/{workspaceId}:move" }, } }, From c2f63e945ec50087dd7dfc449fe941d6d03048a9 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 29 Nov 2024 16:34:16 +0100 Subject: [PATCH 20/29] fix --- .../src/simcore_service_webserver/projects/_groups_api.py | 3 ++- .../src/simcore_service_webserver/projects/_groups_db.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_api.py b/services/web/server/src/simcore_service_webserver/projects/_groups_api.py index 7ae45f0f90c..b32a6d15fa1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_api.py @@ -80,7 +80,8 @@ async def list_project_groups_by_user_and_project( ] = await projects_groups_db.list_project_groups(app=app, project_id=project_id) project_groups_api: list[ProjectGroupGet] = [ - ProjectGroupGet.model_validate(group.model_dump()) for group in project_groups_db + ProjectGroupGet.model_validate(group.model_dump()) + for group in project_groups_db ] return project_groups_api diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py index 0f9169fabea..4355f0c9d92 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py @@ -10,7 +10,7 @@ from aiohttp import web from models_library.projects import ProjectID from models_library.users import GroupID -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy import func, literal_column @@ -34,6 +34,8 @@ class ProjectGroupGetDB(BaseModel): created: datetime modified: datetime + model_config = ConfigDict(from_attributes=True) + ## DB API @@ -89,7 +91,7 @@ async def list_project_groups( async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream(stmt) - rows = await result.first() or [] + rows = await result.all() or [] return TypeAdapter(list[ProjectGroupGetDB]).validate_python(rows) From 6e3ed2a1a7115aa286f16a5f4ce9b30c97041b73 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 14:50:01 +0100 Subject: [PATCH 21/29] fix --- .../projects/_workspaces_api.py | 4 ++-- .../projects/projects_api.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py index 4862c1d047d..1462168fa52 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_api.py @@ -13,7 +13,7 @@ from ..workspaces.api import check_user_workspace_access from . import _folders_db as project_to_folders_db from . import _groups_db as project_groups_db -from . import _projects_db as project_db_v2 +from . import _projects_db from .exceptions import ProjectInvalidRightsError _logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ async def move_project_into_workspace( ) # 4. Update workspace ID on the project resource - await project_db_v2.patch_project( + await _projects_db.patch_project( app=app, connection=conn, project_uuid=project_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 49f6706e342..78c3c1ce483 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -120,8 +120,7 @@ from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError from ..workspaces import _workspaces_db as workspaces_db -from . import _crud_api_delete, _nodes_api -from . import _projects_db as project_db_v2 +from . import _crud_api_delete, _nodes_api, _projects_db from ._access_rights_api import ( check_user_project_permission, has_user_project_access_rights, @@ -254,9 +253,12 @@ async def patch_project( project_patch: ProjectPatch | ProjectPatchExtended, product_name: ProductName, ): - _project_patch_exclude_unset: dict[str, Any] = jsonable_encoder( - project_patch, exclude_unset=True, by_alias=False - ) + _project_patch_exclude_unset = { + key: value + if not isinstance(value, datetime.datetime) + else value # NOTE: Asyncpg needs to have datetime type + for key, value in project_patch.dict(exclude_unset=True, by_alias=False).items() + } db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # 1. Get project @@ -290,7 +292,7 @@ async def patch_project( raise ProjectOwnerNotFoundInTheProjectAccessRightsError # 4. Patch the project - await project_db_v2.patch_project( + await _projects_db.patch_project( app=app, project_uuid=project_uuid, new_partial_project_data=_project_patch_exclude_unset, From d03a4965ef2af99883fb78725b4d75afab489077 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 16:37:11 +0100 Subject: [PATCH 22/29] fix --- .../src/simcore_service_webserver/projects/projects_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 78c3c1ce483..77389fb56f9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -286,9 +286,9 @@ async def patch_project( } user: dict = await get_user(app, project_db.prj_owner) _prj_owner_primary_group = f'{user["primary_gid"]}' - if _prj_owner_primary_group not in new_prj_access_rights: + if _prj_owner_primary_group not in new_prj_access_rights: # type: ignore raise ProjectOwnerNotFoundInTheProjectAccessRightsError - if new_prj_access_rights[_prj_owner_primary_group] != _prj_required_permissions: + if new_prj_access_rights[_prj_owner_primary_group] != _prj_required_permissions: # type: ignore raise ProjectOwnerNotFoundInTheProjectAccessRightsError # 4. Patch the project From 64758ff448642b65b8a75a1fc598b465ad0f4f7c Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 16:58:47 +0100 Subject: [PATCH 23/29] fix --- .../api_schemas_webserver/projects.py | 13 +++++++++++-- .../projects/projects_api.py | 10 ++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 7c4116a136c..1aa6bb33e6a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -10,7 +10,14 @@ from models_library.folders import FolderID from models_library.workspaces import WorkspaceID -from pydantic import BeforeValidator, ConfigDict, Field, HttpUrl, field_validator +from pydantic import ( + BeforeValidator, + ConfigDict, + Field, + HttpUrl, + PlainSerializer, + field_validator, +) from ..api_schemas_long_running_tasks.tasks import TaskGet from ..basic_types import LongTruncatedStr, ShortTruncatedStr @@ -130,7 +137,9 @@ class ProjectPatch(InputSchema): name: ShortTruncatedStr | None = Field(default=None) description: LongTruncatedStr | None = Field(default=None) thumbnail: Annotated[ - HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator) + HttpUrl | None, + BeforeValidator(empty_str_to_none_pre_validator), + PlainSerializer(str), ] = Field(default=None) access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) classifiers: list[ClassifierID] | None = Field(default=None) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 77389fb56f9..6caf436f75a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -253,12 +253,10 @@ async def patch_project( project_patch: ProjectPatch | ProjectPatchExtended, product_name: ProductName, ): - _project_patch_exclude_unset = { - key: value - if not isinstance(value, datetime.datetime) - else value # NOTE: Asyncpg needs to have datetime type - for key, value in project_patch.dict(exclude_unset=True, by_alias=False).items() - } + _project_patch_exclude_unset = project_patch.model_dump( + exclude_unset=True, by_alias=False + ) + db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # 1. Get project From 4dfa541402d086d74046865a9b3baee93e1f8e0a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 17:01:00 +0100 Subject: [PATCH 24/29] fix --- .../src/simcore_service_webserver/projects/projects_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 6caf436f75a..cf9445985c6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -256,7 +256,6 @@ async def patch_project( _project_patch_exclude_unset = project_patch.model_dump( exclude_unset=True, by_alias=False ) - db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # 1. Get project @@ -284,9 +283,9 @@ async def patch_project( } user: dict = await get_user(app, project_db.prj_owner) _prj_owner_primary_group = f'{user["primary_gid"]}' - if _prj_owner_primary_group not in new_prj_access_rights: # type: ignore + if _prj_owner_primary_group not in new_prj_access_rights: raise ProjectOwnerNotFoundInTheProjectAccessRightsError - if new_prj_access_rights[_prj_owner_primary_group] != _prj_required_permissions: # type: ignore + if new_prj_access_rights[_prj_owner_primary_group] != _prj_required_permissions: raise ProjectOwnerNotFoundInTheProjectAccessRightsError # 4. Patch the project From 763586aeaa4e604e53cd35dc54ddcc3e885d4262 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 17:17:10 +0100 Subject: [PATCH 25/29] fix --- .../src/models_library/api_schemas_webserver/projects.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 1aa6bb33e6a..c25bd1dc340 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -9,6 +9,7 @@ from typing import Annotated, Any, Literal, TypeAlias from models_library.folders import FolderID +from models_library.utils._original_fastapi_encoders import jsonable_encoder from models_library.workspaces import WorkspaceID from pydantic import ( BeforeValidator, @@ -144,7 +145,13 @@ class ProjectPatch(InputSchema): access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) classifiers: list[ClassifierID] | None = Field(default=None) dev: dict | None = Field(default=None) - ui: StudyUI | None = Field(default=None) + ui: Annotated[ + StudyUI | None, + BeforeValidator(empty_str_to_none_pre_validator), + PlainSerializer( + lambda obj: jsonable_encoder(obj, exclude_unset=True, by_alias=False) + ), + ] = Field(default=None) quality: dict[str, Any] | None = Field(default=None) From 6e287d9533d755bb2403d11f86b0ea7c4e8e4231 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 2 Dec 2024 17:18:45 +0100 Subject: [PATCH 26/29] fix --- .../src/models_library/api_schemas_webserver/projects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index c25bd1dc340..e8f6f0087ba 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -149,7 +149,9 @@ class ProjectPatch(InputSchema): StudyUI | None, BeforeValidator(empty_str_to_none_pre_validator), PlainSerializer( - lambda obj: jsonable_encoder(obj, exclude_unset=True, by_alias=False) + lambda obj: jsonable_encoder( + obj, exclude_unset=True, by_alias=False + ) # For the sake of backward compatibility ), ] = Field(default=None) quality: dict[str, Any] | None = Field(default=None) From 5c785d24c93171d3befd5c5b972e53cd86333052 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 10:03:23 +0100 Subject: [PATCH 27/29] fix --- .../src/models_library/api_schemas_webserver/projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index e8f6f0087ba..a918ece3b92 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -140,7 +140,7 @@ class ProjectPatch(InputSchema): thumbnail: Annotated[ HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator), - PlainSerializer(str), + PlainSerializer(lambda x: str(x) if x is not None else None), ] = Field(default=None) access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) classifiers: list[ClassifierID] | None = Field(default=None) From a9d5436f64c08e23d8f7159b6796a252a9abe8ed Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 11:58:57 +0100 Subject: [PATCH 28/29] fix --- .../modules/db/repositories/comp_tasks/_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index c5fd0819fcd..dd52f50ac82 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -6,6 +6,7 @@ import aiopg.sa import arrow from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict +from models_library.api_schemas_catalog.services import ServiceGet from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.services import ( NodeRequirements, @@ -89,7 +90,7 @@ async def _get_service_details( node.version, product_name, ) - obj: ServiceMetaDataPublished = ServiceMetaDataPublished(**service_details) + obj: ServiceMetaDataPublished = ServiceGet(**service_details) return obj From eabeafb20daf8d3e2f01c2ed76abe97065767c48 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 14:00:03 +0100 Subject: [PATCH 29/29] fix --- .../test_api_route_computations.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py index 4381c9311d4..6b6084c5895 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py @@ -186,15 +186,29 @@ def _mocked_service_resources(request) -> httpx.Response: def _mocked_services_details( request, service_key: str, service_version: str ) -> httpx.Response: + assert "json_schema_extra" in ServiceGet.model_config + assert isinstance(ServiceGet.model_config["json_schema_extra"], dict) + assert isinstance( + ServiceGet.model_config["json_schema_extra"]["examples"], list + ) + assert isinstance( + ServiceGet.model_config["json_schema_extra"]["examples"][0], dict + ) + data_published = fake_service_details.model_copy( + update={ + "key": urllib.parse.unquote(service_key), + "version": service_version, + } + ).model_dump(by_alias=True) + data = { + **ServiceGet.model_config["json_schema_extra"]["examples"][0], + **data_published, + } + payload = ServiceGet.model_validate(data) return httpx.Response( 200, json=jsonable_encoder( - fake_service_details.model_copy( - update={ - "key": urllib.parse.unquote(service_key), - "version": service_version, - } - ), + payload, by_alias=True, ), )